@naanlang/naan 1.0.2 → 1.0.5

package/frameworks/project/projects.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2016-2021 by Richard C. Zulch
+ * Copyright (c) 2016-2022 by Richard C. Zulch
  *
  */
 
@@ -23,6 +23,7 @@ closure ProjectManager(locate, track, dbname, local error, proj) {
     `(error, proj) = storageResources.ResourceManager(dbname, "Projects/")
     if error
         return(list(error))
+    proj.configLocal = proj.vault("configlocal")                            // local config storage
     proj.locate = locate                                                    // storage locations
     proj.track = track                                                      // execution tracking
     proj.types = []
@@ -150,40 +151,36 @@ closure projConnector(projman, projtype, local connector, watch) {
     connector.label = projtype.classLabel
     connector.vault = projman.vault(projtype.classID)
     watch = projman.watchable(connector)
-        
-    // hashResourceID
-    //
-    // Make a stable hash based on the resource credentials that we can persist or share between
-    // different domains accessing the same resource.
-    connector.hashResourceID = function hashResourceID(resource, local ident) {
-        ident = resource.name.concat(resource.locationID, resource.path)
-        HashSHA256(ident)
-    }
 
     // findResource
     //
-    // Find a resource with the specified contents, returning the resID or false. The loose criteria
-    // is: if added, would the specified contents be redundant to an existing resource?
+    // Find a resource with the specified contents, returning the resID or false.
     connector.findResource = function findResource(resource, local resID, resInfo) {
         for resID in listResources().1 {
             resInfo = access(resID).1
-            if resource.name == resInfo.name
-                && resource.locationID == resInfo.locationID
-                && resource.path == resInfo.path
+            if ((!resource.name || resource.name == resInfo.name)
+                && (!resource.locationID || resource.locationID == resInfo.locationID)
+                && (!resource.path || resource.path == resInfo.path))
                 return (resID)
         }
         false
     }
 
-    // addResource
+    // writeResource
     //
-    // Add credentials for a named resource. The resource is a dictionary comprising these keys:
+    // Add or update credentials for a named resource. If no resID is specified this adds a new
+    // resource; otherwise it updates an existing one. The resource is a dictionary comprising these
+    // keys:
     //      name:       <string>        - the user visible name of the project
     //      locationID: <string>        - resID of location resource where this project resides
     //      path:       <string>        - path to project within location
-    // Error, if non-false, is a dictionary of field keys and error diagnostic strings.
-    connector.addResource = function addResource(resource, local errors, error, resID, data) {
+    // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
+    // keys and error diagnostic strings.
+    connector.writeResource = function writeResource(resource, resID,
+        local errors, error, data, change) {
+        // badField - test for a bad string
         function badField(str) { !string(str) || str.trim().length == 0 }
+
         errors = { }
         if badField(resource.name)
             errors.name = "invalid name"
@@ -197,19 +194,42 @@ closure projConnector(projman, projtype, local connector, watch) {
         if length(errors) > 0
             error = errors
         else {
-            resID =  hashResourceID(resource)
-            `(error, data) = connector.vault.addResource(resID, resource)
+            if resID {
+                `(error, data) = connector.vault.updateResource(resID, resource)
+                change = { changed: [resID] }
+            } else {
+                resID = UUID()
+                `(error, data) = connector.vault.addResource(resID, resource)
+                change = { added: [resID] }
+            }
             if data {
-                watch.notify(projtype.classID, { added: [resID] })
-                data = resID }
+                watch.notify(projtype.classID, change)
+                data = resID
+            }
         }
         list(error, data)
     }
 
+    // addResource
+    //
+    // Add a new resource, returning a standard error tuple per writeResource.
+    connector.addResource = function addResource(resource) {
+        writeResource(resource)
+    }
+
+    // updateResource
+    //
+    // Update an existing resource, returning a standard error tuple per writeResource.
+    connector.updateResource = function updateResource(resID, resource) {
+        writeResource(resource, resID)
+    }
+
     // deleteResource
     //
     // Delete information for a project. The return value is (error, result) tuple.
-    connector.deleteResource = function deleteResource(resID, local error, creds, db) {
+    connector.deleteResource = function deleteResource(resID, local localID) {
+        localID = resID.concat("-data")
+        projman.configLocal.deleteResource(localID)                         // remove local storage if there is any
         connector.vault.deleteResource(resID)
         watch.notify(projtype.classID, { deleted: [resID] })
         list(false, { ok: true})
@@ -276,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.0.2-1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.0.5-1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -289,6 +309,10 @@ closure projConnector(projman, projtype, local connector, watch) {
  *      build:      <build instructions dictionary>
  *  }
  *
+ *     A project configuration also stores local data that is not in any file, which is useful for
+ * items like accounts & locations, or credentials that should not be "on disk". Project data from
+ * the .cfg file is available in config.configDict, while local data is in config.localDict.
+ *
  *     Given a new projConfig instance, you can either load (which fails of the project is not valid)
  * or create (which fails if the project already exists.) The load operation will update the where
  * argument with the actual name and type found. Unless other noted the methods all return standard
@@ -299,9 +323,11 @@ closure projConnector(projman, projtype, local connector, watch) {
 proj_configDataFolder = "Nide.proj/";
 proj_cliserConfigFile = "nide_cliser.cfg";
 
-closure projConfig(projman, where, local procon, fs, configpath) {
+closure projConfig(projman, where, projID, local procon, fs, configpath) {
     procon = new(object, this)
     procon.name = where.name
+    if projID
+        procon.localID = projID.concat("-data")
     
     // validate
     //
@@ -359,6 +385,15 @@ closure projConfig(projman, where, local procon, fs, configpath) {
                 return (list(Error("can't parse project configuration", where.path, exception))) }
         procon.where_name = where.name                                      // copy name to config
         procon.where_typeID = where.typeID                                  // copy typeID to config
+        if procon.localID {
+            `(error, data) = projman.configLocal.accessResource(procon.localID)
+            if data
+                procon.localDict = data
+            else {
+                procon.localDict = { }
+                projman.configLocal.addResource(procon.localID, procon.localDict)
+            }
+        }
         list(false, procon.configDict)
     }
     
@@ -367,6 +402,11 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     // Write procon.configDict to our configuration file, returning a result tuple.
     //
     closure writeConfig(local error, info, data) {
+        if procon.localID {
+            `(error, data) = projman.configLocal.updateResource(procon.localID, procon.localDict)
+            if error
+                debuglog("projConfig.writeConfig cannot write local storage:", ErrorString(error))
+        }
         `(error, data) = fs.writeFile(configpath, JSONstringify(procon.configDict))
         if !error {
             procon.configMD5 = data
@@ -385,6 +425,8 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     // not already be loaded. The provided name overrides any name in the where argument above, but
     // path and locationID must be specified. Returns a standard result tuple.
     //
+    // ### we need to update this for optional multiple-configuraiton builds
+    //
     procon.create = closure create(name, local error, data, configpath) {
         if procon.loaded
             return (list(Error("Project.create: project already loaded")))
@@ -440,7 +482,9 @@ closure projConfig(projman, where, local procon, fs, configpath) {
                 error = Error("project configuration not a directory", where.path)
             else
                 `(error, data) = readConfig() }
-        if error
+        if error.code == "ENOENT"
+            return (list(Error("Project <b>".concat(where.name, "</b>:<br>not found at ", where.path))))
+        else if error
             return (list(Error("Project.load failed", error)))
         procon.loaded = true
         list(false, procon.configDict)
@@ -479,15 +523,52 @@ closure projConfig(projman, where, local procon, fs, configpath) {
             procon.configDict = newconfig 
         `(error, data) = writeConfig()
     }
-    
+        
+    // stages
+    //
+    //     Return a tuple of stages in the project configuration.
+    //
+    procon.stages = function stages() {
+        procon.configDict."build-rules".*
+    }
+
+    // currentStage
+    //
+    //     Validate and return the current stage of the project, or the first one if there isn't a
+    // previous selection, or false if no valid stages exist.
+    //
+    procon.currentStage = function currentStage(local stage, stagelist) {
+        stage = procon.localDict.currentStage
+        stagelist = stages()
+        if stage && !member(stage, stagelist)
+            stage = false
+        if stagelist && !stage
+            stage = stagelist.0
+        stage
+    }
+
     // buildrules
     //
-    //     Get the current build rules.
+    //     Get the build rules for the named stage.
     //
-    function buildrules(local rules) {
-        rules = procon.configDict."build-rules".default                     // "default" will be switchable later
+    function buildrules(stage, local rules, common, rule, key) {
+        rules = procon.configDict."build-rules"[stage]
         if !rules
             rules = procon.configDict.build                                 // fallback to old config structure
+        rules = new(rules)
+        common = procon.configDict."build-common"
+        if common {
+            for rule in common
+                if !rules[rule]
+                    rules[rule] = common[rule]                              // common rule was not overridden
+                else if array(common[rule], rules[rule])
+                    rules[rule] = common[rule].concat(rules[rule])          // merge common array with rule
+                else if dictionary(common[rule], rules[rule]) {
+                    for key in common[rule]                                 // merge common dictionary with rule
+                        if !rules[rule][key]
+                            rules[rule][key] = common[rule][key]
+                }
+        }
         rules
     }
 
@@ -495,16 +576,18 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     //
     //     Run the project by building and then launching.
     //
-    procon.run = closure run(buildonly, local rules, builder, buildpath, error, buildnum) {
+    procon.run = closure run(stage, buildonly, local rules, builder, buildpath, error, buildnum) {
         if !procon.loaded
             return (list(Error("project not loaded")))
-        rules = buildrules()
+        rules = buildrules(stage)
         if !rules || !rules.products
             return (list(Error("build rules missing or incomplete")))
         buildpath = fs.path.join(proj_configDataFolder, rules.products)
         rules.name = procon.where_name
         rules.typeID = procon.where_typeID
-        `(error, builder) = Builder(rules, "debug", fs, "", buildpath)
+        if !rules.operation
+            rules.operation = "debug"
+        `(error, builder) = Builder(rules, rules.operation, fs, "", buildpath)
         if error
             return (list(error))
         `(error, buildnum) = builder.increment()
@@ -530,16 +613,21 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     //
     //     Publish the project to its configured location.
     //
-    procon.publish = closure publish(local rules, publish, error, outfs, inpath) {
-        rules = buildrules()
-        publish = procon.configDict.publish
-        if !rules || !publish.where
+    procon.publish = closure publish(stage, local rules, where, publish, error, outfs, inpath) {
+        rules = buildrules(stage)
+        publish = rules.publish
+        where = procon.localDict.publish[stage].where
+        if !rules || !where
             return (list(Error("no project publish location configured")))
-        `(error, outfs) = projman.locate.connect(publish.where.locationID, "NideFS", publish.where.path)
+        `(error, outfs) = projman.locate.connect(where.locationID, "NideFS", where.path)
         if error
             return (list(Error("can't access publishing filesystem", error)))
         inpath = fs.path.join(proj_configDataFolder, rules.products, publish.input)
-        fs.util.deepcopy(inpath, publish.sources, outfs, publish.output)
+        fs.util.deepcopy(inpath, publish.sources, outfs, publish.output, {
+            erase: true
+            overwrite: true
+            follow: true
+        })
     }
 
     // close

package/frameworks/running/debugnub.nlg

@@ -217,7 +217,7 @@ closure DebugNub(local nub, debug, paused) {
     // constant string, with corresponding arguments defined as follows:
     //
     //  "startpause"        - pause at start of evaluating an expression
-    //      tuple               - expression being evaluated
+    //      tuple               - dictionary of current step info
     //  "breakpoint"        - hit a breakpoint
     //      where               - dictionary
     //      value               - applied value
@@ -237,10 +237,7 @@ closure DebugNub(local nub, debug, paused) {
         if entry != "steptrap"
             lineStepClear()
         if entry == "startpause" {
-            if atom(arg1)
-                msg.error = Error("can't debug atom:", arg1)
-            else
-                msg.expr = arg1
+            msg.expr = arg1
         } else if entry == "breakpoint" {
             if breakpointHit(msg, arg1, arg2)
                 return

package/frameworks/running/executors.nlg

@@ -438,6 +438,7 @@ closure ExecutorTracker(local xtrak) {
     // Notify watchers of an update on an execution instance.
 
     xtrak.update = closure update(executor) {
+        debuglog("xtrack.update", executor.title)
         notify({ updated: [executor] })
     }
     

package/frameworks/storage/csv.nlg

@@ -0,0 +1,120 @@
+/*
+ * csv.nlg
+ * Naanlib/frameworks/storage
+ *
+ *     CSV file parsing and generation.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * csvParse
+ *
+ *     Parse text lines in CSV format into a row array of column arrays. The delimiter defaults to
+ * comma, but can be replaced with anything that's not crazy. Normally missing columns are replaced
+ * with false, but the empty parameter can be used to override this. For example, specify "" to
+ * use an empty string for the empty column.
+ *     The parse regex is derived from the following Stack Overflow article:
+ * https://stackoverflow.com/questions/1293147/example-javascript-code-to-parse-csv-data
+ * Please see the answer by Joshua McLauchlan.
+ *
+ */
+
+function csvParse(text, delim, empty, local res, parseRx, quoteRx, result, matches) {
+    if !string(text)
+        return (false)                                                      // don't hang or be stupid
+    res = '(,|\r?\n|\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\r\n]*))'
+    if delim
+        res = res.replace(RegExp(',', 'g'), delim)
+    else
+        delim = ','
+    parseRx = RegExp(res, 'gi')
+    quoteRx = RegExp('""', 'g')
+    result = [[]]
+    while !!(matches = parseRx.exec(text)) {
+        if !matches[0]
+            break                                                           // should never happen, but don't hang
+        if matches[1].length > 0 && matches[1] !== delim
+            result.push([])                                                 // first column in row
+        result[result.length-1].push(cond(
+            if matches[2] !== undefined
+                matches[2].replace(quoteRx, '"')                            // quoted string, including ""
+            else if matches[3] !== ""
+                matches[3]                                                  // unquoted, non-empty
+            else
+                empty                                                       // empty column
+        ))
+    }
+    result
+};
+
+
+/*
+ * csvPrint
+ *
+ *     Print an array of row arrays of columns to a string, reversing csvParse.
+ *
+ */
+
+function csvPrint(rows, local output, quoter, row, line, delim, col) {
+    quoter = csvQuoteGenerator()
+    output = ""
+    for row in rows {
+        delim = line = ""
+        for col in row {
+            line = line.concat(delim, quoter(col))
+            delim = ","
+        }
+        output = output.concat(line, "\n")
+    }
+    output
+};
+
+
+/*
+ * csvQuoteGenerator
+ *
+ *     Return a function that quotes a string as needed for a CSV file with the given delimiter. This
+ * is a generator because it will likely be needed frequently. Comma is the default delimeter, just 
+ * like God intended for a *C*sv file.
+ *
+ */
+
+closure csvQuoteGenerator(delimiter, local rxComma, rxQuote) {
+    if !delimiter
+        delimiter = ','
+    rxDelimiter = RegExp('['.concat('"\\', delimiter, ']'))
+    rxQuote = RegExp("\"", "g")
+
+    // csvQuote
+    // Quote a string for a CSV file as specified.
+    function csvQuote(str) {
+        if rxDelimiter.test(str)
+            return ("\"".concat(str.replace(rxQuote, "\"\""), "\""))
+        return (str)
+    }
+};
+
+
+/*
+ * csvInit
+ *
+ *     Initialize the component.
+ *
+ */
+
+function csvInit(local manifest) {
+    manifest = `(csvParse, csvPrint, csvQuoteGenerator, csvInit)
+
+    Naan.module.build(module.id, "csv", function(modobj, compobj) {
+        require("./storage.nlg")
+        compobj.manifest = manifest
+        module.exports.csvParse = csvParse
+        module.exports.csvPrint = csvPrint
+        module.exports.csvQuoteGenerator = csvQuoteGenerator
+    })
+}();

package/frameworks/storage/dbt_pouch.nlg

@@ -540,36 +540,26 @@ closure DbConnector(psm, local connClassID, connector, watch) {
             type:           "password" },
         ]
     }
-    
-    // hashResourceID
-    //
-    // Make a stable hash based on the resource credentials that we can persist or share between
-    // different domains accessing the same resource.
-    connector.hashResourceID = function hashResourceID(resource, local ident) {
-        ident = resource.urlName
-        if resource.label
-            ident = ident.concat(resource.label)
-        HashSHA256(ident)
-    }
 
     // findResource
     //
-    // Find a resource with the specified contents, returning the resID or false. The loose criteria
-    // is: if added, would the specified contents be redundant to an existing resource?
+    // Find a resource with the specified contents, returning the resID or false.
     connector.findResource = function findResource(resource, local resID, creds) {
         for resID in listResources().1 {
             creds = access(resID).1
-            if resource.urlName == creds.urlName
+            if ((!resource.urlName ||resource.urlName == creds.urlName)
                 && (!resource.label || resource.label == creds.label)
-                && resource.authName == creds.authName
+                && (!resource.authName || resource.authName == creds.authName))
                 return (resID)
         }
         false
     }
 
-    // addResource
+    // writeResource
     //
-    // Add credentials for a named resource. The resource is a dictionary comprising these keys:
+    // Add or update credentials for a named resource. If no resID is specified this adds a new
+    // resource; otherwise it updates an existing one. The resource is a dictionary comprising these
+    // keys:
     //      label       - [optional] label we use for this resource
     //      urlName     - URL (e.g. "http://domain.com/dbname")
     //                  - filepath (e.g. "/Users/richardz/dbname")  - NodeJS filesystem access
@@ -579,10 +569,13 @@ closure DbConnector(psm, local connClassID, connector, watch) {
     //      hidden      - [optional] don't enumerate to user
     //      locked      - [optional] don't allow user delete
     //      existng     - [optional] true to enable using existing database
-    // The return value is (error, result) tuple. Error, if non-false, is a dictionary of field
-    // keys and error diagnostic strings. The special key "label" refers to the label string
-    connector.addResource = function addResource(resource, local errors, creds, original, error, resID, data) {
+    // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
+    // keys and error diagnostic strings.
+    connector.writeResource = function writeResource(resource, resID, 
+        local errors, creds, original, error, data) {
+        // badField - test for a bad string
         function badField(str) { !string(str) || str.trim().length == 0 }
+
         creds = { urlName: resource.urlName }
         errors = { }
         if resource.label
@@ -591,6 +584,8 @@ closure DbConnector(psm, local connClassID, connector, watch) {
             creds.label = resource.urlName
         if badField(creds.label)
             errors.label = "invalid label"
+        else if !resID && findResource({ label: creds.label })
+            errors.label = "duplicate name"
         if badField(resource.urlName)
             errors.urlName = "required field"
         else if !resource.existing {
@@ -612,15 +607,36 @@ closure DbConnector(psm, local connClassID, connector, watch) {
                 creds.hidden = resource.hidden
             if resource.locked
                 creds.locked = resource.locked
-            resID =  hashResourceID(resource)
-            `(error, data) = connector.vault.addResource(resID, creds)
+            if resID {
+                `(error, data) = connector.vault.updateResource(resID, creds)
+                change = { changed: [resID] }
+            } else {
+                resID = UUID()
+                `(error, data) = connector.vault.addResource(resID, creds)
+                change = { added: [resID] }
+            }
             if data {
-                watch.notify(connClassID, { added: [resID] })
-                data = resID }
+                watch.notify(connClassID, change)
+                data = resID
+            }
         }
         list(error, data)
     }
-    
+
+    // addResource
+    //
+    // Add a new resource, returning a standard error tuple per writeResource.
+    connector.addResource = function addResource(resource) {
+        writeResource(resource)
+    }
+
+    // updateResource
+    //
+    // Update an existing resource, returning a standard error tuple per writeResource.
+    connector.updateResource = function updateResource(resID, resource) {
+        writeResource(resource, resID)
+    }
+
     // getResource
     //
     // Get credentials for a named resource, matching the dictionary semantics of addResource. The