@naanlang/naan 1.0.1 → 1.0.4

package/frameworks/storage/csv.nlg

@@ -0,0 +1,120 @@
+/*
+ * csv.nlg
+ * Naanlib/frameworks/storage
+ *
+ *     CSV file parsing and generation.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * csvParse
+ *
+ *     Parse text lines in CSV format into a row array of column arrays. The delimiter defaults to
+ * comma, but can be replaced with anything that's not crazy. Normally missing columns are replaced
+ * with false, but the empty parameter can be used to override this. For example, specify "" to
+ * use an empty string for the empty column.
+ *     The parse regex is derived from the following Stack Overflow article:
+ * https://stackoverflow.com/questions/1293147/example-javascript-code-to-parse-csv-data
+ * Please see the answer by Joshua McLauchlan.
+ *
+ */
+
+function csvParse(text, delim, empty, local res, parseRx, quoteRx, result, matches) {
+    if !string(text)
+        return (false)                                                      // don't hang or be stupid
+    res = '(,|\r?\n|\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\r\n]*))'
+    if delim
+        res = res.replace(RegExp(',', 'g'), delim)
+    else
+        delim = ','
+    parseRx = RegExp(res, 'gi')
+    quoteRx = RegExp('""', 'g')
+    result = [[]]
+    while !!(matches = parseRx.exec(text)) {
+        if !matches[0]
+            break                                                           // should never happen, but don't hang
+        if matches[1].length > 0 && matches[1] !== delim
+            result.push([])                                                 // first column in row
+        result[result.length-1].push(cond(
+            if matches[2] !== undefined
+                matches[2].replace(quoteRx, '"')                            // quoted string, including ""
+            else if matches[3] !== ""
+                matches[3]                                                  // unquoted, non-empty
+            else
+                empty                                                       // empty column
+        ))
+    }
+    result
+};
+
+
+/*
+ * csvPrint
+ *
+ *     Print an array of row arrays of columns to a string, reversing csvParse.
+ *
+ */
+
+function csvPrint(rows, local output, quoter, row, line, delim, col) {
+    quoter = csvQuoteGenerator()
+    output = ""
+    for row in rows {
+        delim = line = ""
+        for col in row {
+            line = line.concat(delim, quoter(col))
+            delim = ","
+        }
+        output = output.concat(line, "\n")
+    }
+    output
+};
+
+
+/*
+ * csvQuoteGenerator
+ *
+ *     Return a function that quotes a string as needed for a CSV file with the given delimiter. This
+ * is a generator because it will likely be needed frequently. Comma is the default delimeter, just 
+ * like God intended for a *C*sv file.
+ *
+ */
+
+closure csvQuoteGenerator(delimiter, local rxComma, rxQuote) {
+    if !delimiter
+        delimiter = ','
+    rxDelimiter = RegExp('['.concat('"\\', delimiter, ']'))
+    rxQuote = RegExp("\"", "g")
+
+    // csvQuote
+    // Quote a string for a CSV file as specified.
+    function csvQuote(str) {
+        if rxDelimiter.test(str)
+            return ("\"".concat(str.replace(rxQuote, "\"\""), "\""))
+        return (str)
+    }
+};
+
+
+/*
+ * csvInit
+ *
+ *     Initialize the component.
+ *
+ */
+
+function csvInit(local manifest) {
+    manifest = `(csvParse, csvPrint, csvQuoteGenerator, csvInit)
+
+    Naan.module.build(module.id, "csv", function(modobj, compobj) {
+        require("./storage.nlg")
+        compobj.manifest = manifest
+        module.exports.csvParse = csvParse
+        module.exports.csvPrint = csvPrint
+        module.exports.csvQuoteGenerator = csvQuoteGenerator
+    })
+}();

package/frameworks/storage/dbt_pouch.nlg

@@ -540,36 +540,26 @@ closure DbConnector(psm, local connClassID, connector, watch) {
             type:           "password" },
         ]
     }
-    
-    // hashResourceID
-    //
-    // Make a stable hash based on the resource credentials that we can persist or share between
-    // different domains accessing the same resource.
-    connector.hashResourceID = function hashResourceID(resource, local ident) {
-        ident = resource.urlName
-        if resource.label
-            ident = ident.concat(resource.label)
-        HashSHA256(ident)
-    }
 
     // findResource
     //
-    // Find a resource with the specified contents, returning the resID or false. The loose criteria
-    // is: if added, would the specified contents be redundant to an existing resource?
+    // Find a resource with the specified contents, returning the resID or false.
     connector.findResource = function findResource(resource, local resID, creds) {
         for resID in listResources().1 {
             creds = access(resID).1
-            if resource.urlName == creds.urlName
+            if ((!resource.urlName ||resource.urlName == creds.urlName)
                 && (!resource.label || resource.label == creds.label)
-                && resource.authName == creds.authName
+                && (!resource.authName || resource.authName == creds.authName))
                 return (resID)
         }
         false
     }
 
-    // addResource
+    // writeResource
     //
-    // Add credentials for a named resource. The resource is a dictionary comprising these keys:
+    // Add or update credentials for a named resource. If no resID is specified this adds a new
+    // resource; otherwise it updates an existing one. The resource is a dictionary comprising these
+    // keys:
     //      label       - [optional] label we use for this resource
     //      urlName     - URL (e.g. "http://domain.com/dbname")
     //                  - filepath (e.g. "/Users/richardz/dbname")  - NodeJS filesystem access
@@ -579,10 +569,13 @@ closure DbConnector(psm, local connClassID, connector, watch) {
     //      hidden      - [optional] don't enumerate to user
     //      locked      - [optional] don't allow user delete
     //      existng     - [optional] true to enable using existing database
-    // The return value is (error, result) tuple. Error, if non-false, is a dictionary of field
-    // keys and error diagnostic strings. The special key "label" refers to the label string
-    connector.addResource = function addResource(resource, local errors, creds, original, error, resID, data) {
+    // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
+    // keys and error diagnostic strings.
+    connector.writeResource = function writeResource(resource, resID, 
+        local errors, creds, original, error, data) {
+        // badField - test for a bad string
         function badField(str) { !string(str) || str.trim().length == 0 }
+
         creds = { urlName: resource.urlName }
         errors = { }
         if resource.label
@@ -591,6 +584,8 @@ closure DbConnector(psm, local connClassID, connector, watch) {
             creds.label = resource.urlName
         if badField(creds.label)
             errors.label = "invalid label"
+        else if !resID && findResource({ label: creds.label })
+            errors.label = "duplicate name"
         if badField(resource.urlName)
             errors.urlName = "required field"
         else if !resource.existing {
@@ -612,15 +607,36 @@ closure DbConnector(psm, local connClassID, connector, watch) {
                 creds.hidden = resource.hidden
             if resource.locked
                 creds.locked = resource.locked
-            resID =  hashResourceID(resource)
-            `(error, data) = connector.vault.addResource(resID, creds)
+            if resID {
+                `(error, data) = connector.vault.updateResource(resID, creds)
+                change = { changed: [resID] }
+            } else {
+                resID = UUID()
+                `(error, data) = connector.vault.addResource(resID, creds)
+                change = { added: [resID] }
+            }
             if data {
-                watch.notify(connClassID, { added: [resID] })
-                data = resID }
+                watch.notify(connClassID, change)
+                data = resID
+            }
         }
         list(error, data)
     }
-    
+
+    // addResource
+    //
+    // Add a new resource, returning a standard error tuple per writeResource.
+    connector.addResource = function addResource(resource) {
+        writeResource(resource)
+    }
+
+    // updateResource
+    //
+    // Update an existing resource, returning a standard error tuple per writeResource.
+    connector.updateResource = function updateResource(resID, resource) {
+        writeResource(resource, resID)
+    }
+
     // getResource
     //
     // Get credentials for a named resource, matching the dictionary semantics of addResource. The

package/frameworks/storage/psm.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2021 by Richard C. Zulch
+ * Copyright (c) 2019-2022 by Richard C. Zulch
  *
  */
 
@@ -75,7 +75,7 @@ closure MakePSMutil(fs, local util) {
         } (tree, inpath.split(tree.data.pathsep).reverse())
         if !treedir
             return (callback(Error("treeDirUpdate: can't find directory within tree", inpath)))
-        `(error, files) = fs.dirList(dirpath, { head: true })
+        `(error, files) = fs.dirList(dirpath, { stat: true })
         if error
             return (callback(error))
         //
@@ -238,46 +238,126 @@ closure MakePSMutil(fs, local util) {
     //
     // Perform a deep copy of an array of sources from the current filesystem and specified inpath to
     // the outfs filesystem and specified outpath. The input path is used to find the source files,
-    // however it is not added to the output. The result is a standard error tuple, where the errors
-    // are an array if  any occur.
+    // however it is not added to the output. If the source array is false then the file or directory
+    // at the input path is copied to the destination. The result is a standard error tuple, where 
+    // the errors are an array if any occur.
+    //
+    // Supported options are:
+    // {
+    //      overwrite:  <boolean>           -- overwrite existing files
+    //      erase:      <boolean>           -- remove extraneous files in destination folders
+    //      force:      <boolean>           -- copy files even if older (default is only copy if new/newer)
+    //      follow:     <boolean>           -- follow links instead of treating them as files
+    //      preserve:   <boolean>           -- preserve file info where possible
+    //                                         (e.g. mtime/atime/mode/uid/gid on NodeJS)
+    // }
+    //
+    // Limitations:
+    //  - this does not copy info about source folders to destination folders
+    //    (it would need to make destination folder writable, then update after files copied)
 
-    util.deepcopy = closure deepcopy(inpath, sources, outfs, outpath, options, callback, local errors, copies) {
+    util.deepcopy = closure deepcopy(inpath, sources, outfs, outpath, options, callback,
+        local errors, copies, fsopt, folders) {
         if !callback
             return (syncAdapter(deepcopy, inpath, sources, outfs, outpath, options))
-        if !array(sources) || !string(inpath) || !string(outpath)
+        if sources && !array(sources) || !string(inpath) || !string(outpath)
             return (callback(Error("deepcopy: invalid arguments")))
+        if options.follow
+            fsopt = { follow: true }
         errors = []
         copies = []
-        sources = new(sources)                                              // don't modify the original!
-        asyncArray(sources, 10, closure mklist(entry, index, elements, local srcpath, destpath, error, folder, node) {
+        folders = { }
+        if sources
+            sources = new(sources)                                          // don't modify the original!
+        else {
+            sources = [fs.path.basename(inpath)]                            // copy inpath itself
+            inpath = fs.path.dirname(inpath)
+        }
+        //
+        // Pass 1 - Get info on all source files, and make destination directories
+        //
+        asyncArray(sources, 10, closure mklist(entry, index, elements,
+            local srcpath, destpath, epath, error, stat, folder, node) {
             srcpath = fs.path.join(inpath, entry)
-            `(error, folder) = fs.dirList(srcpath)
-            if error && error.code != "ENOTDIR"
-                return (errors.push(error))
-            destpath = outfs.path.join(outpath, entry)                      // "." if both empty strings
-            if !error {
-                if destpath != "."
-                    `(error, node) = outfs.mkdir(destpath)                  // may already exist but we don't care
-                if error
-                    return (errors.push(error))                             // don't do children if we can't form destination
-                for node in folder.children
-                    sources.push(fs.path.join(entry, node.name))            // omit inpath from sources
-            }
-            else
-                copies.push({                                               // copy the file 
+            destpath = outfs.path.join(outpath, entry)
+            if destpath == "."
+                destpath = ""                                               // joined empty strings
+            `(error, stat) = fs.info(srcpath, fsopt)
+            if stat.type != "directory" && (!options.follow || stat.type != "symlink") {
+                copies.push({                                               // copy file or symlink 
                     source: srcpath
                     dest: destpath
                 })
+                return (list(false, { ok: true }))                          // generic success not used
+            }
+            `(error, folder) = fs.dirList(srcpath)
+            if error
+                return (errors.push(error))
+            if destpath == ""
+                epath = ""
+            else
+                epath = outfs.path.join(destpath, outfs.path.sep)           // canonical name
+            folders[epath] = folder                                         // sources for this folder
+            `(error, node) = outfs.mkdir(destpath)                          // may already exist but we don't care
+            if error
+                return (errors.push(error))                                 // don't do children if we can't form destination
+            for node in folder.children
+                sources.push(fs.path.join(entry, node.name))                // add more sources, without inpath
         }).wait()
+        //
+        // Pass 2 - erase extraneous files and folders from destination
+        //
+        if options.erase {
+            asyncArray(keys(folders).toarray, 10, closure eraser(destpath, index, elements
+                local error, outfolder, folder, namesin, removes, delopt) {
+                `(error, outfolder) = outfs.dirList(destpath)               // existing files in our destination
+                if error
+                    return
+                folder = folders[destpath]                                  // our source folders, if any
+                namesin = { }
+                for node in folder.children                                 // make list of top-level dest names
+                    namesin[node.name] = true
+                removes = []
+                for node in outfolder.children
+                    if !namesin[node.name] {
+                        if node.info.type == "directory"
+                            delopt = { recursive: true }                    // deleting a directory
+                        else
+                            delopt = false                                  // deleting a file
+                        removes.push(list(outfs.path.join(destpath, node.name), delopt))
+                    }
+                asyncArray(removes, 10, closure cleardest(delitem, local error, data) {
+                    `(error, data) = outfs.delete(delitem.0, delitem.1)
+                    if error
+                        errors.push(Error("deepcopy: can't remove", error))
+                }).wait()
+            }).wait()
+        }
+        //
+        // Pass 3 - copy files and links
+        //
         asyncArray(copies, 10, closure cpfiles(entry, index, elements, local error, data, instat, outstat) {
-            `(error, instat) = fs.info(entry.source)
+            `(error, instat) = fs.info(entry.source, fsopt)
             if !error {
                 `(error, outstat) = outfs.info(entry.dest)
-                if !error && (instat.md5 && instat.md5 == outstat.md5 || instat.mtimeMs <= outstat.mtimeMs)
-                    return                                                  // file unchanged
-                `(error, data) = fs.readFile(entry.source, { encoding: "binary"})
-                if !error
-                    `(error, data) = outfs.writeFile(entry.dest, data, { encoding: "binary"})
+                if !error &&
+                    (instat.md5 && instat.md5 == outstat.md5
+                    || !options.force && instat.mtimeMs <= outstat.mtimeMs)
+                    return                                                  // file "unchanged"
+                if !error && !options.overwrite
+                    error = Error("deepcopy: can't overwrite", entry.dest)
+                else if instat.type == "symlink" && !options.follow {
+                    `(error, data) = fs.readlink(entry.source)
+                    if !error
+                        `(error, data) = outfs.symlink(data, entry.dest)
+                }
+                else {
+                    `(error, data) = fs.readFile(entry.source, { encoding: "binary"})
+                    if !error
+                        `(error, data) = outfs.writeFile(entry.dest, data, { encoding: "binary"})
+                    if !error && options.preserve
+                        `(error) = outfs.setInfo(entry.dest, instat)
+                }
             }
             if error
                 errors.push(error)

package/frameworks/storage/psm_dbtables.nlg

@@ -287,7 +287,7 @@ closure FSview(table, rootpath, local view, error) {
                             })))
                     if buildpath.slice(-1) != "/"
                         buildpath = buildpath.concat("/")
-                    `(error, data) = view.table.get(buildpath)              // get desired directory
+                    `(error, data) = view.table.head(buildpath)             // get desired directory
                     if error.status == 404
                         `(error, data) = dbWriteRecord(buildpath)           // create desired directory
                     if error
@@ -684,8 +684,12 @@ closure FSview(table, rootpath, local view, error) {
                     if !finfo
                         finfo = { }
                     name = item.key.slice(offset)
-                    if name.slice(-1) == "/"
+                    if name.slice(-1) == "/" {
                         name = name.slice(0,-1)                             // remove trailing "/" from directory
+                        if !finfo.type
+                            finfo.type = "directory"
+                    } else if !finfo.type
+                        finfo.type = "file"                                 // everything is a file in UNIX
                     if item.doc._md5
                         finfo.md5 = item.doc._md5
                     if item.doc._length
@@ -703,7 +707,7 @@ closure FSview(table, rootpath, local view, error) {
     // Read a directory at the specified path
     
     view.readir = closure readir(path, callback, local name, dirdict) {
-        dirList(path, { head: true }, callback)
+        dirList(path, { stat: true }, callback)
     }
 
     // tree

package/frameworks/storage/resources.nlg

@@ -11,7 +11,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2021 by Richard C. Zulch
+ * Copyright (c) 2019-2022 by Richard C. Zulch
  *
  */
 
@@ -38,6 +38,21 @@ closure rsrcVault(db, tablename, local vault) {
         vault.table.add(resID, data)
     }
     
+    // updateResource
+    //
+    // Update credentials for accessing a resource, returning the added credential record.
+    
+    vault.updateResource = function updateResource(resID, data, local error, existing) {
+        `(error, existing) = vault.table.get(resID)
+        if error
+            data = false
+        else {
+            existing.content = data
+            `(error, existing) = vault.table.update(existing)
+        }
+        list(error, data)
+    }
+
     // deleteResource
     //
     // Delete resource credentials.
@@ -111,8 +126,9 @@ closure rsrcVault(db, tablename, local vault) {
  *      conn.uifields()                         - entry fields for add this kind of resource in UI
  *      conn.findResource(resource)             - find a resource by content, returning resID or false
  *      conn.addResource(resource)              - add the resource dict and return `(error, resID)
- *      conn.info(resID)                        - return `(error, infoDictionary) tuple
+ *      conn.updateResource(resID, resource)    - update a resource
  *      conn.deleteResource(resID)              - delete the specified resource, returning a tuple
+ *      conn.info(resID)                        - return `(error, infoDictionary) tuple
  *      conn.connect(resID, serviceID, args)    - connect to resource and return object
  *
  */
@@ -309,6 +325,18 @@ closure ResourceTracker(reman, local retor, watch) {
             reman.conns[classID].addResource(resource)
     }
     
+    // updateResource
+    //
+    // Update credentials for accessing a resource, returning the added credential record.
+    
+    retor.updateResource = function updateResource(resID, resource, local error, classI) {
+        `(error, classID) = resourceClassID(resID)
+        if error
+            list(error)
+        else
+            reman.conns[classID].updateResource(resID, resource)
+    }
+
     // getResource
     //
     // Get a resource for the resID as a standard result tuple.