@naanlang/naan 1.0.3 → 1.0.6

package/frameworks/common/common.nlg

@@ -30,6 +30,49 @@ function EncodeQuery(prefix, items, local result, item) {
 };
 
 
+/*
+ * ContentTypeFromFileExt
+ *
+ *     Given a filename with extension, return a tuple of the recommended contentType and encoding
+ * for that type of data: `(contentType, encoding). Currently the encoding is false for text and
+ * "binary" for everything else.
+ *
+ */
+
+function ContentTypeFromFileExt(filepath, local mimeType, encoding) {
+    mimeType = {
+        "css":  "text/css",
+        "csv":  "text/csv",
+        "gif":  "image/gif",
+        "htm":  "text/html",
+        "html": "text/html",
+        "ico":  "image/x-icon",
+        "ics":  "application/octet-stream",
+        "jpeg": "image/jpeg",
+        "jpg":  "image/jpeg",
+        "js":   "text/javascript",
+        "json": "application/json",
+        "mov":  "video/mp4",
+        "mp3":  "audio/mpeg",
+        "mp4":  "video/mp4",
+        "nlg":  "text/plain",
+        "npk":  "application/json",
+        "ogg":  "video/ogg",
+        "svg":  "image/svg+xml",
+        "text": "text/plain",
+        "txt":  "text/plain",
+        "zip":  "application/octet-stream"
+    }[JSpath.extname(filepath).substring(1)]
+    if !mimeType
+        mimeType = "application/octet-stream"
+    if mimeType.startsWith("text/")
+        mimeType = mimeType.concat("; charset=UTF-8")
+    else
+        encoding = "binary"
+    list(mimeType, encoding)
+};
+
+
 /*
  * UUID
  *
@@ -66,7 +109,7 @@ function JsonParse(data) {
         list(false, data)
     } catch {
         if true
-             list(exception)
+            list(exception)
     }
 };
 
@@ -84,7 +127,7 @@ function JsonStringify(data) {
         list(false, data)
     } catch {
         if true
-             list(exception)
+            list(exception)
     }
 };
 
@@ -125,11 +168,8 @@ function JsLoadScript(libpath, globalID, local scriptTag, fpath) {
         }
         js.w.document.body.appendChild(scriptTag)
         pending.wait()
-    } else if js.g {
-        if !libpath.startsWith("/")
-            fpath = "./".concat(libpath)
-        jsScripts[libpath] = js.r(fpath)                                    // load in NodeJS context
-    }
+    } else if js.g
+        jsScripts[libpath] = js.r(libpath)                                  // load in NodeJS context
     jsScripts[libpath]
 };
 
@@ -260,13 +300,15 @@ EncodeBase64 = false;
 DecodeBase64 = false;
 JSONstringify = false;
 JSONparse = false;
+Uint8ArrayFromString = false;
 
 function comrInit(local manifest) {
-    manifest = `(EncodeQuery, UUID, JsLoadScript, LoadComponentOnDemand, LiveExport, comrInit)
+    manifest = `(EncodeQuery, ContentTypeFromFileExt, UUID, JsLoadScript, LoadComponentOnDemand, LiveExport, comrInit)
 
-    Naan.module.build(module.id, "nideCommon", closure(modobj, compobj) {
+    Naan.module.build(module.id, "common", closure(modobj, compobj) {
         compobj.manifest = manifest
         modobj.exports.EncodeQuery = EncodeQuery
+        modobj.exports.ContentTypeFromFileExt = ContentTypeFromFileExt
         modobj.exports.UUID = UUID
         modobj.exports.JsLoadScript = JsLoadScript
         modobj.exports.LoadComponentOnDemand = LoadComponentOnDemand
@@ -291,14 +333,18 @@ function comrInit(local manifest) {
             EncodeURIComponent = wsg.encodeURIComponent
             DecodeURIComponent = wsg.decodeURIComponent
             //
-            // everywhere: Base65
-            EncodeBase64 = wsg.btoa
-            DecodeBase64 = wsg.atob
-            //
             // everywhere: JSON
             JSONstringify = wsg.JSON.stringify
             JSONparse = wsg.JSON.parse
             //
+            // everywhere: Uint8ArrayFromString
+            //
+            Uint8ArrayFromString = function uint8ArrayFromString(data) {
+                wsg.Uint8Array.from(Array.from(data).map(function(letter) {
+                    letter.charCodeAt(0)
+                }))
+            }
+            //
             // binaryFrom - convert arbitrary item to binary buffer or text.
             //
             function binaryFrom(data) {
@@ -327,22 +373,17 @@ function comrInit(local manifest) {
                     result
             }
             //
-            // base64Format - create base 64 string from binary array
+            // EncodeBase64url - decode base64url to original string
+            // See https://en.wikipedia.org/wiki/Base64 for details on variants
             //
-            function base64Format(data, local blob, reader, pending, result) {
-                blob = xnew(wsg.Blob, [data])
-                reader = xnew(wsg.FileReader)
-                pending = new(nonce)
-                reader.onload = function(event) { pending.signal(event.target.result) }
-                reader.readAsDataURL(blob)
-                result = pending.wait()
-                result.slice(37)
+            EncodeBase64url = function encodeBase64url(data) {
+                data = EncodeBase64(data)
+                data.replace(RegExp("[+]", "g"), '-').replace(RegExp("/", "g"), '_')
             }
             //
             // DecodeBase64url - decode base64url to original string
-            // See https://en.wikipedia.org/wiki/Base64 for details on variants
             //
-            DecodeBase64url = function decodeBase64url(base64url, local padding) {
+            DecodeBase64url = function decodeBase64url(base64url) {
                 base64url = base64url.replace(RegExp("-", "g"), '+').replace(RegExp("_", "g"), '/')
                 DecodeBase64(base64url)
             }
@@ -350,6 +391,31 @@ function comrInit(local manifest) {
             // For browser windows and web workers
             //
             if js.w || js.s {
+                //
+                // base64Format - create base 64 string from binary array
+                //
+                function browserBase64Format(data, local blob, reader, pending, result) {
+                    blob = xnew(wsg.Blob, [data])
+                    reader = xnew(wsg.FileReader)
+                    pending = new(nonce)
+                    reader.onload = function(event) { pending.signal(event.target.result) }
+                    reader.readAsDataURL(blob)
+                    result = pending.wait()
+                    result.slice(37)
+                }
+                //
+                // binary to Base64
+                EncodeBase64 = function browserEncodeBase64(data) {
+                    if xobject(data)
+                        browserBase64Format(data)
+                    else
+                        wsg.btoa(data)
+                }
+                //
+                // Base64 to binary
+                DecodeBase64 = function browserDecodeBase64(data) {
+                    wsg.atob(data)
+                }
                 //
                 // MD5 functions
                 JsLoadScript("frameworks/browser/spark-md5/spark-md5.min.js", "SparkMD5")
@@ -373,7 +439,7 @@ function comrInit(local manifest) {
                         throw(Error("crypto functions unavailable"))
                     data = binaryFrom(data)
                     if string(data)
-                        data = xnew(TextEncoder).encode(data)
+                        data = Uint8ArrayFromString(data)
                     `(error, result) = await(wsg.crypto.subtle.digest(algo, data))
                     if error
                         throw(Error("crypto digest failed", error))
@@ -392,7 +458,7 @@ function comrInit(local manifest) {
                 //
                 // HashSHA-256 in base64
                 HashSHA256_base64 = function browserSHA256_base64(data) {
-                    base64Format(browserDigest(data, "SHA-256"))
+                    browserBase64Format(browserDigest(data, "SHA-256"))
                 }
                //
                 // random bytes as UInt8Array - use toint() for integer
@@ -405,6 +471,16 @@ function comrInit(local manifest) {
             // For NodeJS
             //
             if js.g {
+                //
+                // binary to Base64
+                EncodeBase64 = function nodejsEncodeBase64(data) {
+                    wsg.Buffer.from(data, "binary").toString("base64")
+                }
+                //
+                // Base64 to binary
+                DecodeBase64 = function nodejsDecodeBase64(data) {
+                    wsg.Buffer.from(data, "base64").toString("binary")
+                }
                 if !crypto
                     crypto = js.r("crypto")
                 if crypto {
@@ -456,6 +532,8 @@ function comrInit(local manifest) {
             modobj.exports.JSONparse = JSONparse
             modobj.exports.JsonStringify = JsonStringify
             modobj.exports.JsonParse = JsonParse
+            modobj.exports.Uint8ArrayFromString = Uint8ArrayFromString
+            modobj.exports.EncodeBase64url = EncodeBase64url
             modobj.exports.DecodeBase64url = DecodeBase64url
             modobj.exports.HashMD5 = HashMD5
             modobj.exports.HashMD5_base64 = HashMD5_base64

package/frameworks/common/preferences.nlg

@@ -125,6 +125,7 @@ function prefsInit(local manifest) {
     Naan.module.build(module.id, "preferences", function(modobj, compobj) {
         require("../common/common.nlg")
         compobj.manifest = manifest
+        updateExports()
     })
 
     module.exports.Preferences = Preferences

package/frameworks/common/watching.nlg

@@ -93,5 +93,6 @@ function wachInit(local manifest) {
         require("./common.nlg")
         compobj.manifest = manifest
         module.exports.Watchable = Watchable
+        updateExports()
     })
 } ();

package/frameworks/node/apiserver.nlg

@@ -110,8 +110,12 @@ closure MakeAPI(port, options, callback, local chroot, server) {
 
     server.app.use(function(req,res,next) {
         res.header("Access-Control-Allow-Origin", "*")
-        res.header("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE")
-        next()
+        res.header("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS")
+        res.header("Access-Control-Allow-Headers", "x-naanlang-api-guid")
+        if req.method == "OPTIONS"
+            res.sendStatus(200)
+        else
+            next()
     })
 
     server.app.use("", express.static(chroot))
@@ -167,8 +171,8 @@ closure MakeAPI(port, options, callback, local chroot, server) {
             res.status(403).send("unauthorized")
         timeoutms = toint(req.query.timeout)
         after = toint(req.query.after)
-        if not integer(timeoutms) or timeoutms > 10000
-            res.status(400).send("timeout is integer milliseconds no greater than 10000")
+        if not integer(timeoutms) or timeoutms > 7200000
+            res.status(400).send("timeout is integer milliseconds no greater than 7200000")
         else if not integer(after)
             res.status(400).send("after must be an integer")
         else {
@@ -197,7 +201,7 @@ closure MakeAPI(port, options, callback, local chroot, server) {
 
     server.app.all("/psm", closure(req, res, local guid) {
         guid = req.get("x-naanlang-api-guid")
-        if server.guid && guid != server.guid
+        if server.guid && guid != server.guid && req.method !== "OPTIONS"
             res.status(403).send("unauthorized")
         else if !server.hostfs
             res.status(503).send("host filesystem not available")

package/frameworks/node/filesystem.nlg

@@ -23,7 +23,7 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
     if !rootpath
         return (list(Error("mandatory rootpath missing")))
     else if rootpath == ""
-        rootpath = js.g.process.cwd()                                       // NodeJS uses CWD for empty rootpath
+        rootpath = JSpath.sep
     files = new(object, this)
     files.rootpath = rootpath
     files.path = JSpath
@@ -54,24 +54,31 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
     // Convert a path argument to a local path, returning a standard result tuple. If the path is
     // invalid this will report why. Otherwise this returns the correct localpath, which is the path
     // we use with the underlying API.
+    //     Win32 driver letters are annoying here. If the rootpath specifies a drive letter then the
+    // paths must match it. If the rootpath doesn't specify a drive letter then path can have any
+    // drive letter, or none.
     
     badCreatePathRegEx = RegExp("[:]|[/\\\\][/\\\\]|[/\\\\]$|(^|[/\\\\])[.][.]?$")
 
-    closure localPath(path, create, local pattern, testpath) {
+    files.localPath = closure localPath(path, create, local pattern, testpath, locroot) {
         if !path
             return (list(Error("filesystem path false")))
         else if !string(path)
             return (list(Error("filesystem path invalid:", typeof(path))))
+        if JSpath === JSpath.win32 {
+            locroot = JSpath.resolve(path, rootpath)                        // use path's drive letter if we don't have one
+            pattern = "\\:\\"                                               // unchanged and unique on win32
+        }
+        else {
+            locroot = rootpath
+            pattern = "/:/"                                                 // unchanged and unique on posix
+        }
         if JSpath.isAbsolute(path)
-            path = JSpath.resolve(path)                                     // puts C: in front of \abs\win\path
+            path = JSpath.resolve(locroot, path)                            // add drive letter to absolute path
         else
             path = JSpath.normalize(path)                                   // normalize for current platform
-        if JSpath === JSpath.win32
-            pattern = "\\:\\"                                               // unchanged and unique on win32
-        else
-            pattern = "/:/"                                                 // unchanged and unique on posix
-        if ((testpath = commonParents(path, rootpath)) != "")
-            list(false, JSpath.resolve(rootpath, path))                     // ### shouldn't be so lax on this test
+        if (commonParents(path, locroot) != "")
+            list(false, JSpath.resolve(locroot, path))
         else if ((testpath = JSpath.resolve(pattern, path)).indexOf(pattern.slice(0,-1)) < 0)
             list(Error("filesystem path reaches above root:", path))
         else if create && testpath.indexOf(pattern) < 0
@@ -79,7 +86,31 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
         else if create && badCreatePathRegEx.test(path)
             list(Error("filesystem create path invalid", path))
         else
-            list(false, JSpath.resolve(rootpath, path))
+            list(false, JSpath.resolve(locroot, path))
+    }
+    
+    // folderPath
+    //
+    // Return a folder path for the specified folderID, relative to our root. If the folder path is
+    // not within our root then it is returned as an absolute path.
+    
+    files.folderPath = function folderPath(folderID, callback, local envID, path) {
+        if !callback
+            return (syncAdapter(folderPath, folderID))
+        envID = {
+            "HomeDir":      "HOME",
+            "TempDir":      "TMPDIR",
+            "PackageDir":   "npm_config_prefix"
+        }[folderID]
+        if envID {
+            path = js.g.process.env[envID]
+            if path.startsWith(rootpath)
+                path = path.slice(rootpath.length)                          // remove rootpath from beginning
+        }
+        if path
+            callback(false, path)
+        else
+            callback(Error("unknown folderID:", folderID))
     }
 
     // winDrives
@@ -150,11 +181,11 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
     // readLines
     //
     // Read the contents of a file into an array of lines with the following options:
-    //      pattern:        -- only return lines matching pattern
-    //      fixed           -- pattern is a fixed string, not a regular expression
-    //      ignoreCase:     -- ignore case during match
-    //      invert:         -- return lines that DO NOT match the regex
-    //      returnText:     -- return text of matched lines
+    //      pattern:                            -- only return lines matching pattern
+    //      fixed                               -- pattern is a fixed string, not a regular expression
+    //      ignoreCase:                         -- ignore case during match
+    //      invert:                             -- return lines that DO NOT match the regex
+    //      returnText:                         -- return text of matched lines
     //
 
     files.readLines = closure readLines(path, options, callback,
@@ -639,8 +670,10 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
         // Recursively build the tree
 
         closure recurTree(localpath, depth, parent) {
-            if depthlimit && depth >= depthlimit
+            if depthlimit && depth >= depthlimit {
+                recurNext()
                 return
+            }
             ++pending.active
             fs.readdir(localpath, { }, function(err, nodes, local child, node, stat) {
                 if err.code == "ENOTDIR"
@@ -670,13 +703,21 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
                             parent: child
                         }) } }
                 --pending.active
-                while queue.length > 0 && pending.active < 20 {
-                    node = queue.pop()
-                    recurTree(node.path, node.depth, node.parent) }
-                if pending.active == 0 && queue.length == 0
-                    pending.signal(list(false, root))
+                recurNext()
             })
         }
+        
+        // recurNext
+        //
+        // Pull more queued items and signal when done.
+        
+        function recurNext(local node) {
+            while queue.length > 0 && pending.active < 20 {
+                node = queue.pop()
+                recurTree(node.path, node.depth, node.parent) }
+            if pending.active == 0 && queue.length == 0
+                pending.signal(list(false, root))
+        }
 
         `(error, localpath) = localPath(path)
         if error
@@ -707,21 +748,31 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
     // dirSearch
     //
     // Search the files in a directory for a pattern.
-    
+    //
+    // Most options are handled by readLines, but the following is done here:
+    //      excludeDirs: [dirnames]             -- exclude the specified directory names
+
     files.dirSearch = closure dirSearch(path, options, callback,
-        local error, localpaths, output) {
+        local error, localpaths, dexmatch, item, output) {
         if !callback
             return (syncAdapter(dirSearch, path, options))
         `(error, localpath) = localPath(path)
         if error
             return (asyncResult(callback, error))
+        if options.excludeDirs.0 {                                          // match all provided paths
+            dexmatch = []
+            for item in options.excludeDirs                                 // escape all special characters
+                dexmatch.push(item.replace(RegExp("[^A-Za-z0-9-_\n]","g"), "\\$&"))
+            dexmatch = RegExp(dexmatch.join("|"))
+        }
         output = []
         debuglog("dirSearch begins", localpath)
         asyncArray(localpaths, 10, closure(localpath, index, elements, local error, direnum, node, lines, line) {
             `(error, direnum) = dirList(localpath, { })
             if direnum.children
                 for node in direnum.children
-                    localpaths.push(JSpath.join(localpath, node.name))    // more files or directories to examine
+                    if !dexmatch || !node.name.match(dexmatch)
+                        localpaths.push(JSpath.join(localpath, node.name))  // more files or directories to examine
             else {
                `(error, lines) = readLines(localpath, options)
                if error && localpaths.length == 1 {
@@ -945,22 +996,29 @@ closure Filesystem(rootpath, local files, badCreatePathRegEx) {
     //
     // Return an array of text matches in a file or recursive directory of files with the following
     // options:
-    //      pattern:        -- pattern to find
-    //      ignoreCase:     -- ignore case during match
-    //      invert:         -- return lines that DO NOT match the regex
-    //      returnText:     -- return text of matched lines
-    //      maxCount:       -- integer maximum number of results (default 10,000)
+    //      pattern:                            -- pattern to find
+    //      ignoreCase:                         -- ignore case during match
+    //      invert:                             -- return lines that DO NOT match the regex
+    //      returnText:                         -- return text of matched lines
+    //      excludeDirs: [dirnames]             -- exclude the specified directory names
+    //      maxCount:                           -- integer maximum number of results (default 10,000)
     // grep-specific options:
-    //      ignoreBinary:   -- ignore binary files
+    //      ignoreBinary:                       -- ignore binary files
     //
     files.grepLines = closure grepLines(path, options, callback,
-        local error, localpath, cmdargs, execOptions, grep, result, lineRE, maxcount) {
+        local error, localpath, cmdargs, item, execOptions, grep, result, lineRE, maxcount) {
         if !callback
             return (syncAdapter(grepLines, path, options))
         `(error, localpath) = localPath(path)
         if error
             return (asyncResult(callback, error))
         cmdargs = ["-rn"]                                                   // recursively search directories for files; report line numbers
+        if options.excludeDirs.0 {                                          // exclude a directory
+            for item in options.excludeDirs {
+                cmdargs.push("--exclude-dir")
+                cmdargs.push(item)
+            }
+        }
         if options.ignoreCase
             cmdargs.push("-i")
         if options.invert

package/frameworks/node/gitter.nlg

@@ -23,6 +23,28 @@ closure Gitter(fs, local gitt) {
     gitt = new(object, this)
     gitt.roots = {}
     
+    // gitPath
+    //
+    // Convert a Windows path to a posix (git) path if needed.
+    //
+    function gitPath(fspath) {
+        if fs.path.sep == "/"
+            fspath
+        else
+            fspath.split(fs.path.sep).join("/")
+    }
+        
+    // fsPath
+    //
+    // Convert a posix (git) path to a filesystem path if needed.
+    //
+    function fsPath(gitpath) {
+        if fs.path.sep == "/"
+            gitpath
+        else
+            gitpath.split("/").join(fs.path.sep)
+    }
+
     // repoRoot
     //
     // Compute the path from the fs root to the repo root.
@@ -54,11 +76,10 @@ closure Gitter(fs, local gitt) {
     // Compute the bridge path to an absolute repopath (which can point inside a repo) from the
     // phyiscal repo root. If these are the same, or an error occurs, then the result is "".
     //
-    
     function bridgePath(repopath, local top) {
         top = repoRoot(repopath).1
         if top && repopath.startsWith(top) && repopath.length > top.length
-           top = repopath.slice(top.length+1).concat("/")                   // move leading "/" of bridge path to trailing
+           top = repopath.slice(top.length+1).concat(fs.path.sep)           // move leading "/" of bridge path to trailing
         else
             ""                                                              // we are looking at the root, or an error
     }
@@ -151,12 +172,12 @@ closure Gitter(fs, local gitt) {
             outLE: "\x00"                                                   // lines delimited by nulls
         }
         if options.pathspec
-            execOptions.cmdargs = execOptions.cmdargs.concat("--", options.pathspec)
+            execOptions.cmdargs = execOptions.cmdargs.concat("--", gitPath(options.pathspec))
         result = []
         execOptions.outproc = function(output, liner, kill, local line, path) {
             while (line = liner.read())
                 if line != "" {
-                    path = line.substring(3)
+                    path = fsPath(line.substring(3))
                     if path.startsWith(bridge) {                            // must start with bridge to be within repopath
                         path = path.slice(bridge.length)
                         result.push({
@@ -183,7 +204,7 @@ closure Gitter(fs, local gitt) {
     // Retrieve the content of a specific file and revision. The file pathspec must be the path of
     // the file within the repo.
     
-    gitt.showfile = closure showfile(repopath, options, callback, local execOptions, result) {
+    gitt.showfile = closure showfile(repopath, options, callback, local execOptions) {
         if !callback
             return (syncAdapter(showfile, repopath, options))
         if !string(repopath)
@@ -192,7 +213,7 @@ closure Gitter(fs, local gitt) {
             return (asyncResult(callback, Error("both pathspec and hash required:", options.pathspec, options.hash)))
         repopath = fs.path.resolve(fs.rootpath, repopath)
         execOptions = {
-            cmdargs: ["show", options.hash.concat(":", bridgePath(repopath), options.pathspec)]
+            cmdargs: ["show", options.hash.concat(":", bridgePath(repopath), gitPath(options.pathspec))]
             execops: {
                 cwd: repopath
             }