@tim-greller/xgettext-regex 0.5.2 → 0.8.0

package/README.md

@@ -2,6 +2,24 @@
 
 Minimum viable xgettext .pot file generator. Uses a configurable regex to get translation keys.
 
+___________________
+*Note: This repository was forked from [alanshaw/xgettext-regex](https://github.com/alanshaw/xgettext-regex), which is no longer maintained.*  
+*It provides the following additional features:*
+
+**✨ Improved command line interface (CLI)**
+- **Configurable regex via CLI:** made the regex (and regexTextCaptureIndex) configurable.
+
+**✨ Better support for complex syntax & other languages**
+- Added an **improved default regex** and documentation for it.
+- **Multiline extraction support:** rewrote matching to work across newlines, added regex dotAll support.
+- Added support for **Java text blocks** using """.
+- Fix bug of missing location information if two strings found in same line.
+
+**✨ Better .pot file diffs to use it in version management like GIT**
+- Deterministic order of the keys by file path
+- Switched to **relative paths** for location comments to prohibit commiting personal workspace directory paths.
+___________________
+
 ## Examples
 
 ```sh

package/bin/xgettext-regex.js

@@ -16,7 +16,7 @@ var outFile = argv.o || argv.outfile
 var opts = {
   fn: (argv.f || argv.fn)
 }
-if (argv.r || argv.regex) opts.regex = new RegExp(argv.r || argv.regex, 'g')
+if (argv.r || argv.regex) opts.regex = new RegExp(argv.r || argv.regex, 'gs')
 if (argv.i || argv.index) opts.regexTextCaptureIndex = argv.i || argv.index
 
 if (argv._.length) {

package/index.js

@@ -2,9 +2,7 @@ var fs = require('fs')
 var path = require('path')
 var Readable = require('stream').Readable
 var through = require('through2')
-var readdirp = require('readdirp')
 var once = require('once')
-var xtend = require('xtend')
 
 function createDuplexStream (filename, opts) {
   filename = filename || ''
@@ -50,7 +48,7 @@ function createDuplexStream (filename, opts) {
    *   \)                 closing paranthesis
    * )                    end of positive lookahead
    */
-  opts.regex = opts.regex || new RegExp("(?<=" + opts.fn + "\\(\\s*(?:(?:\"(?:[^\"]|\\\\.)*\"|'(?:[^']|\\\\.)*')\\s*,\\s*)*)(([\"'])(?:(?=(\\\\?))\\3.)*?\\2)(?=(?:\\s*,\\s*(?:\"(?:[^\"]|\\\\.)*\"|'(?:[^']|\\\\.)*'|[^\"')]+))*\\s*\\))", 'g')
+  opts.regex = opts.regex || new RegExp("(?<=" + opts.fn + "\\(\\s*(?:(?:\"(?:[^\"]|\\\\.)*\"|'(?:[^']|\\\\.)*')\\s*,\\s*)*)(([\"'])(?:(?=(\\\\?))\\3.)*?\\2)(?=(?:\\s*,\\s*(?:\"(?:[^\"]|\\\\.)*\"|'(?:[^']|\\\\.)*'|[^\"')]+))*\\s*\\))", 'gs')
   opts.regexTextCaptureIndex = opts.regexTextCaptureIndex || 1
 
   var chunks = []
@@ -83,6 +81,16 @@ function createDuplexStream (filename, opts) {
         text = '"' + text.replace(/"/g, '\\"') + '"'
       }
 
+      // handle triple-quoted string (java code blocks)
+      if (text.slice(0, 3) == '"""') {
+        text = text
+            .slice(3, -3)
+            .replace(/^\r?\n\s*/, '') // remove initial new line
+            .replace(/\\\r?\n\s*/g, '') // escaped linebreak + indent
+            .replace(/\r?\n\s*/g, '\\n') // normal linebreak + indent
+        text = `"${text}"`
+      }
+
       this.push([
         ``,
         `#: ${relativeFilename}:${lineNum}`,
@@ -123,11 +131,68 @@ function getText (filename, opts) {
   return fs.createReadStream(filename).pipe(createDuplexStream(filename, opts))
 }
 
-var READDIRP_OPTS = {
+var DEFAULT_WALK_OPTS = {
   fileFilter: ['!.*', '!*.png', '!*.jpg', '!*.gif', '!*.zip', '!*.gz'],
   directoryFilter: ['!.*', '!node_modules', '!coverage']
 }
 
+function createFilter (patterns) {
+  var compiled = patterns.map(function (pattern) {
+    var isExclude = pattern[0] === '!'
+    var glob = isExclude ? pattern.slice(1) : pattern
+    var re = new RegExp('^' + glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^/]*') + '$')
+    return { re: re, isExclude: isExclude }
+  })
+  return function (name) {
+    for (var i = 0; i < compiled.length; i++) {
+      if (compiled[i].re.test(name)) return !compiled[i].isExclude
+    }
+    return true // include by default when no pattern matches
+  }
+}
+
+function walkDirectorySorted (dir, fileFilter, dirFilter, onFile, cb) {
+  fs.readdir(dir, function (er, entries) {
+    if (er) return cb(er)
+    entries.sort(function (a, b) {
+      return a.localeCompare(b, undefined, {sensitivity: 'base'})
+    })
+
+    var index = 0
+
+    function nextEntry () {
+      if (index >= entries.length) return cb()
+
+      var name = entries[index++]
+      var fullPath = path.join(dir, name)
+
+      fs.stat(fullPath, function (er, stats) {
+        if (er) {
+          console.error('walkDirectorySorted stat error', fullPath, er)
+          return nextEntry()
+        }
+        if (stats.isFile()) {
+          if (fileFilter(name)) {
+            onFile(fullPath, nextEntry)
+          } else {
+            nextEntry()
+          }
+        } else if (stats.isDirectory()) {
+          if (dirFilter(name)) {
+            walkDirectorySorted(fullPath, fileFilter, dirFilter, onFile, nextEntry)
+          } else {
+            nextEntry()
+          }
+        } else {
+          nextEntry()
+        }
+      })
+    }
+
+    nextEntry()
+  })
+}
+
 function createDuplexFileStream (opts) {
   opts = opts || {}
   opts.readdirp = opts.readdirp || {}
@@ -156,32 +221,19 @@ function createDuplexFileStream (opts) {
             cb()
           })
       } else if (stats.isDirectory()) {
-        var total = 0
-        var complete = 0
-        var readdirpComplete = false
-
-        readdirp(filename, xtend(READDIRP_OPTS, opts.readdirp))
-          .on('data', function (entry) {
-            total++
-
-            getText(entry.fullPath, opts)
-              .on('data', function (entry) {self.push(entry)})
-              .on('error', function (er) {
-                console.error('Directory getText error', entry.fullPath, er)
-                cb(er)
-              })
-              .on('end', function () {
-                complete++
-                if (total == complete && readdirpComplete) cb()
-              })
-          })
-          .on('error', function (er) {
-            console.error('Directory error', filename, er)
-            cb(er)
-          })
-          .on('end', function () {
-            readdirpComplete = true
-          })
+        var walkOpts = Object.assign({}, DEFAULT_WALK_OPTS, opts.readdirp)
+        var fileFilter = createFilter(walkOpts.fileFilter)
+        var dirFilter = createFilter(walkOpts.directoryFilter)
+
+        walkDirectorySorted(filename, fileFilter, dirFilter, function (filePath, done) {
+          getText(filePath, opts)
+            .on('data', function (entry) { self.push(entry) })
+            .on('error', function (er) {
+              console.error('Directory getText error', filePath, er)
+              cb(er)
+            })
+            .on('end', done)
+        }, cb)
       }
     })
   })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tim-greller/xgettext-regex",
-  "version": "0.5.2",
+  "version": "0.8.0",
   "description": "Minimum viable xgettext .po file generator. Uses a configurable regex to get translation keys.",
   "main": "index.js",
   "scripts": {
@@ -11,11 +11,9 @@
   "dependencies": {
     "minimist": "^1.1.0",
     "once": "^1.3.1",
-    "readdirp": "^3.1.3",
     "split": "^1.0.1",
     "stream-combiner": "^0.2.1",
-    "through2": "^3.0.1",
-    "xtend": "^4.0.0"
+    "through2": "^3.0.1"
   },
   "bin": {
     "xgettext-regex": "bin/xgettext-regex.js"

package/test/test.js

@@ -164,4 +164,20 @@ test('Can create .pot from multiple files/directories', function (t) {
         t.end()
       }))
   })
-})
+})
+
+test('Emits directory entries in deterministic path order', function (t) {
+  t.plan(3)
+
+  xgettext.createReadStream(__dirname + '/fixtures')
+    .pipe(concat({encoding: 'string'}, function (pot) {
+      var indexJade = pot.indexOf('msgid "index.js"')
+      var indexPhp = pot.indexOf('msgid "php hypertext preprocessor"')
+      var jsApp = pot.indexOf('msgid "app.js"')
+
+      t.ok(indexJade > -1, 'index.jade translation exists')
+      t.ok(indexPhp > indexJade, 'index.php content appears after index.jade content')
+      t.ok(jsApp > indexPhp, 'js/app.js content appears after index.php content')
+      t.end()
+    }))
+})