@tim-greller/xgettext-regex 0.4.0

package/.travis.yml

@@ -0,0 +1,3 @@
+language: node_js
+node_js:
+  - "stable"

package/LICENCE

@@ -0,0 +1,5 @@
+Copyright (c) 2014, Alan Shaw
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -0,0 +1,53 @@
+# xgettext-regex
+
+Minimum viable xgettext .pot file generator. Uses a configurable regex to get translation keys.
+
+## Examples
+
+```sh
+cat foo.js | xgettext-regex # Output to stdout
+xgettext-regex foo.js -o foo.po # Output to foo.po
+xgettext-regex app-dir -o app.po # Recursive read directory
+```
+
+```js
+var fs = require('fs')
+var xgettext = require('xgettext-regex')
+
+var src = '/path/to/file'
+var dest = '/path/to/en-GB.po'
+var opts = {}
+
+fs.createReadStream(src)
+  .pipe(xgettext(src, opts))
+  .pipe(fs.createWriteStream(dest))
+```
+
+```js
+var fs = require('fs')
+var xgettext = require('xgettext-regex')
+
+var files = ['/path/to/file.js', '/path/to/html/dir']
+var opts = {}
+
+xgettext.createReadStream(files, opts))
+  .pipe(fs.createWriteStream('/path/to/en-GB.po'))
+```
+
+## Options
+
+```js
+opts = {
+    /* i18n function name */
+    fn: '_',
+    /* The regex used to match i18n function calls */
+    regex: /_\(((["'])(?:(?=(\\?))\3.)*?\2)\)/g,
+    /* Capture index for the i18n text in the above regex */
+    regexTextCaptureIndex: 1,
+    /* readdirp filters etc. */
+    readdirp: {
+      fileFilter: ['!.*', '!*.png', '!*.jpg', '!*.gif', , '!*.zip', , '!*.gz'],
+      directoryFilter: ['!.*', '!node_modules', '!coverage']
+    }
+}
+```

package/bin/usage.txt

@@ -0,0 +1,17 @@
+Usage: xgettext-regex [files] {OPTIONS}
+
+Standard Options:
+
+    --outfile, -o  Write the .pot formatted output to this file.
+                   If unspecified, xgettext-regex prints to stdout.
+
+         --fn, -f  Name of the i18n translation function.
+                   Defaults to "_" if unspecified.
+
+       --help, -h  Show this message.
+
+      --regex, -r  Complete regex that matches string literals in translation functions.
+                   Defaults to the name of the i18n function followed by "\(((["'])(?:(?=(\\?))\3.)*?\2)\)".
+
+      --index, -i  The index of the capturing group that contains the match's string literal.
+                   Defaults to 1 if unspecified.

package/bin/xgettext-regex.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+var fs = require('fs')
+var argv = require('minimist')(process.argv.slice(2))
+var path = require('path')
+var xgettext = require('../')
+
+if (argv.v || argv.version) {
+  return console.log(require('../package.json').version)
+}
+
+if (argv.h || argv.help) {
+  return fs.createReadStream(__dirname + '/usage.txt').pipe(process.stdout)
+}
+
+var outFile = argv.o || argv.outfile
+var opts = {
+  fn: (argv.f || argv.fn)
+}
+if (argv.r || argv.regex) opts.regex = new RegExp(argv.r || argv.regex, 'g')
+if (argv.i || argv.index) opts.regexTextCaptureIndex = argv.i || argv.index
+
+if (argv._.length) {
+  var files = argv._.map(function (filename) { return path.resolve(filename) })
+  var readable = xgettext.createReadStream(files, opts)
+
+  if (outFile) {
+    readable.pipe(fs.createWriteStream(outFile))
+  } else {
+    readable.pipe(process.stdout)
+  }
+} else {
+  var duplex = xgettext('process.stdin', opts)
+
+  if (outFile) {
+    process.stdin.pipe(duplex).pipe(fs.createWriteStream(outFile))
+  } else {
+    process.stdin.pipe(duplex).pipe(process.stdout)
+  }
+}

package/index.js

@@ -0,0 +1,186 @@
+var fs = require('fs')
+var path = require('path')
+var Readable = require('stream').Readable
+var through = require('through2')
+var readdirp = require('readdirp')
+var once = require('once')
+var split = require('split')
+var xtend = require('xtend')
+var combine = require('stream-combiner')
+
+function createDuplexStream (filename, opts) {
+  filename = filename || ''
+  opts = opts || {}
+  opts.fn = opts.fn || '_'
+  /*
+   * RegExp explaination
+   *
+   * (?<=               begin of a positive lookbehind that makes sure that the string literal is an argument of the i18n function
+   *   opts.fn          i18n function name
+   *   \(\s*            opening paranthesis followed by optional whitespace
+   *   (?:              non capturing group matching individual prior arguments to the function and their delimeters
+   *     (?:                non capturing group matching a single prior argument
+   *       "(?:[^"]|\\.)*"  possible argument: a string literal quoted with double quotes, containing only escaped quotes if any
+   *       |                or
+   *       '(?:[^']|\\.)*'  same with single quotes
+   *     )                  end of single argument
+   *     \s*,\s*          delimeter: the arguments are seperated by commas and optional whitespace
+   *   )*                 end of argument list, ending with a delimeter
+   * )                    end of positive lookbehind and therefore everything that comes before the actual string literal that should be matched
+   *
+   * (                    begin first capturing group containing the string value including its quotes (stripped away later via text.slice(1, -1))
+   *   (["'])             second capturing group containing the quote character
+   *   (?:                begin non capturing group containing a single (escaped or unescaped) character of the string literal's text content
+   *     (?=              begin of a positive lookahead matching and capturing a backslash, if there is one at the current position
+   *       (\\?)          capturing the backslash (or an empty string) in \3
+   *     )
+   *     \3.              matches any character, optionally preceeded by a backslash.
+   *   )*?                end of the character. The lazy repetition ensures, that no quotes (\2) are included. Escaped quotes are included, because the character immediately after a backslash is always included, since \3 is followed by a dot and therefore the string cannot end with a backslash, but automatically matches the next character as well.
+   *   \2                 the quote character, closing the string literal
+   * )                    end of the capturing group containing the string literal
+   *
+   * (?=                  begin of a positive lookahead matching any paramters that might follow after the target string literal
+   *   (?:                analogous to the beginning: non capturing group matching arguments and their delimeters
+   *     \s*,\s*          now we start with the delimeter
+   *     (?:                  match single argument
+   *       "(?:[^"]|\\.)*"    string literal with double quotes
+   *       |'(?:[^']|\\.)*'   or with single quotes
+   *       |[^"')]+           or without any quotes (numbers, object references, variables, ...). Cannot contain any paranthesis either, as it is not possible to ensure that every closing bracket has a preceding opening one with regex (would require at least a context free language)
+   *     )
+   *   )*                 end of argument list
+   *   \s*                optional whitespace
+   *   \)                 closing paranthesis
+   * )                    end of positive lookahead
+   */
+  opts.regex = opts.regex || new RegExp("(?<=" + opts.fn + "\\(\\s*(?:(?:\"(?:[^\"]|\\\\.)*\"|'(?:[^']|\\\\.)*')\\s*,\\s*)*)(([\"'])(?:(?=(\\\\?))\\3.)*?\\2)(?=(?:\\s*,\\s*(?:\"(?:[^\"]|\\\\.)*\"|'(?:[^']|\\\\.)*'|[^\"')]+))*\\s*\\))", 'g')
+  opts.regexTextCaptureIndex = opts.regexTextCaptureIndex || 1
+
+  var lineNum = 1
+
+  return combine(
+    split(),
+    through(function (line, enc, cb) {
+      line = line.toString()
+
+      var matches
+      var first = true
+
+      while ((matches = opts.regex.exec(line)) !== null) {
+        var entry = '\n'
+
+        if (first) {
+          const relativeFilename = path.relative(process.cwd(), filename)
+          entry += '#: ' + relativeFilename + ':' + lineNum + '\n'
+          first = false
+        }
+
+        var text = matches[opts.regexTextCaptureIndex]
+
+        if (text[0] == "'") {
+          text = text.slice(1, -1)
+          text = text.replace(/\\'/g, "'")
+          text = '"' + text.replace(/"/g, '\\"') + '"'
+        }
+
+        entry += 'msgid ' + text + '\n'
+        entry += 'msgstr ' + text + '\n'
+
+        this.push(entry)
+      }
+
+      lineNum++
+      opts.regex.lastIndex = 0
+      cb()
+    })
+  )
+}
+
+module.exports = createDuplexStream
+
+module.exports.createReadStream = function (files, opts) {
+  if (!Array.isArray(files)) files = [files]
+
+  var index = 0
+  var readable = new Readable()
+
+  readable._read = function () {
+    var push = true
+
+    while (push && index < files.length) {
+      push = this.push(files[index])
+      index++
+    }
+
+    this.push(null)
+  }
+
+  return readable.pipe(createDuplexFileStream(opts))
+}
+
+function getText (filename, opts) {
+  return fs.createReadStream(filename).pipe(createDuplexStream(filename, opts))
+}
+
+var READDIRP_OPTS = {
+  fileFilter: ['!.*', '!*.png', '!*.jpg', '!*.gif', '!*.zip', '!*.gz'],
+  directoryFilter: ['!.*', '!node_modules', '!coverage']
+}
+
+function createDuplexFileStream (opts) {
+  opts = opts || {}
+  opts.readdirp = opts.readdirp || {}
+
+  return through(function (filename, enc, cb) {
+    var self = this
+    filename = filename.toString()
+    cb = once(cb)
+
+    self.push('#, fuzzy\n')
+    self.push('msgid ""\n')
+    self.push('msgstr ""\n')
+    self.push('"Content-Type: text/plain; charset=UTF-8\\n"\n')
+
+    fs.stat(filename, function (er, stats) {
+      if (er) return cb(er)
+
+      if (stats.isFile()) {
+        getText(filename, opts)
+          .on('data', function (entry) {self.push(entry)})
+          .on('error', function (er) {
+            console.error('File getText error', filename, er)
+            cb(er)
+          })
+          .on('end', function () {
+            cb()
+          })
+      } else if (stats.isDirectory()) {
+        var total = 0
+        var complete = 0
+        var readdirpComplete = false
+
+        readdirp(filename, xtend(READDIRP_OPTS, opts.readdirp))
+          .on('data', function (entry) {
+            total++
+
+            getText(entry.fullPath, opts)
+              .on('data', function (entry) {self.push(entry)})
+              .on('error', function (er) {
+                console.error('Directory getText error', entry.fullPath, er)
+                cb(er)
+              })
+              .on('end', function () {
+                complete++
+                if (total == complete && readdirpComplete) cb()
+              })
+          })
+          .on('error', function (er) {
+            console.error('Directory error', filename, er)
+            cb(er)
+          })
+          .on('end', function () {
+            readdirpComplete = true
+          })
+      }
+    })
+  })
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@tim-greller/xgettext-regex",
+  "version": "0.4.0",
+  "description": "Minimum viable xgettext .po file generator. Uses a configurable regex to get translation keys.",
+  "main": "index.js",
+  "scripts": {
+    "test": "istanbul cover node_modules/.bin/tape test/test.js"
+  },
+  "author": "Alan Shaw",
+  "license": "ISC",
+  "dependencies": {
+    "minimist": "^1.1.0",
+    "once": "^1.3.1",
+    "readdirp": "^3.1.3",
+    "split": "^1.0.1",
+    "stream-combiner": "^0.2.1",
+    "through2": "^3.0.1",
+    "xtend": "^4.0.0"
+  },
+  "bin": {
+    "xgettext-regex": "bin/xgettext-regex.js"
+  },
+  "devDependencies": {
+    "concat-stream": "^2.0.0",
+    "istanbul": "^0.4.5",
+    "tape": "^4.11.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/alanshaw/xgettext-regex.git"
+  },
+  "keywords": [
+    "xgettext",
+    "gettext",
+    "i18n",
+    "l18n"
+  ],
+  "bugs": {
+    "url": "https://github.com/alanshaw/xgettext-regex/issues"
+  },
+  "homepage": "https://github.com/alanshaw/xgettext-regex",
+  "directories": {
+    "test": "test"
+  }
+}

package/test/fixtures/index.jade

@@ -0,0 +1,2 @@
+div
+  a(href='#')= _("index.js")

package/test/fixtures/index.php

@@ -0,0 +1,3 @@
+<?php
+  $iHaveNoIdeaWhatImDoing = _('php hypertext preprocessor')
+?>

package/test/fixtures/js/app.js

@@ -0,0 +1,5 @@
+for (var i = 0; i < 1000; i++) {
+  console.log(_('app.js'))
+}
+
+alert(_("There's pie in them hills"))

package/test/fixtures/js/plugin.js

@@ -0,0 +1,3 @@
+$.xgettext = function () {
+  return _('w00t!')
+}

package/test/test.js

@@ -0,0 +1,110 @@
+var fs = require('fs')
+var Readable = require('stream').Readable
+var test = require('tape')
+var concat = require('concat-stream')
+var xgettext = require('../')
+
+test('Can create .pot from code', function (t) {
+  t.plan(1)
+
+  var src = new Readable
+  src._read = function () {
+    this.push('_("foobar")\n\n')
+    this.push(null)
+  }
+
+  src
+    .pipe(xgettext())
+    .pipe(concat({encoding: 'string'}, function (pot) {
+      t.ok(pot.indexOf('msgid "foobar"') > -1, 'msgid "foobar" exists')
+      t.end()
+    }))
+})
+
+test('Can deal with single quotes', function (t) {
+  t.plan(1)
+
+  var src = new Readable
+  src._read = function () {
+    this.push("blah;\n_('foobar')")
+    this.push(null)
+  }
+
+  src
+    .pipe(xgettext())
+    .pipe(concat({encoding: 'string'}, function (pot) {
+      t.ok(pot.indexOf('msgid "foobar"') > -1, 'msgid "foobar" exists')
+      t.end()
+    }))
+})
+
+test('Can deal with escaped quotes', function (t) {
+  t.plan(2)
+
+  var src = new Readable
+  src._read = function () {
+    this.push("\n_('foobar\\'s');\nif(true){_(\"air \\\"quotes\\\"\")}")
+    this.push(null)
+  }
+
+  src
+    .pipe(xgettext())
+    .pipe(concat({encoding: 'string'}, function (pot) {
+      t.ok(pot.indexOf('msgid "foobar\'s"') > -1, 'msgid "foobar\'s" exists')
+      t.ok(pot.indexOf('msgid "air \\"quotes\\""') > -1, 'msgid "air \\"quotes\\"" exists')
+      t.end()
+    }))
+})
+
+test('Can create .pot from single file', function (t) {
+  t.plan(1)
+
+  xgettext.createReadStream(__dirname + '/fixtures/index.jade')
+    .pipe(concat({encoding: 'string'}, function (pot) {
+      t.ok(pot.indexOf('msgid "index.js"') > -1, 'msgid "index.js" exists')
+      t.end()
+    }))
+})
+
+test('Can change i18n function', function (t) {
+  t.plan(1)
+
+  var src = new Readable
+  src._read = function () {
+    this.push("blah;\ni18n('foobar')")
+    this.push(null)
+  }
+
+  src
+    .pipe(xgettext('src', {fn: 'i18n'}))
+    .pipe(concat({encoding: 'string'}, function (pot) {
+      t.ok(pot.indexOf('msgid "foobar"') > -1, 'msgid "foobar" exists')
+      t.end()
+    }))
+})
+
+test('Can create .pot from multiple files/directories', function (t) {
+  t.plan(6)
+
+  fs.readdir(__dirname + '/fixtures', function (er, files) {
+    t.ifError(er, 'No error getting fixtures dir listing')
+
+    files = files.map(function (f) {
+      return __dirname + '/fixtures/' + f
+    })
+
+    xgettext.createReadStream(files)
+      .pipe(concat({encoding: 'string'}, function (pot) {
+        // app.js
+        t.ok(pot.indexOf('msgid "app.js"') > -1, 'msgid "app.js" exists')
+        t.ok(pot.indexOf('msgid "There\'s pie in them hills"') > -1, 'msgid "There\'s pie in them hills" exists')
+        // plugin.js
+        t.ok(pot.indexOf('msgid "w00t!"') > -1, 'msgid "w00t!" exists')
+        // index.jade
+        t.ok(pot.indexOf('msgid "index.js"') > -1, 'msgid "index.js" exists')
+        // index.php
+        t.ok(pot.indexOf('msgid "php hypertext preprocessor"') > -1, 'msgid "php hypertext preprocessor" exists')
+        t.end()
+      }))
+  })
+})