comment-parser 0.2.3 → 0.3.2

package/.jshintrc

@@ -0,0 +1,11 @@
+{
+  "node":      true,
+  "strict":    true,
+  "maxlen":    100,
+  "undef":     true,
+  "unused":    true,
+  "onecase":   true,
+  "lastsemic": true,
+  "latedef"  : true,
+  "indent":    2
+}

package/.travis.yml

@@ -0,0 +1,8 @@
+language: node_js
+node_js:
+  - "4.1"
+  - "4.0"
+  - "0.12"
+  - "0.11"
+  - "0.10"
+  - "iojs"

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# v0.3.2
+- fix RegExp for `description` extraction to allow $ char
+
+# v0.3.1
+- use `readable-stream` fro Node 0.8 comatibility
+- allow to pass optional parameters to `parse.file(path [,opts], done)`  
+- allow `parse.stream` to work with Buffers in addition to strings
+
+# v0.3.0
+- `feature` allow to use custom parsers
+- `feature` always include source, no `raw_value` option needed
+- `bugfix` always provide `optional` tag property
+- `refactor` clean up tests
+
 # v0.2.3
 
 - `bugfix` Accept `/** one line */` comments
@@ -28,4 +42,4 @@
 
 # v0.1.0
 
-Initial implementation
+Initial implementation

package/README.md

@@ -1,5 +1,5 @@
-comment-parser
-==============
+# comment-parser
+
 
 Generic JSDoc-like comment parser. This library is not intended to be documentation generator, but rather composite unit for it.
 
@@ -25,50 +25,120 @@ this would be parsed into following
 
 ```javascript
 [{
-  tags: [{
-      tag: "some-tag",
-      type: "Type",
-      name: "name",
-      line: 15,
-      description: "Singleline or multiline description text",
-      tags: [{
-          tag: "some-tag",
-          type: "Type",
-          name: "subname",
-          line: 16,
-          description: "Singleline or multiline description text",
-          tags: [{
-              tag: "some-tag",
-              type: "Type",
-              name: "subsubname",
-              line: 17,
-              description: "Singleline or\nmultiline description text"
-            }]
-        }]
-   }, {
-      tag: "another-tag",
-      type: "",
-      name: "",
-      line: 18,
-      description: ""
-   }],
-  description: "Singleline or multiline description text. Line breaks are preserved."
+  "tags": [{
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name",
+    "optional": false,
+    "description": "Singleline or multiline description text",
+    "line": 3,
+    "source": "@some-tag {Type} name Singleline or multiline description text"
+  }, {
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name.subname",
+    "optional": false,
+    "description": "Singleline or multiline description text",
+    "line": 4,
+    "source": "@some-tag {Type} name.subname Singleline or multiline description text"
+  }, {
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name.subname.subsubname",
+    "optional": false,
+    "description": "Singleline or\nmultiline description text",
+    "line": 5,
+    "source": "@some-tag {Type} name.subname.subsubname Singleline or\nmultiline description text"
+  }, {
+    "tag": "another-tag",
+    "name": "",
+    "optional": false,
+    "type": "",
+    "description": "",
+    "line": 7,
+    "source": "@another-tag"
+  }],
+  "line": 0,
+  "description": "Singleline or multiline description text. Line breaks are preserved.",
+  "source": "Singleline or multiline description text. Line breaks are preserved.\n\n@some-tag {Type} name Singleline or multiline description text\n@some-tag {Type} name.subname Singleline or multiline description text\n@some-tag {Type} name.subname.subsubname Singleline or\nmultiline description text\n@another-tag"
 }]
 ```
 
 By default dotted names like `name.subname.subsubname` will be expanded into nested sections, this can be prevented by passing `opts.dotted_names = false`.
 
-You can also make raw line available in parsed results by passing `opts.raw_value = true`.
-
 Invalid comment blocks are skipped. Comments starting with `/*` and `/***` are considered not valid.
 
 Also you can parse entire file with `parse.file('path/to/file', callback)` or acquire an instance of [Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) stream with `parse.stream()`.
 
-Happy coding :)
+## Custom parsers
+
+
+In case you need to parse tags in different way you can pass `opts.parsers = [parser1, ..., parserN]`, where each parser is `function name(str:String, data:Object):{source:String, data:Object}`.
+
+Each parser function takes string left after previous parsers applied and data produced by them. And returns `null` or `{source: '', data:{}}` where `source` is consumed substring and `data` is a payload with tag node fields.
 
+Tag node data is build by merging result bits from all parsers. Here is some example that is not doing actual parsing but is demonstrating the flow:
 
-Contributors
-============
+```
+/**
+ * Source to be parsed below
+ * @tag {type} name Description
+ */
+parse(source, {parsers: [
+	// takes entire string
+	function parse_tag(str, data) {
+		return {source: ' @tag', data: {tag: 'tag'}};
+	},
+	// parser throwing exception
+	function check_tag(str, data) {
+		if (allowed_tags.indexOf(data.tag) === -1) {
+			throw new Error('Unrecognized tag "' + data.tag + '"');
+		}			
+	},
+	// takes the rest of the string after ' @tag''
+	function parse_name1(str, data) {
+		return {source: ' name', data: {name: 'name1'}};
+	},
+	// alternative name parser
+	function parse_name2(str, data) {
+		return {source: ' name', data: {name: 'name2'}};
+	}
+]});
+```
+
+This would produce following:
 
+```
+[{
+  "tags": [{
+    "tag": "tag",
+    "errors": [
+      "check_tag: Unrecognized tag \\"tag\\""
+    ],
+    "name": "name2",
+    "optional": false,
+    "type": "",
+    "description": "",
+    "line": 2,
+    "source": "@tag {type} name Description"
+  }],
+  "line": 0,
+  "description": "Source to be parsed below",
+  "source": "Source to be parsed below\n@tag {type} name Description"
+}]
+```
+
+## Packaging
+
+`comment-parser` is CommonJS module and was primarely designed to be used with Node. Module `index.js` includes stream and file functionality. Use prser-only module in browser `comment-parser/parse.js`
+
+## Contributors
+
+
+- [Alexej Yaroshevich](https://github.com/zxqfox)
+- [Evgeny Reznichenko](https://github.com/zxcabs)
+- [Jordan Harband](https://github.com/ljharb)
 - [Sergii Iavorskyi](https://github.com/yavorskiy)
-- [Alexej Yaroshevich](https://github.com/zxqfox)
+
+
+Happy coding :)

package/index.js

@@ -1,243 +1,20 @@
 
+'use strict';
+
 var fs     = require('fs');
-var stream = require('stream');
+var stream = require('readable-stream');
 var util   = require('util');
 
-var RE_COMMENT_START = /^\s*\/\*\*\s*$/m;
-var RE_COMMENT_LINE  = /^\s*\*(?:\s|$)/m;
-var RE_COMMENT_END   = /^\s*\*\/\s*$/m;
-var RE_COMMENT_1LINE = /^\s*\/\*\*\s*(.*)\s*\*\/\s*$/;
-
-/**
- * analogue of str.match(/@(\S+)(?:\s+\{([^\}]+)\})?(?:\s+(\S+))?(?:\s+([^$]+))?/);
- * @param {string} str raw jsdoc string
- * @returns {object} parsed tag node
- */
-function parse_tag_line(str) {
-  if (typeof str !== 'string') { return false; }
-
-  if (str[0] !== '@') { return false; }
-
-  var pos = 1;
-  var l = str.length;
-  var error = null;
-  var res = {
-    tag         : _tag(),
-    type        : _type() || '',
-    name        : _name() || '',
-    description : _rest() || ''
-  };
-
-  if (error) {
-    res.error = error;
-  }
-
-  return res;
-
-  function _skipws() {
-    while (str[pos] === ' ' && pos < l) { pos ++; }
-  }
-  function _tag() { // @(\S+)
-    var sp = str.indexOf(' ', pos);
-    sp = sp < 0 ? l : sp;
-    var res = str.substr(pos, sp - pos);
-    pos = sp;
-    return res;
-  }
-  function _type() { // (?:\s+\{([^\}]+)\})?
-    _skipws();
-    if (str[pos] !== '{') { return ''; }
-    var ch;
-    var res = '';
-    var curlies = 0;
-    while (pos < l) {
-      ch = str[pos];
-      curlies += ch === '{' ? 1 : ch === '}' ? -1 : 0;
-      res += ch;
-      pos ++;
-      if (!curlies) {
-        break;
-      }
-    }
-    if (curlies !== 0) {
-      // throw new Error('Unpaired curly in type doc');
-      error = 'Unpaired curly in type doc';
-      pos -= res.length;
-      return '';
-    }
-    return res.substr(1, res.length - 2);
-  }
-  function _name() { // (?:\s+(\S+))?
-    if (error) { return ''; }
-    _skipws();
-    var ch;
-    var res = '';
-    var brackets = 0;
-    var re = /\s/;
-    while (pos < l) {
-      ch = str[pos];
-      brackets += ch === '[' ? 1 : ch === ']' ? -1 : 0;
-      res += ch;
-      pos ++;
-      if (brackets === 0 && re.test(str[pos])) {
-        break;
-      }
-    }
-    if (brackets) {
-      // throw new Error('Unpaired curly in type doc');
-      error = 'Unpaired brackets in type doc';
-      pos -= res.length;
-      return '';
-    }
-    return res;
-  }
-  function _rest() { // (?:\s+([^$]+))?
-    _skipws();
-    return str.substr(pos);
-  }
-}
-
-function parse_chunk(source, opts) {
-  source = source
-    .reduce(function(sections, line) {
-      if (line.value.match(/^@(\w+)/)) { sections.push([]); }
-      var section = sections[sections.length - 1];
-      section.line = section.line || line.line;
-      section.push(line.value);
-      return sections;
-    }, [[]])
-    .map(function(section) {
-      return {value: section.join('\n').trim(), line: section.line};
-    });
-
-  var description = source[0].value.match(/^@(\S+)/) ? {value: '', line: 0} : source.shift();
-
-  var tags = source.reduce(function(tags, tag) {
-    var tag_node = parse_tag_line(tag.value);
-    if (!tag_node) { return tags; }
-
-    tag_node.line = Number(tag.line);
-    if (opts.raw_value) {
-      tag_node.value = tag.value;
-    }
-
-    // used for split results below
-    var parts;
-
-    // parsing optional and default value if exists
-    // probably if should be hidden with option or moved out to some jsdoc standard
-    if (tag_node.name[0] === '[' && tag_node.name[tag_node.name.length - 1] === ']') {
-      tag_node.optional = true;
-      tag_node.name = tag_node.name.substr(1, tag_node.name.length - 2);
-
-      // default value here
-      if (tag_node.name.indexOf('=') !== -1) {
-        parts = tag_node.name.split('=');
-        tag_node.name    = parts[0];
-        tag_node.default = parts[1].replace(/^(["'])(.+)(\1)$/, '$2');
-      }
-    }
-
-    // hidden with `dotted_names` parsing of `obj.value` naming standard
-    if (opts.dotted_names && tag_node.name.indexOf('.') !== -1) {
-      var parent_name;
-      var parent_tag;
-      var parent_tags = tags;
-      parts = tag_node.name.split('.');
-
-      while (parts.length > 1) {
-        parent_name = parts.shift();
-        parent_tag  = _find(parent_tags, {
-          tag  : tag_node.tag,
-          name : parent_name
-        });
-
-        if (!parent_tag) {
-          parent_tag = {
-            tag         : tag_node.tag,
-            line        : Number(tag_node.line),
-            name        : parent_name,
-            type        : '',
-            description : ''
-          };
-          parent_tags.push(parent_tag);
-        }
-
-        parent_tag.tags = parent_tag.tags || [];
-        parent_tags = parent_tag.tags;
-      }
-
-      tag_node.name = parts[0];
-      parent_tags.push(tag_node);
-      return tags;
-    }
-
-    return tags.concat(tag_node);
-  }, []);
-
-  return {
-    tags        : tags,
-    line        : Number(description.line || 0),
-    description : description.value
-  };
-}
+var parse  = require('./parser.js');
 
-function mkextract(opts) {
+module.exports = parse;
 
-  var chunk = null;
-  var line_number = 0;
-
-  return function extract(line) {
-    // if oneliner
-    // then parse it immediately
-    if (!line_number && line.match(RE_COMMENT_1LINE)) {
-      // console.log('line (1)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_1LINE, '$1'));
-      return parse_chunk([{value: line.replace(RE_COMMENT_1LINE, '$1'), line: 0}], opts);
-    }
-
-    line_number += 1;
-
-    // if start of comment
-    // then init the chunk
-    if (line.match(RE_COMMENT_START)) {
-      // console.log('line (1)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_START, ''));
-      chunk = [{value: line.replace(RE_COMMENT_START, ''), line: line_number - 1}];
-      return null;
-    }
-
-    // if comment line and chunk started
-    // then append
-    if (chunk && line.match(RE_COMMENT_LINE)) {
-      // console.log('line (2)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_LINE, ''));
-      chunk.push({value: line.replace(RE_COMMENT_LINE, ''), line: line_number - 1});
-      return null;
-    }
-
-    // if comment end and chunk started
-    // then parse the chunk and push
-    if (chunk && line.match(RE_COMMENT_END)) {
-      // console.log('line (3)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_END, ''));
-      chunk.push({value: line.replace(RE_COMMENT_END, ''), line: line_number - 1});
-      return parse_chunk(chunk, opts);
-    }
-
-    // if non-comment line
-    // then reset the chunk
-    chunk = null;
-    line_number = 0;
-  };
-}
-
-/* ------- Transform strean ------- */
+/* ------- Transform stream ------- */
 
 function Parser(opts) {
   opts = opts || {};
   stream.Transform.call(this, {objectMode: true});
-  this._extract = mkextract(opts);
+  this._extract = parse.mkextract(opts);
 }
 
 util.inherits(Parser, stream.Transform);
@@ -245,7 +22,7 @@ util.inherits(Parser, stream.Transform);
 Parser.prototype._transform = function transform(data, encoding, done) {
 
   var block;
-  var lines = data.split(/\n/);
+  var lines = data.toString().split(/\n/);
 
   while (lines.length) {
     block = this._extract(lines.shift());
@@ -257,34 +34,25 @@ Parser.prototype._transform = function transform(data, encoding, done) {
   done();
 };
 
-/* ------- Public API ------- */
-
-module.exports = function parse(source, opts) {
-  opts = opts || {};
-
-  var block;
-  var blocks  = [];
-  var extract = mkextract(opts);
-  var lines   = source.split(/\n/);
-
-  while (lines.length) {
-    block = extract(lines.shift());
-    if (block) {
-      blocks.push(block);
-    }
-  }
-
-  return blocks;
+module.exports.stream = function stream(opts) {
+  return new Parser(opts);
 };
 
+/* ------- File parser ------- */
+
 module.exports.file = function file(file_path, done) {
 
+  var opts = {};
   var collected = [];
 
+  if (arguments.length === 3) {
+    opts = done;
+    done = arguments[2];
+  }
+
   return fs.createReadStream(file_path, {encoding: 'utf8'})
     .on('error', done)
-
-    .pipe(new Parser())
+    .pipe(new Parser(opts))
     .on('error', done)
     .on('data', function(data) {
       collected.push(data);
@@ -294,22 +62,3 @@ module.exports.file = function file(file_path, done) {
     });
 };
 
-module.exports.stream = function stream(opts) {
-  return new Parser(opts);
-};
-
-function _find(list, filter) {
-  var i, l, k, yes, item;
-  for (i = 0, l = list.length; i < l; i++) {
-    item = list[i];
-    yes = true;
-    for (k in filter) {
-      if (filter.hasOwnProperty(k)) {
-        yes = yes && filter[k] === list[i][k];
-      }
-    }
-    if (yes) {
-      return item;
-    }
-  }
-}

package/package.json

@@ -1,19 +1,24 @@
 {
   "name": "comment-parser",
-  "version": "0.2.3",
+  "version": "0.3.2",
   "description": "Generic JSDoc-like comment parser. ",
   "main": "index.js",
   "directories": {
     "test": "tests"
   },
   "dependencies": {
+    "readable-stream": "^2.0.4"
   },
   "devDependencies": {
+    "chai": "~1.9.0",
+    "jshint": "^2.5.10",
+    "jshint-stylish": "^1.0.0",
     "mocha": "~1.17.1",
-    "chai": "~1.9.0"
+    "nodemon": "^1.2.1"
   },
   "scripts": {
-    "test": "mocha tests/*"
+    "test": "jshint --reporter node_modules/jshint-stylish/stylish.js index.js && mocha tests/*",
+    "watch": "nodemon -q -w index.js -w parser.js -w tests/ -x npm test"
   },
   "repository": {
     "type": "git",