comment-parser 0.2.4 → 0.4.0

package/.jshintrc

@@ -0,0 +1,11 @@
+{
+  "node":      true,
+  "strict":    true,
+  "maxlen":    100,
+  "undef":     true,
+  "unused":    true,
+  "onecase":   true,
+  "lastsemic": true,
+  "latedef"  : true,
+  "indent":    2
+}

package/.travis.yml

@@ -0,0 +1,8 @@
+language: node_js
+node_js:
+  - "4.1"
+  - "4.0"
+  - "0.12"
+  - "0.11"
+  - "0.10"
+  - "iojs"

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# v0.4.0
+- allow to preserve exact source and line numbers with `opts.trim = false`
+
+# v0.3.1
+- use `readable-stream` fro Node 0.8 comatibility
+- allow to pass optional parameters to `parse.file(path [,opts], done)`
+- allow `parse.stream` to work with Buffers in addition to strings
+
+# v0.3.0
+- `feature` allow to use custom parsers
+- `feature` always include source, no `raw_value` option needed
+- `bugfix` always provide `optional` tag property
+- `refactor` clean up tests
+
 # v0.2.3
 
 - `bugfix` Accept `/** one line */` comments
@@ -28,4 +42,4 @@
 
 # v0.1.0
 
-Initial implementation
+Initial implementation

package/README.md

@@ -1,11 +1,11 @@
-comment-parser
-==============
+# comment-parser
+
 
 Generic JSDoc-like comment parser. This library is not intended to be documentation generator, but rather composite unit for it.
 
 `npm install comment-parser`
 
-Module provides `parse(s:String[, opts:Object]):Object` function which takes `/** ... */` comment string and returns array  of objects with parsed data.
+Module provides `parse(s:String[, options:Object]):Object` function which takes `/** ... */` comment string and returns array  of objects with parsed data.
 
 It is not trying to detect relations between tags or somehow recognize their meaning. Any tag can be used, as long as it satisfies the format.
 
@@ -25,50 +25,133 @@ this would be parsed into following
 
 ```javascript
 [{
-  tags: [{
-      tag: "some-tag",
-      type: "Type",
-      name: "name",
-      line: 15,
-      description: "Singleline or multiline description text",
-      tags: [{
-          tag: "some-tag",
-          type: "Type",
-          name: "subname",
-          line: 16,
-          description: "Singleline or multiline description text",
-          tags: [{
-              tag: "some-tag",
-              type: "Type",
-              name: "subsubname",
-              line: 17,
-              description: "Singleline or\nmultiline description text"
-            }]
-        }]
-   }, {
-      tag: "another-tag",
-      type: "",
-      name: "",
-      line: 18,
-      description: ""
-   }],
-  description: "Singleline or multiline description text. Line breaks are preserved."
+  "tags": [{
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name",
+    "optional": false,
+    "description": "Singleline or multiline description text",
+    "line": 3,
+    "source": "@some-tag {Type} name Singleline or multiline description text"
+  }, {
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name.subname",
+    "optional": false,
+    "description": "Singleline or multiline description text",
+    "line": 4,
+    "source": "@some-tag {Type} name.subname Singleline or multiline description text"
+  }, {
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name.subname.subsubname",
+    "optional": false,
+    "description": "Singleline or\nmultiline description text",
+    "line": 5,
+    "source": "@some-tag {Type} name.subname.subsubname Singleline or\nmultiline description text"
+  }, {
+    "tag": "another-tag",
+    "name": "",
+    "optional": false,
+    "type": "",
+    "description": "",
+    "line": 7,
+    "source": "@another-tag"
+  }],
+  "line": 0,
+  "description": "Singleline or multiline description text. Line breaks are preserved.",
+  "source": "Singleline or multiline description text. Line breaks are preserved.\n\n@some-tag {Type} name Singleline or multiline description text\n@some-tag {Type} name.subname Singleline or multiline description text\n@some-tag {Type} name.subname.subsubname Singleline or\nmultiline description text\n@another-tag"
 }]
 ```
 
-By default dotted names like `name.subname.subsubname` will be expanded into nested sections, this can be prevented by passing `opts.dotted_names = false`.
-
-You can also make raw line available in parsed results by passing `opts.raw_value = true`.
-
 Invalid comment blocks are skipped. Comments starting with `/*` and `/***` are considered not valid.
 
 Also you can parse entire file with `parse.file('path/to/file', callback)` or acquire an instance of [Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) stream with `parse.stream()`.
 
-Happy coding :)
 
+### Options
+
+#### dotted_names `true`
+
+Tells parser to expand dotted names like `name.subname.subsubname` into nested sections
+
+#### trim `true`
+
+Makes parser to trim white spaces, set it to `false` to preserve exact source and line numbers
+
+#### parsers, `[PARSERS.{parse_tag, parse_type, parse_name, parse_description}]`
+
+Array of alternative parsers (read below).
+
+
+## Custom parsers
+
+
+In case you need to parse tags in different way you can pass `opts.parsers = [parser1, ..., parserN]`, where each parser is `function name(str:String, data:Object):{source:String, data:Object}`.
+	
+Each parser function takes string left after previous parsers applied and data produced by them. And returns `null` or `{source: '', data:{}}` where `source` is consumed substring and `data` is a payload with tag node fields.
+
+Tag node data is build by merging result bits from all parsers. Here is some example that is not doing actual parsing but is demonstrating the flow:
+
+```
+/**
+ * Source to be parsed below
+ * @tag {type} name Description
+ */
+parse(source, {parsers: [
+	// takes entire string
+	function parse_tag(str, data) { 
+		return {source: ' @tag', data: {tag: 'tag'}}; 
+	}, 
+	// parser throwing exception
+	function check_tag(str, data) {
+		if (allowed_tags.indexOf(data.tag) === -1) { 
+			throw new Error('Unrecognized tag "' + data.tag + '"');
+		}			
+	},
+	// takes the rest of the string after ' @tag''
+	function parse_name1(str, data) { 
+		return {source: ' name', data: {name: 'name1'}}; 
+	},
+	// alternative name parser
+	function parse_name2(str, data) { 
+		return {source: ' name', data: {name: 'name2'}}; 
+	}	
+]});
+```
+
+This would produce following:
+
+```
+[{
+  "tags": [{
+    "tag": "tag",
+    "errors": [
+      "check_tag: Unrecognized tag \\"tag\\""
+    ],
+    "name": "name2",
+    "optional": false,
+    "type": "",
+    "description": "",
+    "line": 2,
+    "source": "@tag {type} name Description"
+  }],
+  "line": 0,
+  "description": "Source to be parsed below",
+  "source": "Source to be parsed below\n@tag {type} name Description"
+}]
+```
+
+## Packaging
+
+`comment-parser` is CommonJS module and was primarely designed to be used with Node. Module `index.js` includes stream and file functionality. Use prser-only module in browser `comment-parser/parse.js`
+
+## Contributors
 
-Contributors
-============
 
 - [Sergii Iavorskyi](https://github.com/yavorskiy)
-- [Alexej Yaroshevich](https://github.com/zxqfox)
+- [Alexej Yaroshevich](https://github.com/zxqfox)
+- [Jordan Harband](https://github.com/ljharb)
+
+
+Happy coding :)

package/index.js

@@ -1,250 +1,20 @@
 
+'use strict';
+
 var fs     = require('fs');
-var stream = require('stream');
+var stream = require('readable-stream');
 var util   = require('util');
 
-var RE_COMMENT_START = /^\s*\/\*\*\s*$/m;
-var RE_COMMENT_LINE  = /^\s*\*(?:\s|$)/m;
-var RE_COMMENT_END   = /^\s*\*\/\s*$/m;
-var RE_COMMENT_1LINE = /^\s*\/\*\*\s*(.*)\s*\*\/\s*$/;
-var RE_SPACE = /\s/;
-
-/**
- * analogue of str.match(/@(\S+)(?:\s+\{([^\}]+)\})?(?:\s+(\S+))?(?:\s+([^$]+))?/);
- * @param {string} str raw jsdoc string
- * @returns {object} parsed tag node
- */
-function parse_tag_line(str) {
-  if (typeof str !== 'string') { return false; }
-
-  if (str[0] !== '@') { return false; }
-
-  var pos = 1;
-  var l = str.length;
-  var error = null;
-  var new_line = false;
-  var res = {
-    tag         : _tag(),
-    type        : !new_line && _type() || '',
-    name        : !new_line && _name() || '',
-    description : _rest() || ''
-  };
-
-  if (error) {
-    res.error = error;
-  }
-
-  return res;
-
-  function _skipws() {
-    var prev_pos = pos;
-    while (pos < l && RE_SPACE.test(str[pos])) {
-      new_line = new_line || str[pos] === '\n';
-      pos++;
-    }
-  }
-  function _tag() { // @(\S+)
-    var sp = str.search(RE_SPACE, pos);
-    sp = sp < 0 ? l : sp;
-    var res = str.substr(pos, sp - pos);
-    pos = sp;
-    return res;
-  }
-  function _type() { // (?:\s+\{([^\}]+)\})?
-    _skipws();
-    if (str[pos] !== '{') { return ''; }
-    var ch;
-    var res = '';
-    var curlies = 0;
-    while (pos < l) {
-      ch = str[pos];
-      curlies += ch === '{' ? 1 : ch === '}' ? -1 : 0;
-      res += ch;
-      pos ++;
-      if (!curlies) {
-        break;
-      }
-    }
-    if (curlies !== 0) {
-      // throw new Error('Unpaired curly in type doc');
-      error = 'Unpaired curly in type doc';
-      pos -= res.length;
-      return '';
-    }
-    return res.substr(1, res.length - 2);
-  }
-  function _name() { // (?:\s+(\S+))?
-    if (error) { return ''; }
-    _skipws();
-    var ch;
-    var res = '';
-    var brackets = 0;
-    var re = /\s/;
-    while (pos < l) {
-      ch = str[pos];
-      brackets += ch === '[' ? 1 : ch === ']' ? -1 : 0;
-      res += ch;
-      pos ++;
-      if (brackets === 0 && re.test(str[pos])) {
-        break;
-      }
-    }
-    if (brackets) {
-      // throw new Error('Unpaired curly in type doc');
-      error = 'Unpaired brackets in type doc';
-      pos -= res.length;
-      return '';
-    }
-    return res;
-  }
-  function _rest() { // (?:\s+([^$]+))?
-    _skipws();
-    return str.substr(pos);
-  }
-}
+var parse  = require('./parser.js');
 
-function parse_chunk(source, base_line_number, opts) {
-  source = source
-    .reduce(function(sections, line) {
-      if (line.value === '' && line.line === base_line_number) return sections;
-      if (line.value.match(/^@(\w+)/)) { sections.push([]); }
-      var section = sections[sections.length - 1];
-      section.line = 'line' in section ? section.line : line.line;
-      section.push(line.value);
-      return sections;
-    }, [[]])
-    .map(function(section) {
-      return {value: section.length ? section.join('\n').trim() : null, line: section.line};
-    });
-
-  var description = source.shift();
-
-  var tags = source.reduce(function(tags, tag) {
-    var tag_node = parse_tag_line(tag.value);
-    if (!tag_node) { return tags; }
-
-    tag_node.line = Number(tag.line);
-    if (opts.raw_value) {
-      tag_node.value = tag.value;
-    }
-
-    // used for split results below
-    var parts;
-
-    // parsing optional and default value if exists
-    // probably if should be hidden with option or moved out to some jsdoc standard
-    if (tag_node.name[0] === '[' && tag_node.name[tag_node.name.length - 1] === ']') {
-      tag_node.optional = true;
-      tag_node.name = tag_node.name.substr(1, tag_node.name.length - 2);
-
-      // default value here
-      if (tag_node.name.indexOf('=') !== -1) {
-        parts = tag_node.name.split('=');
-        tag_node.name    = parts[0];
-        tag_node.default = parts[1].replace(/^(["'])(.+)(\1)$/, '$2');
-      }
-    }
-
-    // hidden with `dotted_names` parsing of `obj.value` naming standard
-    if (opts.dotted_names && tag_node.name.indexOf('.') !== -1) {
-      var parent_name;
-      var parent_tag;
-      var parent_tags = tags;
-      parts = tag_node.name.split('.');
-
-      while (parts.length > 1) {
-        parent_name = parts.shift();
-        parent_tag  = _find(parent_tags, {
-          tag  : tag_node.tag,
-          name : parent_name
-        });
-
-        if (!parent_tag) {
-          parent_tag = {
-            tag         : tag_node.tag,
-            line        : Number(tag_node.line),
-            name        : parent_name,
-            type        : '',
-            description : ''
-          };
-          parent_tags.push(parent_tag);
-        }
-
-        parent_tag.tags = parent_tag.tags || [];
-        parent_tags = parent_tag.tags;
-      }
-
-      tag_node.name = parts[0];
-      parent_tags.push(tag_node);
-      return tags;
-    }
-
-    return tags.concat(tag_node);
-  }, []);
-
-  return {
-    tags        : tags,
-    line        : Number(description.line || 0),
-    description : description.value || ''
-  };
-}
+module.exports = parse;
 
-function mkextract(opts) {
-  var chunk = null;
-  var line_number = 0;
-  var base_line_number = 0;
-
-  return function extract(line) {
-    line_number += 1;
-
-    // if oneliner
-    // then parse it immediately
-    if (!chunk && line.match(RE_COMMENT_1LINE)) {
-      // console.log('line (1)', line, line_number);
-      // console.log('  clean:', line.replace(RE_COMMENT_1LINE, '$1'));
-      return parse_chunk([{value: line.replace(RE_COMMENT_1LINE, '$1'), line: line_number - 1}], line_number - 1, opts);
-    }
-
-    // if start of comment
-    // then init the chunk
-    if (line.match(RE_COMMENT_START)) {
-      // console.log('line (1)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_START, ''));
-      base_line_number = line_number - 1;
-      chunk = [{value: line.replace(RE_COMMENT_START, ''), line: line_number - 1}];
-      return null;
-    }
-
-    // if comment line and chunk started
-    // then append
-    if (chunk && line.match(RE_COMMENT_LINE)) {
-      // console.log('line (2)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_LINE, ''));
-      chunk.push({value: line.replace(RE_COMMENT_LINE, ''), line: line_number - 1});
-      return null;
-    }
-
-    // if comment end and chunk started
-    // then parse the chunk and push
-    if (chunk && line.match(RE_COMMENT_END)) {
-      // console.log('line (3)', line);
-      // console.log('  clean:', line.replace(RE_COMMENT_END, ''));
-      chunk.push({value: line.replace(RE_COMMENT_END, ''), line: line_number - 1});
-      return parse_chunk(chunk, base_line_number, opts);
-    }
-
-    // if non-comment line
-    // then reset the chunk
-    chunk = null;
-  };
-}
-
-/* ------- Transform strean ------- */
+/* ------- Transform stream ------- */
 
 function Parser(opts) {
   opts = opts || {};
   stream.Transform.call(this, {objectMode: true});
-  this._extract = mkextract(opts);
+  this._extract = parse.mkextract(opts);
 }
 
 util.inherits(Parser, stream.Transform);
@@ -252,7 +22,7 @@ util.inherits(Parser, stream.Transform);
 Parser.prototype._transform = function transform(data, encoding, done) {
 
   var block;
-  var lines = data.split(/\n/);
+  var lines = data.toString().split(/\n/);
 
   while (lines.length) {
     block = this._extract(lines.shift());
@@ -264,34 +34,25 @@ Parser.prototype._transform = function transform(data, encoding, done) {
   done();
 };
 
-/* ------- Public API ------- */
-
-module.exports = function parse(source, opts) {
-  opts = opts || {};
-
-  var block;
-  var blocks  = [];
-  var extract = mkextract(opts);
-  var lines   = source.split(/\n/);
-
-  while (lines.length) {
-    block = extract(lines.shift());
-    if (block) {
-      blocks.push(block);
-    }
-  }
-
-  return blocks;
+module.exports.stream = function stream(opts) {
+  return new Parser(opts);
 };
 
+/* ------- File parser ------- */
+
 module.exports.file = function file(file_path, done) {
 
+  var opts = {};
   var collected = [];
 
+  if (arguments.length === 3) {
+    opts = done;
+    done = arguments[2];
+  }
+
   return fs.createReadStream(file_path, {encoding: 'utf8'})
     .on('error', done)
-
-    .pipe(new Parser())
+    .pipe(new Parser(opts))
     .on('error', done)
     .on('data', function(data) {
       collected.push(data);
@@ -301,22 +62,3 @@ module.exports.file = function file(file_path, done) {
     });
 };
 
-module.exports.stream = function stream(opts) {
-  return new Parser(opts);
-};
-
-function _find(list, filter) {
-  var i, l, k, yes, item;
-  for (i = 0, l = list.length; i < l; i++) {
-    item = list[i];
-    yes = true;
-    for (k in filter) {
-      if (filter.hasOwnProperty(k)) {
-        yes = yes && filter[k] === list[i][k];
-      }
-    }
-    if (yes) {
-      return item;
-    }
-  }
-}

package/package.json

@@ -1,19 +1,24 @@
 {
   "name": "comment-parser",
-  "version": "0.2.4",
+  "version": "0.4.0",
   "description": "Generic JSDoc-like comment parser. ",
   "main": "index.js",
   "directories": {
     "test": "tests"
   },
   "dependencies": {
+    "readable-stream": "^2.0.4"
   },
   "devDependencies": {
+    "chai": "~1.9.0",
+    "jshint": "^2.5.10",
+    "jshint-stylish": "^1.0.0",
     "mocha": "~1.17.1",
-    "chai": "~1.9.0"
+    "nodemon": "^1.2.1"
   },
   "scripts": {
-    "test": "mocha tests/*"
+    "test": "jshint --reporter node_modules/jshint-stylish/stylish.js index.js && mocha tests/*",
+    "watch": "nodemon -q -w index.js -w parser.js -w tests/ -x npm test"
   },
   "repository": {
     "type": "git",