comment-parser 0.2.4 → 0.3.0

package/.jshintrc

@@ -0,0 +1,11 @@
+{
+  "node":      true,
+  "strict":    true,
+  "maxlen":    100,
+  "undef":     true,
+  "unused":    true,
+  "onecase":   true,
+  "lastsemic": true,
+  "latedef"  : true,
+  "indent":    2
+}

package/README.md

@@ -25,45 +25,104 @@ this would be parsed into following
 
 ```javascript
 [{
-  tags: [{
-      tag: "some-tag",
-      type: "Type",
-      name: "name",
-      line: 15,
-      description: "Singleline or multiline description text",
-      tags: [{
-          tag: "some-tag",
-          type: "Type",
-          name: "subname",
-          line: 16,
-          description: "Singleline or multiline description text",
-          tags: [{
-              tag: "some-tag",
-              type: "Type",
-              name: "subsubname",
-              line: 17,
-              description: "Singleline or\nmultiline description text"
-            }]
-        }]
-   }, {
-      tag: "another-tag",
-      type: "",
-      name: "",
-      line: 18,
-      description: ""
-   }],
-  description: "Singleline or multiline description text. Line breaks are preserved."
+  "tags": [{
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name",
+    "optional": false,
+    "description": "Singleline or multiline description text",
+    "line": 3,
+    "source": "@some-tag {Type} name Singleline or multiline description text"
+  }, {
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name.subname",
+    "optional": false,
+    "description": "Singleline or multiline description text",
+    "line": 4,
+    "source": "@some-tag {Type} name.subname Singleline or multiline description text"
+  }, {
+    "tag": "some-tag",
+    "type": "Type",
+    "name": "name.subname.subsubname",
+    "optional": false,
+    "description": "Singleline or\nmultiline description text",
+    "line": 5,
+    "source": "@some-tag {Type} name.subname.subsubname Singleline or\nmultiline description text"
+  }, {
+    "tag": "another-tag",
+    "name": "",
+    "optional": false,
+    "type": "",
+    "description": "",
+    "line": 7,
+    "source": "@another-tag"
+  }],
+  "line": 0,
+  "description": "Singleline or multiline description text. Line breaks are preserved.",
+  "source": "Singleline or multiline description text. Line breaks are preserved.\n\n@some-tag {Type} name Singleline or multiline description text\n@some-tag {Type} name.subname Singleline or multiline description text\n@some-tag {Type} name.subname.subsubname Singleline or\nmultiline description text\n@another-tag"
 }]
 ```
 
 By default dotted names like `name.subname.subsubname` will be expanded into nested sections, this can be prevented by passing `opts.dotted_names = false`.
 
-You can also make raw line available in parsed results by passing `opts.raw_value = true`.
-
 Invalid comment blocks are skipped. Comments starting with `/*` and `/***` are considered not valid.
 
 Also you can parse entire file with `parse.file('path/to/file', callback)` or acquire an instance of [Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) stream with `parse.stream()`.
 
+Custom parsers
+==============
+
+In case you need to parse tags in different way you can pass `opts.parsers = [parser1, ..., parserN]`, where each parser is `function name(str:String, data:Object):{source:String, data:Object}`.
+	
+Each parser function takes string left after previous parsers applied and data produced by them. And returns `null` or `{source: '', data:{}}` where `source` is consumed substring and `data` is a payload with tag node fields.
+
+Tag node data is build by merging result bits from all parsers. Here is some example that is not doing actual parsing but is demonstrating the flow:
+
+```
+/**
+ * Source to be parsed below
+ * @tag {type} name Description
+ */
+parse(source, {parsers: [
+	// takes entire string
+	function parse_tag(str, data) { 
+		return {source: ' @tag', data: {tag: 'tag'}}; 
+	}, 
+	// parser throwing exception
+	function check_tag(str, data) {
+		if (allowed_tags.indexOf(data.tag) === -1) { 
+			throw new Error('Unrecognized tag "' + data.tag + '"');
+		}			
+	// takes the rest of the string after ' @tag''
+	function parse_name1(str, data) { 
+		return {source: ' name', data: {name: 'name'}}; 
+	},
+	// alternative name parser
+	function parse_name2(str, data) { 
+		return {source: ' name', data: {name: 'name'}}; 
+	}	
+```
+
+This would produce following:
+
+```
+[{
+  "tags": [{
+    "tag": "tag",
+    "type": "type",
+    "name": "name",
+    "optional": false,
+    "description": "Description",
+    "line": 2,
+    "source": "@tag {type} name Description"
+  }],
+  "line": 0,
+  "description": "Source to be parsed below",
+  "source": "Source to be parsed below\n@tag {type} name Description"
+}]
+```
+
 Happy coding :)
 
 

package/index.js

@@ -1,4 +1,6 @@
 
+'use strict';
+
 var fs     = require('fs');
 var stream = require('stream');
 var util   = require('util');
@@ -7,154 +9,235 @@ var RE_COMMENT_START = /^\s*\/\*\*\s*$/m;
 var RE_COMMENT_LINE  = /^\s*\*(?:\s|$)/m;
 var RE_COMMENT_END   = /^\s*\*\/\s*$/m;
 var RE_COMMENT_1LINE = /^\s*\/\*\*\s*(.*)\s*\*\/\s*$/;
-var RE_SPACE = /\s/;
 
-/**
- * analogue of str.match(/@(\S+)(?:\s+\{([^\}]+)\})?(?:\s+(\S+))?(?:\s+([^$]+))?/);
- * @param {string} str raw jsdoc string
- * @returns {object} parsed tag node
- */
-function parse_tag_line(str) {
-  if (typeof str !== 'string') { return false; }
-
-  if (str[0] !== '@') { return false; }
-
-  var pos = 1;
-  var l = str.length;
-  var error = null;
-  var new_line = false;
-  var res = {
-    tag         : _tag(),
-    type        : !new_line && _type() || '',
-    name        : !new_line && _name() || '',
-    description : _rest() || ''
-  };
+/* ------- util functions ------- */
 
-  if (error) {
-    res.error = error;
+function merge(/* ...objects */) {
+  var k, obj, res = {}, objs = Array.prototype.slice.call(arguments);
+  while (objs.length) {
+    obj = objs.shift();
+    for (k in obj) { if (obj.hasOwnProperty(k)) {
+      res[k] = obj[k];
+    }}
   }
-
   return res;
+}
 
-  function _skipws() {
-    var prev_pos = pos;
-    while (pos < l && RE_SPACE.test(str[pos])) {
-      new_line = new_line || str[pos] === '\n';
-      pos++;
-    }
+function find(list, filter) {
+  var k, i = list.length, matchs = true;
+  while (i--) {
+    for (k in filter) { if (filter.hasOwnProperty(k)) {
+        matchs = (filter[k] === list[i][k]) && matchs;
+    }}
+    if (matchs) { return list[i]; }
   }
-  function _tag() { // @(\S+)
-    var sp = str.search(RE_SPACE, pos);
-    sp = sp < 0 ? l : sp;
-    var res = str.substr(pos, sp - pos);
-    pos = sp;
-    return res;
+  return null;
+}
+
+function skipws(str) {
+  var i = 0;
+  do {
+    if (str[i] !== ' ') { return i; }
+  } while (++i < str.length);
+  return i;
+}
+
+/* ------- default parsers ------- */
+
+var PARSERS = {};
+
+PARSERS.parse_tag = function parse_tag(str) {
+  var result = str.match(/^\s*@(\S+)/);
+
+  if (!result) { throw new Error('Invalid `@tag`, missing @ symbol'); }
+
+  return {
+    source : result[0],
+    data   : {tag: result[1]}
+  };
+};
+
+PARSERS.parse_type = function parse_type(str, data) {
+  if (data.errors && data.errors.length) { return null; }
+
+  var pos = skipws(str);
+  var res = '';
+  var curlies = 0;
+
+  if (str[pos] !== '{') { return null; }
+
+  while (pos < str.length) {
+    curlies += (str[pos] === '{' ? 1 : (str[pos] === '}' ? -1 : 0));
+    res += str[pos];
+    pos ++;
+    if (curlies === 0) { break; }
   }
-  function _type() { // (?:\s+\{([^\}]+)\})?
-    _skipws();
-    if (str[pos] !== '{') { return ''; }
-    var ch;
-    var res = '';
-    var curlies = 0;
-    while (pos < l) {
-      ch = str[pos];
-      curlies += ch === '{' ? 1 : ch === '}' ? -1 : 0;
-      res += ch;
-      pos ++;
-      if (!curlies) {
-        break;
-      }
-    }
-    if (curlies !== 0) {
-      // throw new Error('Unpaired curly in type doc');
-      error = 'Unpaired curly in type doc';
-      pos -= res.length;
-      return '';
-    }
-    return res.substr(1, res.length - 2);
+
+  if (curlies !== 0) { throw new Error('Invalid `{type}`, unpaired curlies'); }
+
+  return {
+    source : str.slice(0, pos),
+    data   : {type: res.slice(1, -1)}
+  };
+};
+
+PARSERS.parse_name = function parse_name(str, data) {
+  if (data.errors && data.errors.length) { return null; }
+
+  var pos      = skipws(str);
+  var name     = '';
+  var brackets = 0;
+
+  while (pos < str.length) {
+    brackets += (str[pos] === '[' ? 1 : (str[pos] === ']' ? -1 : 0));
+    name += str[pos];
+    pos ++;
+    if (brackets === 0 && /\s/.test(str[pos])) { break; }
   }
-  function _name() { // (?:\s+(\S+))?
-    if (error) { return ''; }
-    _skipws();
-    var ch;
-    var res = '';
-    var brackets = 0;
-    var re = /\s/;
-    while (pos < l) {
-      ch = str[pos];
-      brackets += ch === '[' ? 1 : ch === ']' ? -1 : 0;
-      res += ch;
-      pos ++;
-      if (brackets === 0 && re.test(str[pos])) {
-        break;
-      }
-    }
-    if (brackets) {
-      // throw new Error('Unpaired curly in type doc');
-      error = 'Unpaired brackets in type doc';
-      pos -= res.length;
-      return '';
+
+  if (brackets !== 0) { throw new Error('Invalid `name`, unpaired brackets'); }
+
+  var res = {name: name, optional: false};
+
+  if (name[0] === '[' && name[name.length - 1] === ']') {
+    res.optional = true;
+    name = name.slice(1, -1);
+
+    if (name.indexOf('=') !== -1) {
+      var parts = name.split('=');
+      name = parts[0];
+      res.default = parts[1].replace(/^(["'])(.+)(\1)$/, '$2');
     }
-    return res;
   }
-  function _rest() { // (?:\s+([^$]+))?
-    _skipws();
-    return str.substr(pos);
+
+  res.name = name;
+
+  return {
+    source : str.slice(0, pos),
+    data   : res
+  };
+};
+
+PARSERS.parse_description = function parse_description(str, data) {
+  if (data.errors && data.errors.length) { return null; }
+
+  var result = str.match(/^\s+([^$]+)?/);
+
+  if (result) {
+    return {
+      source : result[0],
+      data   : {description: result[1] === undefined ? '' : result[1]}
+    };
   }
+
+  return null;
+};
+
+/* ------- parsing ------- */
+
+/**
+ * Parses "@tag {type} name description"
+ * @param {string} str Raw doc string
+ * @param {Array[function]} parsers Array of parsers to be applied to the source
+ * @returns {object} parsed tag node
+ */
+function parse_tag(str, parsers) {
+  if (typeof str !== 'string' || str[0] !== '@') { return null; }
+
+  var data = parsers.reduce(function(state, parser) {
+    var result;
+
+    try {
+      result = parser(state.source, merge({}, state.data));
+      // console.log('----------------');
+      // console.log(parser.name, ':', result);
+    } catch (err) {
+      // console.warn('Parser "%s" failed: %s', parser.name, err.message);
+      state.data.errors = (state.data.errors || [])
+        .concat(parser.name + ': ' + err.message);
+    }
+
+    if (result) {
+      state.source = state.source.slice(result.source.length);
+      state.data   = merge(state.data, result.data);
+    }
+
+    return state;
+  }, {
+    source : str,
+    data   : {}
+  }).data;
+
+  data.optional    = !!data.optional;
+  data.type        = data.type === undefined        ? '' : data.type;
+  data.name        = data.name === undefined        ? '' : data.name;
+  data.description = data.description === undefined ? '' : data.description;
+
+  return data;
 }
 
-function parse_chunk(source, base_line_number, opts) {
+/**
+ * Parses comment block (array of String lines)
+ */
+function parse_block(source, opts) {
+
+  var source_str = source
+      .map(function(line) { return line.source; })
+      .join('\n')
+      .trim();
+
+  var start = source[0].number;
+
+  // merge source lines into tags
+  // we assume tag starts with "@"
   source = source
-    .reduce(function(sections, line) {
-      if (line.value === '' && line.line === base_line_number) return sections;
-      if (line.value.match(/^@(\w+)/)) { sections.push([]); }
-      var section = sections[sections.length - 1];
-      section.line = 'line' in section ? section.line : line.line;
-      section.push(line.value);
-      return sections;
-    }, [[]])
-    .map(function(section) {
-      return {value: section.length ? section.join('\n').trim() : null, line: section.line};
+    .reduce(function(tags, line) {
+      line.source = line.source.trim();
+      
+      if (line.source.match(/^@(\w+)/)) { 
+        tags.push({source: [line.source], line: line.number});
+      } else {
+        var tag = tags[tags.length - 1];
+        tag.source.push(line.source);
+      }
+
+      return tags;
+    }, [{source: []}])
+    .map(function(tag) {
+      tag.source = tag.source.join('\n').trim();
+      return tag;
     });
 
+  // Block description
   var description = source.shift();
 
-  var tags = source.reduce(function(tags, tag) {
-    var tag_node = parse_tag_line(tag.value);
-    if (!tag_node) { return tags; }
-
-    tag_node.line = Number(tag.line);
-    if (opts.raw_value) {
-      tag_node.value = tag.value;
-    }
+  // skip if no descriptions and no tags
+  if (description.source === '' && source.length === 0) { 
+    return null; 
+  }
 
-    // used for split results below
-    var parts;
+  var tags = source.reduce(function(tags, tag) {
+    var tag_node = parse_tag(tag.source, opts.parsers || [
+      PARSERS.parse_tag,
+      PARSERS.parse_type,
+      PARSERS.parse_name,
+      PARSERS.parse_description
+    ]);
 
-    // parsing optional and default value if exists
-    // probably if should be hidden with option or moved out to some jsdoc standard
-    if (tag_node.name[0] === '[' && tag_node.name[tag_node.name.length - 1] === ']') {
-      tag_node.optional = true;
-      tag_node.name = tag_node.name.substr(1, tag_node.name.length - 2);
+    if (!tag_node) { return tags; }
 
-      // default value here
-      if (tag_node.name.indexOf('=') !== -1) {
-        parts = tag_node.name.split('=');
-        tag_node.name    = parts[0];
-        tag_node.default = parts[1].replace(/^(["'])(.+)(\1)$/, '$2');
-      }
-    }
+    tag_node.line   = tag.line;
+    tag_node.source = tag.source;
 
-    // hidden with `dotted_names` parsing of `obj.value` naming standard
     if (opts.dotted_names && tag_node.name.indexOf('.') !== -1) {
       var parent_name;
       var parent_tag;
       var parent_tags = tags;
-      parts = tag_node.name.split('.');
+      var parts = tag_node.name.split('.');
 
       while (parts.length > 1) {
         parent_name = parts.shift();
-        parent_tag  = _find(parent_tags, {
+        parent_tag  = find(parent_tags, {
           tag  : tag_node.tag,
           name : parent_name
         });
@@ -181,37 +264,50 @@ function parse_chunk(source, base_line_number, opts) {
 
     return tags.concat(tag_node);
   }, []);
-
+  
+  // console.log('-----------');
+  // console.log(description, tags);
+  
   return {
     tags        : tags,
-    line        : Number(description.line || 0),
-    description : description.value || ''
+    line        : start,
+    description : description.source,
+    source      : source_str
   };
 }
 
+/**
+ * Produces `extract` function with internal state initialized
+ */
 function mkextract(opts) {
+
   var chunk = null;
-  var line_number = 0;
-  var base_line_number = 0;
+  var number = 0;
 
+  /**
+   * Cumulatively reading lines until they make one comment block
+   * Returns block object or null.
+   */
   return function extract(line) {
-    line_number += 1;
 
     // if oneliner
     // then parse it immediately
-    if (!chunk && line.match(RE_COMMENT_1LINE)) {
-      // console.log('line (1)', line, line_number);
+    if (line.match(RE_COMMENT_1LINE)) {
+      // console.log('line (1)', line);
       // console.log('  clean:', line.replace(RE_COMMENT_1LINE, '$1'));
-      return parse_chunk([{value: line.replace(RE_COMMENT_1LINE, '$1'), line: line_number - 1}], line_number - 1, opts);
+      return parse_block([{
+        source: line.replace(RE_COMMENT_1LINE, '$1'), 
+        number: number}], opts);
     }
 
+    number += 1;
+
     // if start of comment
     // then init the chunk
     if (line.match(RE_COMMENT_START)) {
       // console.log('line (1)', line);
       // console.log('  clean:', line.replace(RE_COMMENT_START, ''));
-      base_line_number = line_number - 1;
-      chunk = [{value: line.replace(RE_COMMENT_START, ''), line: line_number - 1}];
+      chunk = [{source: line.replace(RE_COMMENT_START, ''), number: number - 1}];
       return null;
     }
 
@@ -220,7 +316,7 @@ function mkextract(opts) {
     if (chunk && line.match(RE_COMMENT_LINE)) {
       // console.log('line (2)', line);
       // console.log('  clean:', line.replace(RE_COMMENT_LINE, ''));
-      chunk.push({value: line.replace(RE_COMMENT_LINE, ''), line: line_number - 1});
+      chunk.push({source: line.replace(RE_COMMENT_LINE, ''), number: number - 1});
       return null;
     }
 
@@ -229,8 +325,8 @@ function mkextract(opts) {
     if (chunk && line.match(RE_COMMENT_END)) {
       // console.log('line (3)', line);
       // console.log('  clean:', line.replace(RE_COMMENT_END, ''));
-      chunk.push({value: line.replace(RE_COMMENT_END, ''), line: line_number - 1});
-      return parse_chunk(chunk, base_line_number, opts);
+      chunk.push({source: line.replace(RE_COMMENT_END, ''), number: number - 1});
+      return parse_block(chunk, opts);
     }
 
     // if non-comment line
@@ -239,7 +335,7 @@ function mkextract(opts) {
   };
 }
 
-/* ------- Transform strean ------- */
+/* ------- Transform stream ------- */
 
 function Parser(opts) {
   opts = opts || {};
@@ -284,6 +380,8 @@ module.exports = function parse(source, opts) {
   return blocks;
 };
 
+module.exports.PARSERS = PARSERS;
+
 module.exports.file = function file(file_path, done) {
 
   var collected = [];
@@ -304,19 +402,3 @@ module.exports.file = function file(file_path, done) {
 module.exports.stream = function stream(opts) {
   return new Parser(opts);
 };
-
-function _find(list, filter) {
-  var i, l, k, yes, item;
-  for (i = 0, l = list.length; i < l; i++) {
-    item = list[i];
-    yes = true;
-    for (k in filter) {
-      if (filter.hasOwnProperty(k)) {
-        yes = yes && filter[k] === list[i][k];
-      }
-    }
-    if (yes) {
-      return item;
-    }
-  }
-}

package/package.json

@@ -1,19 +1,22 @@
 {
   "name": "comment-parser",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "description": "Generic JSDoc-like comment parser. ",
   "main": "index.js",
   "directories": {
     "test": "tests"
   },
-  "dependencies": {
-  },
+  "dependencies": {},
   "devDependencies": {
+    "chai": "~1.9.0",
+    "jshint": "^2.5.10",
+    "jshint-stylish": "^1.0.0",
     "mocha": "~1.17.1",
-    "chai": "~1.9.0"
+    "nodemon": "^1.2.1"
   },
   "scripts": {
-    "test": "mocha tests/*"
+    "test": "jshint --reporter node_modules/jshint-stylish/stylish.js index.js && mocha tests/*",
+    "watch": "nodemon -q -w index.js -w tests/ -x npm test"
   },
   "repository": {
     "type": "git",