comment-parser 1.0.0-beta.2 → 1.1.1

package/src/parser/spec-parser.ts

@@ -1,11 +1,9 @@
-import { splitSpace, isSpace, seedSpec } from '../util';
 import { Line, Spec } from '../primitives';
-import { Spacer } from './spacer';
+import { seedSpec } from '../util';
+import { Tokenizer } from './tokenizers/index';
 
 export type Parser = (source: Line[]) => Spec;
 
-export type Tokenizer = (spec: Spec) => Spec;
-
 export interface Options {
   tokenizers: Tokenizer[];
 }
@@ -20,169 +18,3 @@ export default function getParser({ tokenizers }: Options): Parser {
     return spec;
   };
 }
-
-export function tagTokenizer(): Tokenizer {
-  return (spec: Spec): Spec => {
-    const { tokens } = spec.source[0];
-    const match = tokens.description.match(/\s*(@(\S+))(\s*)/);
-
-    if (match === null) {
-      spec.problems.push({
-        code: 'spec:tag:prefix',
-        message: 'tag should start with "@" symbol',
-        line: spec.source[0].number,
-        critical: true,
-      });
-      return spec;
-    }
-
-    tokens.tag = match[1];
-    tokens.postTag = match[3];
-    tokens.description = tokens.description.slice(match[0].length);
-
-    spec.tag = match[2];
-    return spec;
-  };
-}
-
-export function typeTokenizer(): Tokenizer {
-  return (spec: Spec): Spec => {
-    let res = '';
-    let curlies = 0;
-    const { tokens } = spec.source[0];
-    const source = tokens.description.trimLeft();
-
-    if (source[0] !== '{') return spec;
-
-    for (const ch of source) {
-      if (ch === '{') curlies++;
-      if (ch === '}') curlies--;
-      res += ch;
-      if (curlies === 0) {
-        break;
-      }
-    }
-
-    if (curlies !== 0) {
-      spec.problems.push({
-        code: 'spec:type:unpaired-curlies',
-        message: 'unpaired curlies',
-        line: spec.source[0].number,
-        critical: true,
-      });
-      return spec;
-    }
-
-    spec.type = res.slice(1, -1);
-    tokens.type = res;
-    [tokens.postType, tokens.description] = splitSpace(
-      source.slice(tokens.type.length)
-    );
-
-    return spec;
-  };
-}
-
-export function nameTokenizer(): Tokenizer {
-  return (spec: Spec): Spec => {
-    const { tokens } = spec.source[0];
-    const source = tokens.description.trimLeft();
-
-    const quotedGroups = source.split('"');
-
-    // if it starts with quoted group, assume it is a literal
-    if (
-      quotedGroups.length > 1 &&
-      quotedGroups[0] === '' &&
-      quotedGroups.length % 2 === 1
-    ) {
-      spec.name = quotedGroups[1];
-      tokens.name = `"${quotedGroups[1]}"`;
-      [tokens.postName, tokens.description] = splitSpace(
-        source.slice(tokens.name.length)
-      );
-      return spec;
-    }
-
-    let brackets = 0;
-    let name = '';
-    let optional = false;
-    let defaultValue;
-
-    // assume name is non-space string or anything wrapped into brackets
-    for (const ch of source) {
-      if (brackets === 0 && isSpace(ch)) break;
-      if (ch === '[') brackets++;
-      if (ch === ']') brackets--;
-      name += ch;
-    }
-
-    if (brackets !== 0) {
-      spec.problems.push({
-        code: 'spec:name:unpaired-brackets',
-        message: 'unpaired brackets',
-        line: spec.source[0].number,
-        critical: true,
-      });
-      return spec;
-    }
-
-    const nameToken = name;
-
-    if (name[0] === '[' && name[name.length - 1] === ']') {
-      optional = true;
-      name = name.slice(1, -1);
-
-      const parts = name.split('=');
-      name = parts[0].trim();
-      defaultValue = parts[1]?.trim();
-
-      if (name === '') {
-        spec.problems.push({
-          code: 'spec:name:empty-name',
-          message: 'empty name',
-          line: spec.source[0].number,
-          critical: true,
-        });
-        return spec;
-      }
-
-      if (parts.length > 2) {
-        spec.problems.push({
-          code: 'spec:name:invalid-default',
-          message: 'invalid default value syntax',
-          line: spec.source[0].number,
-          critical: true,
-        });
-        return spec;
-      }
-
-      if (defaultValue === '') {
-        spec.problems.push({
-          code: 'spec:name:empty-default',
-          message: 'empty default value',
-          line: spec.source[0].number,
-          critical: true,
-        });
-        return spec;
-      }
-    }
-
-    spec.optional = optional;
-    spec.name = name;
-    tokens.name = nameToken;
-
-    if (defaultValue !== undefined) spec.default = defaultValue;
-    [tokens.postName, tokens.description] = splitSpace(
-      source.slice(tokens.name.length)
-    );
-    return spec;
-  };
-}
-
-export function descriptionTokenizer(join: Spacer): Tokenizer {
-  return (spec: Spec): Spec => {
-    spec.description = join(spec.source);
-    return spec;
-  };
-}

package/src/parser/tokenizers/description.ts

@@ -0,0 +1,75 @@
+import { Spec, Line, Markers } from '../../primitives';
+import { Tokenizer } from './index';
+
+/**
+ * Walks over provided lines joining description token into a single string.
+ * */
+export type Joiner = (lines: Line[]) => string;
+
+/**
+ * Shortcut for standard Joiners
+ * compact - strip surrounding whitespace and concat lines using a single string
+ * preserve - preserves original whitespace and line breaks as is
+ */
+export type Spacing = 'compact' | 'preserve' | Joiner;
+
+/**
+ * Makes no changes to `spec.lines[].tokens` but joins them into `spec.description`
+ * following given spacing srtategy
+ * @param {Spacing} spacing tells how to handle the whitespace
+ */
+export default function descriptionTokenizer(
+  spacing: Spacing = 'compact'
+): Tokenizer {
+  const join = getJoiner(spacing);
+  return (spec: Spec): Spec => {
+    spec.description = join(spec.source);
+    return spec;
+  };
+}
+
+export function getJoiner(spacing: Spacing): Joiner {
+  if (spacing === 'compact') return compactJoiner;
+  if (spacing === 'preserve') return preserveJoiner;
+  return spacing;
+}
+
+function compactJoiner(lines: Line[]): string {
+  return lines
+    .map(({ tokens: { description } }: Line) => description.trim())
+    .filter((description) => description !== '')
+    .join(' ');
+}
+
+const lineNo = (num: number, { tokens }: Line, i: number) =>
+  tokens.type === '' ? num : i;
+
+const getDescription = ({ tokens }: Line) =>
+  (tokens.delimiter === '' ? tokens.start : tokens.postDelimiter.slice(1)) +
+  tokens.description;
+
+function preserveJoiner(lines: Line[]): string {
+  if (lines.length === 0) return '';
+
+  // skip the opening line with no description
+  if (
+    lines[0].tokens.description === '' &&
+    lines[0].tokens.delimiter === Markers.start
+  )
+    lines = lines.slice(1);
+
+  // skip the closing line with no description
+  const lastLine = lines[lines.length - 1];
+
+  if (
+    lastLine !== undefined &&
+    lastLine.tokens.description === '' &&
+    lastLine.tokens.end.endsWith(Markers.end)
+  )
+    lines = lines.slice(0, -1);
+
+  // description starts at the last line of type definition
+  lines = lines.slice(lines.reduce(lineNo, 0));
+
+  return lines.map(getDescription).join('\n');
+}

package/src/parser/tokenizers/index.ts

@@ -0,0 +1,8 @@
+import { Spec } from '../../primitives';
+
+/**
+ * Splits `spect.lines[].token.description` into other tokens,
+ * and populates the spec.{tag, name, type, description}. Invoked in a chaing
+ * with other tokens, operations listed above can be moved to separate tokenizers
+ */
+export type Tokenizer = (spec: Spec) => Spec;

package/src/parser/tokenizers/name.ts

@@ -0,0 +1,112 @@
+import { Spec, Line } from '../../primitives';
+import { splitSpace, isSpace, seedBlock } from '../../util';
+import { Tokenizer } from './index';
+
+const isQuoted = (s: string) => s && s.startsWith('"') && s.endsWith('"');
+
+/**
+ * Splits remaining `spec.lines[].tokens.description` into `name` and `descriptions` tokens,
+ * and populates the `spec.name`
+ */
+export default function nameTokenizer(): Tokenizer {
+  const typeEnd = (num: number, { tokens }: Line, i: number) =>
+    tokens.type === '' ? num : i;
+
+  return (spec: Spec): Spec => {
+    // look for the name in the line where {type} ends
+    const { tokens } = spec.source[spec.source.reduce(typeEnd, 0)];
+    const source = tokens.description.trimLeft();
+
+    const quotedGroups = source.split('"');
+
+    // if it starts with quoted group, assume it is a literal
+    if (
+      quotedGroups.length > 1 &&
+      quotedGroups[0] === '' &&
+      quotedGroups.length % 2 === 1
+    ) {
+      spec.name = quotedGroups[1];
+      tokens.name = `"${quotedGroups[1]}"`;
+      [tokens.postName, tokens.description] = splitSpace(
+        source.slice(tokens.name.length)
+      );
+      return spec;
+    }
+
+    let brackets = 0;
+    let name = '';
+    let optional = false;
+    let defaultValue: string;
+
+    // assume name is non-space string or anything wrapped into brackets
+    for (const ch of source) {
+      if (brackets === 0 && isSpace(ch)) break;
+      if (ch === '[') brackets++;
+      if (ch === ']') brackets--;
+      name += ch;
+    }
+
+    if (brackets !== 0) {
+      spec.problems.push({
+        code: 'spec:name:unpaired-brackets',
+        message: 'unpaired brackets',
+        line: spec.source[0].number,
+        critical: true,
+      });
+      return spec;
+    }
+
+    const nameToken = name;
+
+    if (name[0] === '[' && name[name.length - 1] === ']') {
+      optional = true;
+      name = name.slice(1, -1);
+
+      const parts = name.split('=');
+      name = parts[0].trim();
+      if (parts[1] !== undefined)
+        defaultValue = parts.slice(1).join('=').trim();
+
+      if (name === '') {
+        spec.problems.push({
+          code: 'spec:name:empty-name',
+          message: 'empty name',
+          line: spec.source[0].number,
+          critical: true,
+        });
+        return spec;
+      }
+
+      if (defaultValue === '') {
+        spec.problems.push({
+          code: 'spec:name:empty-default',
+          message: 'empty default value',
+          line: spec.source[0].number,
+          critical: true,
+        });
+        return spec;
+      }
+
+      // has "=" and is not a string, except for "=>"
+      if (!isQuoted(defaultValue) && /=(?!>)/.test(defaultValue)) {
+        spec.problems.push({
+          code: 'spec:name:invalid-default',
+          message: 'invalid default value syntax',
+          line: spec.source[0].number,
+          critical: true,
+        });
+        return spec;
+      }
+    }
+
+    spec.optional = optional;
+    spec.name = name;
+    tokens.name = nameToken;
+
+    if (defaultValue !== undefined) spec.default = defaultValue;
+    [tokens.postName, tokens.description] = splitSpace(
+      source.slice(tokens.name.length)
+    );
+    return spec;
+  };
+}

package/src/parser/tokenizers/tag.ts

@@ -0,0 +1,30 @@
+import { Spec } from '../../primitives';
+import { Tokenizer } from './index';
+
+/**
+ * Splits the `@prefix` from remaining `Spec.lines[].token.descrioption` into the `tag` token,
+ * and populates `spec.tag`
+ */
+export default function tagTokenizer(): Tokenizer {
+  return (spec: Spec): Spec => {
+    const { tokens } = spec.source[0];
+    const match = tokens.description.match(/\s*(@(\S+))(\s*)/);
+
+    if (match === null) {
+      spec.problems.push({
+        code: 'spec:tag:prefix',
+        message: 'tag should start with "@" symbol',
+        line: spec.source[0].number,
+        critical: true,
+      });
+      return spec;
+    }
+
+    tokens.tag = match[1];
+    tokens.postTag = match[3];
+    tokens.description = tokens.description.slice(match[0].length);
+
+    spec.tag = match[2];
+    return spec;
+  };
+}

package/src/parser/tokenizers/type.ts

@@ -0,0 +1,92 @@
+import { Spec, Line, Tokens } from '../../primitives';
+import { splitSpace } from '../../util';
+import { Tokenizer } from './index';
+
+/**
+ * Joiner is a function taking collected type token string parts,
+ * and joining them together. In most of the cases this will be
+ * a single piece like {type-name}, but type may go over multipe line
+ * ```
+ * @tag {function(
+ *   number,
+ *   string
+ * )}
+ * ```
+ */
+export type Joiner = (parts: string[]) => string;
+
+/**
+ * Shortcut for standard Joiners
+ * compact - trim surrounding space, replace line breaks with a single space
+ * preserve - concat as is
+ */
+export type Spacing = 'compact' | 'preserve' | Joiner;
+
+/**
+ * Sets splits remaining `Spec.lines[].tokes.description` into `type` and `description`
+ * tokens and populates Spec.type`
+ *
+ * @param {Spacing} spacing tells how to deal with a whitespace
+ * for type values going over multiple lines
+ */
+export default function typeTokenizer(spacing: Spacing = 'compact'): Tokenizer {
+  const join = getJoiner(spacing);
+  return (spec: Spec): Spec => {
+    let curlies = 0;
+    let lines: [Tokens, string][] = [];
+
+    for (const [i, { tokens }] of spec.source.entries()) {
+      let type = '';
+      if (i === 0 && tokens.description[0] !== '{') return spec;
+
+      for (const ch of tokens.description) {
+        if (ch === '{') curlies++;
+        if (ch === '}') curlies--;
+        type += ch;
+        if (curlies === 0) break;
+      }
+
+      lines.push([tokens, type]);
+      if (curlies === 0) break;
+    }
+
+    if (curlies !== 0) {
+      spec.problems.push({
+        code: 'spec:type:unpaired-curlies',
+        message: 'unpaired curlies',
+        line: spec.source[0].number,
+        critical: true,
+      });
+      return spec;
+    }
+
+    const parts: string[] = [];
+    const offset = lines[0][0].postDelimiter.length;
+
+    for (const [i, [tokens, type]] of lines.entries()) {
+      if (type === '') continue;
+      tokens.type = type;
+      if (i > 0) {
+        tokens.type = tokens.postDelimiter.slice(offset) + type;
+        tokens.postDelimiter = tokens.postDelimiter.slice(0, offset);
+      }
+      [tokens.postType, tokens.description] = splitSpace(
+        tokens.description.slice(tokens.type.length)
+      );
+      parts.push(tokens.type);
+    }
+
+    parts[0] = parts[0].slice(1);
+    parts[parts.length - 1] = parts[parts.length - 1].slice(0, -1);
+    spec.type = join(parts);
+    return spec;
+  };
+}
+
+const trim = (x: string) => x.trim();
+
+function getJoiner(spacing: Spacing): Joiner {
+  if (spacing === 'compact') return (t: string[]) => t.map(trim).join('');
+  else if (spacing === 'preserve') return (t: string[]) => t.join('\n');
+  else return spacing;
+}

package/src/stringifier/inspect.ts

@@ -0,0 +1,64 @@
+import { Block, Line, Tokens } from '../primitives';
+import { seedTokens, isSpace } from '../util';
+
+interface Width {
+  line: number;
+  start: number;
+  delimiter: number;
+  postDelimiter: number;
+  tag: number;
+  postTag: number;
+  name: number;
+  postName: number;
+  type: number;
+  postType: number;
+  description: number;
+  end: number;
+}
+
+const zeroWidth = {
+  line: 0,
+  start: 0,
+  delimiter: 0,
+  postDelimiter: 0,
+  tag: 0,
+  postTag: 0,
+  name: 0,
+  postName: 0,
+  type: 0,
+  postType: 0,
+  description: 0,
+  end: 0,
+};
+
+const fields = Object.keys(zeroWidth);
+
+const repr = (x: string) => (isSpace(x) ? `{${x.length}}` : x);
+
+const frame = (line: string[]) => '|' + line.join('|') + '|';
+
+const align = (width: Width, tokens: Tokens): string[] =>
+  Object.keys(tokens).map((k) => repr(tokens[k]).padEnd(width[k]));
+
+export default function inspect({ source }: Block): string {
+  if (source.length === 0) return '';
+
+  const width: Width = { ...zeroWidth };
+
+  for (const f of fields) width[f] = f.length;
+  for (const { number, tokens } of source) {
+    width.line = Math.max(width.line, number.toString().length);
+    for (const k in tokens)
+      width[k] = Math.max(width[k], repr(tokens[k]).length);
+  }
+
+  const lines: string[][] = [[], []];
+  for (const f of fields) lines[0].push(f.padEnd(width[f]));
+  for (const f of fields) lines[1].push('-'.padEnd(width[f], '-'));
+  for (const { number, tokens } of source) {
+    const line = number.toString().padStart(width.line);
+    lines.push([line, ...align(width, tokens)]);
+  }
+
+  return lines.map(frame).join('\n');
+}

package/src/transforms/index.ts

@@ -2,9 +2,6 @@ import { Block } from '../primitives';
 
 export type Transform = (Block) => Block;
 
-export { default as indent } from './indent';
-export { default as align } from './align';
-
 export function flow(...transforms: Transform[]): Transform {
   return (block: Block): Block =>
     transforms.reduce((block, t) => t(block), block);

package/src/util.ts

@@ -55,6 +55,11 @@ export function seedTokens(tokens: Partial<Tokens> = {}): Tokens {
   };
 }
 
+/**
+ * Assures Block.tags[].source contains references to the Block.source items,
+ * using Block.source as a source of truth. This is a counterpart of rewireSpecs
+ * @param block parsed coments block
+ */
 export function rewireSource(block: Block): Block {
   const source = block.source.reduce(
     (acc, line) => acc.set(line.number, line),
@@ -65,3 +70,18 @@ export function rewireSource(block: Block): Block {
   }
   return block;
 }
+
+/**
+ * Assures Block.source contains references to the Block.tags[].source items,
+ * using Block.tags[].source as a source of truth. This is a counterpart of rewireSource
+ * @param block parsed coments block
+ */
+export function rewireSpecs(block: Block): Block {
+  const source = block.tags.reduce(
+    (acc, spec) =>
+      spec.source.reduce((acc, line) => acc.set(line.number, line), acc),
+    new Map<number, Line>()
+  );
+  block.source = block.source.map((line) => source.get(line.number) || line);
+  return block;
+}

package/tests/e2e/examples.js

@@ -60,12 +60,16 @@ function parse_spacing(source, parse, stringify, transforms) {
    * over multiple lines followed by
    * @param {string} name the name parameter
    *   with multiline description
+   * @param {function(
+   *   number,
+   *   string
+   * )} options the options
    */
 
   const parsed = parse(source, { spacing: 'preserve' });
   const stringified = parsed[0].tags
-    .map((tag) => `@${tag.tag} - ${tag.description}`)
-    .join('\n');
+    .map((tag) => `@${tag.tag} - ${tag.description}\n\n${tag.type}`)
+    .join('\n----\n');
 }
 
 function parse_escaping(source, parse, stringify, transforms) {
@@ -156,11 +160,44 @@ function parse_source_exploration(source, parse, stringify, transforms) {
     .map((s) => `${pos(s.start)} - ${pos(s.end)}\n${s.source}`);
 }
 
+function parse_advanced_parsing(source, parse, _, _, tokenizers) {
+  // Each '@tag ...' section results into Spec. Spec is computed by
+  // the chain of tokenizers each contributing change to the Spec.* and Spec.tags[].tokens.
+  // Default parse() options come with stadart tokenizers
+  // {
+  //   ...,
+  //   spacing = 'compact',
+  //   tokenizers = [
+  //     tokenizers.tag(),
+  //     tokenizers.type(spacing),
+  //     tokenizers.name(),
+  //     tokenizers.description(spacing),
+  //   ]
+  // }
+  // You can reorder those, or even replace any with a custom function (spec: Spec) => Spec
+  // This example allows to parse "@tag description" comments
+
+  /**
+   * @arg0 my parameter
+   * @arg1
+   *   another parameter
+   *      with a strange formatting
+   */
+
+  const parsed = parse(source, {
+    tokenizers: [tokenizers.tag(), tokenizers.description('preserve')],
+  });
+  const stringified = parsed[0].tags
+    .map((tag) => `@${tag.tag} - ${tag.description}`)
+    .join('\n');
+}
+
 (typeof window === 'undefined' ? module.exports : window).examples = [
   parse_defaults,
   parse_line_numbering,
   parse_escaping,
   parse_spacing,
   parse_source_exploration,
+  parse_advanced_parsing,
   stringify_formatting,
 ];

package/tests/e2e/examples.spec.js

@@ -1,9 +1,11 @@
-const { parse, stringify, transforms } = require('../../lib');
+const { parse, stringify, transforms, tokenizers } = require('../../lib');
 const { examples } = require('./examples');
 
 const table = examples.map((fn) => [fn.name.replace(/_/g, ' '), fn]);
 
 test.each(table)('example - %s', (name, fn) => {
   const source = fn.toString();
-  expect(() => fn(source, parse, stringify, transforms)).not.toThrow();
+  expect(() =>
+    fn(source, parse, stringify, transforms, tokenizers)
+  ).not.toThrow();
 });