comment-parser 1.0.0 → 1.1.2

package/src/parser/spec-parser.ts

@@ -1,11 +1,9 @@
-import { splitSpace, isSpace, seedSpec } from '../util';
 import { Line, Spec } from '../primitives';
-import { Spacer } from './spacer';
+import { seedSpec } from '../util';
+import { Tokenizer } from './tokenizers/index';
 
 export type Parser = (source: Line[]) => Spec;
 
-export type Tokenizer = (spec: Spec) => Spec;
-
 export interface Options {
   tokenizers: Tokenizer[];
 }
@@ -20,169 +18,3 @@ export default function getParser({ tokenizers }: Options): Parser {
     return spec;
   };
 }
-
-export function tagTokenizer(): Tokenizer {
-  return (spec: Spec): Spec => {
-    const { tokens } = spec.source[0];
-    const match = tokens.description.match(/\s*(@(\S+))(\s*)/);
-
-    if (match === null) {
-      spec.problems.push({
-        code: 'spec:tag:prefix',
-        message: 'tag should start with "@" symbol',
-        line: spec.source[0].number,
-        critical: true,
-      });
-      return spec;
-    }
-
-    tokens.tag = match[1];
-    tokens.postTag = match[3];
-    tokens.description = tokens.description.slice(match[0].length);
-
-    spec.tag = match[2];
-    return spec;
-  };
-}
-
-export function typeTokenizer(): Tokenizer {
-  return (spec: Spec): Spec => {
-    let res = '';
-    let curlies = 0;
-    const { tokens } = spec.source[0];
-    const source = tokens.description.trimLeft();
-
-    if (source[0] !== '{') return spec;
-
-    for (const ch of source) {
-      if (ch === '{') curlies++;
-      if (ch === '}') curlies--;
-      res += ch;
-      if (curlies === 0) {
-        break;
-      }
-    }
-
-    if (curlies !== 0) {
-      spec.problems.push({
-        code: 'spec:type:unpaired-curlies',
-        message: 'unpaired curlies',
-        line: spec.source[0].number,
-        critical: true,
-      });
-      return spec;
-    }
-
-    spec.type = res.slice(1, -1);
-    tokens.type = res;
-    [tokens.postType, tokens.description] = splitSpace(
-      source.slice(tokens.type.length)
-    );
-
-    return spec;
-  };
-}
-
-export function nameTokenizer(): Tokenizer {
-  return (spec: Spec): Spec => {
-    const { tokens } = spec.source[0];
-    const source = tokens.description.trimLeft();
-
-    const quotedGroups = source.split('"');
-
-    // if it starts with quoted group, assume it is a literal
-    if (
-      quotedGroups.length > 1 &&
-      quotedGroups[0] === '' &&
-      quotedGroups.length % 2 === 1
-    ) {
-      spec.name = quotedGroups[1];
-      tokens.name = `"${quotedGroups[1]}"`;
-      [tokens.postName, tokens.description] = splitSpace(
-        source.slice(tokens.name.length)
-      );
-      return spec;
-    }
-
-    let brackets = 0;
-    let name = '';
-    let optional = false;
-    let defaultValue;
-
-    // assume name is non-space string or anything wrapped into brackets
-    for (const ch of source) {
-      if (brackets === 0 && isSpace(ch)) break;
-      if (ch === '[') brackets++;
-      if (ch === ']') brackets--;
-      name += ch;
-    }
-
-    if (brackets !== 0) {
-      spec.problems.push({
-        code: 'spec:name:unpaired-brackets',
-        message: 'unpaired brackets',
-        line: spec.source[0].number,
-        critical: true,
-      });
-      return spec;
-    }
-
-    const nameToken = name;
-
-    if (name[0] === '[' && name[name.length - 1] === ']') {
-      optional = true;
-      name = name.slice(1, -1);
-
-      const parts = name.split('=');
-      name = parts[0].trim();
-      defaultValue = parts[1]?.trim();
-
-      if (name === '') {
-        spec.problems.push({
-          code: 'spec:name:empty-name',
-          message: 'empty name',
-          line: spec.source[0].number,
-          critical: true,
-        });
-        return spec;
-      }
-
-      if (parts.length > 2) {
-        spec.problems.push({
-          code: 'spec:name:invalid-default',
-          message: 'invalid default value syntax',
-          line: spec.source[0].number,
-          critical: true,
-        });
-        return spec;
-      }
-
-      if (defaultValue === '') {
-        spec.problems.push({
-          code: 'spec:name:empty-default',
-          message: 'empty default value',
-          line: spec.source[0].number,
-          critical: true,
-        });
-        return spec;
-      }
-    }
-
-    spec.optional = optional;
-    spec.name = name;
-    tokens.name = nameToken;
-
-    if (defaultValue !== undefined) spec.default = defaultValue;
-    [tokens.postName, tokens.description] = splitSpace(
-      source.slice(tokens.name.length)
-    );
-    return spec;
-  };
-}
-
-export function descriptionTokenizer(join: Spacer): Tokenizer {
-  return (spec: Spec): Spec => {
-    spec.description = join(spec.source);
-    return spec;
-  };
-}

package/src/parser/tokenizers/description.ts

@@ -0,0 +1,75 @@
+import { Spec, Line, Markers } from '../../primitives';
+import { Tokenizer } from './index';
+
+/**
+ * Walks over provided lines joining description token into a single string.
+ * */
+export type Joiner = (lines: Line[]) => string;
+
+/**
+ * Shortcut for standard Joiners
+ * compact - strip surrounding whitespace and concat lines using a single string
+ * preserve - preserves original whitespace and line breaks as is
+ */
+export type Spacing = 'compact' | 'preserve' | Joiner;
+
+/**
+ * Makes no changes to `spec.lines[].tokens` but joins them into `spec.description`
+ * following given spacing srtategy
+ * @param {Spacing} spacing tells how to handle the whitespace
+ */
+export default function descriptionTokenizer(
+  spacing: Spacing = 'compact'
+): Tokenizer {
+  const join = getJoiner(spacing);
+  return (spec: Spec): Spec => {
+    spec.description = join(spec.source);
+    return spec;
+  };
+}
+
+export function getJoiner(spacing: Spacing): Joiner {
+  if (spacing === 'compact') return compactJoiner;
+  if (spacing === 'preserve') return preserveJoiner;
+  return spacing;
+}
+
+function compactJoiner(lines: Line[]): string {
+  return lines
+    .map(({ tokens: { description } }: Line) => description.trim())
+    .filter((description) => description !== '')
+    .join(' ');
+}
+
+const lineNo = (num: number, { tokens }: Line, i: number) =>
+  tokens.type === '' ? num : i;
+
+const getDescription = ({ tokens }: Line) =>
+  (tokens.delimiter === '' ? tokens.start : tokens.postDelimiter.slice(1)) +
+  tokens.description;
+
+function preserveJoiner(lines: Line[]): string {
+  if (lines.length === 0) return '';
+
+  // skip the opening line with no description
+  if (
+    lines[0].tokens.description === '' &&
+    lines[0].tokens.delimiter === Markers.start
+  )
+    lines = lines.slice(1);
+
+  // skip the closing line with no description
+  const lastLine = lines[lines.length - 1];
+
+  if (
+    lastLine !== undefined &&
+    lastLine.tokens.description === '' &&
+    lastLine.tokens.end.endsWith(Markers.end)
+  )
+    lines = lines.slice(0, -1);
+
+  // description starts at the last line of type definition
+  lines = lines.slice(lines.reduce(lineNo, 0));
+
+  return lines.map(getDescription).join('\n');
+}

package/src/parser/tokenizers/index.ts

@@ -0,0 +1,8 @@
+import { Spec } from '../../primitives';
+
+/**
+ * Splits `spect.lines[].token.description` into other tokens,
+ * and populates the spec.{tag, name, type, description}. Invoked in a chaing
+ * with other tokens, operations listed above can be moved to separate tokenizers
+ */
+export type Tokenizer = (spec: Spec) => Spec;

package/src/parser/tokenizers/name.ts

@@ -0,0 +1,112 @@
+import { Spec, Line } from '../../primitives';
+import { splitSpace, isSpace, seedBlock } from '../../util';
+import { Tokenizer } from './index';
+
+const isQuoted = (s: string) => s && s.startsWith('"') && s.endsWith('"');
+
+/**
+ * Splits remaining `spec.lines[].tokens.description` into `name` and `descriptions` tokens,
+ * and populates the `spec.name`
+ */
+export default function nameTokenizer(): Tokenizer {
+  const typeEnd = (num: number, { tokens }: Line, i: number) =>
+    tokens.type === '' ? num : i;
+
+  return (spec: Spec): Spec => {
+    // look for the name in the line where {type} ends
+    const { tokens } = spec.source[spec.source.reduce(typeEnd, 0)];
+    const source = tokens.description.trimLeft();
+
+    const quotedGroups = source.split('"');
+
+    // if it starts with quoted group, assume it is a literal
+    if (
+      quotedGroups.length > 1 &&
+      quotedGroups[0] === '' &&
+      quotedGroups.length % 2 === 1
+    ) {
+      spec.name = quotedGroups[1];
+      tokens.name = `"${quotedGroups[1]}"`;
+      [tokens.postName, tokens.description] = splitSpace(
+        source.slice(tokens.name.length)
+      );
+      return spec;
+    }
+
+    let brackets = 0;
+    let name = '';
+    let optional = false;
+    let defaultValue: string;
+
+    // assume name is non-space string or anything wrapped into brackets
+    for (const ch of source) {
+      if (brackets === 0 && isSpace(ch)) break;
+      if (ch === '[') brackets++;
+      if (ch === ']') brackets--;
+      name += ch;
+    }
+
+    if (brackets !== 0) {
+      spec.problems.push({
+        code: 'spec:name:unpaired-brackets',
+        message: 'unpaired brackets',
+        line: spec.source[0].number,
+        critical: true,
+      });
+      return spec;
+    }
+
+    const nameToken = name;
+
+    if (name[0] === '[' && name[name.length - 1] === ']') {
+      optional = true;
+      name = name.slice(1, -1);
+
+      const parts = name.split('=');
+      name = parts[0].trim();
+      if (parts[1] !== undefined)
+        defaultValue = parts.slice(1).join('=').trim();
+
+      if (name === '') {
+        spec.problems.push({
+          code: 'spec:name:empty-name',
+          message: 'empty name',
+          line: spec.source[0].number,
+          critical: true,
+        });
+        return spec;
+      }
+
+      if (defaultValue === '') {
+        spec.problems.push({
+          code: 'spec:name:empty-default',
+          message: 'empty default value',
+          line: spec.source[0].number,
+          critical: true,
+        });
+        return spec;
+      }
+
+      // has "=" and is not a string, except for "=>"
+      if (!isQuoted(defaultValue) && /=(?!>)/.test(defaultValue)) {
+        spec.problems.push({
+          code: 'spec:name:invalid-default',
+          message: 'invalid default value syntax',
+          line: spec.source[0].number,
+          critical: true,
+        });
+        return spec;
+      }
+    }
+
+    spec.optional = optional;
+    spec.name = name;
+    tokens.name = nameToken;
+
+    if (defaultValue !== undefined) spec.default = defaultValue;
+    [tokens.postName, tokens.description] = splitSpace(
+      source.slice(tokens.name.length)
+    );
+    return spec;
+  };
+}

package/src/parser/tokenizers/tag.ts

@@ -0,0 +1,30 @@
+import { Spec } from '../../primitives';
+import { Tokenizer } from './index';
+
+/**
+ * Splits the `@prefix` from remaining `Spec.lines[].token.descrioption` into the `tag` token,
+ * and populates `spec.tag`
+ */
+export default function tagTokenizer(): Tokenizer {
+  return (spec: Spec): Spec => {
+    const { tokens } = spec.source[0];
+    const match = tokens.description.match(/\s*(@(\S+))(\s*)/);
+
+    if (match === null) {
+      spec.problems.push({
+        code: 'spec:tag:prefix',
+        message: 'tag should start with "@" symbol',
+        line: spec.source[0].number,
+        critical: true,
+      });
+      return spec;
+    }
+
+    tokens.tag = match[1];
+    tokens.postTag = match[3];
+    tokens.description = tokens.description.slice(match[0].length);
+
+    spec.tag = match[2];
+    return spec;
+  };
+}

package/src/parser/tokenizers/type.ts

@@ -0,0 +1,92 @@
+import { Spec, Line, Tokens } from '../../primitives';
+import { splitSpace } from '../../util';
+import { Tokenizer } from './index';
+
+/**
+ * Joiner is a function taking collected type token string parts,
+ * and joining them together. In most of the cases this will be
+ * a single piece like {type-name}, but type may go over multipe line
+ * ```
+ * @tag {function(
+ *   number,
+ *   string
+ * )}
+ * ```
+ */
+export type Joiner = (parts: string[]) => string;
+
+/**
+ * Shortcut for standard Joiners
+ * compact - trim surrounding space, replace line breaks with a single space
+ * preserve - concat as is
+ */
+export type Spacing = 'compact' | 'preserve' | Joiner;
+
+/**
+ * Sets splits remaining `Spec.lines[].tokes.description` into `type` and `description`
+ * tokens and populates Spec.type`
+ *
+ * @param {Spacing} spacing tells how to deal with a whitespace
+ * for type values going over multiple lines
+ */
+export default function typeTokenizer(spacing: Spacing = 'compact'): Tokenizer {
+  const join = getJoiner(spacing);
+  return (spec: Spec): Spec => {
+    let curlies = 0;
+    let lines: [Tokens, string][] = [];
+
+    for (const [i, { tokens }] of spec.source.entries()) {
+      let type = '';
+      if (i === 0 && tokens.description[0] !== '{') return spec;
+
+      for (const ch of tokens.description) {
+        if (ch === '{') curlies++;
+        if (ch === '}') curlies--;
+        type += ch;
+        if (curlies === 0) break;
+      }
+
+      lines.push([tokens, type]);
+      if (curlies === 0) break;
+    }
+
+    if (curlies !== 0) {
+      spec.problems.push({
+        code: 'spec:type:unpaired-curlies',
+        message: 'unpaired curlies',
+        line: spec.source[0].number,
+        critical: true,
+      });
+      return spec;
+    }
+
+    const parts: string[] = [];
+    const offset = lines[0][0].postDelimiter.length;
+
+    for (const [i, [tokens, type]] of lines.entries()) {
+      if (type === '') continue;
+      tokens.type = type;
+      if (i > 0) {
+        tokens.type = tokens.postDelimiter.slice(offset) + type;
+        tokens.postDelimiter = tokens.postDelimiter.slice(0, offset);
+      }
+      [tokens.postType, tokens.description] = splitSpace(
+        tokens.description.slice(type.length)
+      );
+      parts.push(tokens.type);
+    }
+
+    parts[0] = parts[0].slice(1);
+    parts[parts.length - 1] = parts[parts.length - 1].slice(0, -1);
+    spec.type = join(parts);
+    return spec;
+  };
+}
+
+const trim = (x: string) => x.trim();
+
+function getJoiner(spacing: Spacing): Joiner {
+  if (spacing === 'compact') return (t: string[]) => t.map(trim).join('');
+  else if (spacing === 'preserve') return (t: string[]) => t.join('\n');
+  else return spacing;
+}

package/src/stringifier/index.ts

@@ -1,5 +1,7 @@
 import { Block, Tokens } from '../primitives';
 
+export type Stringifier = (block: Block) => string;
+
 function join(tokens: Tokens): string {
   return (
     tokens.start +
@@ -16,7 +18,7 @@ function join(tokens: Tokens): string {
   );
 }
 
-export default function getStringifier() {
+export default function getStringifier(): Stringifier {
   return (block: Block): string =>
     block.source.map(({ tokens }) => join(tokens)).join('\n');
 }

package/src/stringifier/inspect.ts

@@ -0,0 +1,64 @@
+import { Block, Line, Tokens } from '../primitives';
+import { seedTokens, isSpace } from '../util';
+
+interface Width {
+  line: number;
+  start: number;
+  delimiter: number;
+  postDelimiter: number;
+  tag: number;
+  postTag: number;
+  name: number;
+  postName: number;
+  type: number;
+  postType: number;
+  description: number;
+  end: number;
+}
+
+const zeroWidth = {
+  line: 0,
+  start: 0,
+  delimiter: 0,
+  postDelimiter: 0,
+  tag: 0,
+  postTag: 0,
+  name: 0,
+  postName: 0,
+  type: 0,
+  postType: 0,
+  description: 0,
+  end: 0,
+};
+
+const fields = Object.keys(zeroWidth);
+
+const repr = (x: string) => (isSpace(x) ? `{${x.length}}` : x);
+
+const frame = (line: string[]) => '|' + line.join('|') + '|';
+
+const align = (width: Width, tokens: Tokens): string[] =>
+  Object.keys(tokens).map((k) => repr(tokens[k]).padEnd(width[k]));
+
+export default function inspect({ source }: Block): string {
+  if (source.length === 0) return '';
+
+  const width: Width = { ...zeroWidth };
+
+  for (const f of fields) width[f] = f.length;
+  for (const { number, tokens } of source) {
+    width.line = Math.max(width.line, number.toString().length);
+    for (const k in tokens)
+      width[k] = Math.max(width[k], repr(tokens[k]).length);
+  }
+
+  const lines: string[][] = [[], []];
+  for (const f of fields) lines[0].push(f.padEnd(width[f]));
+  for (const f of fields) lines[1].push('-'.padEnd(width[f], '-'));
+  for (const { number, tokens } of source) {
+    const line = number.toString().padStart(width.line);
+    lines.push([line, ...align(width, tokens)]);
+  }
+
+  return lines.map(frame).join('\n');
+}

package/src/transforms/align.ts

@@ -23,15 +23,6 @@ const getWidth = (w: Width, { tokens: t }: Line) => ({
   name: Math.max(w.name, t.name.length),
 });
 
-//  /**
-//   * Description may go
-//   * over multiple lines followed by @tags
-//   *
-//* @my-tag {my.type} my-name description line 1
-//      description line 2
-//      * description line 3
-//   */
-
 const space = (len: number) => ''.padStart(len, ' ');
 
 export default function align(): Transform {
@@ -62,15 +53,49 @@ export default function align(): Transform {
         tokens.start = space(w.start + 1);
         break;
       default:
-        tokens.start = space(w.start + 3);
         tokens.delimiter = '';
+        tokens.start = space(w.start + 2); // compensate delimiter
+    }
+
+    if (!intoTags) {
+      tokens.postDelimiter = tokens.description === '' ? '' : ' ';
+      return { ...line, tokens };
+    }
+
+    const nothingAfter = {
+      delim: false,
+      tag: false,
+      type: false,
+      name: false,
+    };
+
+    if (tokens.description === '') {
+      nothingAfter.name = true;
+      tokens.postName = '';
+
+      if (tokens.name === '') {
+        nothingAfter.type = true;
+        tokens.postType = '';
+
+        if (tokens.type === '') {
+          nothingAfter.tag = true;
+          tokens.postTag = '';
+
+          if (tokens.tag === '') {
+            nothingAfter.delim = true;
+          }
+        }
+      }
     }
 
-    if (intoTags) {
+    tokens.postDelimiter = nothingAfter.delim ? '' : ' ';
+
+    if (!nothingAfter.tag)
       tokens.postTag = space(w.tag - tokens.tag.length + 1);
+    if (!nothingAfter.type)
       tokens.postType = space(w.type - tokens.type.length + 1);
+    if (!nothingAfter.name)
       tokens.postName = space(w.name - tokens.name.length + 1);
-    }
 
     return { ...line, tokens };
   }

package/src/transforms/index.ts

@@ -2,9 +2,6 @@ import { Block } from '../primitives';
 
 export type Transform = (Block) => Block;
 
-export { default as indent } from './indent';
-export { default as align } from './align';
-
 export function flow(...transforms: Transform[]): Transform {
   return (block: Block): Block =>
     transforms.reduce((block, t) => t(block), block);

package/src/util.ts

@@ -55,6 +55,11 @@ export function seedTokens(tokens: Partial<Tokens> = {}): Tokens {
   };
 }
 
+/**
+ * Assures Block.tags[].source contains references to the Block.source items,
+ * using Block.source as a source of truth. This is a counterpart of rewireSpecs
+ * @param block parsed coments block
+ */
 export function rewireSource(block: Block): Block {
   const source = block.source.reduce(
     (acc, line) => acc.set(line.number, line),
@@ -65,3 +70,18 @@ export function rewireSource(block: Block): Block {
   }
   return block;
 }
+
+/**
+ * Assures Block.source contains references to the Block.tags[].source items,
+ * using Block.tags[].source as a source of truth. This is a counterpart of rewireSource
+ * @param block parsed coments block
+ */
+export function rewireSpecs(block: Block): Block {
+  const source = block.tags.reduce(
+    (acc, spec) =>
+      spec.source.reduce((acc, line) => acc.set(line.number, line), acc),
+    new Map<number, Line>()
+  );
+  block.source = block.source.map((line) => source.get(line.number) || line);
+  return block;
+}