markdown-magic 2.6.0 → 3.0.0

package/lib/utils/regex-timeout.js

@@ -0,0 +1,83 @@
+// https://stackoverflow.com/questions/38859506/cancel-regex-match-if-timeout
+const util = require('util')
+const vm = require('vm')
+const { getBlockRegex } = require('../block-parser')
+const { getSyntaxInfo } = require('./syntax')
+
+const goodString = `
+# Test Fixture
+
+This is normal text in markdown. <!-- doc-start (wordCount) -->x<!-- doc-end -->
+ Keep it.
+
+<!-- doc-start (wordCount) -->
+THIS CONTENT GETS AUTO GENERATED. Don't directly edit it
+<!-- doc-end -->
+
+This is normal text in markdown. Keep it.
+`
+
+const badString = `
+# Test Fixture
+
+This is normal text in markdown. <!-- doc-start (wordCount) -->x<!-- oc-end -->
+ Keep it.
+
+<!-- doc-start (wordCount) -->
+THIS CONTENT GETS AUTO GENERATED. Don't directly edit it
+<!-- xoc-end -->
+
+This is normal text in markdown. Keep it.
+`
+
+// var pattern = /([ \t]*)(?:<!-{2,}(?:.*|\r?|\n?|\s*)docs-start\s*([(\[\{]*[A-Za-z0-9_$-]*[)\]\}]*)\s*)((?:.*?|.*?\r?\n?)*?)<!-{2,}(?:.*|\r?|\n?|\s*)docs-end(?:.|\r?\n)*?-{2,}>/gim
+
+function safeRegex(str) {
+  const syntaxInfo = getSyntaxInfo('md')
+  if (!syntaxInfo.pattern) {
+    throw new Error(`Unknown syntax "${syntax}"`)
+  }
+  const [ openComment, closeComment ] = syntaxInfo.pattern
+  const pattern = getBlockRegex({
+    openComment,
+    closeComment,
+    openText: 'doc-start',
+    closeText: 'doc-end'
+  })
+  console.log('Pattern', pattern)
+  const sandbox = {
+    // regex: /([ \t]*)(?:<!-{2,}(?:.*|\r?|\n?|\s*)docs-start\s*([(\[\{]*[A-Za-z0-9_$-]*[)\]\}]*)\s*)((?:.*?|.*?\r?\n?)*?)<!-{2,}(?:.*|\r?|\n?|\s*)docs-end(?:.|\r?\n)*?-{2,}>/gim,
+    regexToUse: pattern,
+    string: str,
+    //string: badStringTwo,
+    //string: string,
+  }
+
+  const context = vm.createContext(sandbox)
+  console.log('Sandbox initialized: ' + vm.isContext(sandbox))
+  // var script = new vm.Script('result = regex.exec(string)');
+  const script = new vm.Script(`
+    blocks = []
+    rawMatches = string.match(regexToUse)
+    while ((list = regexToUse.exec(string)) !== null) {
+      const [ block, spaces, __type, params ] = list
+      blocks.push({
+        __type,
+        block
+      })
+      if (list.index === regexToUse.lastIndex) regexToUse.lastIndex++
+    }
+  `);
+  try {
+    // One could argue if a RegExp hasn't processed in a given time.
+    // then, its likely it will take exponential time.
+    script.runInContext(context, { timeout: 1000 }); // milliseconds
+  } catch (e) {
+    console.log('ReDos occurred', e); // Take some remedial action here...
+  }
+  console.log('result:')
+  console.log(util.inspect(sandbox)); // Check the results
+}
+
+// safeRegex(goodString)
+safeRegex(badString)

package/lib/utils/regex.js

@@ -1,19 +1,34 @@
 
-module.exports.matchCommentBlock = function(word) {
+// REGEX to look for regex
+// https://github.com/kgryte/regex-regex/blob/master/lib/index.js
+const REGEX_REGEX = /^\/((?:\\\/|[^\/])+)\/([imgy]*)$/
+
+function escapeRegexString(string) {
+	if (typeof string !== 'string') {
+		throw new TypeError('Expected a string')
+	}
+	// Escape characters with special meaning either inside or outside character sets.
+	// Use a simple backslash escape when it’s always valid, and a `\xnn` escape when the simpler form would be disallowed by Unicode patterns’ stricter grammar.
+	return string
+		.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&')
+		.replace(/-/g, '\\x2d');
+}
+
+function matchCommentBlock(word) {
   //return new RegExp(`(?:\\<\\!--(?:.|\\r?\\n)*?${matchWord}:START(?:.|\\r?\\n)*?\\()(.*)\\)(?:.|\\r?\\n)*?<!--(?:.|\\r?\\n)*?${matchWord}:END(?:.|\\r?\\n)*?--\\>`, 'g')
   return new RegExp(`(.*)(?:\\<\\!--(?:.*|\r?|\n?|\s*)${word}:START(?:.|\\r?\\n)*?\\()(.*)\\)(?:.|\\r?\\n)*?<!--(?:.*|\r?|\n?|\s*)${word}:END(?:.|\\r?\\n)*?--\\>`, 'g')
 }
 
-module.exports.matchOpeningCommentTag = function (word) {
+function matchOpeningCommentTag(word) {
   // return new RegExp(`(\\<\\!--(?:.|\\r?\\n)*?${matchWord}:START)((?:.|\\r?\\n)*?--\\>)`, 'g')
   return new RegExp(`(\\<\\!--(?:.*|\r?|\n?|\s*)${word}:START)((?:.|\\r?\\n)*?--\\>)`, 'g')
 }
 
-module.exports.matchClosingCommentTag = function (word) {
+function matchClosingCommentTag(word) {
   return new RegExp(`--\\>(?:.|\\r?\\n)*?((?:\\<\\!--(?:.*|\\r?\\n)(?:.*|\\r?\\n))*?${word}:END)((?:.|\\r?\\n)*?--\\>)`, 'g')
 }
 
-module.exports.removeLeadingAndTrailingLineBreaks = /^(?:[\t ]*(?:\r?\n|\r))+|\s+$/g
+const removeLeadingAndTrailingLineBreaks = /^(?:[\t ]*(?:\r?\n|\r))+|\s+$/g
 
 const jsRegex = /\/\* Step([\s\S]*?)\*\//g
 const ymlRegex = / *?# Step([\s\S]*?) #\n*?/g
@@ -56,6 +71,15 @@ function getSteps() {
   })
 }
 
+/*
+Alt attribute regex https://regex101.com/r/Ughcb7/1
+const attrNameRegex = "[a-z0-9_]+";
+const attrValueRegex = '(\\\\"|[^"])*';
+const attrRegex = `(${attrNameRegex})\\s*=\\s*"(${attrValueRegex})"`;
+const keyValue = new RegExp(attrRegex, "gi");
+*/
+
+
 // Regex to remove all comment blocks
 const REMOVE_COMMENTS_REGEX = / *?\<\!-- ([\s\S]*?) ?--\>\n\n*?/g
 // content.replace(removeComments, '')
@@ -73,4 +97,13 @@ const TABEL_ROW_REGEX = /^\s*\|.*?\|\s*$/
  * @param {string} text  The text to validate.
  * @returns {boolean}
  */
-const isTableRow = (text) => text.match(TABEL_ROW_REGEX);
+const isTableRow = (text) => text.match(TABEL_ROW_REGEX);
+
+module.exports = {
+  REGEX_REGEX,
+  escapeRegexString,
+  matchCommentBlock,
+  matchOpeningCommentTag,
+  matchClosingCommentTag,
+  removeLeadingAndTrailingLineBreaks,
+}

package/lib/utils/remoteRequest.js

@@ -11,3 +11,57 @@ module.exports = function remoteRequest(url) {
   }
   return body
 }
+
+/*
+TODO add file caching?
+*/
+// const path = require('path')
+// const cacheManager = require('cache-manager')
+// const fsStoreHash = require('cache-manager-fs-hash')
+// const CACHE_KEY = 'foo'
+// const STORAGE_PATH = (process.env.IS_OFFLINE) ? path.join(__dirname, '../tmp') : '/tmp'
+// const SECONDS = 60
+// const MINUTES = 60
+// const ONE_HOUR = SECONDS * MINUTES
+// const mbOfStorage = 512
+// /* initialize caching on disk */
+// const diskCache = cacheManager.caching({
+//   store: fsStoreHash,
+//   options: {
+//     /* TTL in seconds */
+//     ttl: ONE_HOUR,
+//     /* max size in bytes on disk */
+//     // maxsize: mbOfStorage * 1000 * 1000,
+//     path: STORAGE_PATH,
+//   }
+// })
+
+// async function usage() {
+//   const hasCache = await diskCache.get(CACHE_KEY)
+//   if (hasCache && hasCache.length) {
+//     /* If cache NOT empty return it */
+//     // console.log('Using cached value', hasCache)
+//     return hasCache
+//   }
+//   // else do fetch
+//   const data = await getStuff()
+//   // Then save cache
+//   console.log('Saving value')
+//   await diskCache.set(CACHE_KEY, data)
+// }
+
+// function getCacheSize(filePath) {
+//   return new Promise((resolve, reject) => {
+//     fs.stat(filePath, (err, stats) => {
+//       if (err) {
+//         return resolve({ sizeInBytes: 0, sizeInMB: 0 })
+//       }
+//       const byteSize = stats.size
+//       const megaByteSize = byteSize / (1024 * 1024)
+//       return resolve({
+//         sizeInBytes: byteSize,
+//         sizeInMB: megaByteSize
+//       })
+//     })
+//   })
+// }

package/lib/utils/syntax.js

@@ -0,0 +1,79 @@
+const html = {
+  tags: [
+    '<!--', 
+    '-->'
+  ],
+  pattern: [
+    '<!-{2,}', 
+    '-{2,}>' // '-->'
+  ],
+}
+
+// Kramdown https://github.com/HHS/nih-oite-website/blob/480244abfd0f0d741b27d6232273dd03073e06ad/app/javascript/kramdown/parser.js#L12-L48
+
+// JS https://regex101.com/r/XKHU18/5
+const js = {
+  tags: ['/*', '*/'],
+  pattern: [
+    '\/\\*+',
+    // Old ^ '\/\\*{1,}[\n\\*]*',     // '\/\\*+', '\/\*[\*\n\s\t]+', // 
+    '\\*+/'
+  ],
+  /* Match single line JS comment */
+  singleLineTag: '//',
+  singleLinePattern: '//+',
+  singleLine: '\/\/.*$'
+}
+
+const jsx = {
+  tags: [
+    '{/*', 
+    '*/}'
+  ],
+  pattern: [
+    '\{\/\\*+',
+    '\\*+/\}'
+  ]
+}
+
+const yaml = {
+  tags: ['##', '##'],
+  pattern: [
+    '##+',
+    '##+'
+  ],
+  singleLineTag: '#',
+  singleLinePattern: '#+',
+  singleLine: '#.*$',
+  content: '[ \\t\\S]*?',
+  converter: (str) => {
+    return str.split('\n').map((line) => {
+      return line[0] === '#' ? line : `#${line}`
+    }).join()
+  }
+}
+
+const syntaxMap = {
+  // <!-- x -->
+  md: html,
+  markdown: html,
+  // <!-- x -->
+  html: html,
+  // /* x */
+  js: js,
+  // {/* x */}
+  jsx: jsx,
+  mdx: jsx,
+  // ## x ##
+  yaml: yaml,
+  yml: yaml
+}
+
+function getSyntaxInfo(syntax = '') {
+  return syntaxMap[syntax.toLowerCase()] || {}
+}
+
+module.exports = {
+  getSyntaxInfo,
+  syntaxMap
+}

package/lib/utils/text.js

@@ -0,0 +1,260 @@
+const { syntaxMap } = require('./syntax')
+
+function getLines(str = '') {
+  return str.split(/\r\n|\r|\n/)
+}
+
+function getLineCount(str = '') {
+  return getLines(str).length
+}
+
+function getRowAndColumnFromCharPos(input, indexToFind) {
+  const preChunk = input.substr(0, indexToFind);
+  const row = preChunk.split('\n').length - 1
+  const lastIndexOfNewLine = input.lastIndexOf('\n', indexToFind);
+  const col = lastIndexOfNewLine > 0 ? indexToFind - lastIndexOfNewLine - 1 : indexToFind;
+  return { row, col }
+};
+
+function getWordCount(str = '') {
+  return str.trim().split(/\s+/).length
+}
+
+function getFirstCharacter(str) {
+  return str.charAt(0)
+}
+
+function getLastCharacter(str) {
+  return str.substr(-1)
+}
+
+function getLeadingSpaces(text) {
+  const matches = text.match(/^\s*/)
+  return (matches && matches[0]) ? matches[0] : ''
+}
+
+function getTextBetweenChars(text, start, end) {
+  return text.slice(start, end)
+}
+
+function replaceTextBetweenChars(str = '', start, end, newStr) {
+  return str.substring(0, start) + newStr + str.substring(end)
+}
+
+function getTextBetweenLines(content, startLine, endLine) {
+  const startDefined = typeof startLine !== 'undefined'
+  const endDefined = typeof endLine !== 'undefined'
+  if (!startDefined && !endDefined) return
+
+  const lines = getLines(content)
+  if (startDefined && !endDefined) {
+    return lines.slice(startLine - 1, startLine).join('')
+  }
+  if ((startLine) && (endLine) && parseInt(startLine, 10) <= parseInt(endLine, 10)) {
+    return lines.slice(startLine - 1, endLine).join('\n')
+  }
+}
+
+function isUpperCase(str) {
+  return str === str.toUpperCase()
+}
+
+// https://github.com/jamiebuilds/min-indent/blob/master/index.js
+function findMinIndent(string) {
+	const match = string.match(/^[ \t]*(?=\S)/gm)
+	if (!match) return 0
+	return match.reduce((r, a) => Math.min(r, a.length), Infinity)
+}
+
+function stripIndent(string, indentation) {
+  const indent = typeof indentation !== 'undefined' ? indentation : findMinIndent(string);
+	if (indent === 0) {
+		return string
+	}
+	const regex = new RegExp(`^[ \\t]{${indent}}`, 'gm')
+	return string.replace(regex, '')
+}
+
+/**
+ * Trim leading & trailing spaces/line breaks in code and keeps the indentation of the first non-empty line
+ * @param {string} str 
+ * @returns string
+ */
+ function trimString(str = '') {
+  let content = (typeof str === 'number') ? str.toString() : str
+  // console.log('content', `"${content}"`)
+  return content.replace(/^(?:[\t ]*(?:\r?\n|\r))+|\s+$/g, '')
+}
+
+function indentString(string, count = 1, options = {}) {
+	const {
+		indent = ' ',
+		includeEmptyLines = false
+	} = options;
+	if (count === 0) return string
+	const regex = includeEmptyLines ? /^/gm : /^(?!\s*$)/gm
+	return string.replace(regex, indent.repeat(count))
+}
+
+/**
+ * Removes the indentation of multiline strings
+ * @link https://github.com/victornpb/tiny-dedent/
+ * @param  {string} str A template literal string
+ * @return {string} A string without the indentation
+ */
+function dedentString(str) {
+  str = str.replace(/^[ \t]*\r?\n/, ''); // remove leading blank line
+  var indent = /^[ \t]+/m.exec(str); // detected indent
+  if (indent) str = str.replace(new RegExp('^' + indent[0], 'gm'), ''); // remove indent
+  return str.replace(/(\r?\n)[ \t]+$/, '$1'); // remove trailling blank line
+}
+
+/**
+ * Strip out comment blocks
+ * @param {string} str 
+ * @param {'md' | 'js'} syntax 
+ * @returns {string} clean commentless string
+ */
+function stripCommentBlockOld(str, syntax = 'md') {
+  const [ openPattern, closePattern ] = syntaxMap[syntax].pattern
+  const pattern = new RegExp(`^([ \\S]*)${openPattern}(\\s?[\\s\\S]*?)?${closePattern}\n?`, 'gim')
+  // console.log('pattern', pattern)
+  let newString = str
+  let matches
+  while ((matches = pattern.exec(str)) !== null) {
+    if (matches.index === pattern.lastIndex) {
+      pattern.lastIndex++ // avoid infinite loops with zero-width matches
+    }
+    const [ _match, leadingText ] = matches
+    /*
+    console.log('_match', _match)
+    console.log('leadingText', `"${leadingText}"`)
+    console.log('───────────────────────')
+    /**/
+    /* Handle comments that start midway through line after text */
+    if (leadingText) {
+      /* Trim trailing tabs/spaces */
+      const trimmed = leadingText.replace(/([ \t]*)$/, '')
+      const replacement = _match.replace(trimmed, '')
+      // console.log('replacement', `"${replacement}"`)
+      newString = newString.replace(replacement, `\n`)
+      // console.log('new str', newString)
+    }
+  }
+
+  // const pattern = new RegExp(`([ \\t]*)${openPattern}\\s?([\\s\\S]*?)?${closePattern}\n?`, 'gi')
+  return newString.replace(pattern, '')
+}
+
+/**
+ * Strip out comment blocks
+ * @param {string} str 
+ * @param {'md' | 'js'} syntax 
+ * @returns {string} clean commentless string
+ */
+function stripComments(str, { syntax = 'md' }) {
+  const syntaxData = syntaxMap[syntax]
+  const [ openPattern, closePattern ] = syntaxData.pattern
+  const OR = (syntaxData.singleLine) ? `|\\s?[ \\t]*${syntaxData.singleLine}` : ''
+  const CONTENT = syntaxData.content || '[\\s\\S]*?'
+  const pattern = new RegExp(`\\s?[ \\t]*${openPattern}(${CONTENT})?${closePattern}${OR}`, 'gim')
+  console.log('pattern', pattern)
+  return str.replace(pattern, '')
+  // https://regex101.com/r/XKHU18/5
+  return str.replace(/\s?[ \t]*\/\*[\s\S]*?\*\/|\s?[ \t]*\/\/.*$|\/\*{1,}[\n\*]*(\s?[\s\S]*?)?\*+\//gm, '')
+}
+
+// @TODO export as util to import into CODE
+function stripAllComments(block) {
+  // ([^\s]*)?([ \t]*)?(\/\*{1,}[\n\*]*(\s?[\s\S]*?)?\*\/)([^\s<]*)?(\n{1,2})?
+  // https://regex101.com/r/WSioZ7/1
+  const pattern = new RegExp(`([^\\s]*)?([ \\t]*)?(<!-{2,}(\\s?[\\s\\S]*?)?-{2,}>)([^\\s<]*)?(\n{1,2})?`, 'gi')
+  // ALT https://regex101.com/r/hxppia/1
+  // Alt HTML comments https://regex101.com/r/EJyioz/1
+
+  // console.log('closeTagRegex', closeTagRegex)
+  let matches
+  let remove = []
+  while ((matches = pattern.exec(block)) !== null) {
+    if (matches.index === pattern.lastIndex) {
+      pattern.lastIndex++ // avoid infinite loops with zero-width matches
+    }
+    const [ match, leadingText, leadingSpace, comment, insideComment, trailingText, trailingNewLine ] = matches
+    /*
+    console.log('match', match)
+    console.log('leadingText', leadingText)
+    console.log('leadingSpace', leadingSpace)
+    console.log('comment', comment)
+    console.log('insideComment', insideComment)
+    console.log('trailingText', trailingText)
+    console.log('trailingNewLine', trailingNewLine)
+    /** */
+    const newLineCount = (trailingNewLine || '').length
+    const trailing = (!trailingText && newLineCount > 1) ? `${trailingNewLine || ''}` : ''
+    const leading = (leadingSpace) ? leadingSpace.slice(1) : ''
+    remove.push(`${leading}${comment}${trailing}`)
+  }
+  return remove.reduce((acc, curr) => {
+    return acc.replaceAll(curr, '')
+  }, block)
+}
+
+function convertCommentSyntax(str, { from, to }) {
+  const syntaxData = syntaxMap[from]
+  const [ openPattern, closePattern ] = syntaxData.pattern
+  const [ openTag, closeTag ] = syntaxMap[to].tags
+  // const match = ` *?\\${openPattern}([\\s\\S]*?)?${closePattern}\\n\\n*?`
+  const UseLeadingLine = ''// || '|\\s?[ \\t]*'
+  const OR = (syntaxData.singleLine) ? `|${UseLeadingLine}${syntaxData.singleLine}` : ''
+  const CONTENT = syntaxData.content || '[\\s\\S]*?'
+  const match = `${UseLeadingLine}${openPattern}(${CONTENT})?${closePattern}${OR}`
+  // const match = `${openPattern}(.*|\\r?|\\n?|\\s*)*${closePattern}`
+  const regexToUse = new RegExp(match, 'gm')
+  // console.log('regexToUse', regexToUse)
+  const found = str.match(regexToUse)
+  // console.log('found', found)
+  if (!found) {
+    return str
+  }
+
+  const pattern = syntaxData.singleLinePattern ? new RegExp(syntaxData.singleLinePattern) : new RegExp(openPattern)
+  // console.log('pattern', pattern)
+  for (let index = 0; index < found.length; index++) {
+    const comment = found[index]
+    const cleanComment = comment.replace(pattern, '')
+    /*
+    console.log('comment', comment)
+    console.log('cleanComment', cleanComment)
+    /** */
+    str = str.replace(comment, `${openTag}${cleanComment} ${closeTag}`)
+  }
+  return str
+
+  const newComment = found[0].replace(regexToUse, `${openTag}$1${closeTag}`)
+  const converter = syntaxMap[to].converter
+  const newText = (converter) ? converter(newComment) : newComment
+  return str.replace(regexToUse, newText)
+}
+
+module.exports = {
+  getLines,
+  getLineCount,
+  getWordCount,
+  getLeadingSpaces,
+  getFirstCharacter,
+  getLastCharacter,
+  getRowAndColumnFromCharPos,
+  getTextBetweenChars,
+  getTextBetweenLines,
+  replaceTextBetweenChars,
+  stripIndent,
+  indentString,
+  dedentString,
+  stripComments,
+  convertCommentSyntax,
+  // stripCommentBlockJS,
+  trimString,
+  // future https://github.com/junfengliang/autowrap
+  findMinIndent,
+  isUpperCase
+}