markdown-magic 2.6.1 → 3.0.1

package/lib/transforms/remote.js

@@ -1,15 +1,17 @@
-const regexUtils = require('../utils/regex')
 const remoteRequest = require('../utils/remoteRequest')
 
-module.exports = function REMOTE(content, options, config) {
-  const remoteContent = remoteRequest(options.url) || content
+module.exports = function REMOTE(api) {
+  // console.log('REMOTE api', api)
+  const { options, content, settings } = api
+  const { regex } = settings
+  // console.log('MAKE REMOTE REQUEST')
+  const remoteContent = remoteRequest(options.url)
   if (!remoteContent) {
     return content
   }
   if (options.keepComments) {
     return remoteContent
   }
-  const openTagRegex = regexUtils.matchOpeningCommentTag(config.matchWord)
-  const closeTagRegex = regexUtils.matchClosingCommentTag(config.matchWord)
-  return remoteContent.replace(openTagRegex, '').replace(closeTagRegex, '')
+  // console.log('REMOTE', remoteContent)
+  return remoteContent.replace(regex.open, '').replace(regex.close, '')
 }

package/lib/transforms/sectionToc.js

@@ -0,0 +1,18 @@
+const { generateToc } = require('../utils/toc')
+
+module.exports = async function sectionToc(api) {
+  // console.log("sectionToc", api)
+  const { options, fileContent, srcPath, getBlockDetails } = api
+  const opts = options || {}
+  const subToc = await generateToc({
+    options: {
+      ...opts,
+      // Set sub table of contents
+      sub: true
+    },
+    fileContent,
+    srcPath,
+    getBlockDetails: getBlockDetails
+  })
+  return subToc
+}

package/lib/transforms/toc.js

@@ -1,266 +1,13 @@
-const { transform } = require('@technote-space/doctoc')
-const { removeLeadingAndTrailingLineBreaks } = require('../utils/regex')
-
-module.exports = async function TOC(content, options, config) {
-  const opts = options || {}
-  opts.firsth1 = (opts.firsth1) ? true : false
-  let contents = config.outputContent
-  // console.log('contents', contents)
-
-  let debugFileMatch
-  // console.log('config', config.originalPath)
-  // debugFileMatch = config.originalPath.match(/packages\/analytics\/README\.md/)
-
-  // https://www.npmjs.com/package/@technote-space/toc-generator
-  const t = await transform(contents, {
-    // mode: 'github.com', // github.com | bitbucket.org | gitlab.com | nodejs.org | ghost.org (default: github.com)
-    isNotitle: true,
-    //isCustomMode: true,
-    openingComment: '',
-    closingComment: '', 
-    maxHeaderLevel: (opts.maxDepth) ? Number(opts.maxDepth) : 4
-    // maxHeaderLevel: 2, // default: 4
-    // title: '**Table of Contents**',
-    // isNotitle: true,
-    // isFolding: true,
-    // entryPrefix: '*',
-    // processAll: true,
-    // updateOnly: true,
-    // openingComment: '<!-- toc -->',
-    // closingComment: '<!-- tocstop --> ',
-    // checkOpeningComments: ['<!-- toc '],
-    // checkClosingComments: ['<!-- tocstop '],
-    // isCustomMode: false,
-    // customTemplate: '<p align="center">${ITEMS}</p>',
-    // itemTemplate: '<a href="${LINK}">${TEXT}</a>',
-    // separator: '<span>|</span>',
-    // footer: '',
+const { generateToc } = require('../utils/toc')
+
+module.exports = async function TOC(api) {
+  // console.log("TOC API", api)
+  const { options, fileContent, srcPath, getBlockDetails } = api
+  const toc = await generateToc({
+    options: options || {},
+    fileContent,
+    srcPath,
+    getBlockDetails: getBlockDetails
   })
-  let output = t.wrappedToc || ''
-
-  if (debugFileMatch) {
-    console.log('before firsth1 removal', output)
-  }
-
-  /* remove first h1 */
-  if (!opts.firsth1) {
-    // console.log('output', output)
-    const lines = output.split('\n')
-    let firstH1
-    let firstLine
-    const countOfH1s = lines.reduce((acc, line, index) => {
-      if (line !== '' && !firstLine) {
-        firstLine = index
-      }
-      if (line.match(/^-\s+/)) {
-        if (!firstH1) {
-          const rawText = line.match(/\[(.*)\]/)[1]
-          firstH1 = [line, index, firstLine === index, rawText]
-        }
-        acc = acc + 1
-      }
-      return acc
-    }, 0)
-
-    const firstHeadingData =  (firstH1 && Array.isArray(firstH1)) ? firstH1 : []
-    const [ lineText, lineIndex, isFirst, matchText ] = firstHeadingData
-    
-    // verify its h1
-    const matchTextEscaped = escapeStringRegexp(matchText)
-    // console.log('matchTextEscaped', matchTextEscaped)
-    // /^#{1}\s+(.*)/
-    const FIRST_H1_REGEX = new RegExp(`^#\\s*\\[?${matchTextEscaped}\\]?(?:.*)?`, 'gim')
-    // /^<h1\b[^>]*>([\s\S]*?)<\/h1>/
-    const FIRST_HTML_H1_REGEX = new RegExp(`^<h1\\b[^>]*>[\\s]*?(${matchTextEscaped})[\\s]*?<\\/h1>`, 'gim')
-    // /^(.*)\n={3,}/
-    const FIRST_UNDERSCORE_H1 = new RegExp(`^(${matchTextEscaped})\n={3,}`, 'gim')
-    
-    let docHasHeading = false
-    if (contents.match(FIRST_H1_REGEX)) {
-      docHasHeading = matchText
-    } else if (contents.match(FIRST_HTML_H1_REGEX)) {
-      docHasHeading = matchText
-    } else if (contents.match(FIRST_UNDERSCORE_H1)) {
-      docHasHeading = matchText
-    }
-  
-    if (debugFileMatch) {
-      console.log('matchText', matchText)
-      if (docHasHeading) {
-        console.log('Found heading 1', docHasHeading)
-      }
-    }
-
-    if (debugFileMatch) {
-      console.log('top level toc item count', countOfH1s)
-      if (docHasHeading) {
-        console.log('Found heading 1', docHasHeading)
-        console.log('firstH1', firstH1)
-      }
-    }
-
-    let firstItemWithContent
-    let foundFirstH1
-
-    if (docHasHeading) {
-      output = lines.reduce((acc, line, i) => {
-        const isLineEmpty = line === ''
-        if (!isLineEmpty && !firstItemWithContent) {
-          firstItemWithContent = i
-        }
-        if (!foundFirstH1 && (firstItemWithContent === i) && line.match(/^-\s+/)) {
-          foundFirstH1 = true
-          return acc
-        }
-        // Add back line and remove level
-        let newLineValue = line
-        if (lineText) {
-          // const untilFirstH1 = i < lineIndex
-          /* @TODO make option? flatten all non first h1 tags */
-          if (countOfH1s > 0 && !isLineEmpty && newLineValue.match(/^\s+-/)) {
-            newLineValue = line.substring(2)
-          }
-        }
-        
-        acc = acc.concat(newLineValue)
-        return acc
-      }, []).join('\n')
-    }
- 
-    // console.log('output', output)
-    if (debugFileMatch) {
-      console.log('after firsth1 removal', output)
-    }
-  }
-  // console.log(t)
-  // Fix <h1> with multiple lines
-  const spaces = output.match(/\[\n([\s\S]*?)\]/gm)
-  if (spaces) {
-    const fixed = spaces[0]
-      // all new lines
-      .replace(/\n/g, '')
-      // leading spaces
-      .replace(/\[\s+/, '[')
-      .trim()
-    output = output.replace(spaces[0], fixed)
-  }
-  // console.log(output)
-  output = output
-    .replace(/(.*)\[Table of Contents\]\(.*\)\n?/i, '')
-    .replace(/(.*)\[toc\]\(.*\)\n?/i, '')
-
-
-  if (opts.excludeText) {
-    // (\s+)?-(.*)\[Usage\]\(.*\)
-    const regex = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)`, 'i')
-    // /(\s+)?-(.*)\[Usage\]\(.*\)(\n\s+(.*))+/im
-    const nestedRegex = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)\(\\n\\s+\(.*\)\)+`, 'i')
-    
-    const hasNested = nestedRegex.exec(output)
-    if (hasNested) {
-      // Count indentation of spaces
-      const numberOfSpaces = hasNested[1].replace(/\n/g, '').length
-      const subItems = numberOfSpaces + 1
-      // Update regex to only remove sub items
-      const nestedRegexSpaces = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)\(\\n\\s{${subItems},}\(.*\)\)+`, 'i')
-      // console.log('nestedRegexSpaces', nestedRegexSpaces)
-      // If exclude value has nested sections remove them as well.
-      output = output.replace(nestedRegexSpaces, '')
-      output = output.replace(regex, '')
-    } else {
-      output = output.replace(regex, '')
-    }
-  }
-
-  // If collapse wrap in <details>
-  if (opts.collapse) {
-    const text = (opts.collapseText) ? opts.collapseText : 'Table of Contents'
-    return `<details>
-<summary>${text}</summary>
-${output   
-  // Replace leading double spaces
-  .replace(/^\n\n/, '\n')
-  // Replace trailing double spaces
-  .replace(/\n\n$/, '\n')}
-</details>`
-  }
-
-  return output.replace(removeLeadingAndTrailingLineBreaks, '')
-}
-
-const HTML_TAG = /<([a-zA-Z1-6]+)([^<]+)*(?:>(.*)<\/\1>|\s+\/>)/gim
-
-function parseHtmlProps(mdContents) {
-  const htmlTags = mdContents
-    /* Fix non terminating <tags> */
-    .replace(/(['"`]<(.*)>['"`])/gm, '_$2_')
-    .match(HTML_TAG)
-  //console.log('htmlTags', htmlTags)
-
-  let tags = []
-  if (htmlTags) {
-    let propsValues = {}
-    var regexSingleTag = /<([a-zA-Z1-6]+)([^<]+)*(?:>(.*)<\/\1>|\s+\/>)/
-    for (var i = 0; i < htmlTags.length; i++) {
-      var tagMatches = regexSingleTag.exec(htmlTags[i])
-      // console.log('tagMatches', tagMatches)
-      var [ match, tag, props ] = tagMatches
-      // console.log(`Tag #${i} ${tag}`)
-      if (props) {
-        const cleanProps = props
-          // Remove new lines and tabs
-          .replace(/\n\t/g, '')
-          // Remove extra spaces
-          .replace(/\s\s+/g, ' ')
-          .trim()
-
-        propsValues = cleanProps.split(" ").reduce((acc, curr) => {
-          const hasQuotes = curr.match(/=['"]/)
-          // Check key="value" | key='value' |  key={value}
-          const propWithValue = /([A-Za-z-_$]+)=['{"](.*)['}"]/g.exec(curr)
-          if (propWithValue) {
-            return {
-              ...acc,
-              [`${propWithValue[1]}`]: (hasQuotes) ? propWithValue[2] : convert(propWithValue[2])
-            }
-          }
-          // Check isLoading boolean props
-          const booleanProp = curr.match(/([A-Za-z]*)/)
-          if (booleanProp) {
-            return {
-              ...acc,
-              [`${booleanProp[1]}`]: true
-            }
-          }
-          return acc
-        }, {})
-      }
-
-      tags.push({
-        tag: tag,
-        props: propsValues,
-        raw: match
-      })
-    }
-  }
-  return tags
-}
-
-function escapeStringRegexp(string) {
-	if (typeof string !== 'string') {
-		throw new TypeError('Expected a string');
-	}
-
-	// Escape characters with special meaning either inside or outside character sets.
-	// Use a simple backslash escape when it’s always valid, and a `\xnn` escape when the simpler form would be disallowed by Unicode patterns’ stricter grammar.
-	return string
-		.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&')
-		.replace(/-/g, '\\x2d');
-}
-
-function removeUndefined(output) {
-  // remove undefined from new line and start of string if first H1 is missing
-  return output.replace(/\nundefined/g, '\n-').replace(/^undefined/g, '-')
-}
-
-// Alt https://github.com/thlorenz/doctoc
+  return toc
+}

package/lib/transforms/wordCount.js

@@ -0,0 +1,5 @@
+const { getWordCount } = require('../utils/text')
+
+module.exports = function wordCount({ fileContent }) {
+  return getWordCount(fileContent)
+}

package/lib/types.js

@@ -0,0 +1,11 @@
+/**
+ * Allowed file syntaxes
+ * @typedef {'md' | 'js' | 'yml' | 'yaml'} SyntaxType
+ */
+
+/** @type {SyntaxType} */
+const syntaxType = 'md'
+
+module.exports = {
+  syntaxType
+}

package/lib/utils/fs.js

@@ -0,0 +1,342 @@
+const fs = require('fs').promises
+const path = require('path')
+const globrex = require('globrex')
+const isGlob = require('is-glob')
+const isLocalPath = require('is-local-path')
+const { REGEX_REGEX, escapeRegexString } = require('./regex') 
+const { dirname, resolve, join } = require('path')
+const { readdir, stat, readFile } = fs
+
+const IS_HIDDEN_FILE = /(^|[\\\/])\.[^\\\/\.]/g
+
+function isRegex(thing) {
+  return (thing instanceof RegExp)
+}
+
+async function writeFile(filePath, content) {
+  try {
+    await fs.writeFile(filePath, content)
+  } catch(e) {
+    const dirName = path.dirname(filePath)
+    await fs.mkdir(dirName, { recursive: true })
+    await fs.writeFile(filePath, content)
+  }
+}
+
+async function findUp(start, fileName) {
+  const file = await escalade(start, (dir, relativePaths) => {
+    // console.log('~> dir:', dir);
+    // console.log('~> relativePaths:', relativePaths);
+    // console.log('---')
+    if (typeof fileName === 'string' && relativePaths.includes(fileName)) {
+      // will be resolved into absolute
+      return fileName
+    }
+    if (fileName instanceof RegExp) {
+      const found = relativePaths.find((relativePath) => relativePath.match(fileName))
+      if (found) return found
+    }
+  })
+  return file
+}
+
+// https://github.com/lukeed/escalade
+async function escalade(start, callback) {
+	let dir = resolve('.', start)
+	let tmp, stats = await stat(dir)
+
+	if (!stats.isDirectory()) {
+		dir = dirname(dir)
+	}
+
+	while (true) {
+		tmp = await callback(dir, await readdir(dir))
+		if (tmp) return resolve(dir, tmp)
+		dir = dirname(tmp = dir)
+		if (tmp === dir) break;
+	}
+}
+
+// https://github.com/lukeed/totalist
+async function totalist(dir, callback, pre='') {
+	dir = resolve('.', dir)
+	await readdir(dir).then(arr => {
+		return Promise.all(arr.map((str) => {
+      let abs = join(dir, str)
+      return stat(abs).then(stats => {
+        return stats.isDirectory() ? totalist(abs, callback, join(pre, str)) : callback(join(pre, str), abs, stats)
+      })
+    }))
+	})
+}
+
+function combineRegexes(patterns = []) {
+  const string = patterns.map((pat) => {
+    if (isRegex(pat)) {
+      return pat.source
+    } else if (typeof pat === 'string' && REGEX_REGEX.test(pat)) {
+      const regexInfo = pat.match(REGEX_REGEX)
+      console.log('regexInfo', regexInfo)
+      if (regexInfo && regexInfo[1]) {
+        // escapeRegexString
+        return regexInfo[1]
+      }
+    } else if (isGlob(pat)) {
+      console.log('pat', pat)
+      const result = globrex(pat, { globstar: true, extended: true })
+      console.log('result', result)
+      return result.regex.source
+    }
+    return pat
+  }).join('|')
+  console.log('xxxstring', string)
+  return new RegExp(string)
+}
+
+// alt https://github.com/duniul/clean-modules/blob/33b66bcfb7825e1fa1eb1e296e523ddba374f1ae/src/utils/filesystem.ts#L92
+// Alt https://github.com/AvianFlu/ncp
+async function getFilePaths(dirName, {
+  patterns = [],
+  ignore = [],
+  excludeGitIgnore = false,
+  excludeHidden = false
+}) {
+  let findPattern
+  let ignorePattern
+  let filePaths = []
+  let gitIgnoreFiles = []
+  let gitIgnoreGlobs = []
+
+  if (patterns && patterns.length) {
+    findPattern = combineRegexes(patterns)
+    console.log('findPatternfindPatternfindPattern', patterns)
+  }
+  if (ignore && ignore.length) {
+    ignorePattern = combineRegexes(ignore)
+  }
+
+  if (excludeGitIgnore) {
+    const gitIgnoreContents = await getGitignoreContents()
+    for (let index = 0; index < gitIgnoreContents.length; index++) {
+      const ignoreItem = gitIgnoreContents[index]
+      // console.log('ignoreItem', ignoreItem)
+      if (isGlob(ignoreItem)) {
+        gitIgnoreGlobs.push(ignoreItem)
+      } else {
+        gitIgnoreFiles.push(
+          ignoreItem.replace(/^\.\//, '') // normalize relative paths
+        )
+      }
+    }
+  }
+  
+  //*
+  console.log('findPattern', findPattern)
+  console.log('ignorePattern', ignorePattern)
+  console.log('gitIgnoreFiles', gitIgnoreFiles)
+  console.log('gitIgnoreGlobs', gitIgnoreGlobs)
+  // process.exit(1)
+  /** */
+
+  await totalist(dirName, (relativePath, abs, stats) => {
+    const absolutePath = `${dirName}/${relativePath}`
+    const topLevelDir = relativePath.substring(0, relativePath.indexOf('/'))
+    //console.log('absolutePath', absolutePath)
+    // console.log('relativePath', relativePath)
+    // console.log('topLevelDir', topLevelDir)
+
+    /* Remove hidden files */
+    if (excludeHidden && IS_HIDDEN_FILE.test(relativePath)) {
+      return
+    }
+
+    /* Remove files in git ignore */
+    if (excludeGitIgnore && gitIgnoreFiles.length) {
+      if (gitIgnoreFiles.includes(relativePath)) return
+      if (gitIgnoreFiles.includes(path.basename(relativePath))) return
+      //* // slow
+      if (gitIgnoreFiles.some((ignore) => {
+        // console.log('ignore', ignore)
+        // return relativePath.indexOf(ignore) > -1
+        // return relativePath.split('/')[0] === ignore
+        return topLevelDir === ignore || relativePath === ignore
+      })) {
+        return
+      }
+      /** */
+    }
+
+    /* Remove files in ignore array */
+    if (ignorePattern && ignorePattern.test(relativePath)) {
+      // Alt checker https://github.com/axtgr/wildcard-match
+      return
+    }
+
+    /* If no patterns supplied add all files */
+    if (!findPattern) {
+      filePaths.push(absolutePath)
+      return
+    }
+
+    /* If pattern match add file! */
+    // Alt match https://github.com/micromatch/picomatch
+    if (findPattern.test(absolutePath)) {
+      // console.log('Match absolutePath', absolutePath)
+      filePaths.push(absolutePath)
+      return
+    }
+
+    /* If pattern match add file! */
+    if (findPattern.test(relativePath)) {
+      // console.log('Match relativePath', relativePath)
+      filePaths.push(absolutePath)
+      return
+    }
+
+    /*
+    let ignored = false
+    for (let index = 0; index < ignore.length; index++) {
+      const pattern = ignore[index]
+      if (pattern.test(absolutePath)) {
+        ignored = true
+      }
+    }
+    if (!ignored) {
+      filePaths.push(absolutePath)
+    }
+    /** */
+	})
+
+  if (gitIgnoreGlobs && gitIgnoreGlobs.length) {
+    console.log('gitIgnoreGlobs', gitIgnoreGlobs)
+    let removeFiles = []
+    for (let index = 0; index < gitIgnoreGlobs.length; index++) {
+      const glob = gitIgnoreGlobs[index]
+      const result = globrex(glob) // alt lib https://github.com/axtgr/wildcard-match
+      // console.log('result', result)
+      for (let n = 0; n < filePaths.length; n++) {
+        const file = filePaths[n]
+        if (result.regex.test(file)) {
+          removeFiles.push(file)
+        }
+      }
+    }
+    /* Remove files that match glob pattern */
+    if (removeFiles.length) {
+      filePaths = filePaths.filter(function(el) {
+        return removeFiles.indexOf(el) < 0
+      })
+    }
+  }
+
+  return filePaths
+}
+
+function convertToRelativePath(file, cwd) {
+  return file.replace(cwd, '').replace(/^\//, '')
+}
+
+async function getGitignoreContents(filePath = '.gitignore') {
+  try {
+    const gitIgnoreContent = await readFile(filePath, { encoding: 'utf8' })
+    return gitIgnoreContent
+      .split(/\r?\n/)
+      .filter((line) => !/^\s*$/.test(line) && !/^\s*#/.test(line))
+      .map((line) => line.trim().replace(/^\/+|\/+$/g, ''))
+  } catch (_a) {
+    return []
+  }
+}
+
+// slash at the beginning of a filename
+const leadingPathSeparator = new RegExp(`^${escapeRegexString(path.sep)}`)
+const windowsLeadingPathSeparator = new RegExp('^/')
+
+// all slashes in the filename. path.sep is OS agnostic (windows, mac, etc)
+const pathSeparator = new RegExp(escapeRegexString(path.sep), 'g')
+const windowsPathSeparator = new RegExp('/', 'g')
+
+// handle MS Windows style double-backslashed filenames
+const windowsDoubleSlashSeparator = new RegExp('\\\\', 'g')
+
+// derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
+function fileNameToKey(filename) {
+  // const extension = new RegExp(`${path.extname(filename)}$`)
+  const key = filename
+    // .replace(extension, '')
+    .replace(leadingPathSeparator, '')
+    .replace(windowsLeadingPathSeparator, '')
+    .replace(pathSeparator, '.')
+    .replace(windowsPathSeparator, '.')
+    .replace(windowsDoubleSlashSeparator, '.')
+
+  return key
+}
+
+// https://github.com/regexhq/unc-path-regex/blob/master/index.js
+function isUncPath(filepath) {
+  return /^[\\\/]{2,}[^\\\/]+[\\\/]+[^\\\/]+/.test(filepath)
+}
+function isRelative(filepath) {
+  const isRel = !isUncPath(filepath) && !/^([a-z]:)?[\\\/]/i.test(filepath)
+  // console.log(`isRel ${filepath}`, isRel)
+  return isRel
+}
+/* Find common parts of 2 paths */
+function resolveCommonParent(mainDir = '', fileDir = '') {
+  const parts = mainDir.split('/')
+  let acc = ''
+  let value = ''
+  for (let i = 0; i < parts.length; i++) {
+    const element = parts[i];
+    acc+= ((i) ? '/' : '') + element
+    if (fileDir.startsWith(acc)) {
+      value = acc
+    }
+  }
+  return value
+}
+
+function resolveOutputPath(cwd, outputDir, file) {
+  // console.log('file', file)
+  const fileCommon = resolveCommonParent(cwd, file)
+  // console.log('fileCommon', fileCommon)
+  const remove = resolveCommonParent(outputDir, file)
+  const fileName = file.replace(remove, '').replace(fileCommon, '')
+  let outputFilePath = path.join(outputDir, fileName)
+  if (isRelative(outputDir)) {
+    outputFilePath = path.join(cwd, outputDir, fileName)
+  }
+  // console.log('isRelative(outputDir)', isRelative(outputDir))
+  // console.log('outputDir', outputDir)
+  // console.log('fileName', fileName)
+  // console.log('remove', remove)
+  return outputFilePath
+}
+
+function resolveFlatPath(cwd, outputDir, file) {
+  /* old setup */
+  const fileName = path.basename(file)
+  let outputFilePath = path.join(outputDir, fileName)
+  if (isRelative(outputDir)) {
+    outputFilePath = path.join(cwd, outputDir, fileName)
+  }
+  return outputFilePath
+}
+
+function depth(string) {
+  return path.normalize(string).split(path.sep).length - 1;
+}
+
+module.exports = {
+  isLocalPath,
+  writeFile,
+  readFile, 
+  findUp,
+  getFilePaths,
+  resolveOutputPath,
+  resolveFlatPath,
+  resolveCommonParent,
+  getGitignoreContents,
+  convertToRelativePath
+}