markdown-magic 2.6.1 → 3.0.1

package/lib/utils/toc.js

@@ -0,0 +1,315 @@
+const { transform } = require('@technote-space/doctoc')
+const { removeLeadingAndTrailingLineBreaks, escapeRegexString } = require('./regex')
+const { findMinIndent } = require('./text')
+const { readFile } = require('./fs')
+
+// Alt https://github.com/thlorenz/doctoc
+
+/**
+ * @typedef {object} TocOptions
+ * @property {boolean} [collapse = false] - Collapse toc in <details> pane
+ * @property {string} [collapseText] - Text in expand pane
+ * @property {string} [excludeText] - Text to exclude from toc
+ * @property {boolean} [firsth1 = true] - Exclude first heading from toc
+ * @property {boolean} [sub = false] - Mark as sub section table of contents
+ * @property {number} [maxDepth = 4] - Max depth of headings to add to toc.
+ */
+
+/**
+ * @typedef {object} TocAPI
+ * @property {TocOptions} options - Toc generation options
+ * @property {string} fileContent - Markdown contents
+ * @property {string} srcPath - Path to markdown file
+ * @property {function} getBlockDetails - Get block details for sub tables
+ */
+
+/**
+ * Generate a table of contents
+ * @param {TocAPI} api 
+ * @returns 
+ */
+async function generateToc({
+  options = {},
+  fileContent,
+  srcPath,
+  getBlockDetails
+}) {
+  // const currentBlock = api.getCurrentBlock()
+  // console.log('originalBlock', originalBlock)
+
+  // process.exit(1)
+  const opts = options || {}
+  const isSub = opts.sub
+  opts.firsth1 = (opts.firsth1) ? true : false
+  let contents = fileContent
+  // console.log('contents', contents)
+
+  let collapseText = opts.collapseText
+
+  let debugFileMatch
+  // console.log('config', config.originalPath)
+  // debugFileMatch = config.originalPath.match(/packages\/analytics\/README\.md/)
+
+  // https://www.npmjs.com/package/@technote-space/toc-generator
+  const tocOptions = {
+    // mode: 'github.com', // github.com | bitbucket.org | gitlab.com | nodejs.org | ghost.org (default: github.com)
+    isNotitle: true,
+    //isCustomMode: true,
+    openingComment: '',
+    closingComment: '',
+    maxHeaderLevel: (opts.maxDepth) ? Number(opts.maxDepth) : 4,
+    // maxHeaderLevel: 2, // default: 4
+    // title: '**Table of Contents**',
+    // isNotitle: true,
+    // isFolding: true,
+    // entryPrefix: '*',
+    // processAll: true,
+    // updateOnly: true,
+    // openingComment: '<!-- toc -->',
+    // closingComment: '<!-- tocstop --> ',
+    // checkOpeningComments: ['<!-- toc '],
+    // checkClosingComments: ['<!-- tocstop '],
+    // isCustomMode: false,
+    // customTemplate: '<p align="center">${ITEMS}</p>',
+    // itemTemplate: '<a href="${LINK}">${TEXT}</a>',
+    // separator: '<span>|</span>',
+    // footer: 'end',
+  }
+  const t = await transform(contents, tocOptions)
+  let outputText = t.wrappedToc || ''
+
+  if (debugFileMatch) {
+    console.log('before firsth1 removal', outputText)
+  }
+
+  /* remove first h1 */
+  if (!opts.firsth1) {
+    // console.log('outputText', outputText)
+    const lines = outputText.split('\n')
+    let firstH1
+    let firstLine
+    const countOfH1s = lines.reduce((acc, line, index) => {
+      if (line !== '' && !firstLine) {
+        firstLine = index
+      }
+      if (line.match(/^-\s+/)) {
+        if (!firstH1) {
+          const headingText = line.match(/\[(.*)\]/)
+          if (headingText) {
+            const rawText = headingText[1]
+            firstH1 = [line, index, firstLine === index, rawText]
+          }
+        }
+        acc = acc + 1
+      }
+      return acc
+    }, 0)
+
+    const firstHeadingData =  (firstH1 && Array.isArray(firstH1)) ? firstH1 : []
+    const [ lineText, lineIndex, isFirst, matchText ] = firstHeadingData
+    
+    let docHasHeading = false
+    if (matchText) {
+      // verify its h1
+      const matchTextEscaped = escapeRegexString(matchText)
+      // console.log('matchTextEscaped', matchTextEscaped)
+      // /^#{1}\s+(.*)/
+      const FIRST_H1_REGEX = new RegExp(`^#\\s*\\[?${matchTextEscaped}\\]?(?:.*)?`, 'gim')
+      // /^<h1\b[^>]*>([\s\S]*?)<\/h1>/
+      const FIRST_HTML_H1_REGEX = new RegExp(`^<h1\\b[^>]*>[\\s]*?(${matchTextEscaped})[\\s]*?<\\/h1>`, 'gim')
+      // /^(.*)\n={3,}/
+      const FIRST_UNDERSCORE_H1 = new RegExp(`^(${matchTextEscaped})\n={3,}`, 'gim')
+  
+      if (contents.match(FIRST_H1_REGEX)) {
+        docHasHeading = matchText
+      } else if (contents.match(FIRST_HTML_H1_REGEX)) {
+        docHasHeading = matchText
+      } else if (contents.match(FIRST_UNDERSCORE_H1)) {
+        docHasHeading = matchText
+      }
+    }
+  
+    if (debugFileMatch) {
+      console.log('matchText', matchText)
+      if (docHasHeading) {
+        console.log('Found heading 1', docHasHeading)
+      }
+    }
+
+    if (debugFileMatch) {
+      console.log('top level toc item count', countOfH1s)
+      if (docHasHeading) {
+        console.log('Found heading 1', docHasHeading)
+        console.log('firstH1', firstH1)
+      }
+    }
+
+    let firstItemWithContent
+    let foundFirstH1
+
+    if (docHasHeading) {
+      outputText = lines.reduce((acc, line, i) => {
+        const isLineEmpty = line === ''
+        if (!isLineEmpty && !firstItemWithContent) {
+          firstItemWithContent = i
+        }
+        if (!foundFirstH1 && (firstItemWithContent === i) && line.match(/^-\s+/)) {
+          foundFirstH1 = true
+          return acc
+        }
+        // Add back line and remove level
+        let newLineValue = line
+        if (lineText) {
+          // const untilFirstH1 = i < lineIndex
+          /* @TODO make option? flatten all non first h1 tags */
+          if (countOfH1s > 0 && !isLineEmpty && newLineValue.match(/^\s+-/)) {
+            newLineValue = line.substring(2)
+          }
+        }
+        
+        acc = acc.concat(newLineValue)
+        return acc
+      }, []).join('\n')
+    }
+ 
+    // console.log('outputText', outputText)
+    if (debugFileMatch) {
+      console.log('after firsth1 removal', outputText)
+    }
+  }
+  // console.log(t)
+  // Fix <h1> with multiple lines
+  const spaces = outputText.match(/\[\n([\s\S]*?)\]/gm)
+  if (spaces) {
+    const fixed = spaces[0]
+      // all new lines
+      .replace(/\n/g, '')
+      // leading spaces
+      .replace(/\[\s+/, '[')
+      .trim()
+    outputText = outputText.replace(spaces[0], fixed)
+  }
+  // console.log(outputText)
+  outputText = outputText
+    .replace(/(.*)\[Table of Contents\]\(.*\)\n?/i, '')
+    .replace(/(.*)\[toc\]\(.*\)\n?/i, '')
+
+
+  if (opts.excludeText) {
+
+    outputText = excludeTocItem(outputText, opts.excludeText)
+    // // (\s+)?-(.*)\[Usage\]\(.*\)
+    // const regex = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)`, 'i')
+    // // /(\s+)?-(.*)\[Usage\]\(.*\)(\n\s+(.*))+/im
+    // const nestedRegex = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)\(\\n\\s+\(.*\)\)+`, 'i')
+    
+    // const hasNested = nestedRegex.exec(outputText)
+    // if (hasNested) {
+    //   // Count indentation of spaces
+    //   const numberOfSpaces = hasNested[1].replace(/\n/g, '').length
+    //   const subItems = numberOfSpaces + 1
+    //   // Update regex to only remove sub items
+    //   const nestedRegexSpaces = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)\(\\n\\s{${subItems},}\(.*\)\)+`, 'i')
+    //   // console.log('nestedRegexSpaces', nestedRegexSpaces)
+    //   // If exclude value has nested sections remove them as well.
+    //   outputText = outputText.replace(nestedRegexSpaces, '')
+    //   outputText = outputText.replace(regex, '')
+    // } else {
+    //   outputText = outputText.replace(regex, '')
+    // }
+  }
+
+  /* Sub table of contents */
+  if (isSub) {
+    // const start = fileContent.indexOf(open.value)
+    // const linesBefore = fileContent.substr(0, start).split('\n')
+    // const closestHeading = linesBefore.reverse().find((line) => {
+    //   return line.match((/^#+/))
+    // })
+    const originalContents = await readFile(srcPath, { encoding: 'utf8' })
+    const originalBlock = getBlockDetails(originalContents)
+    const linesBefore = originalContents.substr(0, originalBlock.block.start).split('\n')
+    const closestHeading = linesBefore.reverse().find((line) => {
+      return line.match((/^#+/))
+    })
+
+    if (closestHeading) {
+      const headingText = closestHeading.replace(/^#*\s*/, '')
+      // console.log('BEFORE', linesBefore)
+      // console.log('closest parent heading', closestHeading)
+      // console.log('headingText', headingText)
+      collapseText = collapseText || `${headingText} contents`
+      // https://regex101.com/r/MB85zm/1
+
+      const findSubToc = new RegExp(`^\(\\s+\)?-(.*)\\[${headingText}\\]\\(.*\\)(\\n\\s.*)+`, 'gm')
+      // console.log('findSubToc', findSubToc)
+      const single = singleLinePattern(headingText)
+      // console.log('single', single)
+      const subItems = outputText.match(findSubToc)
+      if (subItems) {
+        const items = subItems[0].replace(single, '').split('\n')
+          .filter(Boolean)
+        // console.log('items', items)
+        const finalItems = items // .slice(1, items.length)
+        if (finalItems) {
+          const indent = findMinIndent(finalItems.join('\n'))
+          // console.log('min indent', indent)
+          // console.log('finalItems', finalItems)
+          outputText = finalItems.map((thing) => thing.substring(indent)).join('\n')
+        } else {
+          // console.log('No sub items')
+        }
+      }
+    }
+  }
+
+  // If collapse wrap in <details>
+  if (opts.collapse) {
+    return `<details>
+<summary>${collapseText || 'Table of Contents'}</summary>
+
+${outputText   
+  // Replace leading double spaces
+  .replace(/^\n*/g, '')
+  // Replace trailing double spaces
+  .replace(/\n*$/g, '')
+}
+
+</details>`
+  }
+
+  return outputText.replace(removeLeadingAndTrailingLineBreaks, '')
+}
+
+function singleLinePattern(text) {
+  /* (\s+)?-(.*)\[Usage\]\(.*\) */
+  return new RegExp(`\(\\s+\)?-(.*)\\[${text}\\]\\(.*\\)`, 'i')
+}
+
+function excludeTocItem(str, excludeText) {
+  const matchTextEscaped = escapeRegexString(excludeText)
+  /* (\s+)?-(.*)\[Usage\]\(.*\) */
+  const regex = singleLinePattern(matchTextEscaped) // new RegExp(`\(\\s+\)?-(.*)\\[${matchTextEscaped}\\]\\(.*\\)`, 'i')
+  /* /(\s+)?-(.*)\[Usage\]\(.*\)(\n\s+(.*))+/im */
+  const nestedRegex = new RegExp(`\(\\s+\)?-(.*)\\[${matchTextEscaped}\\]\\(.*\\)\(\\n\\s+\(.*\)\)+`, 'i')
+  
+  const hasNested = nestedRegex.exec(str)
+  if (hasNested) {
+    // Count indentation of spaces
+    const numberOfSpaces = (hasNested[1] || '').replace(/\n/g, '').length
+    const subItems = numberOfSpaces + 1
+    // Update regex to only remove sub items
+    const nestedRegexSpaces = new RegExp(`\(\\s+\)?-(.*)\\[${matchTextEscaped}\\]\\(.*\\)\(\\n\\s{${subItems},}\(.*\)\)+`, 'i')
+    // console.log('nestedRegexSpaces', nestedRegexSpaces)
+    // If exclude value has nested sections remove them as well.
+    str = str.replace(nestedRegexSpaces, '')
+    str = str.replace(regex, '')
+  } else {
+    str = str.replace(regex, '')
+  }
+  return str
+}
+
+module.exports = {
+  generateToc
+}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "markdown-magic",
-  "version": "2.6.1",
+  "version": "3.0.1",
   "description": "Automatically update markdown files with content from external sources",
-  "main": "index.js",
+  "main": "lib/index.js",
   "bin": {
     "markdown": "./cli.js",
     "md-magic": "./cli.js"
@@ -10,18 +10,27 @@
   "files": [
     "README.md",
     "package.json",
-    "index.js",
+    "package-lock.json",
     "cli.js",
-    "cli-utils.js",
     "/lib"
   ],
   "scripts": {
+    "types": "tsc",
+    "emit-types": "tsc --noEmit false --emitDeclarationOnly true",
     "docs": "node examples/generate-readme.js",
-    "uvu": "uvu lib '.test.([mc]js|[jt]sx?)$'",
-    "test": "ava --verbose",
-    "test:watch": "ava --verbose --watch",
+    "test": "npm run test:lib && npm run test:test && echo 'tests done'",
+    "test:lib": "uvu lib '.test.([mc]js|[jt]sx?)$'",
+    "test:test": "uvu test '.test.([mc]js|[jt]sx?)$'",
+    "test:options": "uvu lib 'options-parser.test.([mc]js|[jt]sx?)$'",
+    "test:block": "uvu lib 'block-parser.test.([mc]js|[jt]sx?)$'",
+    "test:cli": "uvu lib 'cli.test.([mc]js|[jt]sx?)$'",
+    "test:md": "uvu lib 'md.test.([mc]js|[jt]sx?)$'",
+    "test:fs": "uvu lib 'fs.test.([mc]js|[jt]sx?)$'",
+    "test:js": "uvu lib 'block-parser-js.test.([mc]js|[jt]sx?)$'",
+    "test:errors": "uvu test 'errors.test.([mc]js|[jt]sx?)$'",
+    "test:transforms": "uvu test 'transforms.test.([mc]js|[jt]sx?)$'",
+    "test:text": "uvu lib 'text.test.([mc]js|[jt]sx?)$'",
     "cli": "node ./cli.js --path 'README.md' --config ./markdown.config.js",
-    "lint": "eslint .",
     "publish": "git push origin && git push origin --tags",
     "release:patch": "npm version patch && npm publish",
     "release:minor": "npm version minor && npm publish",
@@ -36,29 +45,24 @@
   },
   "dependencies": {
     "@technote-space/doctoc": "2.4.7",
-    "commander": "^7.2.0",
-    "deepmerge": "^4.2.2",
-    "find-up": "^5.0.0",
-    "globby": "^10.0.2",
+    "globrex": "^0.1.2",
+    "gray-matter": "^4.0.3",
+    "is-glob": "^4.0.3",
     "is-local-path": "^0.1.6",
-    "json-alexander": "^0.1.8",
-    "mkdirp": "^1.0.4",
+    "is-valid-path": "^0.1.1",
+    "micro-mdx-parser": "^1.1.0",
+    "mri": "^1.2.0",
+    "oparser": "^2.1.1",
+    "smart-glob": "^1.0.2",
     "sync-request": "^6.1.0"
   },
   "devDependencies": {
-    "ava": "^3.15.0",
-    "doxxx": "^1.0.0",
-    "rimraf": "^3.0.2",
-    "sinon": "^11.1.1",
-    "uvu": "^0.5.1",
     "ansi-styles": "^4.2.1",
     "concordance": "^5.0.1",
-    "safe-chalk": "^1.0.0"
-  },
-  "ava": {
-    "files": [
-      "test/**/*.test.js"
-    ],
-    "verbose": true
+    "doxxx": "^2.0.1",
+    "rimraf": "^3.0.2",
+    "safe-chalk": "^1.0.0",
+    "typescript": "^5.0.2",
+    "uvu": "^0.5.1"
   }
 }

package/index.js

@@ -1,46 +0,0 @@
-const globby = require('globby')
-const processFile = require('./lib/processFile')
-/**
- * ### API
- * ```js
- * markdownMagic(filePath, config, callback)
- * ```
- * - `filePaths` - *String or Array* - Path or glob pattern. Uses [globby patterns](https://github.com/sindresorhus/multimatch/blob/master/test.js)
- * - `config` - See configuration options below
- * - `callback` - callback to run after markdown updates
- *
- * @param  {string} filePath - Path to markdown file
- * @param  {object} [config] - configuration object
- * @param  {Function} [callback] - callback function with updated contents
- */
-module.exports = async function markdownMagic(filePaths, config, callback) {
-  const files = globby.sync(filePaths)
-  const configuration = config || {}
-  if (!callback && typeof configuration === 'function') {
-    callback = configuration // eslint-disable-line
-  } else if (typeof config === 'object' && config.callback) {
-    // set callback in config for CLI usage
-    callback = config.callback // eslint-disable-line
-  }
-  if (!files.length) {
-    callback && callback('No files matched')
-    console.log('No files matched pattern', filePaths)
-    return false
-  }
-  configuration.originalFilePaths = files
-
-  const data = files.map(async (file) => {
-    const x = await processFile(file, configuration)
-    return x
-  })
-
-  const values = await Promise.all(data)
-
-  if (callback) {
-    callback(null, values)
-  }
-  return values
-}
-
-// expose globby for use in plugins
-module.exports.globby = globby

package/lib/processFile.js

@@ -1,154 +0,0 @@
-const fs = require('fs')
-const mkdirp = require('mkdirp')
-const path = require('path')
-const merge = require('deepmerge')
-const transforms = require('./transforms')
-const regexUtils = require('./utils/regex')
-const pluginSortOrder = require('./utils/sortOrder')
-const updateContents = require('./updateContents')
-const cwd = process.cwd()
-
-module.exports = async function processFile(filePath, config) {
-  let content
-  try {
-    content = fs.readFileSync(filePath, 'utf8')
-  } catch (e) {
-    console.log(`FILE NOT FOUND ${filePath}`)
-    throw e
-  }
-  /**
-   * ### Configuration Options
-   */
-  const defaultConfig = {
-    /**
-     * - `transforms` - *object* - (optional) Custom commands to transform block contents, see transforms & custom transforms sections below.
-     * @type {Object}
-     */
-    transforms: transforms,
-    /**
-     * - `outputDir` - *string* - (optional) Change output path of new content. Default behavior is replacing the original file
-     * @type {string}
-     */
-    outputDir: path.dirname(filePath),
-    /**
-     * - `matchWord` - *string* - (optional) Comment pattern to look for & replace inner contents. Default `AUTO-GENERATED-CONTENT`
-     * @type {string}
-     * @default [AUTO-GENERATED-CONTENT]
-     */
-    matchWord: 'AUTO-GENERATED-CONTENT',
-    /**
-     * - `DEBUG` - *Boolean* - (optional) set debug flag to `true` to inspect the process
-     * @type {boolean}
-     */
-    DEBUG: false,
-  }
-
-  const userConfig = (config && typeof config === 'object') ? config : {}
-  const mergedConfig = merge(defaultConfig, userConfig)
-
-  const registeredTransforms = Object.keys(mergedConfig.transforms)
-  // Set originalPath constant
-  mergedConfig.originalPath = filePath
-  // contents of original MD file
-  mergedConfig.originalContent = content
-  // set default outputContents for first pass for single commands
-  mergedConfig.outputContent = content
-
-  const regex = regexUtils.matchCommentBlock(mergedConfig.matchWord)
-  // console.log(regex)
-  const match = content.match(regex)
-  const transformsFound = []
-
-  if (match) {
-    let commentMatches
-    let matchIndex = 0
-    while ((commentMatches = regex.exec(content)) !== null) {
-      if (commentMatches.index === regex.lastIndex) {
-        regex.lastIndex++ // This is necessary to avoid infinite loops
-      }
-
-      // console.log('commentMatches', commentMatches)
-      // const command = `Transform ${commentMatches[2]}`
-      // console.log(command)
-      transformsFound.push({
-        spaces: commentMatches[1], // Preserve indentation
-        transform: commentMatches[2],
-        match: match[matchIndex]
-      })
-      // wait
-      matchIndex++
-    }
-
-    // console.log('registeredTransforms', registeredTransforms)
-
-    const transformsToRun = pluginSortOrder(registeredTransforms, transformsFound)
-    if (mergedConfig.DEBUG) {
-      console.log('↓ transformsToRun')
-      console.log(transformsToRun)
-    }
-
-    const fileName = path.basename(filePath)
-    const outputFilePath = path.join(mergedConfig.outputDir, fileName)
-
-    // create folder path if doesnt exist
-    mkdirp.sync(mergedConfig.outputDir)
-
-    // run sort
-    let transformMsg = ''
-    for (const element of transformsToRun) {
-      // console.log('element', element)
-      transformMsg += `  ⁕ ${element.transform} \n`
-      // console.log('order', element.transform)
-      const newContent = await updateContents(element.match, mergedConfig)
-      const firstLineIndentation = element.spaces
-      const contentWithIndentation = newContent.split('\n').join(`\n` + element.spaces)
-      const preserveTabs = `${firstLineIndentation}${contentWithIndentation}`
-
-      content = content.replace(element.match, preserveTabs)
-      mergedConfig.outputContent = content
-    }
-
-    let notFoundNotice
-    if (!transformMsg) {
-      // console.log('config', config)
-      const notFound = transformsFound.map((x) => {
-        return `"${x.transform}"`
-      })
-      notFoundNotice = notFound
-    }
-
-    // console.log('transformMsg', transformMsg)
-
-    const msg = outputFilePath.replace(cwd, '')
-
-    if (transformMsg) {
-      console.log(`✔ ${msg} Updated`)
-      // update file contents
-      fs.writeFileSync(outputFilePath, content)
-      console.log(` Transforms run`)
-      console.log(transformMsg)
-    }
-
-    if (notFoundNotice) {
-      const word = notFoundNotice.length > 1 ? 'transforms' : 'transform'
-      console.log(`ℹ Notice:`)
-      console.log(`  Missing ${word} ${notFoundNotice.join(',')} in ${msg}`)
-      console.log()
-    }
-
-
-    // set return values
-    mergedConfig.outputFilePath = outputFilePath
-    mergedConfig.outputContent = content
-
-    return mergedConfig
-  }
-
-  if (mergedConfig.DEBUG) {
-    console.log(`↓ ${filePath}`)
-    console.log(`[No match] <!-- ${mergedConfig.matchWord} --> comment found`)
-  }
-
-  // no match return original contents
-  return mergedConfig
-}