markdown-magic 2.6.1 → 3.0.1

package/lib/index.js

@@ -1,219 +1,388 @@
 const path = require('path')
-const fs = require('fs').promises
-const { getTextBetween, parseBlocks } = require('./utils/new-parser')
+const { glob, globWithGit } = require('smart-glob')
+const codeTransform = require('./transforms/code')
+const fileTransform = require('./transforms/file')
+const tocTransform = require('./transforms/toc')
+const sectionTocTransform = require('./transforms/sectionToc')
+const wordCountTransform = require('./transforms/wordCount')
+const remoteTransform = require('./transforms/remote')
+const { getSyntaxInfo } = require('./utils/syntax')
+const { onlyUnique, getCodeLocation, pluralize } = require('./utils')
+const { writeFile, resolveOutputPath, resolveFlatPath } = require('./utils/fs')
+const { processFile } = require('./process-file')
+const { processContents } = require('./process-contents')
+const { parseMarkdown } = require('./utils/md/parse')
+const { deepLog, success, error, info } = require('./utils/logs')
+const { OPEN_WORD, CLOSE_WORD, DEFAULT_GLOB_PATTERN } = require('./defaults')
+// const { getBlockRegex } = require('./block-parser')
+// const diff = require('../misc/old-test/utils/diff')
 
-async function processContents({ 
-  filePath, 
-  syntax,
-  open,
-  close,
-  transforms,
-  beforeMiddelwares,
-  afterMiddelwares,
-  DEBUG = false
-}) {
-  const syntaxType = syntax || path.extname(filePath).replace(/^\./, '')
-  const originalContents = await fs.readFile(filePath, 'utf8')
+const LINE = '────────────────────────────────'
+
+const defaultTransforms = {
+  CODE: codeTransform,
+  FILE: fileTransform,
+  TOC: tocTransform,
+  sectionToc: sectionTocTransform,
+  wordCount: wordCountTransform,
+  remote: remoteTransform
+}
+
+/**
+ * Allowed file syntaxes
+ * @typedef {'md' | 'js' | 'yml' | 'yaml'} SyntaxType
+ */
+
+/**
+ * Configuration for markdown magic
+ * 
+ * Below is the main config for markdown magic
+ * 
+ * @typedef {object} MarkdownMagicOptions
+ * @property {Array} [transforms = defaultTransforms] - Custom commands to transform block contents, see transforms & custom transforms sections below.
+ * @property {string} [outputDir] - Change output path of new content. Default behavior is replacing the original file
+ * @property {SyntaxType} [syntax = 'md'] - Syntax to parse
+ * @property {boolean} [outputFlatten] - Flatten files that are output
+ * @property {function} [handleOutputPath] - Custom function for altering output paths
+ * @property {boolean} [useGitGlob] - Use git glob for LARGE file directories
+ * @property {boolean} [dryRun = false] - See planned execution of matched blocks
+ * @property {boolean} [debug = false] - See debug details
+ * @property {boolean} [failOnMissingTransforms = false] - Fail if transform functions are missing. Default skip blocks.
+ */
+
+/**
+ * Markdown Magic
+ * @param {string|MarkdownMagicOptions} globOrOpts - Files to process or config. Uses [globby patterns](https://github.com/sindresorhus/multimatch/blob/master/test.js)
+ * @param {MarkdownMagicOptions} options - Markdown magic config
+ * @returns 
+ * @example
+  markdownMagic(filePath, config)
+ */
+async function markdownMagic(globOrOpts, options = {}) {
+  let opts = options || {}
+  let globPat
+  if (typeof globOrOpts === 'string' || Array.isArray(globOrOpts)) {
+    opts = options
+    globPat = globOrOpts
+  } else if (typeof globOrOpts === 'object') {
+    opts = globOrOpts
+  }
+  const {
+    transforms,
+    // open,
+    outputDir,
+    outputFlatten = false,
+    handleOutputPath,
+    useGitGlob = false, 
+    failOnMissingTransforms = false,
+    dryRun = false,
+    debug = true,
+    syntax = 'md',
+  } = opts
+
+  let open = OPEN_WORD
+  if (opts.open) {
+    open = opts.open
+  } else if (opts.matchWord) {
+    open = `${opts.matchWord}:start`
+  }
+
+  let close = CLOSE_WORD
+  if (opts.close) {
+    close = opts.close
+  } else if (opts.matchWord) {
+    close = `${opts.matchWord}:end`
+  }
+
+  // console.log('outputDir', outputDir)
+  // console.log('outputFlatten', outputFlatten)
+  // return
   
-  const foundBlocks = parseBlocks(originalContents, {
-    syntax: syntaxType,
+  const cwd = opts.cwd || process.cwd()
+  // console.log('cwd', cwd)
+  const globPattern = globPat || globOrOpts.glob || globOrOpts.file || globOrOpts.files
+  // console.log('globPattern', globPattern)
+
+  const useTransforms = Object.assign({}, defaultTransforms, transforms)
+  let globs = []
+  if (!globPattern) {
+    globs = [ DEFAULT_GLOB_PATTERN ]
+  } else if (Array.isArray(globPattern)) {
+    globs = globPattern
+  } else if (typeof globPattern === 'string') {
+    globs = [ globPattern ]
+  }
+  opts.glob = globs
+  // console.log('globs', globs)
+  // return
+
+  info(`Running Markdown Magic:`)
+  logCommentBlockSyntax({
+    syntax: syntax,
     open,
-    close,
+    close
   })
 
-  if (DEBUG) {
-    console.log('foundBlocks')
-    console.log(foundBlocks)
+  if (dryRun || debug) {
+    info(`Glob patterns:`)
+    console.log(globs)
+    console.log()
+    /*
+    process.exit(1)
+    /** */
   }
 
-  const transformsToRun = sortPlugins(foundBlocks.transforms)
-  // console.log('transformsToRun', transformsToRun)
-
-  // if (!transformsToRun.length) {
-  //   process.exit(1)
-  // }
-
-  let missingTransforms = []
-  const updatedContents = await transformsToRun.reduce(async (updatedContent, ogmatch) => {
-    const md = await updatedContent
-    /* Apply Before middleware to all transforms */
-    const match = await applyMiddleware(ogmatch, md, beforeMiddelwares)
-    const { block, raw, transform, args } = match
-    const { openTag, closeTag, content, contentStart, contentEnd, start, end } = block
-    
-    /* Run transform plugins */
-    let tempContent = content
-    if (transforms[transform]) {
-      tempContent = await transforms[transform](match, md)
-    }
+  const globFn = (useGitGlob) ? globWithGit : glob 
 
-    /* Apply After middleware to all transforms */
-    const afterContent = await applyMiddleware({
-      ...match,
-      ...{
-        block: {
-          ...match.block,
-          content: tempContent
-        }
-      }
-    }, md, afterMiddelwares)
-    if (DEBUG) {
-      console.log('afterContent', afterContent)
-    }
-
-    if (!transforms[transform]) {
-      missingTransforms.push(afterContent)
-      console.log(`Missing "${transform}" transform`)
-    }
+  const pathsPromise = globs.map((str) => {
+    // console.log('str', str)
+    // Glob pattern examples https://github.com/terkelg/globrex/blob/master/test/index.js
+    // return globWithGit(str, {
+    // const g = str.replace(/^(?:\.\.\/)+/, "")
+    // console.log('g', g)
+    return globFn(str, {
+      absolute: true,
+      alwaysReturnUnixPaths: true,
+      ignoreGlobs: ['**/node_modules/**'],
+    })
+  })
 
-    const newContent = afterContent.block.content
-    const formattedNewContent = (args.noTrim) ? newContent : smartTrim(newContent)
-    /* Remove any conflicting imported comments */
-    // const fix = removeConflictingComments(formattedNewContent, foundBlocks.commentOpen, foundBlocks.commentClose)
-    const fix = stripAllComments(formattedNewContent, foundBlocks.commentOpen, foundBlocks.commentClose)
-    // console.log('foundBlocks.commentClose', foundBlocks.commentClose)
-    // console.log('formattedNewContent', formattedNewContent)
-    // console.log('fix', fix)
-    const preserveIndent = (true || args.preserveIndent) ? block.indentation.length + block.contentIndent : block.indentation.length
-    const indent = indentString(fix, preserveIndent)
-    const newCont = `${openTag}${indent}${closeTag}`
-    /* Replace original contents */
-    const newContents = md.replace(raw.block, newCont)
-    return Promise.resolve(newContents)
-  }, Promise.resolve(originalContents))
-
-  if (DEBUG) {
-    console.log('Output Markdown')
-    console.log(updatedContents)
+  let files = []
+  try {
+    files = (await Promise.all(pathsPromise)).flat().filter(onlyUnique)
+    opts.files = files
+  } catch (e) {
+    console.log(e.message)
+    throw new Error(e)
   }
+  // console.log('files', files)
+  // return
+  // console.log('globs', globs)
+  // const filex = await globby(globs)
+  // console.log('filex', filex)
 
   /*
-  if (missingTransforms.length) {
-    console.log('missingTransforms', missingTransforms)
-    let matchOne = missingTransforms[1]
-    matchOne = missingTransforms[missingTransforms.length - 1]
-    // console.log('matchOne', matchOne)
-    const { block, transform, args } = matchOne
-    const { openTag, closeTag, content, contentStart, contentEnd, start, end} = block
-
-    // console.log('contentStart', contentStart)
-    // console.log('contentEnd', contentEnd)
-    // console.log('original text between', `"${getTextBetween(md, contentStart, contentEnd)}"`)
-    // console.log('original block between', `"${getTextBetween(md, start, end)}"`)
-  }
+  const relativeFilePaths = files.map((file) => file.replace(cwd, '').replace(/^\//, ''))
+  console.log('relativeFilePaths', relativeFilePaths)
+  process.exit(1)
   /** */
 
-  return {
-    filePath,
-    transforms: transformsToRun,
-    missingTransforms,
-    originalContents,
-    updatedContents
+  if (dryRun || debug) {
+    info(`${files.length} Files found:`)
+    console.log(files)
+    /*
+    process.exit(1)
+    /** */
   }
-}
 
-function applyMiddleware(data, md, middlewares) {
-  return middlewares.reduce(async (acc, curr) => {
-    const realAcc = await acc
-    // console.log(`Running "${curr.name}" Middleware on "${realAcc.transform}" block`)
-    const updatedContent = await curr.transform(realAcc, md)
-    // realAcc.block.content = updatedContent
-    return Promise.resolve({
-      ...realAcc,
-      ...{
-        block: {
-          ...realAcc.block,
-          content: updatedContent
+  const processedFiles = files.map((file) => {
+    // console.log('file', file)
+    let newPath = path.resolve(cwd, file)
+    /* Allow for different output directory */
+    if (outputDir) {
+      newPath = (outputFlatten) ? resolveFlatPath(cwd, outputDir, file) : resolveOutputPath(cwd, outputDir, file) 
+    }
+    /* Allow for custom handling of individual files */
+    if (handleOutputPath) {
+      newPath = handleOutputPath(newPath)
+    }
+    // const cleanerDirPath = path.dirname(file)
+    // const baseDir = cleanerDirPath.replace(cwd, '')
+    // const cleanerDir = baseDir
+    // const resolvedDir = path.join(cleanerDir, outputDir)
+    // const cleanFinalPath = path.resolve(resolvedDir, path.basename(file))
+    /*
+    console.log('cleanerDir', cleanerDir)
+    console.log('resolvedDir', resolvedDir)
+    console.log('cleanFinalPath', cleanFinalPath)
+    /** */
+    // console.log('newPath', newPath)
+    return processFile({
+      ...opts,
+      open,
+      close,
+      srcPath: file,
+      outputPath: newPath,
+      transforms: useTransforms
+    })
+  })
+  // process.exit(1)
+  const plan = (await Promise.all(processedFiles)).filter((file) => {
+    /* Filter out files without transforms */
+    return file.transforms.length
+  })
+
+  if (dryRun || debug) {
+    /* Generate output paths */
+    const outputPaths = plan.map((item) => item.outputPath)
+    info(`Output files:`)
+    console.log(outputPaths)
+    /*
+    process.exit(1)
+    /** */
+  }
+  /*
+  console.log('plan')
+  deepLog(plan)
+  process.exit(1)
+  /** */
+
+  const missing = plan.filter((item) => {
+    return item.missingTransforms.length
+  })
+
+  let errors = []
+  if (missing.length) {
+    errors = missing.map((item, i) => {
+      const errorMessage = `Missing ${item.missingTransforms.length} transforms in ${item.srcPath}`
+      const issues = item.missingTransforms.map((trn) => {
+        // console.log('trn', trn)
+        // const rowData = getRowAndColumnFromCharPos(item.updatedContents, trn.open.start)
+        const location = `${item.srcPath}:${trn.block.lines[0]}:0`
+        const message = `Transform "${trn.transform}" at line ${trn.block.lines[0]} does not exist. → ${location}`
+        return {
+          message,
+          location
         }
+      })
+      return {
+        errorMessage,
+        errors: issues
       }
     })
-  }, Promise.resolve(data))
-}
+  }
 
-/**
- * Trim leading & trailing spaces/line breaks in code and keeps the indentation of the first non-empty line
- * @param {string} str 
- * @returns string
- */
-function smartTrim(str) {
-  return str.replace(/^(?:[\t ]*(?:\r?\n|\r))+|\s+$/g, '')
-}
+  /* If transform errors and should throw, exit early */
+  if (errors.length && failOnMissingTransforms) {
+    throw new Error(logErrors(errors))
+  }
+  
+  /* Log out execution plan */
+  let planTotal = 0
+  let missingTotal = 0
+  const planOutput = plan.map((item, i) => {
+    const transformsToRun = item.transforms.filter((item) => {
+      if (item.context.isMissing) {
+        // console.log('item', item)
+        missingTotal = missingTotal + 1
+      }
+      return !item.context.isMissing
+    })
+    if (!transformsToRun.length) {
+      return
+    }
+    const count = `${i  + 1}.`
+    let planMsg = `${count} Found ${transformsToRun.length} transforms in ${item.srcPath}`
+    planTotal = planTotal + transformsToRun.length
+    // console.log(`Found ${transformsToRun.length} transforms in ${item.srcPath}`)
+    transformsToRun.forEach((trn) => {
+      const line = trn.block.lines[0]
+      const location = getCodeLocation(item.srcPath, line)
+      const planData = ` - "${trn.transform}" at line ${line} → ${location}`
+      planMsg += `\n${planData}`
+      // console.log(` - "${trn.transform}" at line ${trn.block.lines[0]}`)
+    })
+    const newLine = plan.length !== i + 1 ? '\n' : ''
+    return `${planMsg}${newLine}`
+  }).filter(Boolean)
 
-function stripAllComments(block) {
-  const pattern = new RegExp(`([^\\s]*)?([ \\t]*)?(<!-+\\s?([\\s\\S]*?)?-+>)([^\\s<]*)?(\n{1,2})?`, 'gi')
+  if (files.length) {
+    console.log(LINE)
+    info(`Markdown Magic updates`, '')
+    const total = planTotal + missingTotal
+    console.log(`Parsed files:          ${files.length}`)
+    console.log(`Block Open:            ${open}`)
+    console.log(`Block Close:           ${close}`)
+    console.log(`Files w/ transforms:   ${plan.length} / ${files.length}`)
+    console.log(`Total transforms:      ${total}`)
+    console.log(`Valid transforms:      ${planTotal} / ${total}`)
+    console.log(`Invalid transforms:    ${missingTotal} / ${total}`)
+    console.log('Syntax:')
+    logCommentBlockSyntax({
+      syntax: syntax,
+      open,
+      close
+    })
+    console.log(LINE)
+    console.log(planOutput.join('\n'))
+  }
 
-  // console.log('closeTagRegex', closeTagRegex)
-  let matches
-  let remove = []
-  while ((matches = pattern.exec(block)) !== null) {
-    if (matches.index === pattern.lastIndex) {
-      pattern.lastIndex++ // avoid infinite loops with zero-width matches
+  /* Execute updates based on plan */
+  if (!dryRun) {
+    const changed = plan.filter(({ isChanged }) => isChanged)
+    if (changed.length) {
+      console.log(LINE)
+      console.log()
+      const execute = changed.map(({ srcPath, outputPath, updatedContents, originalContents }, i) => {
+        // console.log(`${i + 1}. newPath`, newPath)
+        console.log(`- Update file ${outputPath}`)
+        return writeFile(outputPath, updatedContents)
+      })
+  
+      await Promise.all(execute)
+    } else {
+      console.log(LINE)
+      console.log('No changes. Skipping file writes')
     }
-    const [ match, leadingText, leadingSpace, comment, insideComment, trailingText, trailingNewLine ] = matches
-    /*
-    console.log('match', match)
-    console.log('leadingText', leadingText)
-    console.log('leadingSpace', leadingSpace)
-    console.log('comment', comment)
-    console.log('insideComment', insideComment)
-    console.log('trailingText', trailingText)
-    console.log('trailingNewLine', trailingNewLine)
-    /** */
-    const newLineCount = (trailingNewLine || '').length
-    const trailing = (!trailingText && newLineCount > 1) ? `${trailingNewLine || ''}` : ''
-    const leading = (leadingSpace) ? leadingSpace.slice(1) : ''
-    remove.push(`${leading}${comment}${trailing}`)
   }
-  return remove.reduce((acc, curr) => {
-    return acc.replaceAll(curr, '')
-  }, block)
-}
+  /*
+  TODO:
+    - Output to new file
+    - Clean up default transforms
+    - Expose md utils
+  */
+  if (errors.length) {
+    logErrors(errors)
+  }
 
-/**
- * Remove conflicting comments that might have been inserted from transforms
- * @param {*} content 
- * @param {*} openPattern 
- * @param {*} closePattern 
- * @returns 
- */
-function removeConflictingComments(content, openPattern, closePattern) {
-  const removeOpen = content.replace(openPattern, '')
-  // TODO this probably needs to be a loop for larger blocks
-  closePattern.lastIndex = 0; // reset regex
-  const hasClose = closePattern.exec(content)
-  // console.log('closePattern', closePattern)
-  // console.log('has', content)
-  // console.log('hasClose', hasClose)
-  if (!hasClose) {
-    return removeOpen
+  console.log(LINE)
+  success(`Markdown Magic Done`)
+  console.log(`${LINE}\n`)
+
+  return {
+    errors,
+    changes: plan,
+    data: plan
   }
-  const closeTag = `${hasClose[2]}${hasClose[3] || ''}`
-  // console.log('closeTag', closeTag)
-  return removeOpen
-    .replaceAll(closeTag, '')
-    /* Trailing new line */
-    .replace(/\n$/, '')
 }
- 
-function sortPlugins(data) {
-  return data.sort((a, b) => {
-    // put table of contents (TOC) at end of tranforms
-    if (a.transform === 'TOC') return 1
-    if (b.transform === 'TOC') return -1
-    return 0
+
+function logErrors(allErrors) {
+  const word = pluralize(allErrors, 'error', 'errors')
+  const title = `Markdown Magic ${word}: ${allErrors.length}`
+  console.log('\n──────────────────────────────')
+  error(title)
+  console.log('──────────────────────────────\n')
+  let msg = title
+  allErrors.forEach(({ errorMessage, errors }, i) => {
+    const finalMessage = `${i + 1}. ${errorMessage}`
+    msg += `\n${finalMessage}`
+    error(finalMessage, ``)
+    errors.forEach(({ message }, n) => {
+      const newLineX = errors.length !== n + 1 ? '' : '\n'
+      const lineMessage = ` - ${message}`
+      msg += `\n${lineMessage}`
+      console.log(`${lineMessage}${newLineX}`)
+    })
   })
+  return msg
 }
 
-function indentString(string, count = 1, options = {}) {
-	const {
-		indent = ' ',
-		includeEmptyLines = false
-	} = options;
-	if (count === 0) return string
-	const regex = includeEmptyLines ? /^/gm : /^(?!\s*$)/gm
-	return string.replace(regex, indent.repeat(count))
+function logCommentBlockSyntax({
+  syntax,
+  open,
+  close
+}) {
+  const syntaxDetails = getSyntaxInfo(syntax)
+  console.log(`
+${syntaxDetails.tags[0]} ${open} transformName ${syntaxDetails.tags[1]}
+generated content here
+${syntaxDetails.tags[0]} ${close} ${syntaxDetails.tags[1]}
+`)
 }
 
 module.exports = {
-  processContents
+  markdownMagic,
+  parseMarkdown,
+  processContents,
+  processFile
 }

package/lib/index.test.js

@@ -0,0 +1,11 @@
+const { test } = require('uvu') 
+const assert = require('uvu/assert')
+const { markdownMagic, processContents, processFile } = require('./')
+
+test('Main API', () => {
+  assert.equal(typeof markdownMagic, 'function')
+  assert.equal(typeof processContents, 'function')
+  assert.equal(typeof processFile, 'function')
+})
+
+test.run()