markdown-magic 2.6.0 → 3.0.0

package/lib/process-contents.js

@@ -0,0 +1,330 @@
+const { parseBlocks } = require('./block-parser')
+const { deepLog } = require('./utils/logs')
+const { getCodeLocation } = require('./utils')
+const { indentString, trimString } = require('./utils/text')
+const { OPEN_WORD, CLOSE_WORD } = require('./defaults')
+
+/**
+ * Pull comment blocks out of content and process them
+ * @param {string} text 
+ * @param {object} config 
+ * @returns 
+ */
+ async function processContents(text, config = {}) {
+  const {
+    srcPath,
+    outputPath,
+    open = OPEN_WORD, // 'DOCS:START',
+    close = CLOSE_WORD, // 'DOCS:END',
+    syntax = 'md',
+    transforms,
+    beforeMiddleware = [],
+    afterMiddleware = [],
+    debug = false,
+  } = config
+
+  let foundBlocks = {}
+  try {
+    foundBlocks = parseBlocks(text, {
+      syntax,
+      open,
+      close,
+    })
+  } catch (e) {
+    throw new Error(`${e.message} in file ${srcPath}\n`)
+  }
+
+  if (debug) {
+    console.log(`foundBlocks ${srcPath}`)
+    deepLog(foundBlocks)
+  }
+  /*
+  deepLog(foundBlocks)
+  process.exit(1)
+  /** */
+  const { COMMENT_OPEN_REGEX, COMMENT_CLOSE_REGEX } = foundBlocks
+
+  const blocksWithTransforms = foundBlocks.blocks
+    .filter((block) => block.type)
+    .map((block, i) => {
+      const transform = block.type
+      delete block.type
+      return Object.assign({ transform }, block)
+    })
+
+  
+  const regexInfo = {
+    blocks: foundBlocks.pattern,
+    open: COMMENT_OPEN_REGEX,
+    close: COMMENT_CLOSE_REGEX,
+  }
+  // console.log('blocksWithTransforms', blocksWithTransforms)
+  // process.exit(1)
+
+  const transformsToRun = sortTranforms(blocksWithTransforms, transforms)
+  // .map((transform) => {
+  //   return {
+  //     ...transform,
+  //     srcPath
+  //   }
+  // })
+  // console.log('transformsToRun', transformsToRun)
+
+  // if (!transformsToRun.length) {
+  //   process.exit(1)
+  // }
+  // console.log('transformsToRun', transformsToRun)
+  let missingTransforms = []
+  const updatedContents = await transformsToRun.reduce(async (contentPromise, originalMatch) => {
+    const md = await contentPromise
+    /* Apply Before middleware to all transforms */
+    const match = await applyMiddleware(originalMatch, md, beforeMiddleware)
+    const { block, content, open, close, transform, options, context } = match
+    const closeTag = close.value
+    const openTag = open.value
+    
+    /* Run transform plugins */
+    let tempContent = content.value
+    // console.log('transform', transform)
+    const currentTransform = getTransform(transform, transforms)
+    /* Run each transform */
+    if (currentTransform) {
+      // console.log('context', context)
+      let returnedContent
+      /* DISABLED legacy syntax */
+      /* // Support for legacy syntax... maybe later
+      if (context && context.isLegacy) {
+        console.log(`CALL legacy ${transform}`, srcPath)
+        // backward compat maybe
+        // CODE(content, options, config)
+        returnedContent = await currentTransform(content.value, options, { 
+          originalPath: srcPath
+        })
+      } else {
+        returnedContent = await currentTransform(transformApi({
+          srcPath,
+          ...match,
+          regex: regexInfo,
+          originalContents: text,
+          currentContents: md,
+        }, config))
+      }
+      /** */
+
+      returnedContent = await currentTransform(transformApi({
+        srcPath,
+        ...match,
+        regex: regexInfo,
+        originalContents: text,
+        currentContents: md,
+      }, config))
+
+      /* Run each transform */
+      // console.log('config', config)
+
+      // console.log('returnedContent', returnedContent)
+     
+      if (returnedContent) {
+        tempContent = returnedContent
+      }
+    }
+
+    /* Apply After middleware to all transforms */
+    const afterContent = await applyMiddleware({
+      ...match,
+      ...{
+        content: {
+          ...match.content,
+          value: tempContent
+        }
+      }
+    }, md, afterMiddleware)
+    if (debug) {
+      console.log('afterContent', afterContent)
+    }
+
+    if (!currentTransform) {
+      missingTransforms.push(afterContent)
+      // console.log(`Missing "${transform}" transform`)
+    }
+
+    const newContent = afterContent.content.value
+    const formattedNewContent = (options.noTrim) ? newContent : trimString(newContent)
+    // console.log('formattedNewContent', formattedNewContent)
+    /* Remove any conflicting imported comments */
+    const fix = removeConflictingComments(formattedNewContent, foundBlocks.commentOpen, COMMENT_CLOSE_REGEX)
+    // const fix = stripAllComments(formattedNewContent, foundBlocks.commentOpen, COMMENT_CLOSE_REGEX)
+
+    // console.log('COMMENT_CLOSE_REGEX', COMMENT_CLOSE_REGEX)
+    // console.log('formattedNewContent', formattedNewContent)
+    // console.log('fix', fix)
+    const preserveIndent = (true || options.preserveIndent) ? block.indentation.length + match.content.indentation : block.indentation.length
+    const indent = indentString(fix, preserveIndent)
+    const newCont = `${openTag}${indent}${closeTag}`
+    /* Replace original contents */
+    const newContents = md.replace(block.value, newCont)
+    return Promise.resolve(newContents)
+  }, Promise.resolve(text))
+
+  if (debug) {
+    console.log('Output Markdown')
+    console.log(updatedContents)
+  }
+
+  /*
+  if (missingTransforms.length) {
+    console.log('missingTransforms', missingTransforms)
+    let matchOne = missingTransforms[1]
+    matchOne = missingTransforms[missingTransforms.length - 1]
+    // console.log('matchOne', matchOne)
+    const { block, transform, args } = matchOne
+    const { openTag, closeTag, content, contentStart, contentEnd, start, end} = block
+
+    // console.log('contentStart', contentStart)
+    // console.log('contentEnd', contentEnd)
+    // console.log('original text between', `"${getTextBetweenChars(md, contentStart, contentEnd)}"`)
+    // console.log('original block between', `"${getTextBetweenChars(md, start, end)}"`)
+  }
+  /** */
+
+  // console.log('detect slow srcPath', srcPath)
+
+  const result = {
+    /* Has markdown content changed? */
+    isChanged: text !== updatedContents,
+    isNewPath: srcPath !== outputPath,
+    srcPath,
+    outputPath,
+    // config,
+    transforms: transformsToRun,
+    missingTransforms,
+    originalContents: text,
+    updatedContents
+  }
+
+  // console.log('result')
+  // deepLog(result)
+
+  return result
+}
+
+function transformApi(stuff, opts) {
+  const { transforms, srcPath, outputPath, ...rest } = opts
+  return {
+    srcPath: stuff.srcPath,
+    outputPath: outputPath,
+    transform: stuff.transform,
+    options: stuff.options || {},
+    blockContent: stuff.content.value,
+    fileContent: stuff.currentContents,
+    originalFileContent: stuff.originalContents,
+    settings: {
+      ...rest,
+      regex: stuff.regex,
+    },
+    getOriginalContent: () => stuff.originalContents,
+    getOriginalBlock: () => stuff,
+    getCurrentBlock: () => {
+      /* Re-parse current file for updated positions */
+      const updatedBlocks = parseBlocks(stuff.currentContents, {
+        syntax: opts.syntax,
+        open: opts.open,
+        close: opts.close,
+      })
+
+   
+      const matchingBlocks = updatedBlocks.blocks.filter((block) => {
+        return block.open.value === stuff.open.value
+      })
+
+      if (!matchingBlocks.length) {
+        return {}
+      }
+      let foundBlock = matchingBlocks[0]
+
+      if (matchingBlocks.length > 1) {
+        foundBlock = matchingBlocks.filter((block) => {
+          return block.index === stuff.index
+        })[0]
+      }
+
+      const location = getCodeLocation(stuff.srcPath, foundBlock.block.lines[0])
+
+      return [ foundBlock, location ]
+    },
+  }
+}
+
+/**
+ * Remove conflicting comments that might have been inserted from transforms
+ * @param {*} content 
+ * @param {*} openPattern 
+ * @param {*} closePattern 
+ * @returns
+ */
+ function removeConflictingComments(content, openPattern, closePattern) {
+  const removeOpen = content.replace(openPattern, '')
+  // TODO this probably needs to be a loop for larger blocks
+  closePattern.lastIndex = 0; // reset regex
+  const hasClose = closePattern.exec(content)
+  // console.log('closePattern', closePattern)
+  // console.log('has', content)
+  // console.log('hasClose', hasClose)
+  if (!hasClose) {
+    return removeOpen
+  }
+  const closeTag = `${hasClose[2]}${hasClose[3] || ''}`
+  // console.log('closeTag', closeTag)
+  return removeOpen
+    .replaceAll(closeTag, '')
+    /* Trailing new line */
+    .replace(/\n$/, '')
+}
+
+function applyMiddleware(data, md, middlewares) {
+  return middlewares.reduce(async (acc, curr) => {
+    const realAcc = await acc
+    // console.log(`Running "${curr.name}" Middleware on "${realAcc.transform}" block`)
+    const updatedContent = await curr.transform(realAcc, md)
+    // realAcc.block.content = updatedContent
+    return Promise.resolve({
+      ...realAcc,
+      ...{
+        content: {
+          ...realAcc.content,
+          value: updatedContent
+        }
+      }
+    })
+  }, Promise.resolve(data))
+}
+
+function getTransform(name, transforms = {}) {
+  return transforms[name] || transforms[name.toLowerCase()]
+}
+
+function sortTranforms(foundTransForms, registeredTransforms) {
+  // console.log('transforms', transforms)
+  if (!foundTransForms) return []
+  return foundTransForms.sort((a, b) => {
+    // put table of contents (TOC) at end of tranforms
+    if (a.transform === 'TOC') return 1
+    if (b.transform === 'TOC') return -1
+    return 0
+  }).map((item) => {
+    if (getTransform(item.transform, registeredTransforms)) {
+      return item
+    }
+    return {
+      ...item,
+      context: {
+        ...item.context,
+        isMissing: true,
+      }
+    }
+  })
+}
+
+module.exports = {
+  processContents
+}

package/lib/process-file.js

@@ -0,0 +1,34 @@
+const path = require('path')
+const isValidFile = require('is-valid-path')
+const { readFile } = require('./utils/fs')
+const { processContents } = require('./process-contents')
+
+async function processFile(opts = {}) {
+  const { content, syntax } = opts
+  let srcPath = opts.srcPath
+  if (srcPath && content) throw new Error(`Can't set both "srcPath" & "content"`)
+  let fileContents
+  if (content) {
+    const isFile = isValidFile(content)
+    srcPath = (isFile) ? content : undefined
+    fileContents = (!isFile) ? content : undefined
+  }
+
+  if (!fileContents) {
+    fileContents = await readFile(srcPath || content, 'utf8')
+  }
+  
+  let syntaxType = syntax
+  if (srcPath && !syntaxType) {
+    syntaxType = path.extname(srcPath).replace(/^\./, '')
+  }
+  return processContents(fileContents, {
+    ...opts,
+    srcPath,
+    syntax: syntaxType,
+  })
+}
+
+module.exports = {
+  processFile
+}

package/lib/transforms/code.js

@@ -1,52 +1,97 @@
-const path = require('path')
 const fs = require('fs')
-const isLocalPath = require('is-local-path')
+const path = require('path')
 const remoteRequest = require('../utils/remoteRequest')
+const { isLocalPath } = require('../utils/fs')
+const { deepLog } = require('../utils/logs')
+const { getLineCount, getTextBetweenLines } = require('../utils/text')
 
-module.exports = function CODE(content, options, config) {
+// TODO code sections 
+// https://github.com/linear/linear/blob/94af540244864fbe466fb933256278e04e87513e/docs/transforms/code-section.js
+// https://github.com/linear/linear/blob/bc39d23af232f9fdbe7df458b0aaa9554ca83c57/packages/sdk/src/_tests/readme.test.ts#L133-L140
+// usage https://github.com/linear/linear/blame/93981d3a3db571e2f8efdce9f5271ea678941c43/packages/sdk/README.md#L1
+
+module.exports = function CODE(api) {
+  const { options = {}, blockContent, srcPath } = api
+  // console.log('CODE API', api)
+  // console.log('options', options)
+  const { src, lines } = options
+  const originalContent = blockContent
   let code
   let syntax = options.syntax
-  if (!options.src) {
-    return false
+  if (!src) {
+    deepLog(api.getCurrentBlock())
+    throw new Error('Missing "src" attribute')
   }
-  if (isLocalPath(options.src)) {
-    const fileDir = path.dirname(config.originalPath)
-    const filePath = path.join(fileDir, options.src)
+  let codeFilePath = src
+  if (isLocalPath(src)) {
+    const fileDir = (srcPath) ? path.dirname(srcPath) : process.cwd()
+    codeFilePath = path.resolve(fileDir, src)
     try {
-      code = fs.readFileSync(filePath, 'utf8', (err, contents) => {
+      code = fs.readFileSync(codeFilePath, 'utf8', (err, contents) => {
         if (err) {
-          console.log(`FILE NOT FOUND ${filePath}`)
+          console.log(`FILE NOT FOUND ${codeFilePath}`)
           console.log(err)
            // throw err
         }
         return contents
       })
     } catch (e) {
-      console.log(`FILE NOT FOUND ${filePath}`)
+      console.log(`FILE NOT FOUND ${codeFilePath}`)
       throw e
     }
     if (!syntax) {
-      syntax = path.extname(filePath).replace(/^./, '')
+      syntax = path.extname(codeFilePath).replace(/^./, '')
     }
   } else {
     // do remote request
-    // console.log(options.src)
-    const remoteContent = remoteRequest(options.src)
+    // console.log(src)
+    const remoteContent = remoteRequest(src)
     if (!remoteContent) {
-      console.log(`WARNING: ${options.src} URL NOT FOUND  or internet connection is off`)
-      return content
+      console.log(`WARNING: ${src} URL NOT FOUND  or internet connection is off`)
+      return originalContent
     }
     code = remoteContent
-    syntax = path.extname(options.src).replace(/^./, '')
+    syntax = path.extname(src).replace(/^./, '')
   }
 
-  // handle option `lines`
+  /* handle option `lines` */
   if (options.lines) {
-    const startLine = options.lines.split('-')[0];
-    const endLine = options.lines.split('-')[1];
+    const lineCount = getLineCount(code)
+    // console.log('lineCount', lineCount)
+    // console.log('src', src)
+    // console.log('lines', lines)
+    let startLine
+    let endLine
+    if (Array.isArray(lines)) {
+      startLine = lines[0]
+      endLine = lines[1]
+    } else if (typeof lines === 'string') {
+      const splitLines = lines.split('-')
+      startLine = splitLines[0]
+      endLine = splitLines[1]
+    }
     if ((startLine) && (endLine) && parseInt(startLine, 10) <= parseInt(endLine, 10)) {
-      code = code.split(/\r\n|\n/).slice(startLine - 1, endLine).join('\n');
+      code = getTextBetweenLines(code, startLine, endLine)
+    }
+  }
+
+  /* Check for Id */
+  if (options.id) {
+    const lines = code.split("\n")
+    const startLine = lines.findIndex(line => line.includes(`CODE_SECTION:${options.id}:START`)) ?? 0
+    const endLine = lines.findIndex(line => line.includes(`CODE_SECTION:${options.id}:END`)) ?? lines.length - 1
+    // console.log('startLine', startLine)
+    // console.log('endLineendLine', endLine)
+    if (startLine === -1 && endLine === -1) {
+      throw new Error(`Missing ${options.id} code section from ${codeFilePath}`)
     }
+  
+    const selectedLines = lines.slice(startLine + 1, endLine);
+  
+    const trimBy = selectedLines[0]?.match(/^(\s*)/)?.[1]?.length;
+    const newValue = `${selectedLines.map(line => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, "")).join("\n")}`
+    // console.log('newValue', newValue)
+    code = newValue
   }
 
   // trim leading and trailing spaces/line breaks in code and keeps the indentation of the first non-empty line
@@ -57,7 +102,7 @@ module.exports = function CODE(content, options, config) {
     header = `\n${options.header}`
   }
 
-  return `<!-- The below code snippet is automatically added from ${options.src} -->
+  return `<!-- The below code snippet is automatically added from ${src} -->
 \`\`\`${syntax}${header}
 ${code}
 \`\`\``

package/lib/transforms/file.js

@@ -2,31 +2,29 @@ const path = require('path')
 const fs = require('fs')
 const isLocalPath = require('is-local-path')
 
-module.exports = function FILE(content, options, config) {
+module.exports = function FILE(api) {
+  const { options, srcPath } = api
+  // console.log('FILE API', api)
   let fileContents
-  let syntax = options.syntax
   if (!options.src) {
     return false
   }
   if (isLocalPath(options.src)) {
-    const fileDir = path.dirname(config.originalPath)
-    const filePath = path.join(fileDir, options.src)
+    const fileDir = path.dirname(srcPath)
+    const resolvedFilePath = path.resolve(fileDir, options.src)
     try {
-      fileContents = fs.readFileSync(filePath, 'utf8', (err, contents) => {
+      fileContents = fs.readFileSync(resolvedFilePath, 'utf8', (err, contents) => {
         if (err) {
-          console.log(`FILE NOT FOUND ${filePath}`)
+          console.log(`FILE NOT FOUND ${resolvedFilePath}`)
           console.log(err)
            // throw err
         }
         return contents
       })
     } catch (e) {
-      console.log(`FILE NOT FOUND ${filePath}`)
+      console.log(`FILE NOT FOUND ${resolvedFilePath}`)
       throw e
     }
-    if (!syntax) {
-      syntax = path.extname(filePath).replace(/^./, '')
-    }
   }
 
   // trim leading and trailing spaces/line breaks in code and keeps the indentation of the first non-empty line
@@ -35,3 +33,8 @@ module.exports = function FILE(content, options, config) {
   return `<!-- The below content is automatically added from ${options.src} -->
 ${fileContents}`
 }
+
+// maybe support...
+function legacyCODE(content, options, config) {
+
+}

package/lib/transforms/remote.js

@@ -1,15 +1,18 @@
 const regexUtils = require('../utils/regex')
 const remoteRequest = require('../utils/remoteRequest')
 
-module.exports = function REMOTE(content, options, config) {
-  const remoteContent = remoteRequest(options.url) || content
+module.exports = function REMOTE(api) {
+  // console.log('REMOTE api', api)
+  const { options, blockContent, settings } = api
+  const { regex } = settings
+  // console.log('MAKE REMOTE REQUEST')
+  const remoteContent = remoteRequest(options.url)
   if (!remoteContent) {
-    return content
+    return blockContent
   }
   if (options.keepComments) {
     return remoteContent
   }
-  const openTagRegex = regexUtils.matchOpeningCommentTag(config.matchWord)
-  const closeTagRegex = regexUtils.matchClosingCommentTag(config.matchWord)
-  return remoteContent.replace(openTagRegex, '').replace(closeTagRegex, '')
+  // console.log('REMOTE', remoteContent)
+  return remoteContent.replace(regex.open, '').replace(regex.close, '')
 }