markdown-magic 3.0.9 → 3.1.0

package/lib/index.js

@@ -33,6 +33,11 @@ const defaultTransforms = {
   remote: remoteTransform
 }
 
+const defaultOptions = {
+  failOnMissingTransforms: false,
+  failOnMissingRemote: true,
+}
+
 /**!
  * Allowed file syntaxes
  * @typedef {'md' | 'js' | 'yml' | 'yaml'} SyntaxType
@@ -61,7 +66,9 @@ const defaultTransforms = {
  * @property {boolean} [dryRun = false] - See planned execution of matched blocks
  * @property {boolean} [debug = false] - See debug details
  * @property {boolean} [silent = false] - Silence all console output
+ * @property {boolean} [applyTransformsToSource = true] - Apply transforms to source file. Default is true.
  * @property {boolean} [failOnMissingTransforms = false] - Fail if transform functions are missing. Default skip blocks.
+ * @property {boolean} [failOnMissingRemote = true] - Fail if remote file is missing.
  */
 
 /**
@@ -109,6 +116,9 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
   } else if (typeof globOrOpts === 'object') {
     opts = globOrOpts
   }
+
+  opts = Object.assign({}, defaultOptions, opts)
+
   const {
     transforms,
     // open,
@@ -117,6 +127,7 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
     outputFlatten = false,
     useGitGlob = false, 
     failOnMissingTransforms = false,
+    failOnMissingRemote = true,
     dryRun = false,
     debug = false,
     syntax = 'md',
@@ -128,7 +139,7 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
   const removeComments = output.removeComments || false
   const pathFormatter = output.pathFormatter
   
-  let applyTransformsToSource = false
+  let applyTransformsToSource = true
   if (typeof output.applyTransformsToSource !== 'undefined') {
     applyTransformsToSource = output.applyTransformsToSource
     // @ts-ignore
@@ -137,6 +148,8 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
     applyTransformsToSource = opts.applyTransformsToSource
   }
 
+  opts.applyTransformsToSource = applyTransformsToSource
+
   const logger = (silent) ? () => {} : console.log
 
   let open = OPEN_WORD
@@ -314,6 +327,7 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
     if (pathFormatter) {
       newPath = pathFormatter({ filePath: newPath })
     }
+    // console.log('newPath', newPath)
     // const cleanerDirPath = path.dirname(file)
     // const baseDir = cleanerDirPath.replace(cwd, '')
     // const cleanerDir = baseDir
@@ -628,9 +642,9 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
   logger(`${LINE}`)
   success(`Markdown Magic Done. ${elasped.seconds} seconds`, silent)
   logger(`${LINE}`)
-
   return {
-    filesChanged: plan.filter(({ isChanged }) => isChanged).map(({ outputPath }) => outputPath),
+    // @TODO maybe seperate changed and output files
+    filesChanged: plan.filter(({ isChanged, isNewPath }) => isChanged || isNewPath).map(({ outputPath }) => outputPath),
     results: plan,
     errors,
   }

package/lib/process-file.js

@@ -4,18 +4,9 @@ const { readFile, writeFile } = require('./utils/fs')
 const { processContents } = require('./process-contents')
 
 async function processFile(opts = {}) {
-  const { content, syntax, outputPath, dryRun, patterns, output = {} } = opts
+  const { content, syntax, outputPath, dryRun, patterns, output = {}, applyTransformsToSource } = opts
   const outputDir = output.directory || opts.outputDir
 
-  let applyTransformsToSource = false
-  if (typeof output.applyTransformsToSource !== 'undefined') {
-    applyTransformsToSource = output.applyTransformsToSource
-    // @ts-ignore
-  } else if (typeof opts.applyTransformsToSource !== 'undefined') {
-    // @ts-ignore
-    applyTransformsToSource = opts.applyTransformsToSource
-  }
-
   let srcPath = opts.srcPath
   if (srcPath && content) {
     throw new Error(`Can't set both "srcPath" & "content"`) 
@@ -43,7 +34,9 @@ async function processFile(opts = {}) {
     srcPath,
     syntax: syntaxType,
   })
-  // console.log('result', result)
+  /*
+  console.log('result', result)
+  /** */
 
   if (dryRun) {
     return result
@@ -55,11 +48,11 @@ async function processFile(opts = {}) {
     if (result.stripComments && patterns.openPattern && patterns.closePattern) {
       cleanContents = result.updatedContents.replace(patterns.openPattern, '').replace(patterns.closePattern, '')
     }
-    if (outputDir) {
+    if (outputDir || (srcPath !== outputPath)) {
       // console.log(`- Update output file: ${outputPath}`)
       await writeFile(outputPath, cleanContents)
     }
-    if (!outputDir || applyTransformsToSource) {
+    if (applyTransformsToSource) {
       // console.log(`- Update source file: ${srcPath}`)
       await writeFile(srcPath, result.updatedContents)
     }

package/lib/transforms/code/index.js

@@ -1,6 +1,6 @@
 const fs = require('fs')
 const path = require('path')
-const remoteRequest = require('../../utils/remoteRequest')
+const { remoteRequest } = require('../../utils/remoteRequest')
 const { isLocalPath } = require('../../utils/fs')
 const { deepLog } = require('../../utils/logs')
 const { getLineCount, getTextBetweenLines } = require('../../utils/text')
@@ -8,6 +8,7 @@ const { resolveGithubContents, isGithubLink } = require('./resolve-github-file')
 
 const GITHUB_LINK = /https:\/\/github\.com\/([^/\s]*)\/([^/\s]*)\/blob\//
 const GIST_LINK = /https:\/\/gist\.github\.com\/([^/\s]*)\/([^/\s]*)(\/)?/
+const RAW_URL_LIKE = /^([A-Za-z0-9_]*)\.([A-Za-z0-9_]*)\/(.*)/
 
 /**
  * Options for specifying source code to include in documentation.
@@ -37,7 +38,7 @@ const GIST_LINK = /https:\/\/gist\.github\.com\/([^/\s]*)\/([^/\s]*)(\/)?/
 // usage https://github.com/linear/linear/blame/93981d3a3db571e2f8efdce9f5271ea678941c43/packages/sdk/README.md#L1
 
 module.exports = async function CODE(api) {
-  const { content, srcPath } = api
+  const { content, srcPath, settings } = api
   /** @type {CodeTransformOptions} */
   const options = api.options || {}
   // console.log('CODE API', api)
@@ -57,19 +58,31 @@ module.exports = async function CODE(api) {
     deepLog(api.getCurrentBlock())
     throw new Error('Missing "src" attribute')
   }
+  let remoteContent
   let codeFilePath = src
   if (isLocalPath(src)) {
     const fileDir = (srcPath) ? path.dirname(srcPath) : process.cwd()
     codeFilePath = path.resolve(fileDir, src)
-    try {
-      // console.log('READFILE CODE', codeFilePath)
-      code = fs.readFileSync(codeFilePath, 'utf8')
-    } catch (e) {
-      console.log(`FILE NOT FOUND ${codeFilePath}`)
-      throw e
+    /* If the path looks like a URL, attempt to resolve it */
+    const isUrlLike = src.match(RAW_URL_LIKE)
+    if (isUrlLike) {
+      remoteContent = await resolveRemoteContent(options, settings, srcPath)
+      code = remoteContent
     }
-    if (!syntax) {
-      syntax = path.extname(codeFilePath).replace(/^./, '')
+    if (!remoteContent) {
+      try {
+        // console.log('READFILE CODE', codeFilePath)
+        code = fs.readFileSync(codeFilePath, 'utf8')
+        if (!syntax) {
+          syntax = path.extname(codeFilePath).replace(/^./, '')
+        }
+      } catch (e) {
+        if (isUrlLike) {
+          throw new Error(`Unable to resolve ${src}`)
+        }
+        console.log(`FILE NOT FOUND ${codeFilePath}`)
+        throw e
+      }
     }
   } else {
     /* Automatically get raw code files from github */
@@ -83,23 +96,15 @@ module.exports = async function CODE(api) {
     // https://gist.github.com/DavidWells/7d2e0e1bc78f4ac59a123ddf8b74932d
     // https://gist.githubusercontent.com/DavidWells/7d2e0e1bc78f4ac59a123ddf8b74932d/raw/0808a83de7f07c931fb81ed691c1d6bbafad29d1/aligning-images.md
 
-    let remoteContent
-
-    if (isGithubLink(src)) {
-      remoteContent = await resolveGithubContents({
-        repoFilePath: src,
-        accessToken,
-        // debug: true
-      })
-    }
-
-    // Try initial remote request if public url
-    if (!remoteContent) {
-      remoteContent = remoteRequest(src)
-    }
+    remoteContent = await resolveRemoteContent(options, settings, srcPath)
   
     if (!remoteContent) {
-      console.log(`WARNING: ${src} URL NOT FOUND or internet connection is off or no access to remove URL`)
+      const errorMsg = `WARNING: ${src} URL NOT FOUND or internet connection is off or no access to remove URL`
+      console.log(errorMsg)
+      console.log('settings', settings)
+      if (settings.failOnMissingRemote) {
+        throw new Error(errorMsg)
+      }
       return originalContent
     }
     code = remoteContent
@@ -162,3 +167,24 @@ module.exports = async function CODE(api) {
 ${code}
 \`\`\``
 }
+
+
+async function resolveRemoteContent(options, settings, srcPath) {
+  const { src, accessToken } = options
+  let remoteContent
+
+  if (isGithubLink(src)) {
+    remoteContent = await resolveGithubContents({
+      repoFilePath: src,
+      accessToken,
+      // debug: true
+    })
+  }
+
+  // Try initial remote request if public url
+  if (!remoteContent) {
+    remoteContent = await remoteRequest(src, settings, srcPath)
+  }
+
+  return remoteContent
+}

package/lib/transforms/code/resolve-github-file.js

@@ -5,7 +5,7 @@ const { getTextBetweenLines } = require('../../utils/text')
 const VALID_SLUG_REGEX = /^[A-Z-a-z0-9_-]*$/
 const VALID_FILE_REGEX = /^[^;]*$/
 const GITHUB_LINK_REGEX = /^(?:https:\/\/)?github\.com\/([^/\s]*)\/([^/\s]*)\/blob\/([^/\s]*)\/([^\s]*)/
-const GITHUB_RAW_LINK_REGEX = /^(?:https:\/\/)?raw\.githubusercontent\.com\/([^/\s]*)\/([^/\s]*)\/([^/\s]*)\/([^\s]*)/
+const GITHUB_RAW_LINK_REGEX = /^(?:https:\/\/)?(?:raw\.)?githubusercontent\.com\/([^/\s]*)\/([^/\s]*)\/([^/\s]*)\/([^\s]*)/
 
 function isGithubLink(str = '') {
   return isGithubRepoLink(str) || isGithubRawLink(str)

package/lib/transforms/remote.js

@@ -1,11 +1,11 @@
-const remoteRequest = require('../utils/remoteRequest')
+const { remoteRequest } = require('../utils/remoteRequest')
 
-module.exports = function REMOTE(api) {
+module.exports = async function REMOTE(api) {
   // console.log('REMOTE api', api)
   const { options, content, settings } = api
   const { regex } = settings
   // console.log('MAKE REMOTE REQUEST')
-  const remoteContent = remoteRequest(options.url)
+  const remoteContent = await remoteRequest(options.url, settings, api.srcPath)
   if (!remoteContent) {
     return content
   }

package/lib/utils/remoteRequest.js

@@ -1,19 +1,50 @@
+const fetch = require('node-fetch')
 const request = require('sync-request')
 
-module.exports = function remoteRequest(url) {
+function formatUrl(url = '') {
+  return url.match(/^https?:\/\//) ? url : `https://${url}`
+}
+
+function remoteRequestSync(url, settings = {}, srcPath) {
   let body
-  const finalUrl = (url.match(/^https?:\/\//)) ? url : `https://${url}`
+  const finalUrl = formatUrl(url)
   try {
     // @ts-expect-error
     const res = request('GET', finalUrl)
     body = res.getBody('utf8')
   } catch (e) {
-    console.log(`WARNING: REMOTE URL ${finalUrl} NOT FOUND`)
-    console.log((e.message || '').split('\n')[0])
+    console.log(`⚠️  WARNING: REMOTE URL "${finalUrl}" NOT FOUND`)
+    const msg = (e.message || '').split('\n')[0] + `\nFix "${url}" value in ${srcPath}`
+    console.log(msg)
+    if (settings.failOnMissingRemote) {
+      throw new Error(msg)
+    }
   }
   return body
 }
 
+async function remoteRequest(url, settings = {}, srcPath) {
+  let body
+  const finalUrl = formatUrl(url)
+  try {
+    const res = await fetch(finalUrl)
+    body = await res.text()
+  } catch (e) {
+    console.log(`⚠️  WARNING: REMOTE URL "${finalUrl}" NOT FOUND`)
+    const msg = (e.message || '').split('\n')[0] + `\nFix "${url}" value in ${srcPath}`
+    console.log(msg)
+    if (settings.failOnMissingRemote) {
+      throw new Error(msg)
+    }
+  }
+  return body
+}
+
+module.exports = {
+  remoteRequestSync,
+  remoteRequest
+}
+
 /*
 TODO add file caching?
 */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "markdown-magic",
-  "version": "3.0.9",
+  "version": "3.1.0",
   "description": "Automatically update markdown files with content from external sources",
   "main": "lib/index.js",
   "bin": {
@@ -53,6 +53,7 @@
     "is-valid-path": "^0.1.1",
     "micro-mdx-parser": "^1.1.0",
     "mri": "^1.2.0",
+    "node-fetch": "^2.7.0",
     "oparser": "^3.0.13",
     "smart-glob": "^1.0.2",
     "sync-request": "^6.1.0"