markdown-magic 3.0.3 → 3.0.5

package/lib/utils/fs.js

@@ -57,180 +57,17 @@ async function escalade(start, callback) {
 	}
 }
 
-// https://github.com/lukeed/totalist
-async function totalist(dir, callback, pre='') {
-	dir = resolve('.', dir)
-	await readdir(dir).then(arr => {
-		return Promise.all(arr.map((str) => {
-      let abs = join(dir, str)
-      return stat(abs).then(stats => {
-        return stats.isDirectory() ? totalist(abs, callback, join(pre, str)) : callback(join(pre, str), abs, stats)
-      })
-    }))
-	})
-}
-
-function combineRegexes(patterns = []) {
-  const string = patterns.map((pat) => {
-    if (isRegex(pat)) {
-      return pat.source
-    } else if (typeof pat === 'string' && REGEX_REGEX.test(pat)) {
-      const regexInfo = pat.match(REGEX_REGEX)
-      console.log('regexInfo', regexInfo)
-      if (regexInfo && regexInfo[1]) {
-        // escapeRegexString
-        return regexInfo[1]
-      }
-    } else if (isGlob(pat)) {
-      console.log('pat', pat)
-      const result = globrex(pat, { globstar: true, extended: true })
-      console.log('result', result)
-      return result.regex.source
-    }
-    return pat
-  }).join('|')
-  console.log('xxxstring', string)
-  return new RegExp(string)
-}
-
 // alt https://github.com/duniul/clean-modules/blob/33b66bcfb7825e1fa1eb1e296e523ddba374f1ae/src/utils/filesystem.ts#L92
 // Alt https://github.com/AvianFlu/ncp
-async function getFilePaths(dirName, {
-  patterns = [],
-  ignore = [],
-  excludeGitIgnore = false,
-  excludeHidden = false
-}) {
-  let findPattern
-  let ignorePattern
-  let filePaths = []
-  let gitIgnoreFiles = []
-  let gitIgnoreGlobs = []
-
-  if (patterns && patterns.length) {
-    findPattern = combineRegexes(patterns)
-    console.log('findPatternfindPatternfindPattern', patterns)
-  }
-  if (ignore && ignore.length) {
-    ignorePattern = combineRegexes(ignore)
-  }
-
-  if (excludeGitIgnore) {
-    const gitIgnoreContents = await getGitignoreContents()
-    for (let index = 0; index < gitIgnoreContents.length; index++) {
-      const ignoreItem = gitIgnoreContents[index]
-      // console.log('ignoreItem', ignoreItem)
-      if (isGlob(ignoreItem)) {
-        gitIgnoreGlobs.push(ignoreItem)
-      } else {
-        gitIgnoreFiles.push(
-          ignoreItem.replace(/^\.\//, '') // normalize relative paths
-        )
-      }
-    }
-  }
-  
-  //*
-  console.log('findPattern', findPattern)
-  console.log('ignorePattern', ignorePattern)
-  console.log('gitIgnoreFiles', gitIgnoreFiles)
-  console.log('gitIgnoreGlobs', gitIgnoreGlobs)
-  // process.exit(1)
-  /** */
-
-  await totalist(dirName, (relativePath, abs, stats) => {
-    const absolutePath = `${dirName}/${relativePath}`
-    const topLevelDir = relativePath.substring(0, relativePath.indexOf('/'))
-    //console.log('absolutePath', absolutePath)
-    // console.log('relativePath', relativePath)
-    // console.log('topLevelDir', topLevelDir)
-
-    /* Remove hidden files */
-    if (excludeHidden && IS_HIDDEN_FILE.test(relativePath)) {
-      return
-    }
-
-    /* Remove files in git ignore */
-    if (excludeGitIgnore && gitIgnoreFiles.length) {
-      if (gitIgnoreFiles.includes(relativePath)) return
-      if (gitIgnoreFiles.includes(path.basename(relativePath))) return
-      //* // slow
-      if (gitIgnoreFiles.some((ignore) => {
-        // console.log('ignore', ignore)
-        // return relativePath.indexOf(ignore) > -1
-        // return relativePath.split('/')[0] === ignore
-        return topLevelDir === ignore || relativePath === ignore
-      })) {
-        return
-      }
-      /** */
-    }
+// Moved to https://www.npmjs.com/package/glob-monster
+// async function getFilePaths(dirName, {
+//   patterns = [],
+//   ignore = [],
+//   excludeGitIgnore = false,
+//   excludeHidden = false
+// }) {
 
-    /* Remove files in ignore array */
-    if (ignorePattern && ignorePattern.test(relativePath)) {
-      // Alt checker https://github.com/axtgr/wildcard-match
-      return
-    }
-
-    /* If no patterns supplied add all files */
-    if (!findPattern) {
-      filePaths.push(absolutePath)
-      return
-    }
-
-    /* If pattern match add file! */
-    // Alt match https://github.com/micromatch/picomatch
-    if (findPattern.test(absolutePath)) {
-      // console.log('Match absolutePath', absolutePath)
-      filePaths.push(absolutePath)
-      return
-    }
-
-    /* If pattern match add file! */
-    if (findPattern.test(relativePath)) {
-      // console.log('Match relativePath', relativePath)
-      filePaths.push(absolutePath)
-      return
-    }
-
-    /*
-    let ignored = false
-    for (let index = 0; index < ignore.length; index++) {
-      const pattern = ignore[index]
-      if (pattern.test(absolutePath)) {
-        ignored = true
-      }
-    }
-    if (!ignored) {
-      filePaths.push(absolutePath)
-    }
-    /** */
-	})
-
-  if (gitIgnoreGlobs && gitIgnoreGlobs.length) {
-    console.log('gitIgnoreGlobs', gitIgnoreGlobs)
-    let removeFiles = []
-    for (let index = 0; index < gitIgnoreGlobs.length; index++) {
-      const glob = gitIgnoreGlobs[index]
-      const result = globrex(glob) // alt lib https://github.com/axtgr/wildcard-match
-      // console.log('result', result)
-      for (let n = 0; n < filePaths.length; n++) {
-        const file = filePaths[n]
-        if (result.regex.test(file)) {
-          removeFiles.push(file)
-        }
-      }
-    }
-    /* Remove files that match glob pattern */
-    if (removeFiles.length) {
-      filePaths = filePaths.filter(function(el) {
-        return removeFiles.indexOf(el) < 0
-      })
-    }
-  }
-
-  return filePaths
-}
+// }
 
 function convertToRelativePath(file, cwd) {
   return file.replace(cwd, '').replace(/^\//, '')
@@ -333,7 +170,6 @@ module.exports = {
   writeFile,
   readFile, 
   findUp,
-  getFilePaths,
   resolveOutputPath,
   resolveFlatPath,
   resolveCommonParent,

package/lib/utils/fs.test.js

@@ -3,7 +3,7 @@ const { test } = require('uvu')
 const assert = require('uvu/assert')
 const { glob, globWithGit } = require('smart-glob')
 const GREEN = '\x1b[32m%s\x1b[0m';
-const { findUp, getFilePaths, getGitignoreContents, convertToRelativePath } = require('./fs')
+const { findUp, getGitignoreContents, convertToRelativePath } = require('./fs')
 
 const ROOT_DIR = path.resolve(__dirname, '../../')
 
@@ -25,170 +25,11 @@ test('Finds file from dir', async () => {
   assert.equal(path.basename(file || ''), 'README.md')
 })
 
-test('getFilePaths /\.test\.js?$/', async () => {
-  const files = await getFilePaths(ROOT_DIR, {
-    patterns: [
-      /\.test\.js?$/,
-    ],
-    ignore: [
-      /node_modules/,
-    ],
-  })
-  const foundFiles = convertToRelative(files, ROOT_DIR)
-  console.log('foundFiles', foundFiles)
-  assert.is(Array.isArray(files), true)
-  assert.equal(foundFiles, [
-    'lib/block-parser-js.test.js',
-    'lib/block-parser.test.js',
-    'lib/cli.test.js',
-    'lib/index.test.js',
-    'lib/utils/fs.test.js',
-    "lib/utils/md/find-frontmatter.test.js",
-    "lib/utils/md/md.test.js",
-    "lib/utils/text.test.js",
-    'test/errors.test.js',
-    'test/transforms.test.js'
-  ])
-})
-
-test('getFilePaths /\.mdx?$/, /\.test\.js?$/', async () => {
-  const files = await getFilePaths(ROOT_DIR, {
-    patterns: [
-      /fixtures\/md\/(.*)\.mdx?$/,
-      // /\.js$/,
-      /\.test\.js?$/,
-    ],
-    ignore: [
-      // /^node_modules\//,
-      /node_modules/,
-      // /\.git/, 
-      // /NOTES\.md/
-    ],
-    //excludeGitIgnore: true,
-    excludeHidden: true,
-  })
-  const foundFiles = convertToRelative(files, ROOT_DIR)
-  //console.log('foundFiles', foundFiles)
-  assert.is(Array.isArray(files), true)
-  assert.equal(foundFiles, [
-    'lib/block-parser-js.test.js',
-    'lib/block-parser.test.js',
-    'lib/cli.test.js',
-    'lib/index.test.js',
-    'lib/utils/fs.test.js',
-    "lib/utils/md/find-frontmatter.test.js",
-    "lib/utils/md/md.test.js",
-    "lib/utils/text.test.js",
-    'test/errors.test.js',
-    'test/fixtures/md/basic.md',
-    "test/fixtures/md/broken-inline.md",
-    'test/fixtures/md/error-missing-transforms-two.md',
-    'test/fixtures/md/error-missing-transforms.md',
-    'test/fixtures/md/error-no-block-transform-defined.md',
-    'test/fixtures/md/error-unbalanced.md',
-    'test/fixtures/md/format-inline.md',
-    'test/fixtures/md/format-with-wacky-indentation.md',
-    'test/fixtures/md/inline-two.md',
-    'test/fixtures/md/inline.md',
-    'test/fixtures/md/mdx-file.mdx',
-    'test/fixtures/md/missing-transform.md',
-    'test/fixtures/md/mixed.md',
-    'test/fixtures/md/nested/nested.md',
-    'test/fixtures/md/no-transforms.md',
-    'test/fixtures/md/string.md',
-    'test/fixtures/md/syntax-legacy-colon.md',
-    'test/fixtures/md/syntax-legacy-query.md',
-    'test/fixtures/md/syntax-mixed.md',
-    'test/fixtures/md/transform-code.md',
-    'test/fixtures/md/transform-custom.md',
-    'test/fixtures/md/transform-file.md',
-    'test/fixtures/md/transform-remote.md',
-    'test/fixtures/md/transform-toc.md',
-    'test/fixtures/md/transform-wordCount.md',
-    'test/transforms.test.js'
-  ])
-})
-
-test('getFilePaths glob', async () => {
-  /*
-  const x = await glob('**.md')
-  console.log(x)
-  process.exit(1)
-  /** */
-
-  const files = await getFilePaths(ROOT_DIR, {
-    patterns: [
-      /\.test\.js?$/,
-      // /(.*)\.mdx?$/,
-      'test/fixtures/md/**.{md,mdx}'
-      // /^[^\/]+\.md?$/,
-      // '**.json',
-      // '**/**.js',
-      // '**.md',
-      //'/(.*).md$/',
-      // '/^test/',
-      // 'test/**'
-      ///(.*)\.md/g
-    ],
-    ignore: [
-      // /^node_modules\//,
-      /node_modules/,
-      // /(.*)\.js$/,
-      // /\.git/,
-      // /NOTES\.md/
-    ],
-    excludeGitIgnore: true,
-    excludeHidden: true,
-  })
-  const foundFiles = convertToRelative(files, ROOT_DIR)
-  console.log('foundFiles', foundFiles)
-  /*
-  aggregateReports()
-  process.exit(1)
-  /** */
-  assert.is(Array.isArray(files), true)
-  assert.equal(foundFiles, [
-    'lib/block-parser-js.test.js',
-    'lib/block-parser.test.js',
-    'lib/cli.test.js',
-    'lib/index.test.js',
-    'lib/utils/fs.test.js',
-    "lib/utils/md/find-frontmatter.test.js",
-    "lib/utils/md/md.test.js",
-    'lib/utils/text.test.js',
-    // 'misc/old-test/main.test.js',
-    'test/errors.test.js',
-    'test/fixtures/md/basic.md',
-    "test/fixtures/md/broken-inline.md",
-    'test/fixtures/md/error-missing-transforms-two.md',
-    'test/fixtures/md/error-missing-transforms.md',
-    'test/fixtures/md/error-no-block-transform-defined.md',
-    'test/fixtures/md/error-unbalanced.md',
-    'test/fixtures/md/format-inline.md',
-    'test/fixtures/md/format-with-wacky-indentation.md',
-    'test/fixtures/md/inline-two.md',
-    'test/fixtures/md/inline.md',
-    'test/fixtures/md/mdx-file.mdx',
-    'test/fixtures/md/missing-transform.md',
-    'test/fixtures/md/mixed.md',
-    'test/fixtures/md/no-transforms.md',
-    'test/fixtures/md/string.md',
-    'test/fixtures/md/syntax-legacy-colon.md',
-    'test/fixtures/md/syntax-legacy-query.md',
-    'test/fixtures/md/syntax-mixed.md',
-    'test/fixtures/md/transform-code.md',
-    'test/fixtures/md/transform-custom.md',
-    'test/fixtures/md/transform-file.md',
-    'test/fixtures/md/transform-remote.md',
-    'test/fixtures/md/transform-toc.md',
-    'test/fixtures/md/transform-wordCount.md',
-    'test/transforms.test.js'
-  ])
-})
-
 test('getGitignoreContents', async () => {
   const files = await getGitignoreContents()
+  /*
   console.log('files', files)
+  /** */
   assert.is(Array.isArray(files), true)
   assert.equal(files, [
     'logs',
@@ -200,6 +41,7 @@ test('getGitignoreContents', async () => {
     '_out.md',
     'misc',
     'misc/**/**.js',
+    "__misc",
     '**/old-test/cool.md',
     'pids',
     '*.pid',

package/lib/utils/hash-file.js

@@ -0,0 +1,28 @@
+const crypto = require('crypto')
+const { ReadStream } = require('fs')
+
+function hashFile(filename, algorithm = 'sha1') {
+  return new Promise((resolve, reject) => {
+    // Algorithm depends on availability of OpenSSL on platform
+    // Another algorithms: 'sha1', 'md5', 'sha256', 'sha512' ...
+    let shasum = crypto.createHash(algorithm)
+    try {
+      let s = ReadStream(filename)
+      s.on('data', (data) => {
+        shasum.update(data)
+      })
+      // making digest
+      s.on('end', () => {
+        const hash = shasum.digest('hex')
+        return resolve(hash)
+      })
+    } catch (error) {
+      console.log(error)
+      return reject(new Error('calc fail'))
+    }
+  })
+}
+
+module.exports = {
+  hashFile
+}

package/lib/utils/logs.js

@@ -2,6 +2,19 @@ const util = require('util')
 const ansi = require('ansi-styles') // https://github.com/chalk/ansi-styles/blob/main/index.js
 const process = require('process')
 
+function convertHrtime(hrtime) {
+	const nanoseconds = hrtime;
+	const number = Number(nanoseconds);
+	const milliseconds = number / 1000000;
+	const seconds = number / 1000000000;
+
+	return {
+		seconds: Math.round(seconds * 100) / 100,
+		milliseconds,
+		nanoseconds,
+	};
+}
+
 // via https://github.com/sindresorhus/is-unicode-supported/blob/main/index.js
 function isUnicodeSupported() {
   if (process.platform !== 'win32') return process.env.TERM !== 'linux' // Linux console (kernel)
@@ -47,13 +60,13 @@ const ERROR = allowed ? '✖' : '×'
 
 const colors = {
   default: ['white', ''],
-  success: ['greenBright', `${SUCCESS}${SPACES}`],
+  success: ['greenBright', ` ${SUCCESS}${SPACES}`],
   info: ['cyanBright', `${INFO}${SPACES}`],
   warning: ['yellowBright', `${WARNING}${SPACES}`],
   error: ['redBright', `${ERROR}${SPACES}`]
 }
 
-function log(type, msg, customPrefix, noLog) {
+function log(type, msg, noLog, customPrefix) {
   const [color, prefix] = colors[type] || colors.default
   const finalPrefix = typeof customPrefix !== 'undefined' ? customPrefix : prefix
   const logMsg = `${styles[color].open}${finalPrefix}${msg}${styles[color].close}`
@@ -86,6 +99,7 @@ error('Error: Oh no!')
 /**/
 
 module.exports = {
+  convertHrtime,
   deepLog,
   success,
   info,

package/lib/utils/syntax.js

@@ -61,6 +61,7 @@ const syntaxMap = {
   html: html,
   // /* x */
   js: js,
+  json: js,
   // {/* x */}
   jsx: jsx,
   mdx: jsx,

package/lib/utils/text.js

@@ -177,7 +177,7 @@ function stripComments(str, syntax = 'md') {
   const OR = (syntaxData.singleLine) ? `|\\s?[ \\t]*${syntaxData.singleLine}` : ''
   const CONTENT = syntaxData.content || '[\\s\\S]*?'
   const pattern = new RegExp(`\\s?[ \\t]*${openPattern}(${CONTENT})?${closePattern}${OR}`, 'gim')
-  console.log('pattern', pattern)
+  // console.log('pattern', pattern)
   return str.replace(pattern, '')
   // https://regex101.com/r/XKHU18/5
   return str.replace(/\s?[ \t]*\/\*[\s\S]*?\*\/|\s?[ \t]*\/\/.*$|\/\*{1,}[\n\*]*(\s?[\s\S]*?)?\*+\//gm, '')

package/lib/utils/toposort.js

@@ -0,0 +1,131 @@
+
+// https://github.com/marcelklehr/toposort/tree/master
+/**
+ * Topological sorting function
+ *
+ * @param {Array} edges
+ * @returns {Array}
+ */
+
+module.exports = function(edges) {
+  return toposort(uniqueNodes(edges), edges)
+}
+
+module.exports.array = toposort
+
+function toposort(nodes, edges) {
+  var cursor = nodes.length
+    , sorted = new Array(cursor)
+    , visited = {}
+    , i = cursor
+    // Better data structures make algorithm much faster.
+    , outgoingEdges = makeOutgoingEdges(edges)
+    , nodesHash = makeNodesHash(nodes)
+
+  // check for unknown nodes
+  edges.forEach(function(edge) {
+    if (!nodesHash.has(edge[0]) || !nodesHash.has(edge[1])) {
+      throw new Error('Unknown node. There is an unknown node in the supplied edges.')
+    }
+  })
+
+  while (i--) {
+    if (!visited[i]) visit(nodes[i], i, new Set())
+  }
+
+  return sorted
+
+  function visit(node, i, predecessors) {
+    if(predecessors.has(node)) {
+      var nodeRep
+      try {
+        nodeRep = ", node was:" + JSON.stringify(node)
+      } catch(e) {
+        nodeRep = ""
+      }
+      throw new Error('Cyclic dependency' + nodeRep)
+    }
+
+    if (!nodesHash.has(node)) {
+      throw new Error('Found unknown node. Make sure to provided all involved nodes. Unknown node: '+JSON.stringify(node))
+    }
+
+    if (visited[i]) return;
+    visited[i] = true
+
+    var outgoing = outgoingEdges.get(node) || new Set()
+    outgoing = Array.from(outgoing)
+
+    if (i = outgoing.length) {
+      predecessors.add(node)
+      do {
+        var child = outgoing[--i]
+        visit(child, nodesHash.get(child), predecessors)
+      } while (i)
+      predecessors.delete(node)
+    }
+
+    sorted[--cursor] = node
+  }
+}
+
+function uniqueNodes(arr){
+  var res = new Set()
+  for (var i = 0, len = arr.length; i < len; i++) {
+    var edge = arr[i]
+    res.add(edge[0])
+    res.add(edge[1])
+  }
+  return Array.from(res)
+}
+
+function makeOutgoingEdges(arr){
+  var edges = new Map()
+  for (var i = 0, len = arr.length; i < len; i++) {
+    var edge = arr[i]
+    if (!edges.has(edge[0])) edges.set(edge[0], new Set())
+    if (!edges.has(edge[1])) edges.set(edge[1], new Set())
+    edges.get(edge[0]).add(edge[1])
+  }
+  return edges
+}
+
+function makeNodesHash(arr){
+  var res = new Map()
+  for (var i = 0, len = arr.length; i < len; i++) {
+    res.set(arr[i], i)
+  }
+  return res
+}
+
+
+function topoSortOld(arr) {
+  var map = {} // Creates key value pair of name and object
+  var result = [] // the result array
+  var visited = {} // takes a note of the traversed dependency
+
+  arr.forEach(function(obj) { // build the map
+    map[obj.name]  = obj
+  })
+
+  arr.forEach(function(obj) { // Traverse array
+    if (!visited[obj.name]) { // check for visited object
+      sort_util(obj)
+    }
+  })
+
+  // On visiting object, check for its dependencies and visit them recursively 
+  function sort_util(obj) {
+    if (!obj) return;
+    // console.log('obj', obj)
+    visited[obj.name] = true
+    obj.dependencies.forEach(function(dep){
+      if (!visited[dep]) {
+        sort_util(map[dep])
+      } 
+    })
+    result.push(obj)
+  }
+
+  console.log(result);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "markdown-magic",
-  "version": "3.0.3",
+  "version": "3.0.5",
   "description": "Automatically update markdown files with content from external sources",
   "main": "lib/index.js",
   "bin": {
@@ -44,6 +44,7 @@
     "url": "https://github.com/DavidWells/markdown-magic"
   },
   "dependencies": {
+    "@davidwells/md-utils": "^0.0.40",
     "@technote-space/doctoc": "2.4.7",
     "globrex": "^0.1.2",
     "gray-matter": "^4.0.3",
@@ -52,14 +53,14 @@
     "is-valid-path": "^0.1.1",
     "micro-mdx-parser": "^1.1.0",
     "mri": "^1.2.0",
-    "oparser": "^2.1.1",
+    "oparser": "^3.0.13",
     "smart-glob": "^1.0.2",
     "sync-request": "^6.1.0"
   },
   "devDependencies": {
     "ansi-styles": "^4.2.1",
     "concordance": "^5.0.1",
-    "doxxx": "^2.0.1",
+    "doxxx": "^2.0.7",
     "rimraf": "^3.0.2",
     "safe-chalk": "^1.0.0",
     "typescript": "^5.0.2",

package/lib/utils/md/filters.js

@@ -1,20 +0,0 @@
-const IMAGE_POSTFIX_REGEX = /\.(png|apng|jpe?g|gif|webp|svg|avif)$/
-const RELATIVE_LINK_REGEX = /^(?!(?:(?:https?|ftp):\/\/|data:))((?:\.\.?\/)*)*([\w\d\-_./?=#%:+&]+)/
-
-function onlyUnique(value, index, self) {
-  return self.indexOf(value) === index
-}
-
-function isImage(link = '') {
-  return link.match(IMAGE_POSTFIX_REGEX)
-}
-
-function isRelative(filepath = '') {
-  return RELATIVE_LINK_REGEX.test(filepath)
-}
-
-module.exports = {
-  onlyUnique,
-  isImage,
-  isRelative
-}