markdown-magic 4.7.0 → 4.9.0

package/src/index.js

@@ -153,6 +153,7 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
   const outputDir = output.directory || opts.outputDir
   const removeComments = output.removeComments || false
   const forceRemoveComments = output.forceRemoveComments || false
+  const normalizeBlankLines = output.normalizeBlankLines || false
   const pathFormatter = output.pathFormatter
   
   let applyTransformsToSource = true
@@ -343,22 +344,24 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
       name: file,
       id: file,
       srcPath: file,
-      blocks: foundBlocks.blocks
+      blocks: foundBlocks.blocks,
+      parsedBlocks: foundBlocks
     }
   })
 
   const blocks = blocksByPath.map((item) => {
     const dir = path.dirname(item.srcPath)
-    item.dependencies = []
+    const dependencySet = new Set()
     item.blocks.forEach((block) => {
       if (block.options && block.options.src) {
         const resolvedPath = path.resolve(dir, block.options.src)
         // if (resolvedPath.match(/\.md$/)) {
           // console.log('resolvedPath', resolvedPath)
-          item.dependencies = item.dependencies.concat(resolvedPath)
+          dependencySet.add(resolvedPath)
         //}
       }
     })
+    item.dependencies = Array.from(dependencySet)
     return item
   })
 
@@ -380,23 +383,23 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
   /** */
 
   // Convert items into a format suitable for toposort
-  const graph = blocks
-    .filter((item) => item.blocks && item.blocks.length)
-    .map((item) => {
-      return [ item.id, ...item.dependencies ]
-    })
+  const { itemsWithBlocks, itemIds, graph } = createDependencyGraph(blocks)
   // console.log('graph', graph)
   // Perform the topological sort and reverse for execution order
-  const sortedIds = toposort(graph).reverse();
+  const sortedIds = graph.length ? toposort.array(itemIds, graph).reverse() : itemIds
 
   // Reorder items based on sorted ids
-  const sortedItems = sortedIds.map(id => blocks.find(item => item.id === id)).filter(Boolean);
+  const itemById = new Map(itemsWithBlocks.map((item) => [item.id, item]))
+  const sortedItems = sortedIds.map((id) => itemById.get(id)).filter(Boolean)
 
   // topoSort(blocks)
   const orderedFiles = sortedItems.map((block) => block.id)
   // console.log('sortedItems', sortedItems)
   // console.log('orderedFiles', orderedFiles)
 
+  const fileContentByPath = new Map(files.map((file, i) => [file, fileContents[i]]))
+  const parsedBlocksByPath = new Map(blocksByPath.map((item) => [item.id, item.parsedBlocks]))
+
   const processedFiles = []
   await asyncForEach(orderedFiles, async (file) => {
     // logger('file', file)
@@ -423,6 +426,8 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
     // logger('newPath', newPath)
     const result = await processFile({
       ...opts,
+      content: fileContentByPath.get(file),
+      parsedBlocks: parsedBlocksByPath.get(file),
       patterns,
       open,
       close,
@@ -431,6 +436,7 @@ async function markdownMagic(globOrOpts = {}, options = {}) {
       transforms: useTransforms,
       removeComments,
       forceRemoveComments,
+      normalizeBlankLines,
       processedFiles
     })
 
@@ -785,6 +791,29 @@ function changedFiles(files) {
   return files.filter(({ isChanged }) => isChanged)
 }
 
+/**
+ * Create graph data for deterministic dependency ordering
+ * @param {Array<{id: string, blocks: Array<any>, dependencies?: Array<string>}>} blockItems
+ * @returns {{itemsWithBlocks: Array<any>, itemIds: Array<string>, graph: Array<[string, string]>}}
+ */
+function createDependencyGraph(blockItems = []) {
+  const itemsWithBlocks = blockItems.filter((item) => item.blocks && item.blocks.length)
+  const itemIds = itemsWithBlocks.map((item) => item.id)
+  const itemIdSet = new Set(itemIds)
+  const graph = itemsWithBlocks
+    .flatMap((item) => {
+      return (item.dependencies || [])
+        .filter((dependency) => itemIdSet.has(dependency))
+        .map((dependency) => [item.id, dependency])
+    })
+
+  return {
+    itemsWithBlocks,
+    itemIds,
+    graph
+  }
+}
+
 async function asyncForEach(array, callback) {
   for (let index = 0; index < array.length; index++) {
     await callback(array[index], index, array)
@@ -800,5 +829,8 @@ module.exports = {
   parseMarkdown,
   blockTransformer,
   processFile,
-  stringUtils
+  stringUtils,
+  __private: {
+    createDependencyGraph
+  }
 }

package/src/transforms/code/index.js

@@ -158,23 +158,29 @@ module.exports = async function CODE(api) {
 
   /* Check for Id */
   if (id) {
-    const lines = code.split("\n")
-    const startLineIndex = lines.findIndex(line => line.includes(`CODE_SECTION:${id}:START`));
-    const startLine = startLineIndex !== -1 ? startLineIndex : 0;
+    const lines = code.split('\n')
+    const startLineIndex = lines.findIndex((line) => line.includes(`CODE_SECTION:${id}:START`))
+    const endLineIndex = lines.findIndex((line) => line.includes(`CODE_SECTION:${id}:END`))
+
+    if (startLineIndex === -1 || endLineIndex === -1) {
+      throw new Error(`Missing ${id} code section from ${codeFilePath}`)
+    }
+
+    if (endLineIndex <= startLineIndex) {
+      throw new Error(`Invalid ${id} code section in ${codeFilePath}. End marker must be after start marker`)
+    }
 
-    const endLineIndex = lines.findIndex(line => line.includes(`CODE_SECTION:${id}:END`));
-    const endLine = endLineIndex !== -1 ? endLineIndex : lines.length - 1;
     // console.log('startLine', startLine)
     // console.log('endLine', endLine)
-    if (startLine === -1 && endLine === -1) {
-      throw new Error(`Missing ${id} code section from ${codeFilePath}`)
+    const selectedLines = lines.slice(startLineIndex + 1, endLineIndex)
+
+    if (!selectedLines.length) {
+      throw new Error(`Empty ${id} code section in ${codeFilePath}`)
     }
-  
-    const selectedLines = lines.slice(startLine + 1, endLine)
-  
-    const firstMatch = selectedLines[0] && selectedLines[0].match(/^(\s*)/);
-    const trimBy = firstMatch && firstMatch[1] ? firstMatch[1].length : 0;
-    const newValue = `${selectedLines.map(line => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, "")).join("\n")}`
+
+    const firstMatch = selectedLines[0] && selectedLines[0].match(/^(\s*)/)
+    const trimBy = firstMatch && firstMatch[1] ? firstMatch[1].length : 0
+    const newValue = `${selectedLines.map((line) => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, '')).join('\n')}`
     // console.log('newValue', newValue)
     code = newValue
   }

package/src/utils/remoteRequest.js

@@ -1,28 +1,52 @@
 const fetch = require('node-fetch')
 
 function formatUrl(url = '') {
-  return url.match(/^https?:\/\//) ? url : `https://${url}`
+  if (typeof url !== 'string') return ''
+  const trimmed = url.trim()
+  if (!trimmed) return ''
+  return trimmed.match(/^https?:\/\//) ? trimmed : `https://${trimmed}`
 }
 
 async function remoteRequest(url, settings = {}, srcPath) {
-  let body
   const finalUrl = formatUrl(url)
+  const fixText = srcPath ? `\nFix "${url}" value in ${srcPath}` : ''
+  if (!finalUrl) {
+    const msg = `Invalid URL "${url}"${fixText}`
+    if (settings.failOnMissingRemote) {
+      throw new Error(msg)
+    }
+    console.log(msg)
+    return
+  }
+
   // ignore demo url todo remove one day
   if (finalUrl === 'http://url-to-raw-md-file.md') {
     return
   }
+
+  let response
   try {
-    const res = await fetch(finalUrl)
-    body = await res.text()
+    response = await fetch(finalUrl)
   } catch (e) {
     console.log(`⚠️  WARNING: REMOTE URL "${finalUrl}" NOT FOUND`)
-    const msg = (e.message || '').split('\n')[0] + `\nFix "${url}" value in ${srcPath}`
+    const msg = (e.message || '').split('\n')[0] + fixText
     console.log(msg)
     if (settings.failOnMissingRemote) {
       throw new Error(msg)
     }
+    return
+  }
+
+  if (!response.ok) {
+    const msg = `Remote request failed with status ${response.status} (${response.statusText}) for "${finalUrl}"${fixText}`
+    console.log(`⚠️  WARNING: ${msg}`)
+    if (settings.failOnMissingRemote) {
+      throw new Error(msg)
+    }
+    return
   }
-  return body
+
+  return response.text()
 }
 
 module.exports = {

package/types/src/argparse/argparse.d.ts

@@ -1,33 +1,3 @@
-export function uxParse(_rawArgv?: any[], opts?: {}): {
-    leadingCommands: string[];
-    extraParse: {};
-    mriOptionsOriginal: mri.Argv<{
-        [x: string]: any;
-    }>;
-    globGroups: any[];
-    rawArgv?: undefined;
-    mriOptionsClean?: undefined;
-    mriDiff?: undefined;
-    yargsParsed?: undefined;
-    mergedOptions?: undefined;
-} | {
-    rawArgv: string;
-    leadingCommands: string[];
-    globGroups: any[];
-    extraParse: {};
-    mriOptionsOriginal: mri.Argv<{
-        [x: string]: any;
-    }>;
-    mriOptionsClean: {
-        [x: string]: any;
-    } & {
-        _: string[];
-    };
-    mriDiff: boolean;
-    yargsParsed: string;
-    mergedOptions: {
-        _: string[];
-    };
-};
-import mri = require("mri");
+export { dxParse };
+import { dxParse } from "@davidwells/dx-args";
 //# sourceMappingURL=argparse.d.ts.map

package/types/src/argparse/argparse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"argparse.d.ts","sourceRoot":"","sources":["../../../src/argparse/argparse.js"],"names":[],"mappings":"AAoCA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2QC"}
+{"version":3,"file":"argparse.d.ts","sourceRoot":"","sources":["../../../src/argparse/argparse.js"],"names":[],"mappings":""}

package/types/src/argparse/index.d.ts

@@ -1,3 +1,3 @@
-export { uxParse };
-import { uxParse } from "./argparse";
+export { dxParse };
+import { dxParse } from "./argparse";
 //# sourceMappingURL=index.d.ts.map

package/types/src/argparse/splitOutsideQuotes.d.ts

@@ -1,2 +1,3 @@
-export function splitOutsideQuotes(str: any): any[];
+export { splitOutsideQuotes };
+import { splitOutsideQuotes } from "@davidwells/dx-args";
 //# sourceMappingURL=splitOutsideQuotes.d.ts.map

package/types/src/argparse/splitOutsideQuotes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"splitOutsideQuotes.d.ts","sourceRoot":"","sources":["../../../src/argparse/splitOutsideQuotes.js"],"names":[],"mappings":"AAEA,oDAuEC"}
+{"version":3,"file":"splitOutsideQuotes.d.ts","sourceRoot":"","sources":["../../../src/argparse/splitOutsideQuotes.js"],"names":[],"mappings":""}

package/types/src/cli-run.d.ts

@@ -1,4 +1,6 @@
-import { getGlobGroupsFromArgs } from "./globparse";
-export function runCli(options: {}, rawArgv: any): Promise<import("./").MarkdownMagicResult>;
+import { getGlobGroupsFromArgs } from "@davidwells/dx-args";
+export function parseCliArgv(rawArgv?: any[]): any;
+export function normalizeCliOptions(parsed: any): any;
+export function runCli(options: {}, rawArgv: any, deps?: {}): Promise<any>;
 export { getGlobGroupsFromArgs };
 //# sourceMappingURL=cli-run.d.ts.map

package/types/src/cli-run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli-run.d.ts","sourceRoot":"","sources":["../../src/cli-run.js"],"names":[],"mappings":";AA0GA,6FA4QC"}
+{"version":3,"file":"cli-run.d.ts","sourceRoot":"","sources":["../../src/cli-run.js"],"names":[],"mappings":";AA8OA,mDAIC;AAED,sDAgDC;AAjLD,2EAyHC"}

package/types/src/globparse.d.ts

@@ -1,18 +1,12 @@
-export function getGlobGroupsFromArgs(args: any, opts?: {}): {
-    globGroups: {
-        key: any;
-        rawKey: any;
-        values: any;
-    }[];
-    otherOpts: any[];
-};
-export function isArrayLike(str: any): boolean;
-export function stringLooksLikeFile(value: any): boolean;
-export function getValue(val: any): any;
-export function trimLeadingDashes(value: any): any;
-export function removeNodeModules(value?: string): boolean;
-export function coerceStringToRegex(str: any): any;
-export function convertToArray(str?: string): any;
-export function addValue(value: any, currentCollection: any): any;
-export function customIsGlob(arg: any): any;
+import { getGlobGroupsFromArgs } from "@davidwells/dx-args";
+import { isArrayLike } from "@davidwells/dx-args";
+import { stringLooksLikeFile } from "@davidwells/dx-args";
+import { getValue } from "@davidwells/dx-args";
+import { trimLeadingDashes } from "@davidwells/dx-args";
+import { removeNodeModules } from "@davidwells/dx-args";
+import { coerceStringToRegex } from "@davidwells/dx-args";
+import { convertToArray } from "@davidwells/dx-args";
+import { addValue } from "@davidwells/dx-args";
+import { customIsGlob } from "@davidwells/dx-args";
+export { getGlobGroupsFromArgs, isArrayLike, stringLooksLikeFile, getValue, trimLeadingDashes, removeNodeModules, coerceStringToRegex, convertToArray, addValue, customIsGlob };
 //# sourceMappingURL=globparse.d.ts.map

package/types/src/globparse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"globparse.d.ts","sourceRoot":"","sources":["../../src/globparse.js"],"names":[],"mappings":"AA6EA;;;;;;;EA8EC;AA5ID,+CAGC;AAED,yDAOC;AAED,wCAEC;AAED,mDAEC;AAED,2DAKC;AAED,mDAOC;AAED,kDAGC;AAED,kEASC;AAED,4CAMC"}
+{"version":3,"file":"globparse.d.ts","sourceRoot":"","sources":["../../src/globparse.js"],"names":[],"mappings":""}

package/types/src/index.d.ts

@@ -1,18 +1,32 @@
+declare namespace _exports {
+    export { SyntaxType, FilePathsOrGlobs, ProcessFileOptions, MarkdownMagicOptions, OutputConfig, MarkdownMagicResult };
+}
+declare namespace _exports {
+    export { markdownMagic };
+    export { parseMarkdown };
+    export { blockTransformer };
+    export { processFile };
+    export { stringUtils };
+    export namespace __private {
+        export { createDependencyGraph };
+    }
+}
+export = _exports;
 /**
  * Allowed file syntaxes
  */
-export type SyntaxType = "md" | "js" | "yml" | "yaml";
+type SyntaxType = "md" | "js" | "yml" | "yaml";
 /**
  * Path to file, files or Glob string or Glob Array
  */
-export type FilePathsOrGlobs = string | Array<string>;
-export type ProcessFileOptions = import("comment-block-replacer").ProcessFileOptions;
+type FilePathsOrGlobs = string | Array<string>;
+type ProcessFileOptions = import("comment-block-replacer").ProcessFileOptions;
 /**
  * Configuration for markdown magic
  *
  * Below is the main config for `markdown-magic`
  */
-export type MarkdownMagicOptions = {
+type MarkdownMagicOptions = {
     /**
      * - Files to process.
      */
@@ -81,7 +95,7 @@ export type MarkdownMagicOptions = {
 /**
  * Optional output configuration
  */
-export type OutputConfig = {
+type OutputConfig = {
     /**
      * - Change output path of new content. Default behavior is replacing the original file
      */
@@ -102,7 +116,7 @@ export type OutputConfig = {
 /**
  * Result of markdown processing
  */
-export type MarkdownMagicResult = {
+type MarkdownMagicResult = {
     /**
      * - Any errors encountered.
      */
@@ -183,12 +197,25 @@ export type MarkdownMagicResult = {
     console.log(`Processing complete`, result)
   })
  */
-export function markdownMagic(globOrOpts?: FilePathsOrGlobs | MarkdownMagicOptions, options?: MarkdownMagicOptions): Promise<MarkdownMagicResult>;
+declare function markdownMagic(globOrOpts?: FilePathsOrGlobs | MarkdownMagicOptions, options?: MarkdownMagicOptions): Promise<MarkdownMagicResult>;
 import { blockTransformer } from "comment-block-transformer";
 import { processFile } from "comment-block-replacer";
-export namespace stringUtils {
+declare namespace stringUtils {
     export { stringBreak };
 }
+/**
+ * Create graph data for deterministic dependency ordering
+ * @param {Array<{id: string, blocks: Array<any>, dependencies?: Array<string>}>} blockItems
+ * @returns {{itemsWithBlocks: Array<any>, itemIds: Array<string>, graph: Array<[string, string]>}}
+ */
+declare function createDependencyGraph(blockItems?: Array<{
+    id: string;
+    blocks: Array<any>;
+    dependencies?: Array<string>;
+}>): {
+    itemsWithBlocks: Array<any>;
+    itemIds: Array<string>;
+    graph: Array<[string, string]>;
+};
 import stringBreak = require("./utils/string-break");
-export { parseMarkdown, blockTransformer, processFile };
 //# sourceMappingURL=index.d.ts.map

package/types/src/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.js"],"names":[],"mappings":";;;yBAkDa,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM;;;;+BAK5B,MAAM,GAAC,KAAK,CAAC,MAAM,CAAC;iCAIpB,OAAO,wBAAwB,EAAE,kBAAkB;;;;;;;;;;YASlD,gBAAgB;;;;;;;;aAEhB,YAAY,GAAC,MAAM;;;;aACnB,UAAU;;;;WACV,MAAM;;;;YACN,MAAM;;;;UACN,MAAM;;;;oBACN,OAAO;;;;iBACP,OAAO;;;;UACP,OAAO;;;;YACP,OAAO;;;;aACP,OAAO;;;;8BACP,OAAO;;;;8BACP,OAAO;;;;0BACP,OAAO;;;;gBACP,MAAM;;;;;;;;;gBAMN,MAAM;;;;qBACN,OAAO;;;;;;;;8BAEP,OAAO;;;;;;;;;;;;;kBAQP,KAAK,CAAC,MAAM,CAAC;;;;;;AApD3B;;;GAGG;AAEH;;;GAGG;AAEH;;GAEG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH;;;;;;;GAOG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;;;;;;;;;GAgBG;AACH,2CARW,gBAAgB,GAAC,oBAAoB,YACrC,oBAAoB,GAClB,OAAO,CAAC,mBAAmB,CAAC,CAqmBxC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;kBAkDa,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM;;;;wBAK5B,MAAM,GAAC,KAAK,CAAC,MAAM,CAAC;0BAIpB,OAAO,wBAAwB,EAAE,kBAAkB;;;;;;;;;;YASlD,gBAAgB;;;;;;;;aAEhB,YAAY,GAAC,MAAM;;;;aACnB,UAAU;;;;WACV,MAAM;;;;YACN,MAAM;;;;UACN,MAAM;;;;oBACN,OAAO;;;;iBACP,OAAO;;;;UACP,OAAO;;;;YACP,OAAO;;;;aACP,OAAO;;;;8BACP,OAAO;;;;8BACP,OAAO;;;;0BACP,OAAO;;;;gBACP,MAAM;;;;;;;;;gBAMN,MAAM;;;;qBACN,OAAO;;;;;;;;8BAEP,OAAO;;;;;;;;;;;;;kBAQP,KAAK,CAAC,MAAM,CAAC;;;;;;AApD3B;;;GAGG;AAEH;;;GAGG;AAEH;;GAEG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH;;;;;;;GAOG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;;;;;;;;;GAgBG;AACH,4CARW,gBAAgB,GAAC,oBAAoB,YACrC,oBAAoB,GAClB,OAAO,CAAC,mBAAmB,CAAC,CA+nBxC;;;;;;AAuCD;;;;GAIG;AACH,oDAHW,KAAK,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAAC,YAAY,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CAAC,CAAC,GACnE;IAAC,eAAe,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAAC,CAkBjG"}

package/types/src/transforms/code/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/transforms/code/index.js"],"names":[],"mappings":";;;AA+CiB,kDA+JhB;;;;;;;;;SAzLa,MAAM;;;;aACN,MAAM;;;;aACN,MAAM;;;;YACN,MAAM;;;;cACN,MAAM;;;;mBACN,OAAO"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/transforms/code/index.js"],"names":[],"mappings":";;;AA+CiB,kDAqKhB;;;;;;;;;SA/La,MAAM;;;;aACN,MAAM;;;;aACN,MAAM;;;;YACN,MAAM;;;;cACN,MAAM;;;;mBACN,OAAO"}

package/types/src/utils/remoteRequest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"remoteRequest.d.ts","sourceRoot":"","sources":["../../../src/utils/remoteRequest.js"],"names":[],"mappings":"AAMA,kFAmBC"}
+{"version":3,"file":"remoteRequest.d.ts","sourceRoot":"","sources":["../../../src/utils/remoteRequest.js"],"names":[],"mappings":"AASA,kFAwCC"}