@mintlify/scraping 4.0.5 → 4.0.6

package/src/utils/emptyParagraphs.ts

@@ -7,7 +7,7 @@ export function unifiedRemoveEmptyParagraphs() {
   };
 }
 
-export function removeEmptyParagraphs(node: MdxJsxFlowElement) {
+function removeEmptyParagraphs(node: MdxJsxFlowElement) {
   return visit(node, 'paragraph', function (subNode, index, parent) {
     let emptyChildrenCount = 0;
     for (const child of subNode.children) {

package/src/utils/formatEmphasis.ts

@@ -0,0 +1,37 @@
+import type { Root as MdastRoot } from 'mdast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+export function remarkProperlyFormatEmphasis() {
+  return function (root: MdastRoot) {
+    return properlyFormatEmphasis(root);
+  };
+}
+
+const spaceNode = {
+  type: 'text' as const,
+  value: ' ',
+};
+
+function properlyFormatEmphasis(root: MdastRoot) {
+  visit(root, ['emphasis', 'strong'], function (node, index, parent) {
+    if (node.type !== 'emphasis' && node.type !== 'strong') return CONTINUE;
+    if (
+      node.children.length !== 1 ||
+      !node.children[0] ||
+      node.children[0].type !== 'text' ||
+      !parent ||
+      typeof index !== 'number'
+    )
+      return CONTINUE;
+
+    const child = node.children[0];
+    if (child.value.startsWith(' ') || child.value.endsWith(' ')) {
+      if (index !== 0 && child.value.startsWith(' ')) parent.children.splice(index, 0, spaceNode);
+      if (parent.children.length > index + 1 && child.value.endsWith(' '))
+        parent.children.splice(index + 1, 0, spaceNode);
+
+      child.value = child.value.trim();
+      node.children[0] = child;
+    }
+  });
+}

package/src/utils/images.ts

@@ -84,7 +84,7 @@ export function getFilenameBeforeMetadata(src: string, ext: string): string {
   return src.slice(0, lengthUntilMetadata);
 }
 
-export function removeMetadataFromImageSrc(src: string) {
+export function removeMetadataFromImageSrc(src: string): string {
   let filename = '';
   if (src.includes('gitbook/image')) {
     for (const ext of SUPPORTED_MEDIA_EXTENSIONS) {
@@ -93,8 +93,19 @@ export function removeMetadataFromImageSrc(src: string) {
       }
     }
   }
+
   if (!filename) {
-    return src.split('#')[0]!.split('?')[0]!;
+    if (src.startsWith('http')) {
+      src = new URL(src).pathname;
+    }
+    filename =
+      decodeURIComponent(
+        src
+          .split('#')[0]!
+          .split('?')[0]!
+          .replace(/[\/]{2,}/g, '/')
+      ).replace(/(?:_{2,}|[\s%#&{}\\<>*?$!'":@+`|=])/g, '-') || 'image';
+    return filename;
   }
 
   return filename.split('%2F').slice(4).join('%2F');

package/src/utils/lists.ts

@@ -0,0 +1,22 @@
+import type { Root as MdastRoot } from 'mdast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+export function remarkSpaceListsOut() {
+  return function (root: MdastRoot) {
+    return spaceListsOut(root);
+  };
+}
+
+// ReadMe-specific function since they sometimes stack `<ol>`
+// elements right on top of each other
+function spaceListsOut(root: MdastRoot) {
+  return visit(root, 'list', (_, index, parent) => {
+    if (!parent || typeof index !== 'number') return CONTINUE;
+    if (index && index > 0 && parent.children[index - 1]?.type === 'list') {
+      parent.children.splice(index, 0, {
+        type: 'paragraph',
+        children: [{ type: 'text', value: '' }],
+      });
+    }
+  });
+}

package/src/utils/log.ts

@@ -7,8 +7,21 @@ export const colors = {
   cyan: '\x1b[36m',
   default: '\x1b[0m',
 } as const;
+
+const noColors = {
+  red: '',
+  green: '',
+  yellow: '',
+  blue: '',
+  magenta: '',
+  cyan: '',
+  default: '',
+} as const;
+
 const statuses = ['success', 'failure', 'error', 'warn', 'warning', 'info'] as const;
 
+export const activeColors = process.stdout.isTTY ? colors : noColors;
+
 export type Status = (typeof statuses)[number];
 export type Color = keyof typeof colors;
 
@@ -18,7 +31,7 @@ export const infoIcon = 'ⓘ ' as const;
 export const warningIcon = '⚠ ' as const;
 
 export function log(message: string | unknown, statusOrColor?: Status): void {
-  let color: (typeof colors)[Color] = colors.blue;
+  let color: (typeof activeColors)[Color] = activeColors.blue;
   let statusMsg: string = 'INFO   ';
   let icon: string = infoIcon;
 
@@ -49,28 +62,28 @@ export function log(message: string | unknown, statusOrColor?: Status): void {
       break;
 
     case 'success':
-      color = colors.green;
+      color = activeColors.green;
       statusMsg = 'SUCCESS';
       icon = checkIcon;
       break;
 
     case 'warn':
     case 'warning':
-      color = colors.yellow;
+      color = activeColors.yellow;
       statusMsg = 'WARNING';
       icon = warningIcon;
       break;
 
     case 'failure':
     case 'error':
-      color = colors.red;
+      color = activeColors.red;
       statusMsg = 'ERROR  ';
       icon = xIcon;
       break;
   }
 
   console.log(
-    `${color}${icon} ${statusMsg}${colors.default} - ${
+    `${color}${icon} ${statusMsg}${activeColors.default} - ${
       typeof message === 'string' ||
       typeof message === 'bigint' ||
       typeof message === 'number' ||

package/src/utils/metadata.ts

@@ -0,0 +1,26 @@
+import type { Root as MdastRoot } from 'mdast';
+import { visit } from 'unist-util-visit';
+
+export function remarkRemoveBottomMetadata() {
+  return function (root: MdastRoot) {
+    return removeBottomMetadata(root);
+  };
+}
+
+function removeBottomMetadata(root: MdastRoot) {
+  if (root.children.at(-1)?.type === 'thematicBreak') {
+    root.children.pop();
+  }
+  if (root.children.at(-1)?.type === 'paragraph') {
+    let shouldDelete = false as boolean;
+    visit(root, 'text', function (node) {
+      if (
+        (node.value.startsWith('Updated') || node.value.startsWith('Last updated')) &&
+        node.value.endsWith('ago')
+      ) {
+        shouldDelete = true;
+      }
+    });
+    if (shouldDelete) root.children.pop();
+  }
+}

package/src/utils/nestedRoots.ts

@@ -11,7 +11,7 @@ export function unifiedRemoveNestedRoots() {
   };
 }
 
-export function removeNestedRoots(root: MdastRoot) {
+function removeNestedRoots(root: MdastRoot) {
   visit(root, 'root', function (node, _, parent) {
     // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
     if (!parent) return CONTINUE;

package/src/utils/position.ts

@@ -7,7 +7,7 @@ export function unifiedRemovePositions() {
   };
 }
 
-export function removePositions(node: Element) {
+function removePositions(node: Element) {
   return visit(node, function (subNode) {
     delete subNode.position;
   });

package/src/utils/tableCells.ts

@@ -0,0 +1,23 @@
+import type { Code, Parent, InlineCode, Root as MdastRoot } from 'mdast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+export function remarkRemoveCodeBlocksInCells() {
+  return function (root: MdastRoot) {
+    return removeCodeBlocksInCells(root);
+  };
+}
+
+// ReadMe specific, since they allow for `<pre>` blocks inside
+// of table cells, could be supported elsewhere, but haven't seen it
+function removeCodeBlocksInCells(root: MdastRoot) {
+  visit(root, 'tableCell', function (node) {
+    visit(node, 'code', function (subNode: Code, index, parent: Parent | undefined) {
+      if (!parent || typeof index !== 'number') return CONTINUE;
+      const newNode: InlineCode = {
+        type: 'inlineCode',
+        value: subNode.value,
+      };
+      (parent as Parent).children[index] = newNode;
+    });
+  });
+}

package/src/utils/title.ts

@@ -4,7 +4,7 @@ import { visit, CONTINUE, EXIT } from 'unist-util-visit';
 
 export function findTitle(
   node: Element | ElementContent | BlockContent | MdastRoot | HastRoot | undefined,
-  opts: { delete: boolean; nodeType?: string; tagName?: string } = {
+  opts: { delete: boolean; nodeType?: string; tagName?: string; escaped?: boolean } = {
     delete: true,
     nodeType: undefined,
     tagName: undefined,
@@ -25,7 +25,13 @@ export function findTitle(
       }
     });
   });
-  return title.trim();
+
+  title = title.trim();
+  if (opts.escaped) {
+    return title.replace(/"/g, '\\"');
+  } else {
+    return title;
+  }
 }
 
 export function getTitleFromHeading(root: MdastRoot): string {
@@ -37,7 +43,7 @@ export function getTitleFromHeading(root: MdastRoot): string {
     }
     return EXIT;
   });
-  return findTitle(headingElement);
+  return findTitle(headingElement, { delete: true, escaped: true });
 }
 
 export function getDescriptionFromRoot(root: MdastRoot): string {
@@ -51,7 +57,7 @@ export function getDescriptionFromRoot(root: MdastRoot): string {
     }
     return EXIT;
   });
-  return findTitle(descriptionElement);
+  return findTitle(descriptionElement, { delete: true, escaped: true });
 }
 
 export function getTitleFromLink(url: string): string {

package/src/utils/updatedAt.ts

@@ -0,0 +1,25 @@
+import type { Root as MdastRoot } from 'mdast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+export function remarkRemoveUpdatedAt() {
+  return function (root: MdastRoot) {
+    return removeUpdatedAt(root);
+  };
+}
+
+function removeUpdatedAt(root: MdastRoot) {
+  visit(root, 'paragraph', function (node) {
+    visit(node, 'text', function (subNode, index, parent) {
+      if (
+        (subNode.value.trim().startsWith('Updated') ||
+          subNode.value.trim().startsWith('Last updated')) &&
+        subNode.value.endsWith('ago')
+      ) {
+        if (parent && typeof index === 'number') {
+          parent.children.splice(index, 1);
+          return [CONTINUE, index];
+        }
+      }
+    });
+  });
+}

package/bin/utils/escape.d.ts

@@ -1,2 +0,0 @@
-import type { Root } from 'hast';
-export declare function escapeCharactersOutsideCodeBlocks(): (tree: Root) => void;

package/bin/utils/escape.js

@@ -1,25 +0,0 @@
-import { CONTINUE, SKIP, visit } from 'unist-util-visit';
-export function escapeCharactersOutsideCodeBlocks() {
-    const charsToEscape = ['<', '{', '`'];
-    const escapeRegex = new RegExp(`[${charsToEscape.join('')}]`, 'g');
-    return function (tree) {
-        visit(tree, function (node) {
-            if (node.type === 'element') {
-                if (node.tagName === 'pre' || node.tagName === 'code') {
-                    return SKIP;
-                }
-            }
-            if (node.type === 'text') {
-                const escapedText = node.value.replace(escapeRegex, (match) => {
-                    return `\\${match}`;
-                });
-                if (escapedText !== node.value) {
-                    node.value = escapedText;
-                    return CONTINUE;
-                }
-            }
-            return CONTINUE;
-        });
-    };
-}
-//# sourceMappingURL=escape.js.map

package/bin/utils/escape.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"escape.js","sourceRoot":"","sources":["../../src/utils/escape.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,UAAU,iCAAiC;IAC/C,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnE,OAAO,UAAU,IAAU;QACzB,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI;YACxB,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;oBACtD,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE;oBAC5D,OAAO,KAAK,KAAK,EAAE,CAAC;gBACtB,CAAC,CAAC,CAAC;gBAEH,IAAI,WAAW,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC/B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;oBACzB,OAAO,QAAQ,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC"}

package/src/utils/escape.ts

@@ -1,30 +0,0 @@
-import type { Root } from 'hast';
-import { CONTINUE, SKIP, visit } from 'unist-util-visit';
-
-export function escapeCharactersOutsideCodeBlocks() {
-  const charsToEscape = ['<', '{', '`'];
-  const escapeRegex = new RegExp(`[${charsToEscape.join('')}]`, 'g');
-
-  return function (tree: Root) {
-    visit(tree, function (node) {
-      if (node.type === 'element') {
-        if (node.tagName === 'pre' || node.tagName === 'code') {
-          return SKIP;
-        }
-      }
-
-      if (node.type === 'text') {
-        const escapedText = node.value.replace(escapeRegex, (match) => {
-          return `\\${match}`;
-        });
-
-        if (escapedText !== node.value) {
-          node.value = escapedText;
-          return CONTINUE;
-        }
-      }
-
-      return CONTINUE;
-    });
-  };
-}