@mintlify/scraping 3.0.186 → 3.0.188

package/src/utils/emptyParagraphs.ts

@@ -0,0 +1,21 @@
+import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import { visit } from 'unist-util-visit';
+
+export function unifiedRemoveEmptyParagraphs() {
+  return function (node: MdxJsxFlowElement) {
+    return removeEmptyParagraphs(node);
+  };
+}
+
+export function removeEmptyParagraphs(node: MdxJsxFlowElement) {
+  return visit(node, 'paragraph', function (subNode, index, parent) {
+    let emptyChildrenCount = 0;
+    for (const child of subNode.children) {
+      if ('children' in child && child.children.length === 0) emptyChildrenCount++;
+    }
+
+    if (emptyChildrenCount === subNode.children.length && parent && typeof index === 'number') {
+      parent.children.splice(index, 1);
+    }
+  });
+}

package/src/utils/errors.ts

@@ -0,0 +1,24 @@
+import type { Result } from '../types/result.js';
+import { log } from './log.js';
+
+export function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? `: ${error.message}` : '';
+}
+
+export function logErrorResults(
+  actionThatFailed: string,
+  results: Array<Result<[string, string]>>
+) {
+  const sumOfErrors = results.reduce(
+    (prev, curr) => (Number.isNaN(Number(curr.success)) ? prev : prev + Number(!curr.success)),
+    0
+  );
+
+  if (sumOfErrors > 0) {
+    log(`We encountered ${sumOfErrors} errors when ${actionThatFailed}`, 'warn');
+    results.forEach((result) => {
+      if (result.success) return;
+      log(result.message, 'failure');
+    });
+  }
+}

package/src/utils/escape.ts

@@ -0,0 +1,30 @@
+import type { Root } from 'hast';
+import { CONTINUE, SKIP, visit } from 'unist-util-visit';
+
+export function escapeCharactersOutsideCodeBlocks() {
+  const charsToEscape = ['<', '{', '`'];
+  const escapeRegex = new RegExp(`[${charsToEscape.join('')}]`, 'g');
+
+  return function (tree: Root) {
+    visit(tree, function (node) {
+      if (node.type === 'element') {
+        if (node.tagName === 'pre' || node.tagName === 'code') {
+          return SKIP;
+        }
+      }
+
+      if (node.type === 'text') {
+        const escapedText = node.value.replace(escapeRegex, (match) => {
+          return `\\${match}`;
+        });
+
+        if (escapedText !== node.value) {
+          node.value = escapedText;
+          return CONTINUE;
+        }
+      }
+
+      return CONTINUE;
+    });
+  };
+}

package/src/utils/extension.ts

@@ -0,0 +1,19 @@
+export function addMdx(filename: string): string {
+  if (filename.endsWith('.mdx')) {
+    return filename;
+  }
+  return filename + '.mdx';
+}
+
+export function getFileExtension(filename: string): string | undefined {
+  const extBeginsIndex = filename.lastIndexOf('.') + 1;
+  const ext = filename.substring(extBeginsIndex);
+
+  if (filename === ext) return undefined;
+  return ext.toLowerCase();
+}
+
+export function fileBelongsInPagesFolder(filename: string): boolean {
+  const extension = getFileExtension(filename);
+  return !!extension && (extension === 'mdx' || extension === 'md' || extension === 'tsx');
+}

package/src/utils/file.ts

@@ -0,0 +1,58 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+import { getErrorMessage } from './errors.js';
+import { log } from './log.js';
+import { createFilename } from './path.js';
+
+export function write(filename: string, data: string | NodeJS.TypedArray) {
+  writeFileSync(filename, data);
+}
+
+export function toFilename(title: string) {
+  // Gets rid of special characters at the start and end
+  // of the name by converting to spaces then using trim.
+  //
+  // I did not write this, I can feel the regex jokes
+  // coming already. - Ricardo
+  return title
+    .replace(/[^a-z0-9]/gi, ' ')
+    .trim()
+    .replace(/ /g, '-')
+    .toLowerCase();
+}
+
+export function writePage(
+  filename: string | URL = '',
+  title: string = '',
+  description: string = '',
+  markdown: string = '',
+  url?: string
+): void {
+  const rootPath = process.cwd();
+  const writePath = createFilename(rootPath, filename, title);
+  if (!writePath) return;
+
+  const cleanedWritePath = writePath.replace(rootPath, '.');
+
+  try {
+    mkdirSync(dirname(writePath), { recursive: true });
+    write(writePath, formatPageWithFrontmatter(title, description, markdown, url));
+    log(`${cleanedWritePath} written to disk`, 'success');
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    log(`${cleanedWritePath} couldn't download to disk${errorMessage}`);
+  }
+}
+
+export function formatPageWithFrontmatter(
+  title: string = '',
+  description: string = '',
+  markdown: string = '',
+  url: string = ''
+) {
+  const optionalTitle = title ? `\ntitle: "${title}"` : '';
+  const optionalDescription = description ? `\ndescription: "${description}"` : '';
+  const optionalUrl = url ? `\nurl: "${url}"` : '';
+  return `---${optionalTitle}${optionalDescription}${optionalUrl}\n---\n\n${markdown}`;
+}

package/src/utils/firstChild.ts

@@ -0,0 +1,13 @@
+import type { Element } from 'hast';
+import { EXIT, visit } from 'unist-util-visit';
+
+export function findFirstChild(node: Element, tagName: string): Element | undefined {
+  let element: Element | undefined = undefined;
+  visit(node, 'element', function (subNode) {
+    if (subNode.tagName === tagName) {
+      element = subNode;
+      return EXIT;
+    }
+  });
+  return element;
+}

package/src/utils/images.ts

@@ -0,0 +1,101 @@
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+import { SUPPORTED_MEDIA_EXTENSIONS } from '../constants.js';
+import type { Result } from '../types/result.js';
+import { getErrorMessage } from './errors.js';
+import { getFileExtension } from './extension.js';
+import { write } from './file.js';
+import { log } from './log.js';
+import { fetchImage } from './network.js';
+
+export async function downloadImage(
+  src: string,
+  rootPath: string
+): Promise<Result<[string, string]>> {
+  try {
+    let filename = await writeImageToFile(src, rootPath);
+    filename = filename.replace(process.cwd(), '');
+
+    const shortenedFilename = filename.length > 36 ? `...${filename.slice(-36)}` : filename;
+    log(`${shortenedFilename} downloaded and written to disk`, 'success');
+
+    return { success: true, data: [src, filename] };
+  } catch (error) {
+    if (error instanceof Error) {
+      return { success: false, message: error.message };
+    } else {
+      return {
+        success: false,
+        message: `${src} - an unknown error occurred downloading this image`,
+      };
+    }
+  }
+}
+
+async function writeImageToFile(src: string, rootPath: string): Promise<string> {
+  const filename = removeMetadataFromImageSrc(src);
+  const imagePath = join(rootPath, filename);
+  const shortenedFilename = filename.length > 36 ? `...${filename.slice(-36)}` : filename;
+
+  if (!isValidImageSrc(filename)) {
+    throw new Error(`${shortenedFilename} - file extension not supported`);
+  }
+
+  if (existsSync(imagePath)) {
+    return imagePath;
+  }
+
+  try {
+    mkdirSync(dirname(imagePath), { recursive: true });
+  } catch (error) {
+    throw new Error(`${imagePath} - failed to create directory`);
+  }
+
+  try {
+    const imageData = await fetchImage(src);
+    write(imagePath, imageData);
+    return imagePath;
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    throw new Error(`${shortenedFilename} - failed to download file from source${errorMessage}`);
+  }
+}
+
+export function isValidImageSrc(src: string) {
+  if (!src) {
+    return false;
+  }
+
+  if (src.startsWith('data:')) {
+    return false;
+  }
+
+  const ext = getFileExtension(src);
+  if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
+    return false;
+  }
+
+  return true;
+}
+
+export function getFilenameBeforeMetadata(src: string, ext: string): string {
+  const lengthUntilMetadata = src.indexOf(`.${ext}`) + `.${ext}`.length;
+  return src.slice(0, lengthUntilMetadata);
+}
+
+export function removeMetadataFromImageSrc(src: string) {
+  let filename = '';
+  if (src.includes('gitbook/image')) {
+    for (const ext of SUPPORTED_MEDIA_EXTENSIONS) {
+      if (src.includes(`.${ext}`)) {
+        filename = getFilenameBeforeMetadata(src, ext);
+      }
+    }
+  }
+  if (!filename) {
+    return src.split('#')[0]!.split('?')[0]!;
+  }
+
+  return filename.split('%2F').slice(4).join('%2F');
+}

package/src/utils/img.ts

@@ -0,0 +1,17 @@
+import type { Element } from 'hast';
+import { visit, CONTINUE, EXIT } from 'unist-util-visit';
+
+export function findImg(node: Element): string | undefined {
+  let imgSrc: string | undefined = undefined;
+  visit(node, 'element', function (subNode, index, parent) {
+    if (subNode.tagName !== 'img') return CONTINUE;
+
+    if (parent && typeof index === 'number') {
+      parent.children.splice(index, 1);
+    }
+
+    imgSrc = (subNode.properties.src as string) || undefined;
+    return EXIT;
+  });
+  return imgSrc;
+}

package/src/utils/log.ts

@@ -0,0 +1,82 @@
+export const colors = {
+  red: '\x1b[31m',
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  magenta: '\x1b[35m',
+  cyan: '\x1b[36m',
+  default: '\x1b[0m',
+} as const;
+const statuses = ['success', 'failure', 'error', 'warn', 'warning', 'info'] as const;
+
+export type Status = (typeof statuses)[number];
+export type Color = keyof typeof colors;
+
+export const checkIcon = '✔ ' as const;
+export const xIcon = '✘ ' as const;
+export const infoIcon = 'ⓘ ' as const;
+export const warningIcon = '⚠ ' as const;
+
+export function log(message: string | unknown, statusOrColor?: Status): void {
+  let color: (typeof colors)[Color] = colors.blue;
+  let statusMsg: string = 'INFO   ';
+  let icon: string = infoIcon;
+
+  const msg = typeof message === 'string' ? message.toLowerCase() : '';
+  if (!statusOrColor) {
+    statusOrColor =
+      msg.includes('fail') ||
+      msg.includes('error') ||
+      msg.includes('could not') ||
+      msg.includes("couldn't") ||
+      msg.includes('did not') ||
+      msg.includes('invalid') ||
+      msg.includes("didn't")
+        ? 'error'
+        : msg.includes('success') ||
+            msg.includes('downloaded') ||
+            msg.includes('written') ||
+            msg.includes('added')
+          ? 'success'
+          : msg.includes('warn')
+            ? 'warn'
+            : undefined;
+  }
+
+  switch (statusOrColor) {
+    case undefined:
+    case 'info':
+      break;
+
+    case 'success':
+      color = colors.green;
+      statusMsg = 'SUCCESS';
+      icon = checkIcon;
+      break;
+
+    case 'warn':
+    case 'warning':
+      color = colors.yellow;
+      statusMsg = 'WARNING';
+      icon = warningIcon;
+      break;
+
+    case 'failure':
+    case 'error':
+      color = colors.red;
+      statusMsg = 'ERROR  ';
+      icon = xIcon;
+      break;
+  }
+
+  console.log(
+    `${color}${icon} ${statusMsg}${colors.default} - ${
+      typeof message === 'string' ||
+      typeof message === 'bigint' ||
+      typeof message === 'number' ||
+      typeof message === 'boolean'
+        ? message
+        : JSON.stringify(message, undefined, 2)
+    }`
+  );
+}

package/src/utils/nestedRoots.ts

@@ -0,0 +1,20 @@
+import type { Root as MdastRoot } from 'mdast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+/** There should only be one root; this replaces any inner roots
+ * that we might get because of us converting the content manually
+ * to `contentAsRoot` from `content`
+ */
+export function unifiedRemoveNestedRoots() {
+  return function (node: MdastRoot) {
+    return removeNestedRoots(node);
+  };
+}
+
+export function removeNestedRoots(root: MdastRoot) {
+  visit(root, 'root', function (node, _, parent) {
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    if (!parent) return CONTINUE;
+    (parent as MdastRoot).children = node.children;
+  });
+}

package/src/utils/network.ts

@@ -0,0 +1,95 @@
+import { Browser, launch } from 'puppeteer';
+
+import { getErrorMessage } from './errors.js';
+import { log } from './log.js';
+
+async function exponentialBackoff<T>(
+  operation: () => Promise<T>,
+  retries: number = 3,
+  delay: number = 1000,
+  factor: number = 2
+): Promise<T> {
+  try {
+    return await operation();
+  } catch (error) {
+    if (retries > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+      return exponentialBackoff(operation, retries - 1, delay * factor, factor);
+    } else {
+      throw error;
+    }
+  }
+}
+
+export async function startPuppeteer() {
+  try {
+    return await launch({
+      headless: true,
+      ignoreHTTPSErrors: true,
+    });
+  } catch (error) {
+    if (error instanceof Error) {
+      log(`Could not create a browser instance: ${error.message}`);
+    }
+  }
+}
+
+export async function getHtmlWithPuppeteer(
+  browser: Browser,
+  url: string | URL
+): Promise<string | undefined> {
+  const page = await browser.newPage();
+  await page.goto(url.toString(), {
+    waitUntil: 'networkidle2',
+  });
+  const html = await exponentialBackoff(() => page.content());
+  return html;
+}
+
+async function fetchPageResponse(url: string | URL): Promise<string> {
+  try {
+    const res = await fetch(url);
+    if (!res.ok) {
+      throw new Error(`${res.status} ${res.statusText}`);
+    }
+    return await res.text();
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    throw new Error(`${url}\n\t- failed to fetch page from source${errorMessage}`);
+  }
+}
+
+export async function fetchPageHtml(
+  url: string | URL,
+  browser: Browser | undefined = undefined
+): Promise<string> {
+  if (browser) {
+    try {
+      const res = await getHtmlWithPuppeteer(browser, url);
+      if (res) return res;
+      throw new Error('an unknown error occured');
+    } catch (error) {
+      const errorMessage = getErrorMessage(error);
+      throw new Error(`${url}\n\t- Puppeteer failed to retrieve page from source${errorMessage}`);
+    }
+  }
+
+  return await exponentialBackoff(() => fetchPageResponse(url));
+}
+
+export async function fetchImage(url: string): Promise<NodeJS.TypedArray> {
+  try {
+    const res = await exponentialBackoff(() => fetch(url));
+    if (!res.ok) {
+      throw new Error(`${res.status} ${res.statusText}`);
+    }
+
+    const imageBuffer = await res.arrayBuffer();
+    const imageData = new Uint8Array(imageBuffer);
+
+    return imageData;
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    throw new Error(`${url}\n\t- failed to retrieve image from source${errorMessage}`);
+  }
+}

package/src/utils/path.ts

@@ -0,0 +1,27 @@
+import { join } from 'path';
+
+import { addMdx } from './extension.js';
+import { toFilename } from './file.js';
+import { log } from './log.js';
+
+export function createFilename(
+  rootPath: string = process.cwd(),
+  filename: string | URL,
+  title?: string
+): string | undefined {
+  if (typeof filename === 'string' && filename.startsWith('http')) {
+    const url = new URL(filename);
+    filename = url.pathname;
+  } else if (typeof filename === 'object') {
+    filename = (filename as URL).pathname;
+  } else {
+    filename = filename as string;
+  }
+
+  if (filename === '') {
+    log(`Invalid file name provided: ${filename}`, 'error');
+    return undefined;
+  }
+
+  return join(rootPath, addMdx(filename || toFilename(title ?? '')));
+}

package/src/utils/position.ts

@@ -0,0 +1,14 @@
+import type { Element } from 'hast';
+import { visit } from 'unist-util-visit';
+
+export function unifiedRemovePositions() {
+  return function (node: Element) {
+    return removePositions(node);
+  };
+}
+
+export function removePositions(node: Element) {
+  return visit(node, function (subNode) {
+    delete subNode.position;
+  });
+}

package/src/utils/reservedNames.ts

@@ -0,0 +1,31 @@
+export const blacklistedNames = new Map<string, string>([['api', 'api-reference']]);
+
+export const INDEX_NAMES = [
+  'home',
+  'introduction',
+  'getting-started',
+  'get-started',
+  'welcome',
+  'start',
+];
+
+export const GROUP_NAMES = [
+  'Home',
+  'Introduction',
+  'Getting Started',
+  'Get Started',
+  'Welcome',
+  'Start',
+  'Docs',
+];
+
+export function iterateThroughReservedNames(
+  namesToUse: Array<string>,
+  namesInUse: Array<string>
+): string {
+  for (const name of namesToUse) {
+    if (namesInUse.includes(name)) continue;
+    return name;
+  }
+  return '';
+}

package/src/utils/strings.ts

@@ -0,0 +1,7 @@
+export function removeTrailingSlash(str: string): string {
+  return str.endsWith('/') ? str.substring(0, str.length - 1) : str;
+}
+
+export function removeLeadingSlash(str: string): string {
+  return str.startsWith('/') ? str.substring(1) : str;
+}

package/src/utils/text.ts

@@ -0,0 +1,11 @@
+import type { Element } from 'hast';
+import { visit } from 'unist-util-visit';
+
+export function getText(element: Element | undefined): string {
+  if (!element) return '';
+  let text = '';
+  visit(element, 'text', function (node) {
+    text += node.value;
+  });
+  return text;
+}

package/src/utils/title.ts

@@ -0,0 +1,68 @@
+import type { Element, ElementContent, Root as HastRoot } from 'hast';
+import type { Root as MdastRoot, BlockContent } from 'mdast';
+import { visit, CONTINUE, EXIT } from 'unist-util-visit';
+
+export function findTitle(
+  node: Element | ElementContent | BlockContent | MdastRoot | HastRoot | undefined,
+  opts: { delete: boolean; nodeType?: string; tagName?: string } = {
+    delete: true,
+    nodeType: undefined,
+    tagName: undefined,
+  }
+): string {
+  let title = '';
+  if (!node) return title;
+
+  visit(node, opts.nodeType ? opts.nodeType : node, function (subNode) {
+    if (opts.tagName && subNode.type === 'element' && subNode.tagName !== opts.tagName) {
+      return CONTINUE;
+    }
+
+    visit(subNode, 'text', function (textNode, index, parent) {
+      title += textNode.value;
+      if (opts.delete && parent && typeof index === 'number') {
+        parent.children.splice(index, 1);
+      }
+    });
+  });
+  return title.trim();
+}
+
+export function getTitleFromHeading(root: MdastRoot): string {
+  let headingElement: BlockContent | undefined = undefined;
+  visit(root, 'heading', function (subNode, index, parent) {
+    headingElement = subNode;
+    if (parent && typeof index === 'number') {
+      parent.children.splice(index, 1);
+    }
+    return EXIT;
+  });
+  return findTitle(headingElement);
+}
+
+export function getDescriptionFromRoot(root: MdastRoot): string {
+  let descriptionElement: BlockContent | undefined = undefined;
+  visit(root, 'paragraph', function (subNode, index, parent) {
+    if (typeof index !== 'number' || index !== 0 || !parent || parent.type !== 'root') return EXIT;
+
+    descriptionElement = subNode;
+    if (typeof index === 'number') {
+      parent.children.splice(index, 1);
+    }
+    return EXIT;
+  });
+  return findTitle(descriptionElement);
+}
+
+export function getTitleFromLink(url: string): string {
+  if (url.startsWith('http')) {
+    url = new URL(url).pathname;
+  }
+  const lastPathname = url.split('/').at(-1) ?? '';
+  const dashSplitPathname = lastPathname.split('-').flatMap((i) => i.split('_'));
+  dashSplitPathname.forEach((str, index) => {
+    dashSplitPathname[index] = str[0] ? `${str[0].toUpperCase()}${str.substring(1)}` : str;
+  });
+
+  return dashSplitPathname.join(' ');
+}

package/src/utils/url.ts

@@ -0,0 +1,8 @@
+export function checkUrl({ url }: { url: string }): boolean {
+  try {
+    new URL(url);
+  } catch {
+    throw Error(`Invalid link: ${url}\nMake sure the link starts with http:// or https://`);
+  }
+  return true;
+}

package/bin/browser.d.ts

@@ -1,2 +0,0 @@
-export declare function startBrowser(): Promise<import("puppeteer").Browser>;
-export declare function getHtmlWithPuppeteer(href: string): Promise<string>;