@mintlify/scraping 3.0.14 → 3.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/.prettierrc +1 -0
  2. package/bin/browser.js +3 -3
  3. package/bin/constants.js +23 -23
  4. package/bin/constants.js.map +1 -1
  5. package/bin/downloadImage.js +18 -18
  6. package/bin/downloadImage.js.map +1 -1
  7. package/bin/scraping/detectFramework.js +13 -13
  8. package/bin/scraping/detectFramework.js.map +1 -1
  9. package/bin/scraping/downloadAllImages.js +5 -5
  10. package/bin/scraping/downloadAllImages.js.map +1 -1
  11. package/bin/scraping/downloadLogoImage.js +4 -4
  12. package/bin/scraping/downloadLogoImage.js.map +1 -1
  13. package/bin/scraping/getSitemapLinks.js +4 -4
  14. package/bin/scraping/scrapeFileGettingFileNameFromUrl.js +10 -10
  15. package/bin/scraping/scrapeFileGettingFileNameFromUrl.js.map +1 -1
  16. package/bin/scraping/scrapeGettingFileNameFromUrl.js +2 -2
  17. package/bin/scraping/scrapeGettingFileNameFromUrl.js.map +1 -1
  18. package/bin/scraping/scrapePage.js +3 -3
  19. package/bin/scraping/scrapePage.js.map +1 -1
  20. package/bin/scraping/scrapePageCommands.d.ts +1 -1
  21. package/bin/scraping/scrapePageCommands.js +15 -15
  22. package/bin/scraping/scrapePageCommands.js.map +1 -1
  23. package/bin/scraping/scrapeSection.js +6 -6
  24. package/bin/scraping/scrapeSection.js.map +1 -1
  25. package/bin/scraping/scrapeSectionCommands.d.ts +1 -1
  26. package/bin/scraping/scrapeSectionCommands.js +14 -14
  27. package/bin/scraping/scrapeSectionCommands.js.map +1 -1
  28. package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js +11 -11
  29. package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js.map +1 -1
  30. package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js +12 -14
  31. package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js.map +1 -1
  32. package/bin/scraping/site-scrapers/alternateGroupTitle.js +1 -1
  33. package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js +8 -11
  34. package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js.map +1 -1
  35. package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js +6 -6
  36. package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js +4 -4
  37. package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.d.ts +1 -1
  38. package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js +4 -4
  39. package/bin/scraping/site-scrapers/openNestedGitbookMenus.d.ts +1 -1
  40. package/bin/scraping/site-scrapers/openNestedGitbookMenus.js.map +1 -1
  41. package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js +14 -14
  42. package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js.map +1 -1
  43. package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js +9 -9
  44. package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map +1 -1
  45. package/bin/scraping/site-scrapers/scrapeGitBookPage.js +12 -14
  46. package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map +1 -1
  47. package/bin/scraping/site-scrapers/scrapeGitBookSection.js +10 -15
  48. package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map +1 -1
  49. package/bin/scraping/site-scrapers/scrapeReadMePage.js +15 -15
  50. package/bin/scraping/site-scrapers/scrapeReadMePage.js.map +1 -1
  51. package/bin/scraping/site-scrapers/scrapeReadMeSection.js +11 -15
  52. package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map +1 -1
  53. package/bin/tsconfig.tsbuildinfo +1 -1
  54. package/bin/util.d.ts +1 -1
  55. package/bin/util.js +23 -26
  56. package/bin/util.js.map +1 -1
  57. package/bin/validation/stopIfInvalidLink.js +3 -3
  58. package/package.json +8 -3
  59. package/src/browser.ts +3 -3
  60. package/src/constants.ts +23 -23
  61. package/src/downloadImage.ts +21 -26
  62. package/src/scraping/detectFramework.ts +18 -18
  63. package/src/scraping/downloadAllImages.ts +7 -9
  64. package/src/scraping/downloadLogoImage.ts +5 -4
  65. package/src/scraping/getSitemapLinks.ts +4 -4
  66. package/src/scraping/scrapeFileGettingFileNameFromUrl.ts +12 -18
  67. package/src/scraping/scrapeGettingFileNameFromUrl.ts +7 -5
  68. package/src/scraping/scrapePage.ts +4 -3
  69. package/src/scraping/scrapePageCommands.ts +17 -18
  70. package/src/scraping/scrapeSection.ts +8 -16
  71. package/src/scraping/scrapeSectionCommands.ts +19 -34
  72. package/src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts +12 -11
  73. package/src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts +23 -24
  74. package/src/scraping/site-scrapers/alternateGroupTitle.ts +1 -1
  75. package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts +8 -11
  76. package/src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts +6 -6
  77. package/src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts +4 -4
  78. package/src/scraping/site-scrapers/openNestedDocusaurusMenus.ts +5 -5
  79. package/src/scraping/site-scrapers/openNestedGitbookMenus.ts +2 -4
  80. package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts +15 -18
  81. package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts +11 -14
  82. package/src/scraping/site-scrapers/scrapeGitBookPage.ts +13 -14
  83. package/src/scraping/site-scrapers/scrapeGitBookSection.ts +11 -15
  84. package/src/scraping/site-scrapers/scrapeReadMePage.ts +17 -22
  85. package/src/scraping/site-scrapers/scrapeReadMeSection.ts +27 -31
  86. package/src/util.ts +25 -36
  87. package/src/validation/stopIfInvalidLink.ts +3 -3
package/.prettierrc ADDED
@@ -0,0 +1 @@
1
+ "@mintlify/prettier-config/config.js"
package/bin/browser.js CHANGED
@@ -1,4 +1,4 @@
1
- import { launch } from "puppeteer";
1
+ import { launch } from 'puppeteer';
2
2
  export async function startBrowser() {
3
3
  try {
4
4
  return await launch({
@@ -7,7 +7,7 @@ export async function startBrowser() {
7
7
  });
8
8
  }
9
9
  catch (err) {
10
- console.log("Could not create a browser instance: ", err);
10
+ console.log('Could not create a browser instance: ', err);
11
11
  process.exit(1);
12
12
  }
13
13
  }
@@ -15,7 +15,7 @@ export async function getHtmlWithPuppeteer(href) {
15
15
  const browser = await startBrowser();
16
16
  const page = await browser.newPage();
17
17
  await page.goto(href, {
18
- waitUntil: "networkidle2",
18
+ waitUntil: 'networkidle2',
19
19
  });
20
20
  const html = await page.content();
21
21
  browser.close();
package/bin/constants.js CHANGED
@@ -1,30 +1,30 @@
1
- import path from "path";
2
- import * as url from "url";
3
- import os from "os";
1
+ import os from 'os';
2
+ import path from 'path';
3
+ import * as url from 'url';
4
4
  // package installation location
5
- export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
5
+ export const INSTALL_PATH = url.fileURLToPath(new URL('.', import.meta.url));
6
6
  export const HOME_DIR = os.homedir();
7
- export const DOT_MINTLIFY = path.join(HOME_DIR, ".mintlify");
8
- export const VERSION_PATH = path.join(DOT_MINTLIFY, "mint", "mint-version.txt");
9
- export const CLIENT_PATH = path.join(DOT_MINTLIFY, "mint", "client");
10
- export const MINT_PATH = path.join(DOT_MINTLIFY, "mint");
7
+ export const DOT_MINTLIFY = path.join(HOME_DIR, '.mintlify');
8
+ export const VERSION_PATH = path.join(DOT_MINTLIFY, 'mint', 'mint-version.txt');
9
+ export const CLIENT_PATH = path.join(DOT_MINTLIFY, 'mint', 'client');
10
+ export const MINT_PATH = path.join(DOT_MINTLIFY, 'mint');
11
11
  // command execution location
12
12
  export const CMD_EXEC_PATH = process.cwd();
13
13
  export const SUPPORTED_MEDIA_EXTENSIONS = [
14
- "jpeg",
15
- "jpg",
16
- "jfif",
17
- "pjpeg",
18
- "pjp",
19
- "png",
20
- "svg",
21
- "svgz",
22
- "ico",
23
- "webp",
24
- "gif",
25
- "apng",
26
- "avif",
27
- "bmp",
28
- "mp4",
14
+ 'jpeg',
15
+ 'jpg',
16
+ 'jfif',
17
+ 'pjpeg',
18
+ 'pjp',
19
+ 'png',
20
+ 'svg',
21
+ 'svgz',
22
+ 'ico',
23
+ 'webp',
24
+ 'gif',
25
+ 'apng',
26
+ 'avif',
27
+ 'bmp',
28
+ 'mp4',
29
29
  ];
30
30
  //# sourceMappingURL=constants.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gCAAgC;AAChC,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;AAErC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AAE7D,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAErE,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAEzD,6BAA6B;AAC7B,MAAM,CAAC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAE3C,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;CACN,CAAC"}
1
+ {"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAE3B,gCAAgC;AAChC,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;AAErC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AAE7D,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAErE,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAEzD,6BAA6B;AAC7B,MAAM,CAAC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAE3C,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;CACN,CAAC"}
@@ -1,13 +1,13 @@
1
- import { existsSync, mkdirSync, createWriteStream } from "fs";
2
- import path from "path";
3
- import axios from "axios";
4
- import { getFileExtension } from "./util.js";
5
- import { SUPPORTED_MEDIA_EXTENSIONS } from "./constants.js";
1
+ import axios from 'axios';
2
+ import { existsSync, mkdirSync, createWriteStream } from 'fs';
3
+ import path from 'path';
4
+ import { SUPPORTED_MEDIA_EXTENSIONS } from './constants.js';
5
+ import { getFileExtension } from './util.js';
6
6
  async function writeImageToFile(imageSrc, writePath, overwrite) {
7
7
  // Avoid unnecessary downloads
8
8
  if (existsSync(writePath) && !overwrite) {
9
9
  return Promise.reject({
10
- code: "EEXIST",
10
+ code: 'EEXIST',
11
11
  });
12
12
  }
13
13
  // Create the folders needed if they're missing
@@ -15,21 +15,21 @@ async function writeImageToFile(imageSrc, writePath, overwrite) {
15
15
  const writer = createWriteStream(writePath);
16
16
  try {
17
17
  const response = await axios.get(imageSrc, {
18
- responseType: "stream",
18
+ responseType: 'stream',
19
19
  });
20
20
  // wx prevents overwriting an image with the exact same name
21
21
  // being created in the time we were downloading
22
22
  response.data.pipe(writer, {
23
- flag: "wx",
23
+ flag: 'wx',
24
24
  });
25
25
  return new Promise((resolve, reject) => {
26
- writer.on("finish", resolve);
27
- writer.on("error", reject);
26
+ writer.on('finish', resolve);
27
+ writer.on('error', reject);
28
28
  });
29
29
  }
30
30
  catch (e) {
31
31
  return Promise.reject({
32
- code: "ENOTFOUND",
32
+ code: 'ENOTFOUND',
33
33
  });
34
34
  }
35
35
  }
@@ -38,20 +38,20 @@ export function isValidImageSrc(src) {
38
38
  return false;
39
39
  }
40
40
  // We do not support downloading base64 in-line images.
41
- if (src.startsWith("data:")) {
41
+ if (src.startsWith('data:')) {
42
42
  return false;
43
43
  }
44
44
  const imageHref = removeMetadataFromImageSrc(src);
45
45
  const ext = getFileExtension(imageHref);
46
46
  if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
47
- console.error("🚨 We do not support the file extension: " + ext);
47
+ console.error('🚨 We do not support the file extension: ' + ext);
48
48
  return false;
49
49
  }
50
50
  return true;
51
51
  }
52
52
  export function removeMetadataFromImageSrc(src) {
53
53
  // Part of the URL standard
54
- const metadataSymbols = ["?", "#"];
54
+ const metadataSymbols = ['?', '#'];
55
55
  metadataSymbols.forEach((dividerSymbol) => {
56
56
  // Some frameworks add metadata after the file extension, we need to remove that.
57
57
  src = src.split(dividerSymbol)[0];
@@ -60,18 +60,18 @@ export function removeMetadataFromImageSrc(src) {
60
60
  }
61
61
  export function cleanImageSrc(src, origin) {
62
62
  // Add origin if the image tags are using relative sources
63
- return src.startsWith("http") ? src : new URL(src, origin).href;
63
+ return src.startsWith('http') ? src : new URL(src, origin).href;
64
64
  }
65
65
  export default async function downloadImage(imageSrc, writePath, overwrite = false) {
66
66
  await writeImageToFile(imageSrc, writePath, overwrite)
67
67
  .then(() => {
68
- console.log("🖼️ - " + writePath);
68
+ console.log('🖼️ - ' + writePath);
69
69
  })
70
70
  .catch((e) => {
71
- if (e.code === "EEXIST") {
71
+ if (e.code === 'EEXIST') {
72
72
  console.log(`❌ Skipping existing image ${writePath}`);
73
73
  }
74
- else if (e.code === "ENOTFOUND") {
74
+ else if (e.code === 'ENOTFOUND') {
75
75
  console.error(`🚨 Cannot download the image, address not found ${imageSrc}`);
76
76
  }
77
77
  else {
@@ -1 +1 @@
1
- {"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAE5D,KAAK,UAAU,gBAAgB,CAC7B,QAAgB,EAChB,SAAiB,EACjB,SAAkB;IAElB,8BAA8B;IAC9B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;KACJ;IAED,+CAA+C;IAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;YACzC,YAAY,EAAE,QAAQ;SACvB,CAAC,CAAC;QACH,4DAA4D;QAC5D,gDAAgD;QAChD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACzB,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;KACJ;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE;QACR,OAAO,KAAK,CAAC;KACd;IACD,uDAAuD;IACvD,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC3B,OAAO,KAAK,CAAC;KACd;IAED,MAAM,SAAS,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAExC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACpD,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACjE,OAAO,KAAK,CAAC;KACd;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,2BAA2B;IAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnC,eAAe,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;QACxC,iFAAiF;QACjF,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAc;IACvD,0DAA0D;IAC1D,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,aAAa,CACzC,QAAgB,EAChB,SAAiB,EACjB,SAAS,GAAG,KAAK;IAEjB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;SACnD,IAAI,CAAC,GAAG,EAAE;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IACpC,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACX,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;SACvD;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE;YACjC,OAAO,CAAC,KAAK,CACX,mDAAmD,QAAQ,EAAE,CAC9D,CAAC;SACH;aAAM;YACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClB;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
1
+ {"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,SAAiB,EAAE,SAAkB;IACrF,8BAA8B;IAC9B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;KACJ;IAED,+CAA+C;IAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;YACzC,YAAY,EAAE,QAAQ;SACvB,CAAC,CAAC;QACH,4DAA4D;QAC5D,gDAAgD;QAChD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACzB,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;KACJ;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE;QACR,OAAO,KAAK,CAAC;KACd;IACD,uDAAuD;IACvD,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC3B,OAAO,KAAK,CAAC;KACd;IAED,MAAM,SAAS,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAExC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACpD,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACjE,OAAO,KAAK,CAAC;KACd;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,2BAA2B;IAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnC,eAAe,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;QACxC,iFAAiF;QACjF,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAc;IACvD,0DAA0D;IAC1D,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,aAAa,CACzC,QAAgB,EAChB,SAAiB,EACjB,SAAS,GAAG,KAAK;IAEjB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;SACnD,IAAI,CAAC,GAAG,EAAE;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IACpC,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACX,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;SACvD;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE;YACjC,OAAO,CAAC,KAAK,CAAC,mDAAmD,QAAQ,EAAE,CAAC,CAAC;SAC9E;aAAM;YACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClB;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
@@ -1,4 +1,4 @@
1
- import cheerio from "cheerio";
1
+ import cheerio from 'cheerio';
2
2
  export var Frameworks;
3
3
  (function (Frameworks) {
4
4
  Frameworks["DOCUSAURUS"] = "DOCUSAURUS";
@@ -10,22 +10,22 @@ export function detectFramework(html) {
10
10
  const $ = cheerio.load(html);
11
11
  const docusaurusMeta = $('meta[name="generator"]');
12
12
  if (docusaurusMeta.length > 0 &&
13
- docusaurusMeta.attr("content") &&
14
- typeof docusaurusMeta.attr("content") === "string" &&
15
- docusaurusMeta.attr("content").includes("Docusaurus")) {
16
- const metaAttrString = docusaurusMeta.attr("content");
17
- if (metaAttrString.includes("v3")) {
18
- return { framework: Frameworks.DOCUSAURUS, version: "3" };
13
+ docusaurusMeta.attr('content') &&
14
+ typeof docusaurusMeta.attr('content') === 'string' &&
15
+ docusaurusMeta.attr('content').includes('Docusaurus')) {
16
+ const metaAttrString = docusaurusMeta.attr('content');
17
+ if (metaAttrString.includes('v3')) {
18
+ return { framework: Frameworks.DOCUSAURUS, version: '3' };
19
19
  }
20
- if (metaAttrString.includes("v2")) {
21
- return { framework: Frameworks.DOCUSAURUS, version: "2" };
20
+ if (metaAttrString.includes('v2')) {
21
+ return { framework: Frameworks.DOCUSAURUS, version: '2' };
22
22
  }
23
- else if (metaAttrString.includes("v1")) {
24
- console.warn("WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.");
25
- return { framework: Frameworks.DOCUSAURUS, version: "1" };
23
+ else if (metaAttrString.includes('v1')) {
24
+ console.warn('WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.');
25
+ return { framework: Frameworks.DOCUSAURUS, version: '1' };
26
26
  }
27
27
  }
28
- const isGitBook = $(".gitbook-root").length > 0;
28
+ const isGitBook = $('.gitbook-root').length > 0;
29
29
  if (isGitBook) {
30
30
  return { framework: Frameworks.GITBOOK };
31
31
  }
@@ -1 +1 @@
1
- {"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,CAAN,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,uCAAyB,CAAA;IACzB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,mCAAqB,CAAA;AACvB,CAAC,EALW,UAAU,KAAV,UAAU,QAKrB;AAED,MAAM,UAAU,eAAe,CAAC,IAAI;IAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ;QACjD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACjE;QACA,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC;QAChE,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACxC,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;KACF;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;KAC1C;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE;QACZ,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;KACzC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE;QACd,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;KAC3C;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAC,CAAC;AACjC,CAAC"}
1
+ {"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,CAAN,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,uCAAyB,CAAA;IACzB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,mCAAqB,CAAA;AACvB,CAAC,EALW,UAAU,KAAV,UAAU,QAKrB;AAED,MAAM,UAAU,eAAe,CAAC,IAAI;IAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ;QACjD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACjE;QACA,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC;QAChE,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACxC,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;KACF;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;KAC1C;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE;QACZ,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;KACzC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE;QACd,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;KAC3C;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC"}
@@ -1,17 +1,17 @@
1
- import path from "path";
2
- import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from "../downloadImage.js";
1
+ import path from 'path';
2
+ import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from '../downloadImage.js';
3
3
  // To Do: Use CheerioElement instead of any when we bump the cheerio version
4
4
  export default async function downloadAllImages($, content, origin, baseDir, overwrite, modifyFileName) {
5
5
  if (!baseDir) {
6
- console.debug("Skipping image downloading");
6
+ console.debug('Skipping image downloading');
7
7
  return;
8
8
  }
9
9
  // We remove duplicates because some frameworks duplicate img tags
10
10
  // to show the image larger when clicked on.
11
11
  const imageSrcs = [
12
12
  ...new Set(content
13
- .find("img[src]")
14
- .map((i, image) => $(image).attr("src"))
13
+ .find('img[src]')
14
+ .map((i, image) => $(image).attr('src'))
15
15
  .toArray()),
16
16
  ];
17
17
  // Wait to all images to download before continuing
@@ -1 +1 @@
1
- {"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE;YAC3C,OAAO,EAAE,CAAC;SACX;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAC1B,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnD,EAAE,CACH,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE;YAC3C,OAAO,EAAE,CAAC;SACX;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;AACxF,CAAC"}
@@ -1,13 +1,13 @@
1
- import path from "path";
2
- import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from "../downloadImage.js";
3
- import { getFileExtension } from "../util.js";
1
+ import path from 'path';
2
+ import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from '../downloadImage.js';
3
+ import { getFileExtension } from '../util.js';
4
4
  // To Do: Use CheerioElement instead of any when we bump the cheerio version
5
5
  export default async function downloadLogoImage(imageSrc, imageBaseDir, origin, overwrite) {
6
6
  if (!imageSrc || !isValidImageSrc(imageSrc))
7
7
  return;
8
8
  const imageHref = cleanImageSrc(imageSrc, origin);
9
9
  const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
10
- const imagePath = path.join(imageBaseDir, "logo", "logo-light-mode." + ext);
10
+ const imagePath = path.join(imageBaseDir, 'logo', 'logo-light-mode.' + ext);
11
11
  await downloadImage(imageHref, imagePath, overwrite);
12
12
  }
13
13
  //# sourceMappingURL=downloadLogoImage.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAA4B,EAC5B,YAAoB,EACpB,MAAc,EACd,SAAkB;IAElB,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO;IAEpD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}
1
+ {"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAA4B,EAC5B,YAAoB,EACpB,MAAc,EACd,SAAkB;IAElB,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO;IAEpD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}
@@ -1,9 +1,9 @@
1
- import axios from "axios";
1
+ import axios from 'axios';
2
2
  // Not in use.
3
3
  // Gets all links in a sitemap.
4
4
  export const getSitemapLinks = async (url) => {
5
- const hostname = url.hostname.replace(".", "\\.");
6
- const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
5
+ const hostname = url.hostname.replace('.', '\\.');
6
+ const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, 'gmi');
7
7
  try {
8
8
  const indexData = (await axios.get(url.href)).data;
9
9
  const array = indexData.match(regex);
@@ -11,7 +11,7 @@ export const getSitemapLinks = async (url) => {
11
11
  }
12
12
  catch (err) {
13
13
  console.error(err);
14
- console.log("Skipping sitemap links because we encountered an error.");
14
+ console.log('Skipping sitemap links because we encountered an error.');
15
15
  return [];
16
16
  }
17
17
  };
@@ -1,22 +1,22 @@
1
- import path from "path";
2
- import axios from "axios";
3
- import { getHtmlWithPuppeteer } from "../browser.js";
4
- import { createPage } from "../util.js";
1
+ import axios from 'axios';
2
+ import path from 'path';
3
+ import { getHtmlWithPuppeteer } from '../browser.js';
4
+ import { createPage } from '../util.js';
5
5
  export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
6
6
  // Skip scraping external links
7
- if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
7
+ if (pathname.startsWith('https://') || pathname.startsWith('http://')) {
8
8
  return pathname;
9
9
  }
10
10
  // Removes file name from the end
11
- const splitSubpath = pathname.split("/");
12
- let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
11
+ const splitSubpath = pathname.split('/');
12
+ let folders = splitSubpath.slice(0, splitSubpath.length - 1).join('/');
13
13
  // Remove base dir if passed in
14
14
  if (baseToRemove && folders.startsWith(baseToRemove)) {
15
- folders = folders.replace(baseToRemove, "");
15
+ folders = folders.replace(baseToRemove, '');
16
16
  }
17
17
  // TO DO: Improve this by putting each page's images in a separate
18
18
  // folder named after the title of the page.
19
- const imageBaseDir = path.join(cliDir, "images", folders);
19
+ const imageBaseDir = path.join(cliDir, 'images', folders);
20
20
  // Scrape each page separately
21
21
  const href = new URL(pathname, origin).href;
22
22
  let html;
@@ -34,7 +34,7 @@ export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin,
34
34
  }
35
35
  const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
36
36
  // Default to introduction.mdx if we encountered index.html
37
- const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
37
+ const fileName = splitSubpath[splitSubpath.length - 1] || 'introduction';
38
38
  // Will create subfolders as needed
39
39
  createPage(title ?? '', description, markdown, overwrite, newFileLocation, fileName);
40
40
  // Removes first slash if we are in a folder, Mintlify doesn't need it
@@ -1 +1 @@
1
- {"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;KACtB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CACR,KAAK,IAAI,EAAE,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,QAAQ,CACT,CAAC;IAEF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
1
+ {"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;KACtB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAC;IAErF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
@@ -1,6 +1,6 @@
1
- import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
1
+ import { scrapeFileGettingFileNameFromUrl } from './scrapeFileGettingFileNameFromUrl.js';
2
2
  export async function scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
3
- if (typeof navEntry !== "string") {
3
+ if (typeof navEntry !== 'string') {
4
4
  const newPages = [];
5
5
  for (const nestedNavEntry of navEntry.pages) {
6
6
  newPages.push(await scrapeGettingFileNameFromUrl(nestedNavEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, version, baseToRemove));
@@ -1 +1 @@
1
- {"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAA6B,EAC7B,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,cAAc,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC3C,QAAQ,CAAC,IAAI,CAAC,MAAM,4BAA4B,CAC5C,cAAc,EACd,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACf,CAAC,CAAA;SACH;QACD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC1B,OAAO,QAAQ,CAAC;KACjB;IAED,OAAO,MAAM,gCAAgC,CAC3C,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAA6B,EAC7B,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAC3C,KAAK,MAAM,cAAc,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC3C,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,cAAc,EACd,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CACF,CAAC;SACH;QACD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC1B,OAAO,QAAQ,CAAC;KACjB;IAED,OAAO,MAAM,gCAAgC,CAC3C,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CAAC;AACJ,CAAC"}
@@ -1,9 +1,9 @@
1
- import path from "path";
2
- import { createPage, getOrigin } from "../util.js";
1
+ import path from 'path';
2
+ import { createPage, getOrigin } from '../util.js';
3
3
  export async function scrapePage(scrapeFunc, href, html, overwrite, version) {
4
4
  const origin = getOrigin(href);
5
5
  const cwd = process.cwd();
6
- const imageBaseDir = path.join(cwd, "images");
6
+ const imageBaseDir = path.join(cwd, 'images');
7
7
  const { title, description, markdown } = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
8
8
  createPage(title, description, markdown, overwrite, process.cwd());
9
9
  }
@@ -1 +1 @@
1
- {"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAwB,EACxB,IAAY,EACZ,IAAY,EACZ,SAAkB,EAClB,OAA2B;IAE3B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CACvD,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AACrE,CAAC"}
1
+ {"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAwB,EACxB,IAAY,EACZ,IAAY,EACZ,SAAkB,EAClB,OAA2B;IAE3B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CACvD,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AACrE,CAAC"}
@@ -1,4 +1,4 @@
1
- import { ArgumentsCamelCase } from "yargs";
1
+ import { ArgumentsCamelCase } from 'yargs';
2
2
  export declare function scrapePageWrapper(argv: ArgumentsCamelCase, scrapeFunc: ScrapePageFn, options?: {
3
3
  version?: string;
4
4
  puppeteer?: boolean;
@@ -1,19 +1,19 @@
1
- import axios from "axios";
2
- import { scrapePage } from "./scrapePage.js";
3
- import { scrapeDocusaurusPage } from "./site-scrapers/scrapeDocusaurusPage.js";
4
- import { scrapeGitBookPage } from "./site-scrapers/scrapeGitBookPage.js";
5
- import { scrapeReadMePage } from "./site-scrapers/scrapeReadMePage.js";
6
- import { detectFramework, Frameworks } from "./detectFramework.js";
7
- import { getHrefFromArgs } from "../util.js";
8
- import { getHtmlWithPuppeteer } from "../browser.js";
9
- import { scrapeIntercomPage } from "./site-scrapers/Intercom/scrapeIntercomPage.js";
1
+ import axios from 'axios';
2
+ import { getHtmlWithPuppeteer } from '../browser.js';
3
+ import { getHrefFromArgs } from '../util.js';
4
+ import { detectFramework, Frameworks } from './detectFramework.js';
5
+ import { scrapePage } from './scrapePage.js';
6
+ import { scrapeIntercomPage } from './site-scrapers/Intercom/scrapeIntercomPage.js';
7
+ import { scrapeDocusaurusPage } from './site-scrapers/scrapeDocusaurusPage.js';
8
+ import { scrapeGitBookPage } from './site-scrapers/scrapeGitBookPage.js';
9
+ import { scrapeReadMePage } from './site-scrapers/scrapeReadMePage.js';
10
10
  function validateFramework(framework) {
11
11
  if (!framework) {
12
- console.log("Could not detect the framework automatically. Please use one of:");
13
- console.log("scrape-page-docusaurus");
14
- console.log("scrape-page-gitbook");
15
- console.log("scrape-page-readme");
16
- console.log("scrape-page-intercom");
12
+ console.log('Could not detect the framework automatically. Please use one of:');
13
+ console.log('scrape-page-docusaurus');
14
+ console.log('scrape-page-gitbook');
15
+ console.log('scrape-page-readme');
16
+ console.log('scrape-page-intercom');
17
17
  return process.exit(1);
18
18
  }
19
19
  }
@@ -36,7 +36,7 @@ export async function scrapePageAutomatically(argv) {
36
36
  const html = res.data;
37
37
  const { framework, version } = detectFramework(html);
38
38
  validateFramework(framework);
39
- console.log("Detected framework: " + framework);
39
+ console.log('Detected framework: ' + framework);
40
40
  switch (framework) {
41
41
  case Frameworks.DOCUSAURUS:
42
42
  await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
@@ -1 +1 @@
1
- {"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAEpF,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAwB,EACxB,UAAwB,EACxB,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE;QACtB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjE,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAChD,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAClD,MAAM;KACT;AACH,CAAC"}
1
+ {"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAEvE,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAwB,EACxB,UAAwB,EACxB,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE;QACtB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjE,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAChD,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAClD,MAAM;KACT;AACH,CAAC"}
@@ -1,12 +1,12 @@
1
- import path from "path";
2
- import { objToReadableString } from "../util.js";
1
+ import path from 'path';
2
+ import { objToReadableString } from '../util.js';
3
3
  export async function scrapeSection(scrapeFunc, html, origin, overwrite, version) {
4
- console.log(`Started scraping${overwrite ? ", overwrite mode is on" : ""}...`);
4
+ console.log(`Started scraping${overwrite ? ', overwrite mode is on' : ''}...`);
5
5
  const cwd = process.cwd();
6
- const imageBaseDir = path.join(cwd, "images");
6
+ const imageBaseDir = path.join(cwd, 'images');
7
7
  const groupsConfig = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
8
- console.log("Finished scraping.");
9
- console.log("Add the following to your navigation in mint.json:");
8
+ console.log('Finished scraping.');
9
+ console.log('Add the following to your navigation in mint.json:');
10
10
  console.log(objToReadableString(groupsConfig));
11
11
  }
12
12
  //# sourceMappingURL=scrapeSection.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA2B,EAC3B,IAAY,EACZ,MAAc,EACd,SAAkB,EAClB,OAA2B;IAE3B,OAAO,CAAC,GAAG,CACT,mBAAmB,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,KAAK,CAClE,CAAC;IACF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;AACjD,CAAC"}
1
+ {"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA2B,EAC3B,IAAY,EACZ,MAAc,EACd,SAAkB,EAClB,OAA2B;IAE3B,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;AACjD,CAAC"}
@@ -1,4 +1,4 @@
1
- import { ArgumentsCamelCase } from "yargs";
1
+ import { ArgumentsCamelCase } from 'yargs';
2
2
  export declare function scrapeSectionAxiosWrapper(argv: ArgumentsCamelCase, scrapeFunc: ScrapeSectionFn): Promise<void>;
3
3
  export declare function scrapeDocusaurusSectionCommand(argv: any, version: string | undefined): Promise<void>;
4
4
  export declare function scrapeGitbookSectionCommand(argv: any): Promise<void>;
@@ -1,14 +1,14 @@
1
- import axios from "axios";
2
- import { detectFramework, Frameworks } from "./detectFramework.js";
3
- import { getHrefFromArgs, getOrigin } from "../util.js";
4
- import { scrapeSection } from "./scrapeSection.js";
5
- import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
6
- import openNestedDocusaurusMenus from "./site-scrapers/openNestedDocusaurusMenus.js";
7
- import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
8
- import openNestedGitbookMenus from "./site-scrapers/openNestedGitbookMenus.js";
9
- import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
10
- import { startBrowser } from "../browser.js";
11
- import { scrapeIntercomSection } from "./site-scrapers/Intercom/scrapeIntercomSection.js";
1
+ import axios from 'axios';
2
+ import { startBrowser } from '../browser.js';
3
+ import { getHrefFromArgs, getOrigin } from '../util.js';
4
+ import { detectFramework, Frameworks } from './detectFramework.js';
5
+ import { scrapeSection } from './scrapeSection.js';
6
+ import { scrapeIntercomSection } from './site-scrapers/Intercom/scrapeIntercomSection.js';
7
+ import openNestedDocusaurusMenus from './site-scrapers/openNestedDocusaurusMenus.js';
8
+ import openNestedGitbookMenus from './site-scrapers/openNestedGitbookMenus.js';
9
+ import { scrapeDocusaurusSection } from './site-scrapers/scrapeDocusaurusSection.js';
10
+ import { scrapeGitBookSection } from './site-scrapers/scrapeGitBookSection.js';
11
+ import { scrapeReadMeSection } from './site-scrapers/scrapeReadMeSection.js';
12
12
  export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
13
13
  const href = getHrefFromArgs(argv);
14
14
  const res = await axios.get(href);
@@ -28,7 +28,7 @@ async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc, versio
28
28
  const browser = await startBrowser();
29
29
  const page = await browser.newPage();
30
30
  await page.goto(href, {
31
- waitUntil: "networkidle2",
31
+ waitUntil: 'networkidle2',
32
32
  });
33
33
  const html = await openLinks(page);
34
34
  browser.close();
@@ -41,7 +41,7 @@ export async function scrapeSectionAutomatically(argv) {
41
41
  const html = res.data;
42
42
  const { framework, version } = detectFramework(html);
43
43
  validateFramework(framework);
44
- console.log("Detected framework: " + framework);
44
+ console.log('Detected framework: ' + framework);
45
45
  switch (framework) {
46
46
  case Frameworks.DOCUSAURUS:
47
47
  await scrapeDocusaurusSectionCommand(argv, version);
@@ -59,7 +59,7 @@ export async function scrapeSectionAutomatically(argv) {
59
59
  }
60
60
  function validateFramework(framework) {
61
61
  if (!framework) {
62
- console.log("Could not detect the framework automatically. We only support Docusaurus (V2 and V3), GitBook, and ReadMe.");
62
+ console.log('Could not detect the framework automatically. We only support Docusaurus (V2 and V3), GitBook, and ReadMe.');
63
63
  process.exit();
64
64
  }
65
65
  }
@@ -1 +1 @@
1
- {"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAE1F,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAwB,EACxB,UAA2B;IAE3B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,CAAC,SAAS,EAChB,SAAS,CACV,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAA2B,CAAC,kBAAkB;;IAE9C,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CACjC,IAAI,EACJ,sBAAsB,EACtB,oBAAoB,CACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,CAAC,SAAS,EAChB,OAAO,CACR,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,yBAAyB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAC7D,MAAM;KACT;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;KAChB;AACH,CAAC"}
1
+ {"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAC1F,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAwB,EACxB,UAA2B;IAE3B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAA2B,CAAC,kBAAkB;;IAE9C,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CAAC,IAAI,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;AAC1F,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,yBAAyB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAC7D,MAAM;KACT;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;KAChB;AACH,CAAC"}