@mintlify/scraping 4.0.428 → 4.0.430

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/bin/nav/root.js +24 -7
  2. package/bin/nav/root.js.map +1 -1
  3. package/bin/tsconfig.build.tsbuildinfo +1 -1
  4. package/bin/utils/detectFramework.js +15 -9
  5. package/bin/utils/detectFramework.js.map +1 -1
  6. package/package.json +4 -4
  7. package/src/nav/root.ts +24 -7
  8. package/src/utils/detectFramework.ts +18 -11
package/bin/utils/detectFramework.js CHANGED
@@ -6,10 +6,17 @@ export const framework = {
6
6
  };
7
7
  export function detectFramework(rootHast) {
8
8
  visit(rootHast, 'element', function (node) {
9
- if (node.tagName === 'link' &&
10
- Array.isArray(node.properties.rel) &&
11
- node.properties.rel.includes('preconnect') &&
12
- node.properties.href === 'https://api.gitbook.com') {
9
+ const { content: rawContent, rel, name, href } = node.properties;
10
+ const content = (Array.isArray(rawContent) ? rawContent.join(' ') : String(rawContent)).toLowerCase();
11
+ if ((node.tagName === 'link' &&
12
+ Array.isArray(rel) &&
13
+ rel.includes('preconnect') &&
14
+ href === 'https://api.gitbook.com') ||
15
+ (node.tagName === 'link' &&
16
+ Array.isArray(rel) &&
17
+ rel.includes('preconnect') &&
18
+ href === 'https://api.gitbook.com/cache') ||
19
+ (node.tagName === 'meta' && name === 'generator' && content.includes('gitbook'))) {
13
20
  framework.vendor = 'gitbook';
14
21
  framework.version = undefined;
15
22
  return EXIT;
@@ -17,9 +24,9 @@ export function detectFramework(rootHast) {
17
24
  // All of the other docs vendors rely on
18
25
  // the `meta` element as well as a `name`
19
26
  // property which should be a string
20
- if (node.tagName !== 'meta' || typeof node.properties.name !== 'string')
27
+ if (node.tagName !== 'meta' || typeof name !== 'string')
21
28
  return CONTINUE;
22
- switch (node.properties.name) {
29
+ switch (name) {
23
30
  case 'readme-deploy':
24
31
  framework.vendor = 'readme';
25
32
  framework.version = undefined;
@@ -29,10 +36,9 @@ export function detectFramework(rootHast) {
29
36
  // framework.version = undefined;
30
37
  // return EXIT;
31
38
  case 'generator':
32
- if (typeof node.properties.content === 'string' &&
33
- node.properties.content.includes('Docusaurus')) {
39
+ if (content.includes('docusaurus')) {
34
40
  framework.vendor = 'docusaurus';
35
- const meta = node.properties.content;
41
+ const meta = content;
36
42
  if (meta.includes('v3')) {
37
43
  framework.version = 3;
38
44
  }
package/bin/utils/detectFramework.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/utils/detectFramework.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,SAAS,GAAc;IAClC,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,SAAS;CACnB,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,QAAc;IAC5C,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,IACE,IAAI,CAAC,OAAO,KAAK,MAAM;YACvB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC1C,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,yBAAyB,EAClD,CAAC;YACD,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC;YAC7B,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wCAAwC;QACxC,yCAAyC;QACzC,oCAAoC;QACpC,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAEzF,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAC7B,KAAK,eAAe;gBAClB,SAAS,CAAC,MAAM,GAAG,QAAQ,CAAC;gBAC5B,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;gBAC9B,OAAO,IAAI,CAAC;YAEd,kCAAkC;YAClC,mCAAmC;YACnC,mCAAmC;YACnC,iBAAiB;YAEjB,KAAK,WAAW;gBACd,IACE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,KAAK,QAAQ;oBAC3C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAC9C,CAAC;oBACD,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;oBAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;oBACrC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBACxB,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC;oBACxB,CAAC;yBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC/B,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC;oBACxB,CAAC;yBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC/B,GAAG,CACD,gFAAgF,EAChF,OAAO,CACR,CAAC;wBACF,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC;wBAC7B,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;oBAChC,CAAC;oBACD,OAAO,IAAI,CAAC;gBACd,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,8CAA8C,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACzE,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,4EAA4E,CAAC,CAAC;QAClF,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/utils/detectFramework.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,SAAS,GAAc;IAClC,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,SAAS;CACnB,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,QAAc;IAC5C,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;QACjE,MAAM,OAAO,GAAG,CACd,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CACtE,CAAC,WAAW,EAAE,CAAC;QAEhB,IACE,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM;YACtB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;YAClB,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC1B,IAAI,KAAK,yBAAyB,CAAC;YACrC,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM;gBACtB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;gBAClB,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC1B,IAAI,KAAK,+BAA+B,CAAC;YAC3C,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAChF,CAAC;YACD,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC;YAC7B,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wCAAwC;QACxC,yCAAyC;QACzC,oCAAoC;QACpC,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAEzE,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,eAAe;gBAClB,SAAS,CAAC,MAAM,GAAG,QAAQ,CAAC;gBAC5B,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;gBAC9B,OAAO,IAAI,CAAC;YAEd,kCAAkC;YAClC,mCAAmC;YACnC,mCAAmC;YACnC,iBAAiB;YAEjB,KAAK,WAAW;gBACd,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACnC,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;oBAChC,MAAM,IAAI,GAAG,OAAO,CAAC;oBACrB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBACxB,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC;oBACxB,CAAC;yBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC/B,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC;oBACxB,CAAC;yBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC/B,GAAG,CACD,gFAAgF,EAChF,OAAO,CACR,CAAC;wBACF,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC;wBAC7B,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;oBAChC,CAAC;oBACD,OAAO,IAAI,CAAC;gBACd,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,8CAA8C,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACzE,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,4EAA4E,CAAC,CAAC;QAClF,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mintlify/scraping",
3
- "version": "4.0.428",
3
+ "version": "4.0.430",
4
4
  "description": "Scrape documentation frameworks to Mintlify docs",
5
5
  "engines": {
6
6
  "node": ">=18.0.0"
@@ -38,7 +38,7 @@
38
38
  "format:check": "prettier . --check"
39
39
  },
40
40
  "dependencies": {
41
- "@mintlify/common": "1.0.569",
41
+ "@mintlify/common": "1.0.571",
42
42
  "@mintlify/openapi-parser": "^0.0.8",
43
43
  "fs-extra": "^11.1.1",
44
44
  "hast-util-to-mdast": "^10.1.0",
@@ -61,7 +61,7 @@
61
61
  "@mintlify/models": "0.0.233",
62
62
  "@mintlify/prettier-config": "1.0.4",
63
63
  "@mintlify/ts-config": "2.0.2",
64
- "@mintlify/validation": "0.1.494",
64
+ "@mintlify/validation": "0.1.496",
65
65
  "@trivago/prettier-plugin-sort-imports": "^4.2.1",
66
66
  "@tsconfig/recommended": "1.x",
67
67
  "@types/hast": "^3.0.4",
@@ -77,5 +77,5 @@
77
77
  "typescript": "^5.5.3",
78
78
  "vitest": "^2.0.4"
79
79
  },
80
- "gitHead": "bdea2ca03aaeb0a6327c4f51bfb57410960760f6"
80
+ "gitHead": "3b688e5b1316e7fc551542e114d487924123e76c"
81
81
  }
package/src/nav/root.ts CHANGED
@@ -17,26 +17,31 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
17
17
  break;
18
18
  }
19
19
 
20
- let rootSelector = 'page-no-toc:hidden';
20
+ let rootSelectorSet = new Set('page-no-toc:hidden');
21
21
  switch (framework.vendor) {
22
22
  case 'docusaurus':
23
- rootSelector = 'menu';
23
+ rootSelectorSet = new Set('menu');
24
24
  break;
25
25
  case 'gitbook':
26
- rootSelector = 'page-no-toc:lg:hidden';
26
+ rootSelectorSet = new Set([
27
+ 'lg:page-no-toc:hidden',
28
+ 'page-no-toc:hidden',
29
+ 'page-no-toc:lg:hidden',
30
+ ]);
27
31
  break;
28
32
  case 'readme':
29
- rootSelector = 'rm-Sidebar';
33
+ rootSelectorSet = new Set('rm-Sidebar');
30
34
  break;
31
35
  }
32
36
 
33
37
  let element: Element | undefined = undefined;
34
38
  visit(rootNode, 'element', function (node) {
39
+ const { className } = node.properties;
35
40
  if (
36
41
  node.tagName === rootTagName &&
37
- node.properties.className &&
38
- Array.isArray(node.properties.className) &&
39
- node.properties.className.includes(rootSelector)
42
+ className &&
43
+ Array.isArray(className) &&
44
+ !!intersection(className, rootSelectorSet).size
40
45
  ) {
41
46
  element = node;
42
47
  return EXIT;
@@ -45,3 +50,15 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
45
50
 
46
51
  return element;
47
52
  }
53
+
54
+ function intersection<T>(set1: Set<T> | Array<T>, set2: Set<T> | Array<T>): Set<T> {
55
+ if (Array.isArray(set1)) set1 = new Set(set1);
56
+ if (Array.isArray(set2)) set2 = new Set(set2);
57
+
58
+ const intersectedSet = new Set<T>();
59
+ for (const el of set1) {
60
+ if (set2.has(el)) intersectedSet.add(el);
61
+ }
62
+
63
+ return intersectedSet;
64
+ }
package/src/utils/detectFramework.ts CHANGED
@@ -11,11 +11,21 @@ export const framework: Framework = {
11
11
 
12
12
  export function detectFramework(rootHast: Root): void {
13
13
  visit(rootHast, 'element', function (node) {
14
+ const { content: rawContent, rel, name, href } = node.properties;
15
+ const content = (
16
+ Array.isArray(rawContent) ? rawContent.join(' ') : String(rawContent)
17
+ ).toLowerCase();
18
+
14
19
  if (
15
- node.tagName === 'link' &&
16
- Array.isArray(node.properties.rel) &&
17
- node.properties.rel.includes('preconnect') &&
18
- node.properties.href === 'https://api.gitbook.com'
20
+ (node.tagName === 'link' &&
21
+ Array.isArray(rel) &&
22
+ rel.includes('preconnect') &&
23
+ href === 'https://api.gitbook.com') ||
24
+ (node.tagName === 'link' &&
25
+ Array.isArray(rel) &&
26
+ rel.includes('preconnect') &&
27
+ href === 'https://api.gitbook.com/cache') ||
28
+ (node.tagName === 'meta' && name === 'generator' && content.includes('gitbook'))
19
29
  ) {
20
30
  framework.vendor = 'gitbook';
21
31
  framework.version = undefined;
@@ -25,9 +35,9 @@ export function detectFramework(rootHast: Root): void {
25
35
  // All of the other docs vendors rely on
26
36
  // the `meta` element as well as a `name`
27
37
  // property which should be a string
28
- if (node.tagName !== 'meta' || typeof node.properties.name !== 'string') return CONTINUE;
38
+ if (node.tagName !== 'meta' || typeof name !== 'string') return CONTINUE;
29
39
 
30
- switch (node.properties.name) {
40
+ switch (name) {
31
41
  case 'readme-deploy':
32
42
  framework.vendor = 'readme';
33
43
  framework.version = undefined;
@@ -39,12 +49,9 @@ export function detectFramework(rootHast: Root): void {
39
49
  // return EXIT;
40
50
 
41
51
  case 'generator':
42
- if (
43
- typeof node.properties.content === 'string' &&
44
- node.properties.content.includes('Docusaurus')
45
- ) {
52
+ if (content.includes('docusaurus')) {
46
53
  framework.vendor = 'docusaurus';
47
- const meta = node.properties.content;
54
+ const meta = content;
48
55
  if (meta.includes('v3')) {
49
56
  framework.version = 3;
50
57
  } else if (meta.includes('v2')) {