@mintlify/scraping 4.0.52 → 4.0.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +4 -4
- package/__test__/getOpenApiDefinition.test.ts +154 -0
- package/bin/cli.js +8 -4
- package/bin/cli.js.map +1 -1
- package/bin/openapi/generateOpenApiPages.d.ts +6 -1
- package/bin/openapi/generateOpenApiPages.js +32 -7
- package/bin/openapi/generateOpenApiPages.js.map +1 -1
- package/bin/tsconfig.build.tsbuildinfo +1 -1
- package/bin/utils/network.d.ts +2 -0
- package/bin/utils/network.js +15 -0
- package/bin/utils/network.js.map +1 -1
- package/package.json +3 -3
- package/src/cli.ts +8 -4
- package/src/openapi/generateOpenApiPages.ts +38 -7
- package/src/utils/network.ts +16 -0
package/CONTRIBUTING.md
CHANGED
|
@@ -14,15 +14,15 @@ The CLI has many commands. This doc focuses on how we coded scraping websites.
|
|
|
14
14
|
|
|
15
15
|
## User Interface
|
|
16
16
|
|
|
17
|
-
There are
|
|
17
|
+
There are three main commands:
|
|
18
18
|
|
|
19
19
|
`mintlify-scrape page [url]`
|
|
20
20
|
|
|
21
|
-
and
|
|
22
|
-
|
|
23
21
|
`mintlify-scrape section [url]`
|
|
24
22
|
|
|
25
|
-
|
|
23
|
+
`mintlify-scrape openapi-file [url]`
|
|
24
|
+
|
|
25
|
+
Scraping a page downloads a single page’s content. Scraping a section goes through the navigation and scrapes each page. The code for downloading a page’s content is shared between the two commands. Scraping an OpenAPI file supports either a file path or an HTTPS URL.
|
|
26
26
|
|
|
27
27
|
Important files: `scraping/scrapePageCommands.ts`, `scraping/scrapeSectionAutomatically.ts`
|
|
28
28
|
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import yaml from 'js-yaml';
|
|
3
|
+
import { OpenAPI } from 'openapi-types';
|
|
4
|
+
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
5
|
+
|
|
6
|
+
import { getOpenApiDefinition } from '../src/openapi/generateOpenApiPages.js';
|
|
7
|
+
|
|
8
|
+
vi.mock('fs/promises');
|
|
9
|
+
vi.mock('node-fetch');
|
|
10
|
+
|
|
11
|
+
const mockOpenApiDoc: OpenAPI.Document = {
|
|
12
|
+
openapi: '3.0.0',
|
|
13
|
+
info: {
|
|
14
|
+
title: 'Test API',
|
|
15
|
+
version: '1.0.0',
|
|
16
|
+
},
|
|
17
|
+
paths: {},
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
describe('getOpenApiDefinition', () => {
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
vi.resetAllMocks();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('should load OpenAPI doc from a local file path', async () => {
|
|
26
|
+
const mockYaml = yaml.dump(mockOpenApiDoc);
|
|
27
|
+
vi.mocked(fs.readFile).mockResolvedValue(mockYaml);
|
|
28
|
+
|
|
29
|
+
const result = await getOpenApiDefinition('test.yaml');
|
|
30
|
+
|
|
31
|
+
expect(fs.readFile).toHaveBeenCalledWith(expect.stringContaining('test.yaml'), 'utf-8');
|
|
32
|
+
expect(result).toEqual({ document: mockOpenApiDoc, isUrl: false });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('should accept OpenAPI document directly', async () => {
|
|
36
|
+
const result = await getOpenApiDefinition(mockOpenApiDoc);
|
|
37
|
+
expect(result).toEqual({ document: mockOpenApiDoc, isUrl: false });
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('should fetch yaml OpenAPI doc from URL', async () => {
|
|
41
|
+
const mockYaml = yaml.dump(mockOpenApiDoc);
|
|
42
|
+
global.fetch = vi.fn().mockResolvedValue({
|
|
43
|
+
ok: true,
|
|
44
|
+
text: () => Promise.resolve(mockYaml),
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const url = new URL('https://example.com/openapi.yaml');
|
|
48
|
+
const result = await getOpenApiDefinition(url);
|
|
49
|
+
|
|
50
|
+
expect(fetch).toHaveBeenCalledWith(url);
|
|
51
|
+
expect(result).toEqual({ document: mockOpenApiDoc, isUrl: true });
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('should fetch and parse valid JSON OpenAPI doc from URL', async () => {
|
|
55
|
+
const mockJson = JSON.stringify(mockOpenApiDoc);
|
|
56
|
+
global.fetch = vi.fn().mockResolvedValue({
|
|
57
|
+
ok: true,
|
|
58
|
+
status: 200,
|
|
59
|
+
text: () => Promise.resolve(mockJson),
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const url = new URL('https://example.com/openapi.yaml');
|
|
63
|
+
const result = await getOpenApiDefinition(url);
|
|
64
|
+
|
|
65
|
+
expect(fetch).toHaveBeenCalledWith(url);
|
|
66
|
+
expect(result).toEqual({ document: mockOpenApiDoc, isUrl: true });
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('should fetch OpenAPI doc from URL string', async () => {
|
|
70
|
+
const mockYaml = yaml.dump(mockOpenApiDoc);
|
|
71
|
+
global.fetch = vi.fn().mockResolvedValue({
|
|
72
|
+
ok: true,
|
|
73
|
+
status: 200,
|
|
74
|
+
text: () => Promise.resolve(mockYaml),
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const urlString = 'https://example.com/openapi.yaml';
|
|
78
|
+
const result = await getOpenApiDefinition(urlString);
|
|
79
|
+
|
|
80
|
+
expect(fetch).toHaveBeenCalledWith(new URL(urlString));
|
|
81
|
+
expect(result).toEqual({ document: mockOpenApiDoc, isUrl: true });
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('should throw error when local file read fails', async () => {
|
|
85
|
+
vi.mocked(fs.readFile).mockRejectedValue(new Error('File read error'));
|
|
86
|
+
|
|
87
|
+
await expect(getOpenApiDefinition('test.yaml')).rejects.toThrow('File read error');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('should throw error with URL and status code when fetch fails', async () => {
|
|
91
|
+
global.fetch = vi.fn().mockResolvedValue({
|
|
92
|
+
ok: false,
|
|
93
|
+
status: 404,
|
|
94
|
+
statusText: 'Not Found',
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
const urlString = 'https://petstore3.swagger.io/api/v3/openapi.doesnotexist';
|
|
98
|
+
|
|
99
|
+
await expect(getOpenApiDefinition(urlString)).rejects.toThrow(
|
|
100
|
+
'https://petstore3.swagger.io/api/v3/openapi.doesnotexist - failed to retrieve OpenAPI file from source: 404 Not Found'
|
|
101
|
+
);
|
|
102
|
+
|
|
103
|
+
expect(fetch).toHaveBeenCalledWith(new URL(urlString));
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('should throw error when HTTP URL is provided', async () => {
|
|
107
|
+
const httpUrl = new URL('http://example.com/openapi.yaml');
|
|
108
|
+
await expect(getOpenApiDefinition(httpUrl)).rejects.toThrow(
|
|
109
|
+
'Only HTTPS URLs are supported. Please provide an HTTPS URL'
|
|
110
|
+
);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('should throw error when HTTP URL string is provided', async () => {
|
|
114
|
+
const httpUrlString = 'http://example.com/openapi.yaml';
|
|
115
|
+
await expect(getOpenApiDefinition(httpUrlString)).rejects.toThrow(
|
|
116
|
+
'Only HTTPS URLs are supported. Please provide an HTTPS URL'
|
|
117
|
+
);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('should throw error when non-HTTPS URL string is provided', async () => {
|
|
121
|
+
const httpUrlString = 'ftp://example.com/openapi.yaml';
|
|
122
|
+
await expect(getOpenApiDefinition(httpUrlString)).rejects.toThrow(
|
|
123
|
+
'Only HTTPS URLs are supported. Please provide an HTTPS URL'
|
|
124
|
+
);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it('should throw error when URL response is invalid YAML', async () => {
|
|
128
|
+
global.fetch = vi.fn().mockResolvedValue({
|
|
129
|
+
text: () => Promise.resolve('invalid: yaml: content'),
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
await expect(getOpenApiDefinition('https://example.com/openapi.yaml')).rejects.toThrow();
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('should throw error when URL response is invalid JSON', async () => {
|
|
136
|
+
global.fetch = vi.fn().mockResolvedValue({
|
|
137
|
+
text: () => Promise.resolve('{"invalid": "yaml", "invalid": "content"}'),
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
await expect(getOpenApiDefinition('https://example.com/openapi.json')).rejects.toThrow();
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it('should throw error when local file contains invalid YAML', async () => {
|
|
144
|
+
vi.mocked(fs.readFile).mockResolvedValue('invalid: yaml: content');
|
|
145
|
+
|
|
146
|
+
await expect(getOpenApiDefinition('test.yaml')).rejects.toThrow();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('should throw error when local file contains invalid JSON', async () => {
|
|
150
|
+
vi.mocked(fs.readFile).mockResolvedValue('{"invalid": "yaml", "invalid": "content"}');
|
|
151
|
+
|
|
152
|
+
await expect(getOpenApiDefinition('test.json')).rejects.toThrow();
|
|
153
|
+
});
|
|
154
|
+
});
|
package/bin/cli.js
CHANGED
|
@@ -15,9 +15,9 @@ import { checkUrl } from './utils/url.js';
|
|
|
15
15
|
await yargs(hideBin(process.argv))
|
|
16
16
|
.command('page <url>', 'Scrapes the docs page for the URL provided', (yargs) => yargs.positional('url', { type: 'string', demandOption: true }).check(checkUrl), async ({ url }) => await page(url))
|
|
17
17
|
.command('section <url>', 'Scrapes the entire docs site based on the URL provided', (yargs) => yargs.positional('url', { type: 'string', demandOption: true }).check(checkUrl), async ({ url }) => await site(url))
|
|
18
|
-
.command('openapi-file <
|
|
19
|
-
.positional('
|
|
20
|
-
describe: 'The filename of the OpenAPI spec',
|
|
18
|
+
.command('openapi-file <openapiLocation>', 'Creates MDX files from an OpenAPI spec', (yargs) => yargs
|
|
19
|
+
.positional('openapiLocation', {
|
|
20
|
+
describe: 'The filename or URL location of the OpenAPI spec',
|
|
21
21
|
type: 'string',
|
|
22
22
|
demandOption: true,
|
|
23
23
|
})
|
|
@@ -38,7 +38,7 @@ await yargs(hideBin(process.argv))
|
|
|
38
38
|
type: 'boolean',
|
|
39
39
|
}), async (argv) => {
|
|
40
40
|
try {
|
|
41
|
-
const { nav } = await generateOpenApiPages(argv.
|
|
41
|
+
const { nav, isUrl } = await generateOpenApiPages(argv.openapiLocation, {
|
|
42
42
|
openApiFilePath: undefined,
|
|
43
43
|
version: undefined,
|
|
44
44
|
writeFiles: argv.writeFiles,
|
|
@@ -47,6 +47,10 @@ await yargs(hideBin(process.argv))
|
|
|
47
47
|
});
|
|
48
48
|
console.log('navigation object suggestion:');
|
|
49
49
|
console.log(JSON.stringify(nav, undefined, 2));
|
|
50
|
+
if (isUrl) {
|
|
51
|
+
console.log('openapi location suggestion:');
|
|
52
|
+
console.log(`openapi: ${argv.openapiLocation}`);
|
|
53
|
+
}
|
|
50
54
|
}
|
|
51
55
|
catch (error) {
|
|
52
56
|
if (error instanceof Error) {
|
package/bin/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,4CAA4C,EAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,eAAe,EACf,wDAAwD,EACxD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,iBAAiB,EAAE;IAC7B,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,4CAA4C,EAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,eAAe,EACf,wDAAwD,EACxD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,iBAAiB,EAAE;IAC7B,QAAQ,EAAE,kDAAkD;IAC5D,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;KACD,MAAM,CAAC,YAAY,EAAE;IACpB,QAAQ,EAAE,+CAA+C;IACzD,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,QAAQ,EAAE;IAChB,QAAQ,EAAE,4DAA4D;IACtE,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,WAAW,EAAE;IACnB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,EAAE,KAAK;IACd,IAAI,EAAE,SAAS;CAChB,CAAC,EACN,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,eAAe,EAAE;YACtE,eAAe,EAAE,SAAS;YAC1B,OAAO,EAAE,SAAS;YAClB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CACF;KAEA,cAAc,EAAE;KAChB,aAAa,CAAC,CAAC,EAAE,gEAAgE,CAAC;KAClF,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,KAAK,EAAE,CAAC;AAEX,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC9B,eAAe,CAAC,IAAI,CAAC,CAAC;QAEtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI;YAC3B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,2CAA2C,GAAG,EAAE;SAC1D,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,GAAG,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9D,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import type { DecoratedNavigation, Navigation, DecoratedNavigationPage } from '@mintlify/models';
|
|
2
2
|
import { OpenAPI } from 'openapi-types';
|
|
3
|
-
export declare const
|
|
3
|
+
export declare const getOpenApiDefinition: (pathOrDocumentOrUrl: string | OpenAPI.Document | URL) => Promise<{
|
|
4
|
+
document: OpenAPI.Document;
|
|
5
|
+
isUrl: boolean;
|
|
6
|
+
}>;
|
|
7
|
+
export declare const generateOpenApiPages: (pathOrDocumentOrUrl: string | OpenAPI.Document | URL, opts?: {
|
|
4
8
|
openApiFilePath?: string | undefined;
|
|
5
9
|
version?: string | undefined;
|
|
6
10
|
writeFiles?: boolean;
|
|
@@ -12,5 +16,6 @@ export declare const generateOpenApiPages: (pathOrDocument: string | OpenAPI.Doc
|
|
|
12
16
|
decoratedNav: DecoratedNavigation;
|
|
13
17
|
spec: OpenAPI.Document;
|
|
14
18
|
pagesAcc: Record<string, DecoratedNavigationPage>;
|
|
19
|
+
isUrl: boolean;
|
|
15
20
|
}>;
|
|
16
21
|
export declare const prepareStringToBeValidFilename: (str?: string) => string | undefined;
|
|
@@ -5,17 +5,41 @@ import yaml from 'js-yaml';
|
|
|
5
5
|
import { OpenAPIV3 } from 'openapi-types';
|
|
6
6
|
import { join, resolve, parse } from 'path';
|
|
7
7
|
import path from 'path';
|
|
8
|
-
|
|
8
|
+
import { fetchOpenApi } from '../utils/network.js';
|
|
9
|
+
export const getOpenApiDefinition = async (pathOrDocumentOrUrl) => {
|
|
10
|
+
if (typeof pathOrDocumentOrUrl === 'string') {
|
|
11
|
+
if (pathOrDocumentOrUrl.startsWith('http://')) {
|
|
12
|
+
// This is an invalid location either for a file or a URL
|
|
13
|
+
throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
try {
|
|
17
|
+
const url = new URL(pathOrDocumentOrUrl);
|
|
18
|
+
pathOrDocumentOrUrl = url;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
const pathname = path.join(process.cwd(), pathOrDocumentOrUrl.toString());
|
|
22
|
+
const file = await fs.readFile(pathname, 'utf-8');
|
|
23
|
+
pathOrDocumentOrUrl = yaml.load(file);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const isUrl = pathOrDocumentOrUrl instanceof URL;
|
|
28
|
+
if (pathOrDocumentOrUrl instanceof URL) {
|
|
29
|
+
if (pathOrDocumentOrUrl.protocol !== 'https:') {
|
|
30
|
+
throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
|
|
31
|
+
}
|
|
32
|
+
pathOrDocumentOrUrl = await fetchOpenApi(pathOrDocumentOrUrl);
|
|
33
|
+
}
|
|
34
|
+
return { document: pathOrDocumentOrUrl, isUrl };
|
|
35
|
+
};
|
|
36
|
+
export const generateOpenApiPages = async (pathOrDocumentOrUrl, opts) => {
|
|
9
37
|
const { openApiFilePath, version, writeFiles, outDir, outDirBasePath } = opts ?? {};
|
|
10
38
|
const openApiFilePathFromRoot = openApiFilePath
|
|
11
39
|
? optionallyAddLeadingSlash(openApiFilePath)
|
|
12
40
|
: undefined;
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const file = await fs.readFile(pathname, 'utf-8');
|
|
16
|
-
pathOrDocument = yaml.load(file);
|
|
17
|
-
}
|
|
18
|
-
const { schema } = await validate(pathOrDocument);
|
|
41
|
+
const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl);
|
|
42
|
+
const { schema } = await validate(document);
|
|
19
43
|
if (schema?.paths === undefined || Object.keys(schema.paths).length === 0) {
|
|
20
44
|
throw new Error('No paths defined.');
|
|
21
45
|
}
|
|
@@ -72,6 +96,7 @@ export const generateOpenApiPages = async (pathOrDocument, opts) => {
|
|
|
72
96
|
decoratedNav,
|
|
73
97
|
spec: schema,
|
|
74
98
|
pagesAcc,
|
|
99
|
+
isUrl,
|
|
75
100
|
};
|
|
76
101
|
};
|
|
77
102
|
// returns the group with the given group name, or the top-level group if no group name is provided
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generateOpenApiPages.js","sourceRoot":"","sources":["../../src/openapi/generateOpenApiPages.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,6BAA6B,EAC7B,yBAAyB,EACzB,WAAW,EACX,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAS1B,OAAO,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,EAAW,SAAS,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC5C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,
|
|
1
|
+
{"version":3,"file":"generateOpenApiPages.js","sourceRoot":"","sources":["../../src/openapi/generateOpenApiPages.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,6BAA6B,EAC7B,yBAAyB,EACzB,WAAW,EACX,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAS1B,OAAO,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,EAAW,SAAS,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC5C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,mBAAoD,EACK,EAAE;IAC3D,IAAI,OAAO,mBAAmB,KAAK,QAAQ,EAAE,CAAC;QAC5C,IAAI,mBAAmB,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9C,yDAAyD;YACzD,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAChF,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAC;gBACzC,mBAAmB,GAAG,GAAG,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,mBAAmB,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1E,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAClD,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAqB,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,YAAY,GAAG,CAAC;IACjD,IAAI,mBAAmB,YAAY,GAAG,EAAE,CAAC;QACvC,IAAI,mBAAmB,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAChF,CAAC;QACD,mBAAmB,GAAG,MAAM,YAAY,CAAC,mBAAmB,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,mBAAuC,EAAE,KAAK,EAAE,CAAC;AACtE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,mBAAoD,EACpD,IAOC,EAOA,EAAE;IACH,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;IACpF,MAAM,uBAAuB,GAAG,eAAe;QAC7C,CAAC,CAAC,yBAAyB,CAAC,eAAe,CAAC;QAC5C,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,oBAAoB,CAAC,mBAAmB,CAAC,CAAC;IAE5E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5C,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAwB,EAAE,CAAC;IAC7C,MAAM,aAAa,GAAoB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAA4C,EAAE,CAAC;IAC7D,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE;QAC9D,IAAI,CAAC,cAAc,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;YAC1D,OAAO;QACT,CAAC;QACD,MAAM,mBAAmB,GAAG,cAA0C,CAAC;QACvE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACtD,IAAI,MAAM,IAAI,mBAAmB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACvC,MAAM,KAAK,GACT,8BAA8B,CAAC,SAAS,EAAE,OAAO,CAAC;oBAClD,GAAG,MAAM,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtD,MAAM,MAAM,GAAG,8BAA8B,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;gBAE/C,MAAM,QAAQ,GAAG,YAAY,CAAkB,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC/D,MAAM,iBAAiB,GAAG,YAAY,CAA2B,YAAY,EAAE,SAAS,CAAC,CAAC;gBAE1F,MAAM,wBAAwB,GAAG,sCAAsC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACxF,MAAM,cAAc,GAAG,GACrB,uBAAuB,CAAC,CAAC,CAAC,GAAG,uBAAuB,GAAG,CAAC,CAAC,CAAC,EAC5D,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,6BAA6B,CACpE;oBACE;wBACE,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB;wBAC3E,IAAI,EAAE,MAA0B;wBAChC,oBAAoB,EAAE,eAAe;qBACtC;iBACF,EACD,cAAc,CACf,CAAC;gBACF,QAAQ,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACxC,MAAM,IAAI,GAA4B;oBACpC,OAAO,EAAE,cAAc;oBACvB,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE,wBAAwB,CAAC;oBAC5C,KAAK,EAAE,QAAQ,IAAI,WAAW,CAAC,wBAAwB,CAAC;oBACxD,WAAW;oBACX,OAAO;iBACR,CAAC;gBACF,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,QAAQ,CAAC,wBAAwB,CAAC,GAAG,IAAI,CAAC;gBAC1C,MAAM,UAAU,GAAG,cAAc;oBAC/B,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,wBAAwB,MAAM,CAAC;oBACzD,CAAC,CAAC,GAAG,wBAAwB,MAAM,CAAC;gBACtC,IAAI,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;oBACvE,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAEjC,OAAO;QACL,GAAG;QACH,YAAY;QACZ,IAAI,EAAE,MAA0B;QAChC,QAAQ;QACR,KAAK;KACN,CAAC;AACJ,CAAC,CAAC;AAEF,mGAAmG;AACnG,MAAM,YAAY,GAAG,CACnB,GAAyB,EACzB,SAAkB,EACI,EAAE;IACxB,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,SAAS,GAAG,eAAe,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CACpB,CAAC,WAAW,EAAE,EAAE,CACd,OAAO,WAAW,KAAK,QAAQ,IAAI,OAAO,IAAI,WAAW,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,CAC9E,CAAC;IACnB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG;YACf,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,EAAE;SACV,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnB,OAAO,QAAQ,CAAC,KAAK,CAAC;IACxB,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;AACH,CAAC,CAAC;AAEF,oEAAoE;AACpE,MAAM,sCAAsC,GAAG,CAAC,KAAwB,EAAE,IAAY,EAAU,EAAE;IAChG,IAAI,QAAQ,GAAG,IAAI,CAAC;IACpB,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,QAAQ,GAAG,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,SAAS,IAAI,CAAC,CAAC;YACf,QAAQ,GAAG,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,KAAK,EACpC,QAAgB,EAChB,cAAsB,EACtB,OAAgB,EAChB,EAAE;IACF,MAAM,IAAI,GAAG;WACJ,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;IAC9D,CAAC;IAEH,MAAM,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,GAAY,EAAE,EAAE,CAC7D,GAAG;IACD,CAAC,CAAC,GAAG;SACA,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,yBAAyB;SAClD,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;SACjB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;SACjB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,4BAA4B;SAC1D,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,yBAAyB;SAChD,WAAW,EAAE;IAClB,CAAC,CAAC,SAAS,CAAC"}
|