@nanocollective/get-md 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +36 -0
- package/README.md +205 -0
- package/bin/get-md.js +4 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +91 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.spec.d.ts +2 -0
- package/dist/cli.spec.d.ts.map +1 -0
- package/dist/cli.spec.js +278 -0
- package/dist/cli.spec.js.map +1 -0
- package/dist/config.d.ts +5 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +6 -0
- package/dist/config.js.map +1 -0
- package/dist/extractors/metadata-extractor.d.ts +6 -0
- package/dist/extractors/metadata-extractor.d.ts.map +1 -0
- package/dist/extractors/metadata-extractor.js +131 -0
- package/dist/extractors/metadata-extractor.js.map +1 -0
- package/dist/index.d.ts +44 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +70 -0
- package/dist/index.js.map +1 -0
- package/dist/optimizers/html-cleaner.d.ts +12 -0
- package/dist/optimizers/html-cleaner.d.ts.map +1 -0
- package/dist/optimizers/html-cleaner.js +228 -0
- package/dist/optimizers/html-cleaner.js.map +1 -0
- package/dist/optimizers/llm-formatter.d.ts +8 -0
- package/dist/optimizers/llm-formatter.d.ts.map +1 -0
- package/dist/optimizers/llm-formatter.js +94 -0
- package/dist/optimizers/llm-formatter.js.map +1 -0
- package/dist/optimizers/structure-enhancer.d.ts +8 -0
- package/dist/optimizers/structure-enhancer.d.ts.map +1 -0
- package/dist/optimizers/structure-enhancer.js +92 -0
- package/dist/optimizers/structure-enhancer.js.map +1 -0
- package/dist/parsers/markdown-parser.d.ts +16 -0
- package/dist/parsers/markdown-parser.d.ts.map +1 -0
- package/dist/parsers/markdown-parser.js +369 -0
- package/dist/parsers/markdown-parser.js.map +1 -0
- package/dist/types.d.ts +115 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/url-fetcher.d.ts +10 -0
- package/dist/utils/url-fetcher.d.ts.map +1 -0
- package/dist/utils/url-fetcher.js +54 -0
- package/dist/utils/url-fetcher.js.map +1 -0
- package/dist/utils/validators.d.ts +5 -0
- package/dist/utils/validators.d.ts.map +1 -0
- package/dist/utils/validators.js +23 -0
- package/dist/utils/validators.js.map +1 -0
- package/package.json +104 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# License
|
|
2
|
+
|
|
3
|
+
MIT License with Attribution
|
|
4
|
+
|
|
5
|
+
Copyright (c) 2025 Nano Collective
|
|
6
|
+
|
|
7
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
8
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
9
|
+
in the Software without restriction, including without limitation the rights
|
|
10
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
12
|
+
furnished to do so, subject to the following conditions:
|
|
13
|
+
|
|
14
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
## Attribution Requirement
|
|
17
|
+
|
|
18
|
+
Any use, reproduction, or distribution of the Software, or any derivative works based on the Software, must include clear and prominent attribution to the original creators and contributors of Nanocoder. This attribution must include:
|
|
19
|
+
|
|
20
|
+
1. The name "Nanocoder" and a reference to the original project
|
|
21
|
+
2. A link to the original repository: https://github.com/nano-collective/nanocoder
|
|
22
|
+
3. Credit to "Nano Collective and contributors"
|
|
23
|
+
|
|
24
|
+
This attribution should be included in:
|
|
25
|
+
|
|
26
|
+
- User-facing documentation (README, About pages, etc.)
|
|
27
|
+
- Source code comments or LICENSE files in derivative works
|
|
28
|
+
- Any public presentation, publication, or distribution of the Software
|
|
29
|
+
|
|
30
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
31
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
32
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
33
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
34
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
35
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
36
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# get-md
|
|
2
|
+
|
|
3
|
+
A fast, lightweight HTML to Markdown converter optimized for LLM consumption. Uses proven parsing libraries to deliver clean, well-structured markdown with intelligent content extraction and noise filtering.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Lightning-fast**: Converts HTML to Markdown in <100ms
|
|
8
|
+
- **Intelligent extraction**: Uses Mozilla Readability to extract main content
|
|
9
|
+
- **LLM-optimized**: Consistent formatting perfect for AI consumption
|
|
10
|
+
- **CLI included**: Use from the command line or as a library
|
|
11
|
+
- **TypeScript**: Full type definitions included
|
|
12
|
+
- **Zero downloads**: No models to download, works instantly
|
|
13
|
+
- **Lightweight**: Small package size (~10MB)
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
npm install @nanocollective/get-md
|
|
19
|
+
# or
|
|
20
|
+
pnpm add @nanocollective/get-md
|
|
21
|
+
# or
|
|
22
|
+
yarn add @nanocollective/get-md
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Quick Start
|
|
26
|
+
|
|
27
|
+
### As a Library
|
|
28
|
+
|
|
29
|
+
```typescript
|
|
30
|
+
import { convertToMarkdown } from "@nanocollective/get-md";
|
|
31
|
+
|
|
32
|
+
// From HTML string
|
|
33
|
+
const result = await convertToMarkdown("<h1>Hello</h1><p>World</p>");
|
|
34
|
+
console.log(result.markdown);
|
|
35
|
+
// # Hello
|
|
36
|
+
//
|
|
37
|
+
// World
|
|
38
|
+
|
|
39
|
+
// From URL (auto-detected)
|
|
40
|
+
const result = await convertToMarkdown("https://example.com");
|
|
41
|
+
console.log(result.metadata.title);
|
|
42
|
+
|
|
43
|
+
// From URL with custom timeout and headers
|
|
44
|
+
const result = await convertToMarkdown("https://example.com", {
|
|
45
|
+
timeout: 10000,
|
|
46
|
+
headers: { Authorization: "Bearer token" },
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// Force URL mode if auto-detection fails
|
|
50
|
+
const result = await convertToMarkdown("example.com", { isUrl: true });
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### As a CLI
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
# From stdin
|
|
57
|
+
echo '<h1>Hello</h1>' | getmd
|
|
58
|
+
|
|
59
|
+
# From file
|
|
60
|
+
getmd input.html
|
|
61
|
+
|
|
62
|
+
# From URL
|
|
63
|
+
getmd https://example.com
|
|
64
|
+
|
|
65
|
+
# Save to file
|
|
66
|
+
getmd input.html -o output.md
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## API
|
|
70
|
+
|
|
71
|
+
### `convertToMarkdown(html, options?)`
|
|
72
|
+
|
|
73
|
+
Convert HTML to clean, LLM-optimized Markdown.
|
|
74
|
+
|
|
75
|
+
**Parameters:**
|
|
76
|
+
|
|
77
|
+
- `html` (string): Raw HTML string or URL to fetch
|
|
78
|
+
- `options` (MarkdownOptions): Conversion options
|
|
79
|
+
|
|
80
|
+
**Returns:** `Promise<MarkdownResult>`
|
|
81
|
+
|
|
82
|
+
**Options:**
|
|
83
|
+
|
|
84
|
+
````typescript
|
|
85
|
+
{
|
|
86
|
+
// Content options
|
|
87
|
+
extractContent?: boolean; // Use Readability extraction (default: true)
|
|
88
|
+
includeMeta?: boolean; // Include YAML frontmatter (default: true)
|
|
89
|
+
includeImages?: boolean; // Include images (default: true)
|
|
90
|
+
includeLinks?: boolean; // Include links (default: true)
|
|
91
|
+
includeTables?: boolean; // Include tables (default: true)
|
|
92
|
+
aggressiveCleanup?: boolean; // Remove ads, nav, etc. (default: true)
|
|
93
|
+
maxLength?: number; // Max output length (default: 1000000)
|
|
94
|
+
baseUrl?: string; // Base URL for resolving relative links
|
|
95
|
+
|
|
96
|
+
// URL fetch options (only used when input is a URL)
|
|
97
|
+
isUrl?: boolean; // Force treat input as URL (default: auto-detect)
|
|
98
|
+
timeout?: number; // Request timeout in ms (default: 15000)
|
|
99
|
+
followRedirects?: boolean; // Follow redirects (default: true)
|
|
100
|
+
maxRedirects?: number; // Max redirects to follow (default: 5)
|
|
101
|
+
headers?: Record<string, string>; // Custom HTTP headers
|
|
102
|
+
userAgent?: string; // Custom user agent
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## CLI Usage
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
getmd [input] [options]
|
|
110
|
+
|
|
111
|
+
Options:
|
|
112
|
+
-o, --output <file> Output file (default: stdout)
|
|
113
|
+
--no-extract Disable Readability content extraction
|
|
114
|
+
--no-frontmatter Exclude metadata from YAML frontmatter
|
|
115
|
+
--no-images Remove images from output
|
|
116
|
+
--no-links Remove links from output
|
|
117
|
+
--no-tables Remove tables from output
|
|
118
|
+
--max-length <n> Maximum output length (default: 1000000)
|
|
119
|
+
--base-url <url> Base URL for resolving relative links
|
|
120
|
+
-v, --verbose Verbose output
|
|
121
|
+
-h, --help Display help
|
|
122
|
+
````
|
|
123
|
+
|
|
124
|
+
## Examples
|
|
125
|
+
|
|
126
|
+
### Basic Conversion
|
|
127
|
+
|
|
128
|
+
```typescript
|
|
129
|
+
import { convertToMarkdown } from "@nanocollective/getmd";
|
|
130
|
+
|
|
131
|
+
const html = `
|
|
132
|
+
<article>
|
|
133
|
+
<h1>My Article</h1>
|
|
134
|
+
<p>This is a <strong>test</strong>.</p>
|
|
135
|
+
</article>
|
|
136
|
+
`;
|
|
137
|
+
|
|
138
|
+
const result = await convertToMarkdown(html);
|
|
139
|
+
console.log(result.markdown);
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### With Metadata
|
|
143
|
+
|
|
144
|
+
```typescript
|
|
145
|
+
// Metadata is included by default
|
|
146
|
+
const result = await convertToMarkdown(html);
|
|
147
|
+
console.log(result.markdown);
|
|
148
|
+
// ---
|
|
149
|
+
// title: "My Article"
|
|
150
|
+
// author: "John Doe"
|
|
151
|
+
// readingTime: 5
|
|
152
|
+
// ---
|
|
153
|
+
//
|
|
154
|
+
// # My Article
|
|
155
|
+
// ...
|
|
156
|
+
|
|
157
|
+
// To exclude metadata:
|
|
158
|
+
const resultNoMeta = await convertToMarkdown(html, { includeMeta: false });
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### CLI Examples
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
# Convert HTML file (frontmatter included by default)
|
|
165
|
+
getmd article.html -o article.md
|
|
166
|
+
|
|
167
|
+
# Fetch from URL
|
|
168
|
+
getmd https://blog.example.com/post -o post.md
|
|
169
|
+
|
|
170
|
+
# Remove images and links
|
|
171
|
+
getmd article.html --no-images --no-links -o clean.md
|
|
172
|
+
|
|
173
|
+
# Exclude frontmatter metadata
|
|
174
|
+
getmd article.html --no-frontmatter -o clean.md
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## Why get-md?
|
|
178
|
+
|
|
179
|
+
### For LLMs
|
|
180
|
+
|
|
181
|
+
- **Consistent output**: Deterministic markdown formatting helps LLMs learn patterns
|
|
182
|
+
- **Clean structure**: Proper heading hierarchy, list formatting, and spacing
|
|
183
|
+
- **Noise removal**: Automatically removes ads, navigation, footers, etc.
|
|
184
|
+
- **Fast processing**: <100ms per document enables real-time workflows
|
|
185
|
+
|
|
186
|
+
### vs Other Tools
|
|
187
|
+
|
|
188
|
+
- **Faster than LLM-based extractors**: No model inference overhead
|
|
189
|
+
- **More reliable**: Deterministic output, no hallucinations
|
|
190
|
+
- **Cheaper**: No API costs
|
|
191
|
+
- **Privacy-friendly**: Runs locally, no data sent to third parties
|
|
192
|
+
|
|
193
|
+
## Community
|
|
194
|
+
|
|
195
|
+
We're a small community-led team building local and privacy-first AI solutions under the [Nano Collective](https://nanocollective.org) and would love your help! Whether you're interested in contributing code, documentation, or just being part of our community, there are several ways to get involved.
|
|
196
|
+
|
|
197
|
+
**If you want to contribute to the code:**
|
|
198
|
+
|
|
199
|
+
- Read our detailed [CONTRIBUTING.md](CONTRIBUTING.md) guide for information on development setup, coding standards, and how to submit your changes.
|
|
200
|
+
|
|
201
|
+
**If you want to be part of our community or help with other aspects like design or marketing:**
|
|
202
|
+
|
|
203
|
+
- Join our Discord server to connect with other users, ask questions, share ideas, and get help: [Join our Discord server](https://discord.gg/ktPDV6rekE)
|
|
204
|
+
|
|
205
|
+
- Head to our GitHub issues or discussions to open and join current conversations with others in the community.
|
package/bin/get-md.js
ADDED
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// src/cli.ts
|
|
3
|
+
import { Command } from "commander";
|
|
4
|
+
import fs from "fs/promises";
|
|
5
|
+
import { convertToMarkdown } from "./index.js";
|
|
6
|
+
const program = new Command();
|
|
7
|
+
program
|
|
8
|
+
.name("get-md")
|
|
9
|
+
.description("Convert HTML to LLM-optimized Markdown or extract structured JSON")
|
|
10
|
+
.version("1.0.0");
|
|
11
|
+
// Main conversion command
|
|
12
|
+
program
|
|
13
|
+
.argument("[input]", "HTML file path, URL, or stdin")
|
|
14
|
+
.option("-o, --output <file>", "Output file (default: stdout)")
|
|
15
|
+
.option("--no-extract", "Disable Readability content extraction")
|
|
16
|
+
.option("--no-frontmatter", "Exclude metadata from YAML frontmatter")
|
|
17
|
+
.option("--no-images", "Remove images from output")
|
|
18
|
+
.option("--no-links", "Remove links from output")
|
|
19
|
+
.option("--no-tables", "Remove tables from output")
|
|
20
|
+
.option("--max-length <n>", "Maximum output length", "1000000")
|
|
21
|
+
.option("--base-url <url>", "Base URL for resolving relative links")
|
|
22
|
+
.option("-v, --verbose", "Verbose output")
|
|
23
|
+
.action(async (input, options) => {
|
|
24
|
+
try {
|
|
25
|
+
// Get input HTML
|
|
26
|
+
const html = await getInput(input);
|
|
27
|
+
// Markdown conversion mode
|
|
28
|
+
await handleMarkdownConversion(html, options);
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
console.error("Error:", error.message);
|
|
32
|
+
if (options.verbose) {
|
|
33
|
+
console.error(error.stack);
|
|
34
|
+
}
|
|
35
|
+
process.exit(1);
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
async function getInput(input) {
|
|
39
|
+
// Read from URL
|
|
40
|
+
if (input && input.startsWith("http")) {
|
|
41
|
+
const response = await fetch(input, {
|
|
42
|
+
signal: AbortSignal.timeout(15000),
|
|
43
|
+
});
|
|
44
|
+
if (!response.ok) {
|
|
45
|
+
throw new Error(`Failed to fetch ${input}: ${response.statusText}`);
|
|
46
|
+
}
|
|
47
|
+
return await response.text();
|
|
48
|
+
}
|
|
49
|
+
// Read from file
|
|
50
|
+
if (input && input !== "-") {
|
|
51
|
+
return await fs.readFile(input, "utf-8");
|
|
52
|
+
}
|
|
53
|
+
// Read from stdin
|
|
54
|
+
if (!process.stdin.isTTY) {
|
|
55
|
+
const chunks = [];
|
|
56
|
+
for await (const chunk of process.stdin) {
|
|
57
|
+
chunks.push(Buffer.from(chunk));
|
|
58
|
+
}
|
|
59
|
+
return Buffer.concat(chunks).toString("utf-8");
|
|
60
|
+
}
|
|
61
|
+
throw new Error("No input provided. Provide a file path, URL, or pipe to stdin.");
|
|
62
|
+
}
|
|
63
|
+
async function handleMarkdownConversion(html, options) {
|
|
64
|
+
const conversionOptions = {
|
|
65
|
+
extractContent: options.extract,
|
|
66
|
+
includeMeta: options.frontmatter,
|
|
67
|
+
includeImages: options.images,
|
|
68
|
+
includeLinks: options.links,
|
|
69
|
+
includeTables: options.tables,
|
|
70
|
+
maxLength: parseInt(options.maxLength, 10),
|
|
71
|
+
baseUrl: options.baseUrl,
|
|
72
|
+
};
|
|
73
|
+
const result = await convertToMarkdown(html, conversionOptions);
|
|
74
|
+
// Write output
|
|
75
|
+
if (options.output) {
|
|
76
|
+
await fs.writeFile(options.output, result.markdown, "utf-8");
|
|
77
|
+
if (process.stdout.isTTY) {
|
|
78
|
+
console.error(`✓ Written to ${options.output}`);
|
|
79
|
+
if (options.verbose) {
|
|
80
|
+
console.error(` Input: ${result.stats.inputLength} chars`);
|
|
81
|
+
console.error(` Output: ${result.stats.outputLength} chars`);
|
|
82
|
+
console.error(` Time: ${result.stats.processingTime}ms`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
console.log(result.markdown);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
program.parse();
|
|
91
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,aAAa;AAEb,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAe/C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,QAAQ,CAAC;KACd,WAAW,CACV,mEAAmE,CACpE;KACA,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,0BAA0B;AAC1B,OAAO;KACJ,QAAQ,CAAC,SAAS,EAAE,+BAA+B,CAAC;KACpD,MAAM,CAAC,qBAAqB,EAAE,+BAA+B,CAAC;KAC9D,MAAM,CAAC,cAAc,EAAE,wCAAwC,CAAC;KAChE,MAAM,CAAC,kBAAkB,EAAE,wCAAwC,CAAC;KACpE,MAAM,CAAC,aAAa,EAAE,2BAA2B,CAAC;KAClD,MAAM,CAAC,YAAY,EAAE,0BAA0B,CAAC;KAChD,MAAM,CAAC,aAAa,EAAE,2BAA2B,CAAC;KAClD,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,SAAS,CAAC;KAC9D,MAAM,CAAC,kBAAkB,EAAE,uCAAuC,CAAC;KACnE,MAAM,CAAC,eAAe,EAAE,gBAAgB,CAAC;KACzC,MAAM,CAAC,KAAK,EAAE,KAAyB,EAAE,OAAmB,EAAE,EAAE;IAC/D,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEnC,2BAA2B;QAC3B,MAAM,wBAAwB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAE,KAAe,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,KAAK,UAAU,QAAQ,CAAC,KAAc;IACpC,gBAAgB;IAChB,IAAI,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE;YAClC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED,iBAAiB;IACjB,IAAI,KAAK,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QAC3B,OAAO,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,kBAAkB;IAClB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,IAAY,EACZ,OAAmB;IAEnB,MAAM,iBAAiB,GAAoB;QACzC,cAAc,EAAE,OAAO,CAAC,OAAO;QAC/B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,YAAY,EAAE,OAAO,CAAC,KAAK;QAC3B,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;QAC1C,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAEhE,eAAe;IACf,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7D,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,gBAAgB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAChD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,OAAO,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC;gBAC5D,OAAO,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,KAAK,CAAC,YAAY,QAAQ,CAAC,CAAC;gBAC9D,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.spec.d.ts","sourceRoot":"","sources":["../src/cli.spec.ts"],"names":[],"mappings":""}
|
package/dist/cli.spec.js
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
// src/cli.spec.ts
|
|
2
|
+
import test from "ava";
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import fs from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
// Path to the compiled CLI binary
|
|
7
|
+
const CLI_PATH = path.join(process.cwd(), "bin", "get-md.js");
|
|
8
|
+
// Test fixtures
|
|
9
|
+
const SIMPLE_HTML = `
|
|
10
|
+
<!DOCTYPE html>
|
|
11
|
+
<html>
|
|
12
|
+
<head>
|
|
13
|
+
<title>Test Page</title>
|
|
14
|
+
<meta name="description" content="A test description">
|
|
15
|
+
</head>
|
|
16
|
+
<body>
|
|
17
|
+
<h1>Hello World</h1>
|
|
18
|
+
<p>This is a test paragraph.</p>
|
|
19
|
+
<img src="/test.jpg" alt="Test image">
|
|
20
|
+
<a href="/link">Test link</a>
|
|
21
|
+
<table>
|
|
22
|
+
<tr><td>Cell 1</td><td>Cell 2</td></tr>
|
|
23
|
+
</table>
|
|
24
|
+
</body>
|
|
25
|
+
</html>
|
|
26
|
+
`;
|
|
27
|
+
const COMPLEX_HTML = `
|
|
28
|
+
<!DOCTYPE html>
|
|
29
|
+
<html>
|
|
30
|
+
<head>
|
|
31
|
+
<title>Article Title</title>
|
|
32
|
+
<meta name="author" content="John Doe">
|
|
33
|
+
<meta property="og:site_name" content="Test Site">
|
|
34
|
+
</head>
|
|
35
|
+
<body>
|
|
36
|
+
<nav>Navigation content</nav>
|
|
37
|
+
<article>
|
|
38
|
+
<h1>Main Article</h1>
|
|
39
|
+
<p>This is the main content that should be extracted.</p>
|
|
40
|
+
</article>
|
|
41
|
+
<footer>Footer content</footer>
|
|
42
|
+
</body>
|
|
43
|
+
</html>
|
|
44
|
+
`;
|
|
45
|
+
// Helper to create a temporary file
|
|
46
|
+
async function createTempFile(content) {
|
|
47
|
+
const tmpDir = await fs.mkdtemp(path.join(process.cwd(), "tmp-test-"));
|
|
48
|
+
const filePath = path.join(tmpDir, "test.html");
|
|
49
|
+
await fs.writeFile(filePath, content, "utf-8");
|
|
50
|
+
return filePath;
|
|
51
|
+
}
|
|
52
|
+
// Helper to clean up temp files
|
|
53
|
+
async function cleanupTempFile(filePath) {
|
|
54
|
+
const dir = path.dirname(filePath);
|
|
55
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
56
|
+
}
|
|
57
|
+
// Helper to run CLI command
|
|
58
|
+
async function runCli(args, input) {
|
|
59
|
+
return new Promise((resolve, reject) => {
|
|
60
|
+
const child = spawn("node", [CLI_PATH, ...args], {
|
|
61
|
+
timeout: 5000,
|
|
62
|
+
});
|
|
63
|
+
let stdout = "";
|
|
64
|
+
let stderr = "";
|
|
65
|
+
child.stdout.on("data", (data) => {
|
|
66
|
+
stdout += data.toString();
|
|
67
|
+
});
|
|
68
|
+
child.stderr.on("data", (data) => {
|
|
69
|
+
stderr += data.toString();
|
|
70
|
+
});
|
|
71
|
+
child.on("error", (error) => {
|
|
72
|
+
reject(error);
|
|
73
|
+
});
|
|
74
|
+
child.on("close", (code) => {
|
|
75
|
+
resolve({ stdout, stderr, exitCode: code });
|
|
76
|
+
});
|
|
77
|
+
// Write input to stdin if provided
|
|
78
|
+
if (input !== undefined) {
|
|
79
|
+
child.stdin.write(input);
|
|
80
|
+
child.stdin.end();
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
test("CLI: converts HTML from file to markdown", async (t) => {
|
|
85
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
86
|
+
try {
|
|
87
|
+
const { stdout } = await runCli([inputFile]);
|
|
88
|
+
t.true(stdout.includes("# Hello World"));
|
|
89
|
+
t.true(stdout.includes("This is a test paragraph"));
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
await cleanupTempFile(inputFile);
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
test("CLI: writes output to file with -o flag", async (t) => {
|
|
96
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
97
|
+
const outputFile = path.join(path.dirname(inputFile), "output.md");
|
|
98
|
+
try {
|
|
99
|
+
const { exitCode } = await runCli([inputFile, "-o", outputFile]);
|
|
100
|
+
// Check the command succeeded
|
|
101
|
+
t.is(exitCode, 0);
|
|
102
|
+
// Verify the output file was created with correct content
|
|
103
|
+
const content = await fs.readFile(outputFile, "utf-8");
|
|
104
|
+
t.true(content.includes("# Hello World"));
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
await cleanupTempFile(inputFile);
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
test("CLI: reads from stdin when input is -", async (t) => {
|
|
111
|
+
const { stdout } = await runCli(["-"], SIMPLE_HTML);
|
|
112
|
+
t.true(stdout.includes("# Hello World"));
|
|
113
|
+
t.true(stdout.includes("This is a test paragraph"));
|
|
114
|
+
});
|
|
115
|
+
test("CLI: --no-extract flag disables Readability extraction", async (t) => {
|
|
116
|
+
const inputFile = await createTempFile(COMPLEX_HTML);
|
|
117
|
+
try {
|
|
118
|
+
const { stdout } = await runCli([inputFile, "--no-extract"]);
|
|
119
|
+
// Should include navigation and footer when not extracting
|
|
120
|
+
t.true(stdout.includes("Navigation") || stdout.includes("Footer"));
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
await cleanupTempFile(inputFile);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
test("CLI: --no-frontmatter flag excludes metadata", async (t) => {
|
|
127
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
128
|
+
try {
|
|
129
|
+
const { stdout } = await runCli([inputFile, "--no-frontmatter"]);
|
|
130
|
+
// Should not include YAML frontmatter markers
|
|
131
|
+
const lines = stdout.trim().split("\n");
|
|
132
|
+
// First line should be content, not frontmatter delimiter
|
|
133
|
+
t.false(lines[0] === "---");
|
|
134
|
+
// Should not have title in frontmatter format
|
|
135
|
+
t.false(stdout.startsWith("---\ntitle:"));
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
await cleanupTempFile(inputFile);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
test("CLI: --no-images flag removes images", async (t) => {
|
|
142
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
143
|
+
try {
|
|
144
|
+
const { stdout } = await runCli([inputFile, "--no-images"]);
|
|
145
|
+
// Should not include markdown image syntax
|
|
146
|
+
t.false(stdout.includes("!["));
|
|
147
|
+
t.false(stdout.includes("test.jpg"));
|
|
148
|
+
}
|
|
149
|
+
finally {
|
|
150
|
+
await cleanupTempFile(inputFile);
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
test("CLI: --no-links flag removes links", async (t) => {
|
|
154
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
155
|
+
try {
|
|
156
|
+
const { stdout } = await runCli([inputFile, "--no-links"]);
|
|
157
|
+
// Should not include markdown link syntax (but may include text)
|
|
158
|
+
t.false(stdout.includes("[Test link]"));
|
|
159
|
+
t.false(stdout.includes("](/link)"));
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
await cleanupTempFile(inputFile);
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
test("CLI: --no-tables flag removes tables", async (t) => {
|
|
166
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
167
|
+
try {
|
|
168
|
+
const { stdout } = await runCli([inputFile, "--no-tables"]);
|
|
169
|
+
// Should not include markdown table syntax
|
|
170
|
+
t.false(stdout.includes("|"));
|
|
171
|
+
t.false(stdout.includes("Cell 1") && stdout.includes("Cell 2"));
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
await cleanupTempFile(inputFile);
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
test("CLI: --base-url flag resolves relative URLs", async (t) => {
|
|
178
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
179
|
+
try {
|
|
180
|
+
const { stdout } = await runCli([
|
|
181
|
+
inputFile,
|
|
182
|
+
"--base-url",
|
|
183
|
+
"https://example.com",
|
|
184
|
+
]);
|
|
185
|
+
// Should resolve relative URL to absolute
|
|
186
|
+
t.true(stdout.includes("https://example.com/test.jpg") ||
|
|
187
|
+
stdout.includes("example.com/test.jpg"));
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
await cleanupTempFile(inputFile);
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
test("CLI: --verbose flag shows detailed stats", async (t) => {
|
|
194
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
195
|
+
const outputFile = path.join(path.dirname(inputFile), "output.md");
|
|
196
|
+
try {
|
|
197
|
+
const { exitCode } = await runCli([
|
|
198
|
+
inputFile,
|
|
199
|
+
"-o",
|
|
200
|
+
outputFile,
|
|
201
|
+
"--verbose",
|
|
202
|
+
]);
|
|
203
|
+
// The verbose flag is accepted without error
|
|
204
|
+
t.is(exitCode, 0);
|
|
205
|
+
// Verify the output file was created
|
|
206
|
+
const content = await fs.readFile(outputFile, "utf-8");
|
|
207
|
+
t.true(content.length > 0);
|
|
208
|
+
// Note: In test environment (non-TTY), stats are not printed to stderr
|
|
209
|
+
// The verbose flag only affects TTY output (see cli.ts:109)
|
|
210
|
+
}
|
|
211
|
+
finally {
|
|
212
|
+
await cleanupTempFile(inputFile);
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
test("CLI: --max-length flag limits output length", async (t) => {
|
|
216
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
217
|
+
try {
|
|
218
|
+
const { stdout } = await runCli([inputFile, "--max-length", "50"]);
|
|
219
|
+
// Output should be truncated to around 50 characters
|
|
220
|
+
t.true(stdout.length < 200); // Some margin for frontmatter
|
|
221
|
+
}
|
|
222
|
+
finally {
|
|
223
|
+
await cleanupTempFile(inputFile);
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
test("CLI: handles missing input file gracefully", async (t) => {
|
|
227
|
+
const { stderr } = await runCli(["/nonexistent/file.html"]);
|
|
228
|
+
t.true(stderr.includes("Error:"));
|
|
229
|
+
t.true(stderr.includes("ENOENT") ||
|
|
230
|
+
stderr.includes("no such file") ||
|
|
231
|
+
stderr.includes("cannot find"));
|
|
232
|
+
});
|
|
233
|
+
test("CLI: shows version with --version flag", async (t) => {
|
|
234
|
+
const { stdout } = await runCli(["--version"]);
|
|
235
|
+
t.true(stdout.includes("1.0.0"));
|
|
236
|
+
});
|
|
237
|
+
test("CLI: shows help with --help flag", async (t) => {
|
|
238
|
+
const { stdout } = await runCli(["--help"]);
|
|
239
|
+
t.true(stdout.includes("get-md"));
|
|
240
|
+
t.true(stdout.includes("Convert HTML to LLM-optimized Markdown"));
|
|
241
|
+
t.true(stdout.includes("--output"));
|
|
242
|
+
t.true(stdout.includes("--no-extract"));
|
|
243
|
+
});
|
|
244
|
+
test("CLI: handles verbose error output", async (t) => {
|
|
245
|
+
const { stderr } = await runCli(["/nonexistent/file.html", "--verbose"]);
|
|
246
|
+
t.true(stderr.includes("Error:"));
|
|
247
|
+
// Verbose mode should show stack trace
|
|
248
|
+
t.true(stderr.includes("at ") || stderr.includes("ENOENT"));
|
|
249
|
+
});
|
|
250
|
+
test("CLI: processes multiple options together", async (t) => {
|
|
251
|
+
const inputFile = await createTempFile(SIMPLE_HTML);
|
|
252
|
+
try {
|
|
253
|
+
const { stdout } = await runCli([
|
|
254
|
+
inputFile,
|
|
255
|
+
"--no-images",
|
|
256
|
+
"--no-links",
|
|
257
|
+
"--no-frontmatter",
|
|
258
|
+
"--base-url",
|
|
259
|
+
"https://example.com",
|
|
260
|
+
]);
|
|
261
|
+
// All options should be applied
|
|
262
|
+
const lines = stdout.trim().split("\n");
|
|
263
|
+
t.false(lines[0] === "---"); // No frontmatter at start
|
|
264
|
+
t.false(stdout.includes("); // No links
|
|
266
|
+
t.true(stdout.includes("Hello World")); // Content still present
|
|
267
|
+
}
|
|
268
|
+
finally {
|
|
269
|
+
await cleanupTempFile(inputFile);
|
|
270
|
+
}
|
|
271
|
+
});
|
|
272
|
+
test("CLI: handles stdin with no TTY", async (t) => {
|
|
273
|
+
// Simulate piping HTML via stdin
|
|
274
|
+
const { stdout } = await runCli([], SIMPLE_HTML);
|
|
275
|
+
t.true(stdout.includes("# Hello World"));
|
|
276
|
+
t.true(stdout.includes("This is a test paragraph"));
|
|
277
|
+
});
|
|
278
|
+
//# sourceMappingURL=cli.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.spec.js","sourceRoot":"","sources":["../src/cli.spec.ts"],"names":[],"mappings":"AAAA,kBAAkB;AAElB,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,kCAAkC;AAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;AAE9D,gBAAgB;AAChB,MAAM,WAAW,GAAG;;;;;;;;;;;;;;;;;CAiBnB,CAAC;AAEF,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;CAiBpB,CAAC;AAEF,oCAAoC;AACpC,KAAK,UAAU,cAAc,CAAC,OAAe;IAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAChD,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,gCAAgC;AAChC,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,4BAA4B;AAC5B,KAAK,UAAU,MAAM,CACnB,IAAc,EACd,KAAc;IAEd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE;YAC/C,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,mCAAmC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,IAAI,CAAC,0CAA0C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAE7C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;QACzC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,CAAC;IACtD,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC1D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,WAAW,CAAC,CAAC;IAEnE,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;QAEjE,8BAA8B;QAC9B,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAElB,0DAA0D;QAC1D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;IAC5C,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;IAEpD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACzE,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,YAAY,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;QAE7D,2DAA2D;QAC3D,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrE,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAC;QAEjE,8CAA8C;QAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,0DAA0D;QAC1D,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QAC5B,8CAA8C;QAC9C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5C,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvD,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;QAE5D,2CAA2C;QAC3C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IACvC,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrD,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;QAE3D,iEAAiE;QACjE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IACvC,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvD,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;QAE5D,2CAA2C;QAC3C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC9D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC;YAC9B,SAAS;YACT,YAAY;YACZ,qBAAqB;SACtB,CAAC,CAAC;QAEH,0CAA0C;QAC1C,CAAC,CAAC,IAAI,CACJ,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAAC;YAC7C,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAC1C,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,WAAW,CAAC,CAAC;IAEnE,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC;YAChC,SAAS;YACT,IAAI;YACJ,UAAU;YACV,WAAW;SACZ,CAAC,CAAC;QAEH,6CAA6C;QAC7C,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAElB,qCAAqC;QACrC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE3B,uEAAuE;QACvE,4DAA4D;IAC9D,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC9D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;QAEnE,qDAAqD;QACrD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,8BAA8B;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAE5D,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,IAAI,CACJ,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACvB,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CACjC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACzD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAE/C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACnC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kCAAkC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE5C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,wCAAwC,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mCAAmC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACpD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC,wBAAwB,EAAE,WAAW,CAAC,CAAC,CAAC;IAEzE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClC,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC;YAC9B,SAAS;YACT,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,YAAY;YACZ,qBAAqB;SACtB,CAAC,CAAC;QAEH,gCAAgC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,0BAA0B;QACvD,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY;QAC5C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW;QAC3C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,wBAAwB;IAClE,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gCAAgC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACjD,iCAAiC;IACjC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IAEjD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Default user agent for fetching URLs */
|
|
2
|
+
export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; get-md/1.0; +https://github.com/nano-collective/get-md)";
|
|
3
|
+
/** Default timeout for URL fetching (15 seconds) */
|
|
4
|
+
export declare const DEFAULT_FETCH_TIMEOUT = 15000;
|
|
5
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,2CAA2C;AAC3C,eAAO,MAAM,kBAAkB,qFACqD,CAAC;AAErF,oDAAoD;AACpD,eAAO,MAAM,qBAAqB,QAAQ,CAAC"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// src/config.ts
|
|
2
|
+
/** Default user agent for fetching URLs */
|
|
3
|
+
export const DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; get-md/1.0; +https://github.com/nano-collective/get-md)";
|
|
4
|
+
/** Default timeout for URL fetching (15 seconds) */
|
|
5
|
+
export const DEFAULT_FETCH_TIMEOUT = 15000;
|
|
6
|
+
//# sourceMappingURL=config.js.map
|