llms-generator-cli 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +59 -0
- package/llms-full.txt +1774 -0
- package/llms.txt +235 -0
- package/package.json +16 -0
package/index.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { Command } from 'commander';
|
|
4
|
+
import axios from 'axios';
|
|
5
|
+
import * as cheerio from 'cheerio';
|
|
6
|
+
import TurndownService from 'turndown';
|
|
7
|
+
import fs from 'fs';
|
|
8
|
+
import path from 'path';
|
|
9
|
+
|
|
10
|
+
const program = new Command();
|
|
11
|
+
const turndownService = new TurndownService();
|
|
12
|
+
|
|
13
|
+
program
|
|
14
|
+
.name('llms-gen')
|
|
15
|
+
.description('Generate llms.txt and advanced context from a URL')
|
|
16
|
+
.version('1.0.0')
|
|
17
|
+
.argument('<url>', 'The target website URL')
|
|
18
|
+
.option('-a, --advanced', 'Generate extended llms-full.txt with deep context')
|
|
19
|
+
.action(async (url, options) => {
|
|
20
|
+
try {
|
|
21
|
+
console.log(`đ Fetching and analyzing: ${url}...`);
|
|
22
|
+
|
|
23
|
+
// ā§§. āĻā§ā§āĻŦāĻ¸āĻžāĻāĻā§āĻ° HTML āĻĄā§āĻāĻž āĻĄāĻžāĻāĻ¨āĻ˛ā§āĻĄ āĻāĻ°āĻž
|
|
24
|
+
const { data } = await axios.get(url);
|
|
25
|
+
|
|
26
|
+
// ā§¨. HTML āĻĄā§āĻāĻž āĻ°āĻŋāĻĄ (Parse) āĻāĻ°āĻž
|
|
27
|
+
const $ = cheerio.load(data);
|
|
28
|
+
|
|
29
|
+
// āĻŽā§āĻāĻžāĻĄā§āĻāĻž (āĻāĻžāĻāĻā§āĻ˛ āĻ āĻĄā§āĻ¸āĻā§āĻ°āĻŋāĻĒāĻļāĻ¨) āĻā§āĻāĻā§ āĻŦā§āĻ° āĻāĻ°āĻž
|
|
30
|
+
const title = $('title').text() || 'Documentation';
|
|
31
|
+
const description = $('meta[name="description"]').attr('content') || 'No description available.';
|
|
32
|
+
|
|
33
|
+
// āĻŽā§āĻāĻ¨ āĻāĻ¨ā§āĻā§āĻ¨ā§āĻ āĻ°āĻŋāĻĄ āĻāĻ°āĻž (āĻ
āĻĒā§āĻ°ā§ā§āĻāĻ¨ā§ā§ āĻšā§āĻĄāĻžāĻ°/āĻĢā§āĻāĻžāĻ° āĻŦāĻžāĻĻ āĻĻā§āĻā§āĻžāĻ° āĻāĻ¨ā§āĻ¯)
|
|
34
|
+
const mainContentHtml = $('main').html() || $('body').html();
|
|
35
|
+
|
|
36
|
+
// ā§Š. HTML-āĻā§ āĻā§āĻ˛āĻŋāĻ¨ Markdown-āĻ āĻ°ā§āĻĒāĻžāĻ¨ā§āĻ¤āĻ° āĻāĻ°āĻž
|
|
37
|
+
const markdownContent = turndownService.turndown(mainContentHtml);
|
|
38
|
+
|
|
39
|
+
// ā§Ē. āĻ¸ā§āĻā§āĻ¯āĻžāĻ¨ā§āĻĄāĻžāĻ°ā§āĻĄ llms.txt āĻĢāĻ°āĻŽā§āĻ¯āĻžāĻ āĻ¤ā§āĻ°āĻŋ āĻāĻ°āĻž
|
|
40
|
+
const llmsTxtContent = `# ${title}\n\n> ${description}\n\n## Info\n- URL: ${url}\n\n## Core Content\n${markdownContent.substring(0, 2000)}`;
|
|
41
|
+
|
|
42
|
+
// ā§Ģ. āĻĢāĻžāĻāĻ˛āĻāĻŋ āĻāĻāĻāĻžāĻ°ā§āĻ° āĻāĻžāĻ°ā§āĻ¨ā§āĻ āĻĢā§āĻ˛ā§āĻĄāĻžāĻ°ā§ āĻ¸ā§āĻ āĻāĻ°āĻž
|
|
43
|
+
fs.writeFileSync(path.join(process.cwd(), 'llms.txt'), llmsTxtContent);
|
|
44
|
+
console.log(`â
Successfully generated llms.txt!`);
|
|
45
|
+
|
|
46
|
+
// ā§Ŧ. āĻāĻāĻāĻžāĻ° āĻ¯āĻĻāĻŋ āĻ
ā§āĻ¯āĻžāĻĄāĻāĻžāĻ¨ā§āĻ¸āĻĄ āĻĢā§āĻ˛ā§āĻ¯āĻžāĻ (-a) āĻŦā§āĻ¯āĻŦāĻšāĻžāĻ° āĻāĻ°ā§
|
|
47
|
+
if (options.advanced) {
|
|
48
|
+
console.log(`đ Generating advanced llms-full.txt...`);
|
|
49
|
+
const advancedContent = `# Full Context: ${title}\n\n${markdownContent}`;
|
|
50
|
+
fs.writeFileSync(path.join(process.cwd(), 'llms-full.txt'), advancedContent);
|
|
51
|
+
console.log(`â
Successfully generated llms-full.txt!`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
} catch (error) {
|
|
55
|
+
console.error(`â Error: ${error.message}`);
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
program.parse();
|