agentsite-kit 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -0
- package/README.zh-CN.md +226 -0
- package/bin/agentsite.js +2 -0
- package/dist/chunk-AYLZUDQP.js +281 -0
- package/dist/chunk-AYLZUDQP.js.map +1 -0
- package/dist/chunk-YWR5EH3F.js +339 -0
- package/dist/chunk-YWR5EH3F.js.map +1 -0
- package/dist/chunk-YWUDTSOR.js +360 -0
- package/dist/chunk-YWUDTSOR.js.map +1 -0
- package/dist/generate-V5JMMT4J.js +10 -0
- package/dist/generate-V5JMMT4J.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1238 -0
- package/dist/index.js.map +1 -0
- package/dist/scan-QWNB65DB.js +10 -0
- package/dist/scan-QWNB65DB.js.map +1 -0
- package/package.json +66 -0
- package/templates/agentsite.config.yaml +32 -0
- package/templates/presets/api-docs.yaml +29 -0
- package/templates/presets/blog.yaml +26 -0
- package/templates/presets/community.yaml +31 -0
- package/templates/presets/docs-site.yaml +26 -0
- package/templates/presets/ecommerce.yaml +32 -0
- package/templates/presets/knowledge-base.yaml +28 -0
- package/templates/presets/portfolio.yaml +27 -0
- package/templates/presets/saas.yaml +30 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1238 @@
|
|
|
1
|
+
import {
|
|
2
|
+
classifyPage,
|
|
3
|
+
crawlSite,
|
|
4
|
+
normalizeUrl,
|
|
5
|
+
parseSitemap,
|
|
6
|
+
registerScanCommand,
|
|
7
|
+
urlToId
|
|
8
|
+
} from "./chunk-YWR5EH3F.js";
|
|
9
|
+
import {
|
|
10
|
+
generateAgentIndex,
|
|
11
|
+
generateAgentSitemap,
|
|
12
|
+
generateLlmsTxt,
|
|
13
|
+
generateStructuredExports,
|
|
14
|
+
registerGenerateCommand
|
|
15
|
+
} from "./chunk-AYLZUDQP.js";
|
|
16
|
+
import {
|
|
17
|
+
extractContent,
|
|
18
|
+
loadConfig,
|
|
19
|
+
log,
|
|
20
|
+
sha256,
|
|
21
|
+
spinner,
|
|
22
|
+
toSlug
|
|
23
|
+
} from "./chunk-YWUDTSOR.js";
|
|
24
|
+
|
|
25
|
+
// src/index.ts
|
|
26
|
+
import { Command } from "commander";
|
|
27
|
+
|
|
28
|
+
// src/commands/init.ts
|
|
29
|
+
import { writeFileSync, mkdirSync, existsSync as existsSync2 } from "fs";
|
|
30
|
+
import { createInterface } from "readline";
|
|
31
|
+
import yaml2 from "js-yaml";
|
|
32
|
+
|
|
33
|
+
// src/templates/registry.ts
|
|
34
|
+
import { readFileSync, existsSync } from "fs";
|
|
35
|
+
import { fileURLToPath } from "url";
|
|
36
|
+
import { dirname, resolve } from "path";
|
|
37
|
+
import yaml from "js-yaml";
|
|
38
|
+
var __filename = fileURLToPath(import.meta.url);
|
|
39
|
+
var __dirname = dirname(__filename);
|
|
40
|
+
var PRESETS_DIR = existsSync(resolve(__dirname, "../../templates/presets")) ? resolve(__dirname, "../../templates/presets") : resolve(__dirname, "../templates/presets");
|
|
41
|
+
var BUILTIN_TEMPLATES = ["docs-site", "blog", "saas", "knowledge-base", "ecommerce", "portfolio", "api-docs", "community"];
|
|
42
|
+
function listTemplates() {
|
|
43
|
+
const templates = [];
|
|
44
|
+
for (const id of BUILTIN_TEMPLATES) {
|
|
45
|
+
const preset = loadTemplate(id);
|
|
46
|
+
if (preset) templates.push(preset);
|
|
47
|
+
}
|
|
48
|
+
return templates;
|
|
49
|
+
}
|
|
50
|
+
function loadTemplate(id) {
|
|
51
|
+
const filePath = resolve(PRESETS_DIR, `${id}.yaml`);
|
|
52
|
+
if (!existsSync(filePath)) return null;
|
|
53
|
+
const raw = readFileSync(filePath, "utf-8");
|
|
54
|
+
const parsed = yaml.load(raw);
|
|
55
|
+
return {
|
|
56
|
+
id,
|
|
57
|
+
name: parsed.name ?? id,
|
|
58
|
+
description: parsed.description ?? "",
|
|
59
|
+
config: parsed.config ?? {}
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function applyTemplate(baseConfig, template) {
|
|
63
|
+
return deepMerge(baseConfig, template.config);
|
|
64
|
+
}
|
|
65
|
+
function deepMerge(target, source) {
|
|
66
|
+
const result = { ...target };
|
|
67
|
+
for (const key of Object.keys(source)) {
|
|
68
|
+
const sv = source[key];
|
|
69
|
+
const tv = target[key];
|
|
70
|
+
if (sv && typeof sv === "object" && !Array.isArray(sv) && tv && typeof tv === "object" && !Array.isArray(tv)) {
|
|
71
|
+
result[key] = deepMerge(tv, sv);
|
|
72
|
+
} else {
|
|
73
|
+
result[key] = sv;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return result;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// src/commands/init.ts
|
|
80
|
+
function ask(question) {
|
|
81
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
82
|
+
return new Promise((resolve2) => {
|
|
83
|
+
rl.question(question, (answer) => {
|
|
84
|
+
rl.close();
|
|
85
|
+
resolve2(answer.trim());
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
function registerInitCommand(program2) {
|
|
90
|
+
program2.command("init").description("Initialize AgentSite config for your website").option("-t, --template <name>", "Use an industry template (docs-site, blog, saas, knowledge-base, ecommerce, portfolio, api-docs, community)").option("--list-templates", "List available templates").action(async (opts) => {
|
|
91
|
+
if (opts.listTemplates) {
|
|
92
|
+
const templates = listTemplates();
|
|
93
|
+
if (templates.length === 0) {
|
|
94
|
+
log.warn("No templates found.");
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
log.info("Available templates:");
|
|
98
|
+
for (const t of templates) {
|
|
99
|
+
console.log(` ${t.id.padEnd(20)} ${t.description}`);
|
|
100
|
+
}
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (existsSync2("agentsite.config.yaml")) {
|
|
104
|
+
log.warn("agentsite.config.yaml already exists. Overwrite? (y/N)");
|
|
105
|
+
const confirm = await ask("");
|
|
106
|
+
if (confirm.toLowerCase() !== "y") {
|
|
107
|
+
log.info("Aborted.");
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const url = normalizeUrl(await ask("Site URL: "));
|
|
112
|
+
const name = await ask("Site name: ");
|
|
113
|
+
const description = await ask("Site description: ");
|
|
114
|
+
let config = {
|
|
115
|
+
site: { url, name, description },
|
|
116
|
+
scan: {
|
|
117
|
+
maxPages: 100,
|
|
118
|
+
concurrency: 3,
|
|
119
|
+
delayMs: 200,
|
|
120
|
+
include: ["**"],
|
|
121
|
+
exclude: [],
|
|
122
|
+
respectRobotsTxt: true
|
|
123
|
+
},
|
|
124
|
+
output: {
|
|
125
|
+
dir: ".agentsite",
|
|
126
|
+
formats: ["llms-txt", "agent-sitemap", "agent-index", "structured"]
|
|
127
|
+
},
|
|
128
|
+
server: {
|
|
129
|
+
port: 3141,
|
|
130
|
+
rateLimit: { max: 60, timeWindow: "1 minute" },
|
|
131
|
+
accessLog: true
|
|
132
|
+
},
|
|
133
|
+
access: {
|
|
134
|
+
allowedPages: ["**"],
|
|
135
|
+
blockedPages: [],
|
|
136
|
+
allowedTypes: ["docs", "faq", "blog", "product", "pricing", "about", "contact", "changelog"],
|
|
137
|
+
summaryOnly: false,
|
|
138
|
+
allowSearch: true
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
if (opts.template) {
|
|
142
|
+
const template = loadTemplate(opts.template);
|
|
143
|
+
if (!template) {
|
|
144
|
+
log.error(`Template "${opts.template}" not found. Use --list-templates to see available templates.`);
|
|
145
|
+
process.exit(1);
|
|
146
|
+
}
|
|
147
|
+
config = applyTemplate(config, template);
|
|
148
|
+
log.info(`Applied template: ${template.name}`);
|
|
149
|
+
}
|
|
150
|
+
const yamlContent = yaml2.dump(config, { lineWidth: -1, noRefs: true });
|
|
151
|
+
writeFileSync("agentsite.config.yaml", yamlContent, "utf-8");
|
|
152
|
+
mkdirSync(".agentsite/cache/pages", { recursive: true });
|
|
153
|
+
mkdirSync(".agentsite/data", { recursive: true });
|
|
154
|
+
log.success("Created agentsite.config.yaml");
|
|
155
|
+
log.success("Created .agentsite/ directory");
|
|
156
|
+
log.info("Next: run `agentsite scan` to scan your site.");
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// src/commands/serve.ts
|
|
161
|
+
import { readFileSync as readFileSync3, existsSync as existsSync4 } from "fs";
|
|
162
|
+
|
|
163
|
+
// src/server/app.ts
|
|
164
|
+
import Fastify from "fastify";
|
|
165
|
+
import cors from "@fastify/cors";
|
|
166
|
+
import rateLimit from "@fastify/rate-limit";
|
|
167
|
+
|
|
168
|
+
// src/utils/access-control.ts
|
|
169
|
+
import { minimatch } from "minimatch";
|
|
170
|
+
function isPageAllowed(page, config) {
|
|
171
|
+
const { allowedPages, blockedPages, allowedTypes } = config.access;
|
|
172
|
+
for (const pattern of blockedPages) {
|
|
173
|
+
if (minimatch(page.url, pattern)) return false;
|
|
174
|
+
}
|
|
175
|
+
let matchesAllowed = false;
|
|
176
|
+
for (const pattern of allowedPages) {
|
|
177
|
+
if (minimatch(page.url, pattern)) {
|
|
178
|
+
matchesAllowed = true;
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (!matchesAllowed) return false;
|
|
183
|
+
return allowedTypes.includes(page.type);
|
|
184
|
+
}
|
|
185
|
+
function filterPageContent(page, config) {
|
|
186
|
+
if (config.access.summaryOnly) {
|
|
187
|
+
return {
|
|
188
|
+
url: page.url,
|
|
189
|
+
title: page.title,
|
|
190
|
+
type: page.type,
|
|
191
|
+
summary: page.summary,
|
|
192
|
+
updatedAt: page.updatedAt
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
return page;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// src/web/dashboard.ts
|
|
199
|
+
var dashboardHtml = `<!DOCTYPE html>
|
|
200
|
+
<html>
|
|
201
|
+
<head>
|
|
202
|
+
<meta charset="UTF-8">
|
|
203
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
204
|
+
<title>AgentSite Kit Dashboard</title>
|
|
205
|
+
<style>
|
|
206
|
+
*{margin:0;padding:0;box-sizing:border-box}
|
|
207
|
+
body{font-family:system-ui,sans-serif;background:#f5f5f5;display:flex;min-height:100vh}
|
|
208
|
+
.sidebar{width:220px;background:#1a1a2e;color:#fff;padding:20px 0;position:fixed;height:100vh;overflow-y:auto}
|
|
209
|
+
.sidebar h1{font-size:16px;padding:0 20px 20px;border-bottom:1px solid #333}
|
|
210
|
+
.sidebar nav{padding:10px 0}
|
|
211
|
+
.sidebar a{display:block;padding:10px 20px;color:#aaa;text-decoration:none;font-size:14px;transition:all .2s}
|
|
212
|
+
.sidebar a:hover,.sidebar a.active{background:#16213e;color:#fff}
|
|
213
|
+
.sidebar a .icon{margin-right:8px}
|
|
214
|
+
.main{margin-left:220px;flex:1;padding:20px;min-height:100vh}
|
|
215
|
+
.header{background:#fff;padding:20px;border-radius:8px;margin-bottom:20px;box-shadow:0 2px 4px rgba(0,0,0,0.1)}
|
|
216
|
+
h2{font-size:20px;margin-bottom:5px}
|
|
217
|
+
.subtitle{color:#666;font-size:14px}
|
|
218
|
+
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(200px,1fr));gap:16px;margin-bottom:20px}
|
|
219
|
+
.card{background:#fff;padding:20px;border-radius:8px;box-shadow:0 2px 4px rgba(0,0,0,0.1)}
|
|
220
|
+
.card h3{font-size:14px;margin-bottom:8px;color:#666}
|
|
221
|
+
.stat{font-size:28px;font-weight:bold;color:#0066cc}
|
|
222
|
+
.table-wrap{background:#fff;border-radius:8px;overflow:hidden;box-shadow:0 2px 4px rgba(0,0,0,0.1);margin-bottom:20px}
|
|
223
|
+
table{width:100%;border-collapse:collapse}
|
|
224
|
+
th,td{padding:10px 12px;text-align:left;border-bottom:1px solid #eee;font-size:13px}
|
|
225
|
+
th{background:#f9f9f9;font-weight:600}
|
|
226
|
+
.badge{display:inline-block;padding:3px 8px;border-radius:4px;font-size:11px;font-weight:500}
|
|
227
|
+
.badge-docs{background:#e3f2fd;color:#1976d2}
|
|
228
|
+
.badge-faq{background:#f3e5f5;color:#7b1fa2}
|
|
229
|
+
.badge-blog{background:#e8f5e9;color:#388e3c}
|
|
230
|
+
.badge-product{background:#fff3e0;color:#f57c00}
|
|
231
|
+
.badge-pricing{background:#fce4ec;color:#c2185b}
|
|
232
|
+
.badge-changelog{background:#e0f2f1;color:#00796b}
|
|
233
|
+
.badge-homepage{background:#f5f5f5;color:#616161}
|
|
234
|
+
.badge-about{background:#ede7f6;color:#512da8}
|
|
235
|
+
.badge-contact{background:#e8eaf6;color:#283593}
|
|
236
|
+
.badge-unknown{background:#eceff1;color:#546e7a}
|
|
237
|
+
.toolbar{display:flex;gap:10px;margin-bottom:16px;flex-wrap:wrap}
|
|
238
|
+
.toolbar input,.toolbar select{padding:8px 12px;border:1px solid #ddd;border-radius:6px;font-size:13px}
|
|
239
|
+
.toolbar input{flex:1;min-width:200px}
|
|
240
|
+
.pre-wrap{background:#f8f8f8;border:1px solid #eee;border-radius:6px;padding:16px;font-family:monospace;font-size:12px;white-space:pre-wrap;word-break:break-all;max-height:500px;overflow-y:auto;margin-bottom:16px}
|
|
241
|
+
.file-card{background:#fff;border-radius:8px;box-shadow:0 2px 4px rgba(0,0,0,0.1);margin-bottom:16px;overflow:hidden}
|
|
242
|
+
.file-card .file-header{padding:12px 16px;background:#f9f9f9;font-weight:600;font-size:13px;display:flex;justify-content:space-between;cursor:pointer}
|
|
243
|
+
.file-card .file-body{padding:16px;display:none}
|
|
244
|
+
.file-card.open .file-body{display:block}
|
|
245
|
+
.log-entry{display:grid;grid-template-columns:180px 60px 1fr 60px 80px;gap:8px;padding:8px 12px;border-bottom:1px solid #f0f0f0;font-size:12px;align-items:center}
|
|
246
|
+
.log-entry:hover{background:#f9f9f9}
|
|
247
|
+
.status-2xx{color:#388e3c}.status-4xx{color:#f57c00}.status-5xx{color:#c62828}
|
|
248
|
+
.page-view{display:none}
|
|
249
|
+
.page-view.active{display:block}
|
|
250
|
+
.tab-bar{display:flex;gap:4px;margin-bottom:16px;border-bottom:2px solid #eee;padding-bottom:-2px}
|
|
251
|
+
.tab{padding:8px 16px;cursor:pointer;font-size:13px;color:#666;border-bottom:2px solid transparent;margin-bottom:-2px}
|
|
252
|
+
.tab.active{color:#0066cc;border-bottom-color:#0066cc}
|
|
253
|
+
.site-switcher{padding:10px 20px;margin-top:10px}
|
|
254
|
+
.site-switcher select{width:100%;padding:6px 8px;border-radius:4px;border:1px solid #444;background:#16213e;color:#fff;font-size:12px}
|
|
255
|
+
</style>
|
|
256
|
+
</head>
|
|
257
|
+
<body>
|
|
258
|
+
<div class="sidebar">
|
|
259
|
+
<h1>AgentSite Kit</h1>
|
|
260
|
+
<div class="site-switcher" id="siteSwitcher" style="display:none">
|
|
261
|
+
<select id="siteSelect"></select>
|
|
262
|
+
</div>
|
|
263
|
+
<nav>
|
|
264
|
+
<a href="#/" class="active"><span class="icon">■</span> Overview</a>
|
|
265
|
+
<a href="#/pages"><span class="icon">☰</span> Pages</a>
|
|
266
|
+
<a href="#/files"><span class="icon">📄</span> Files</a>
|
|
267
|
+
<a href="#/config"><span class="icon">⚙</span> Config</a>
|
|
268
|
+
<a href="#/logs"><span class="icon">📋</span> Access Logs</a>
|
|
269
|
+
<a href="#/sites"><span class="icon">🌐</span> Sites</a>
|
|
270
|
+
<a href="#/export"><span class="icon">▶</span> Operations</a>
|
|
271
|
+
</nav>
|
|
272
|
+
</div>
|
|
273
|
+
<div class="main">
|
|
274
|
+
|
|
275
|
+
<!-- Overview Page -->
|
|
276
|
+
<div class="page-view active" id="page-overview">
|
|
277
|
+
<div class="header"><h2>Overview</h2><p class="subtitle">Site health at a glance</p></div>
|
|
278
|
+
<div class="grid" id="statsGrid"></div>
|
|
279
|
+
<div class="table-wrap">
|
|
280
|
+
<table><thead><tr><th>Title</th><th>Type</th><th>URL</th><th>Words</th></tr></thead>
|
|
281
|
+
<tbody id="overviewTable"></tbody></table>
|
|
282
|
+
</div>
|
|
283
|
+
</div>
|
|
284
|
+
|
|
285
|
+
<!-- Pages Page -->
|
|
286
|
+
<div class="page-view" id="page-pages">
|
|
287
|
+
<div class="header"><h2>Pages</h2><p class="subtitle">All scanned pages</p></div>
|
|
288
|
+
<div class="toolbar">
|
|
289
|
+
<input type="text" id="pageSearch" placeholder="Search pages...">
|
|
290
|
+
<select id="typeFilter"><option value="">All types</option></select>
|
|
291
|
+
</div>
|
|
292
|
+
<div class="table-wrap">
|
|
293
|
+
<table><thead><tr><th>Title</th><th>Type</th><th>URL</th><th>Words</th><th>Tags</th></tr></thead>
|
|
294
|
+
<tbody id="pagesTable"></tbody></table>
|
|
295
|
+
</div>
|
|
296
|
+
</div>
|
|
297
|
+
|
|
298
|
+
<!-- Files Page -->
|
|
299
|
+
<div class="page-view" id="page-files">
|
|
300
|
+
<div class="header"><h2>Generated Files</h2><p class="subtitle">Preview output files</p></div>
|
|
301
|
+
<div id="filesList"></div>
|
|
302
|
+
</div>
|
|
303
|
+
|
|
304
|
+
<!-- Config Page -->
|
|
305
|
+
<div class="page-view" id="page-config">
|
|
306
|
+
<div class="header"><h2>Configuration</h2><p class="subtitle">Current AgentSite config (API keys hidden)</p></div>
|
|
307
|
+
<div class="pre-wrap" id="configContent">Loading...</div>
|
|
308
|
+
</div>
|
|
309
|
+
|
|
310
|
+
<!-- Logs Page -->
|
|
311
|
+
<div class="page-view" id="page-logs">
|
|
312
|
+
<div class="header"><h2>Access Logs</h2><p class="subtitle">Recent API requests</p></div>
|
|
313
|
+
<div class="grid" id="logStats"></div>
|
|
314
|
+
<div class="table-wrap" style="margin-top:16px">
|
|
315
|
+
<div class="log-entry" style="font-weight:600;background:#f9f9f9">
|
|
316
|
+
<span>Timestamp</span><span>Method</span><span>Path</span><span>Status</span><span>Time</span>
|
|
317
|
+
</div>
|
|
318
|
+
<div id="logEntries"></div>
|
|
319
|
+
</div>
|
|
320
|
+
</div>
|
|
321
|
+
|
|
322
|
+
<!-- Sites Page -->
|
|
323
|
+
<div class="page-view" id="page-sites">
|
|
324
|
+
<div class="header"><h2>Sites</h2><p class="subtitle">Multi-site management</p></div>
|
|
325
|
+
<div id="sitesContent">
|
|
326
|
+
<div class="card"><p>Configure multiple sites in your <code>agentsite.config.yaml</code> using the <code>sites</code> array.</p></div>
|
|
327
|
+
</div>
|
|
328
|
+
</div>
|
|
329
|
+
|
|
330
|
+
<!-- Export/Operations Page -->
|
|
331
|
+
<div class="page-view" id="page-export">
|
|
332
|
+
<div class="header"><h2>Operations</h2><p class="subtitle">Trigger scan and generation tasks</p></div>
|
|
333
|
+
<div class="grid">
|
|
334
|
+
<div class="card">
|
|
335
|
+
<h3>Re-scan Site</h3>
|
|
336
|
+
<p style="font-size:13px;color:#666;margin-bottom:12px">Crawl the site again and update scan results.</p>
|
|
337
|
+
<button id="btnRescan" style="padding:8px 20px;background:#0066cc;color:#fff;border:none;border-radius:6px;cursor:pointer;font-size:13px">Start Rescan</button>
|
|
338
|
+
<div id="rescanStatus" style="margin-top:8px;font-size:12px;color:#666"></div>
|
|
339
|
+
</div>
|
|
340
|
+
<div class="card">
|
|
341
|
+
<h3>Regenerate Files</h3>
|
|
342
|
+
<p style="font-size:13px;color:#666;margin-bottom:12px">Regenerate all output files from scan results.</p>
|
|
343
|
+
<button id="btnRegenerate" style="padding:8px 20px;background:#388e3c;color:#fff;border:none;border-radius:6px;cursor:pointer;font-size:13px">Regenerate</button>
|
|
344
|
+
<div id="regenStatus" style="margin-top:8px;font-size:12px;color:#666"></div>
|
|
345
|
+
</div>
|
|
346
|
+
</div>
|
|
347
|
+
</div>
|
|
348
|
+
|
|
349
|
+
</div>
|
|
350
|
+
<script>
|
|
351
|
+
// Router
|
|
352
|
+
var allPages=[];
|
|
353
|
+
function navigate(){
|
|
354
|
+
var hash=location.hash||'#/';
|
|
355
|
+
var route=hash.replace('#','');
|
|
356
|
+
document.querySelectorAll('.page-view').forEach(function(el){el.classList.remove('active')});
|
|
357
|
+
document.querySelectorAll('.sidebar a').forEach(function(el){el.classList.remove('active')});
|
|
358
|
+
var pageMap={'/':"page-overview",'/pages':"page-pages",'/files':"page-files",'/config':"page-config",'/logs':"page-logs",'/sites':"page-sites",'/export':"page-export"};
|
|
359
|
+
var pageId=pageMap[route]||'page-overview';
|
|
360
|
+
var el=document.getElementById(pageId);
|
|
361
|
+
if(el)el.classList.add('active');
|
|
362
|
+
var link=document.querySelector('.sidebar a[href="'+hash+'"]');
|
|
363
|
+
if(link)link.classList.add('active');
|
|
364
|
+
if(route==='/logs')loadLogs();
|
|
365
|
+
if(route==='/files')loadFiles();
|
|
366
|
+
if(route==='/config')loadConfig();
|
|
367
|
+
}
|
|
368
|
+
window.addEventListener('hashchange',navigate);
|
|
369
|
+
|
|
370
|
+
// Load overview + stats
|
|
371
|
+
function loadOverview(){
|
|
372
|
+
fetch('/api/stats').then(function(r){return r.json()}).then(function(s){
|
|
373
|
+
var g=document.getElementById('statsGrid');
|
|
374
|
+
g.innerHTML='<div class="card"><h3>Total Pages</h3><div class="stat">'+s.totalPages+'</div></div>'
|
|
375
|
+
+'<div class="card"><h3>Total Words</h3><div class="stat">'+s.totalWords.toLocaleString()+'</div></div>'
|
|
376
|
+
+'<div class="card"><h3>Docs</h3><div class="stat">'+(s.pageTypes.docs||0)+'</div></div>'
|
|
377
|
+
+'<div class="card"><h3>FAQ</h3><div class="stat">'+(s.pageTypes.faq||0)+'</div></div>'
|
|
378
|
+
+'<div class="card"><h3>Blog</h3><div class="stat">'+(s.pageTypes.blog||0)+'</div></div>'
|
|
379
|
+
+'<div class="card"><h3>Products</h3><div class="stat">'+(s.pageTypes.product||0)+'</div></div>';
|
|
380
|
+
});
|
|
381
|
+
fetch('/api/pages-data').then(function(r){return r.json()}).then(function(d){
|
|
382
|
+
allPages=d.pages||[];
|
|
383
|
+
renderOverviewTable(allPages.slice(0,20));
|
|
384
|
+
renderPagesTable(allPages);
|
|
385
|
+
populateTypeFilter(allPages);
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function renderOverviewTable(pages){
|
|
390
|
+
var tb=document.getElementById('overviewTable');
|
|
391
|
+
tb.innerHTML=pages.map(function(p){
|
|
392
|
+
return '<tr><td>'+esc(p.title)+'</td><td><span class="badge badge-'+p.type+'">'+p.type+'</span></td><td style="font-size:12px;color:#666">'+esc(p.url)+'</td><td>'+p.wordCount+'</td></tr>';
|
|
393
|
+
}).join('');
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// Pages
|
|
397
|
+
function renderPagesTable(pages){
|
|
398
|
+
var tb=document.getElementById('pagesTable');
|
|
399
|
+
tb.innerHTML=pages.map(function(p){
|
|
400
|
+
var tags=(p.tags||[]).join(', ');
|
|
401
|
+
return '<tr><td>'+esc(p.title)+'</td><td><span class="badge badge-'+p.type+'">'+p.type+'</span></td><td style="font-size:12px;color:#666">'+esc(p.url)+'</td><td>'+p.wordCount+'</td><td style="font-size:11px;color:#888">'+esc(tags)+'</td></tr>';
|
|
402
|
+
}).join('');
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function populateTypeFilter(pages){
|
|
406
|
+
var types={};pages.forEach(function(p){types[p.type]=1});
|
|
407
|
+
var sel=document.getElementById('typeFilter');
|
|
408
|
+
Object.keys(types).sort().forEach(function(t){
|
|
409
|
+
var o=document.createElement('option');o.value=t;o.textContent=t;sel.appendChild(o);
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
document.getElementById('pageSearch').addEventListener('input',filterPages);
|
|
414
|
+
document.getElementById('typeFilter').addEventListener('change',filterPages);
|
|
415
|
+
function filterPages(){
|
|
416
|
+
var q=document.getElementById('pageSearch').value.toLowerCase();
|
|
417
|
+
var t=document.getElementById('typeFilter').value;
|
|
418
|
+
var filtered=allPages.filter(function(p){
|
|
419
|
+
if(t&&p.type!==t)return false;
|
|
420
|
+
if(q&&p.title.toLowerCase().indexOf(q)===-1&&p.url.toLowerCase().indexOf(q)===-1)return false;
|
|
421
|
+
return true;
|
|
422
|
+
});
|
|
423
|
+
renderPagesTable(filtered);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
// Files
|
|
427
|
+
function loadFiles(){
|
|
428
|
+
fetch('/api/files').then(function(r){return r.json()}).then(function(d){
|
|
429
|
+
var container=document.getElementById('filesList');
|
|
430
|
+
container.innerHTML=d.files.map(function(f){
|
|
431
|
+
var preview=f.content?f.content.substring(0,2000):'';
|
|
432
|
+
return '<div class="file-card"><div class="file-header" onclick="this.parentElement.classList.toggle(\\'open\\')"><span>'+esc(f.name)+'</span><span style="color:#888">'+formatSize(f.size)+'</span></div><div class="file-body"><pre class="pre-wrap">'+esc(preview)+(f.size>2000?'\\n...truncated':'')+'</pre></div></div>';
|
|
433
|
+
}).join('');
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// Config
|
|
438
|
+
function loadConfig(){
|
|
439
|
+
fetch('/api/config').then(function(r){return r.json()}).then(function(c){
|
|
440
|
+
document.getElementById('configContent').textContent=JSON.stringify(c,null,2);
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// Logs
|
|
445
|
+
function loadLogs(){
|
|
446
|
+
fetch('/api/access-log?limit=200').then(function(r){return r.json()}).then(function(d){
|
|
447
|
+
var entries=d.entries||[];
|
|
448
|
+
// Stats
|
|
449
|
+
var total=entries.length;
|
|
450
|
+
var ok=entries.filter(function(e){return e.statusCode<400}).length;
|
|
451
|
+
var avgTime=total?Math.round(entries.reduce(function(s,e){return s+e.responseTimeMs},0)/total):0;
|
|
452
|
+
document.getElementById('logStats').innerHTML=
|
|
453
|
+
'<div class="card"><h3>Recent Requests</h3><div class="stat">'+total+'</div></div>'
|
|
454
|
+
+'<div class="card"><h3>Success Rate</h3><div class="stat">'+(total?Math.round(ok/total*100):0)+'%</div></div>'
|
|
455
|
+
+'<div class="card"><h3>Avg Response</h3><div class="stat">'+avgTime+'ms</div></div>';
|
|
456
|
+
// Entries
|
|
457
|
+
var container=document.getElementById('logEntries');
|
|
458
|
+
container.innerHTML=entries.slice(0,100).map(function(e){
|
|
459
|
+
var cls=e.statusCode<400?'status-2xx':e.statusCode<500?'status-4xx':'status-5xx';
|
|
460
|
+
return '<div class="log-entry"><span>'+new Date(e.timestamp).toLocaleString()+'</span><span><b>'+e.method+'</b></span><span style="color:#666">'+esc(e.path+(e.query?'?'+e.query:''))+'</span><span class="'+cls+'">'+e.statusCode+'</span><span>'+e.responseTimeMs+'ms</span></div>';
|
|
461
|
+
}).join('');
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// Operations
|
|
466
|
+
document.getElementById('btnRescan').addEventListener('click',function(){
|
|
467
|
+
var btn=this;btn.disabled=true;btn.textContent='Scanning...';
|
|
468
|
+
document.getElementById('rescanStatus').textContent='Running...';
|
|
469
|
+
fetch('/api/rescan',{method:'POST'}).then(function(r){return r.json()}).then(function(d){
|
|
470
|
+
document.getElementById('rescanStatus').textContent=d.message||'Done';
|
|
471
|
+
btn.disabled=false;btn.textContent='Start Rescan';
|
|
472
|
+
loadOverview();
|
|
473
|
+
}).catch(function(e){
|
|
474
|
+
document.getElementById('rescanStatus').textContent='Error: '+e.message;
|
|
475
|
+
btn.disabled=false;btn.textContent='Start Rescan';
|
|
476
|
+
});
|
|
477
|
+
});
|
|
478
|
+
document.getElementById('btnRegenerate').addEventListener('click',function(){
|
|
479
|
+
var btn=this;btn.disabled=true;btn.textContent='Generating...';
|
|
480
|
+
document.getElementById('regenStatus').textContent='Running...';
|
|
481
|
+
fetch('/api/regenerate',{method:'POST'}).then(function(r){return r.json()}).then(function(d){
|
|
482
|
+
document.getElementById('regenStatus').textContent=d.message||'Done';
|
|
483
|
+
btn.disabled=false;btn.textContent='Regenerate';
|
|
484
|
+
}).catch(function(e){
|
|
485
|
+
document.getElementById('regenStatus').textContent='Error: '+e.message;
|
|
486
|
+
btn.disabled=false;btn.textContent='Regenerate';
|
|
487
|
+
});
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
// Utils
|
|
491
|
+
function esc(s){if(!s)return '';return s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"')}
|
|
492
|
+
function formatSize(b){if(b<1024)return b+'B';if(b<1048576)return (b/1024).toFixed(1)+'KB';return (b/1048576).toFixed(1)+'MB'}
|
|
493
|
+
|
|
494
|
+
// Init
|
|
495
|
+
loadOverview();
|
|
496
|
+
navigate();
|
|
497
|
+
</script>
|
|
498
|
+
</body>
|
|
499
|
+
</html>`;
|
|
500
|
+
|
|
501
|
+
// src/server/access-log.ts
|
|
502
|
+
import { appendFileSync, readFileSync as readFileSync2, existsSync as existsSync3 } from "fs";
|
|
503
|
+
var AccessLogger = class {
|
|
504
|
+
logPath;
|
|
505
|
+
constructor(outDir) {
|
|
506
|
+
this.logPath = `${outDir}/access-log.jsonl`;
|
|
507
|
+
}
|
|
508
|
+
log(entry) {
|
|
509
|
+
appendFileSync(this.logPath, JSON.stringify(entry) + "\n", "utf-8");
|
|
510
|
+
}
|
|
511
|
+
getRecent(limit = 100) {
|
|
512
|
+
if (!existsSync3(this.logPath)) return [];
|
|
513
|
+
const lines = readFileSync2(this.logPath, "utf-8").split("\n").filter((l) => l.trim().length > 0);
|
|
514
|
+
return lines.slice(-limit).reverse().map((l) => JSON.parse(l));
|
|
515
|
+
}
|
|
516
|
+
register(app) {
|
|
517
|
+
app.addHook("onResponse", async (request, reply) => {
|
|
518
|
+
const entry = {
|
|
519
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
520
|
+
method: request.method,
|
|
521
|
+
path: request.url.split("?")[0],
|
|
522
|
+
query: request.url.includes("?") ? request.url.split("?")[1] : "",
|
|
523
|
+
ip: request.ip,
|
|
524
|
+
userAgent: request.headers["user-agent"] ?? "",
|
|
525
|
+
statusCode: reply.statusCode,
|
|
526
|
+
responseTimeMs: Math.round(reply.elapsedTime)
|
|
527
|
+
};
|
|
528
|
+
this.log(entry);
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
};
|
|
532
|
+
|
|
533
|
+
// src/server/routes/search.ts
|
|
534
|
+
function registerSearchRoute(app, data, searchIndex) {
|
|
535
|
+
app.get(
|
|
536
|
+
"/api/search",
|
|
537
|
+
async (req) => {
|
|
538
|
+
const q = (req.query.q ?? "").toLowerCase().trim();
|
|
539
|
+
const page = Math.max(1, parseInt(req.query.page ?? "1", 10));
|
|
540
|
+
const limit = Math.min(50, Math.max(1, parseInt(req.query.limit ?? "10", 10)));
|
|
541
|
+
if (!q) return { ok: true, data: [], total: 0, page };
|
|
542
|
+
const words = q.split(/\W+/).filter((w) => w.length > 2);
|
|
543
|
+
const scoreMap = /* @__PURE__ */ new Map();
|
|
544
|
+
for (const word of words) {
|
|
545
|
+
const matches = searchIndex.get(word);
|
|
546
|
+
if (matches) {
|
|
547
|
+
for (const idx of matches) {
|
|
548
|
+
scoreMap.set(idx, (scoreMap.get(idx) ?? 0) + 1);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
const sorted = [...scoreMap.entries()].sort((a, b) => b[1] - a[1]);
|
|
553
|
+
const total = sorted.length;
|
|
554
|
+
const start = (page - 1) * limit;
|
|
555
|
+
const results = sorted.slice(start, start + limit).map(([idx]) => data.pages[idx]);
|
|
556
|
+
return { ok: true, data: results, total, page };
|
|
557
|
+
}
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// src/server/routes/pages.ts
|
|
562
|
+
function registerPagesRoute(app, data, config) {
|
|
563
|
+
app.get("/api/pages/:id", async (req, reply) => {
|
|
564
|
+
const page = data.pages.find((p) => urlToId(p.url) === req.params.id);
|
|
565
|
+
if (!page) {
|
|
566
|
+
reply.code(404);
|
|
567
|
+
return { ok: false, error: "Page not found" };
|
|
568
|
+
}
|
|
569
|
+
return { ok: true, data: filterPageContent(page, config) };
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// src/server/routes/faq.ts
|
|
574
|
+
function registerFaqRoute(app, data) {
|
|
575
|
+
app.get("/api/faq", async (req) => {
|
|
576
|
+
let items = data.faq;
|
|
577
|
+
const category = req.query.category;
|
|
578
|
+
if (category) {
|
|
579
|
+
items = items.filter((f) => f.category?.toLowerCase() === category.toLowerCase());
|
|
580
|
+
}
|
|
581
|
+
return { ok: true, data: items, total: items.length };
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// src/server/routes/products.ts
|
|
586
|
+
function registerProductsRoute(app, data) {
|
|
587
|
+
app.get("/api/products", async () => {
|
|
588
|
+
return { ok: true, data: data.products, total: data.products.length };
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// src/server/routes/docs.ts
|
|
593
|
+
function registerDocsRoute(app, data) {
|
|
594
|
+
app.get("/api/docs", async (req) => {
|
|
595
|
+
let items = data.docs;
|
|
596
|
+
const section = req.query.section;
|
|
597
|
+
if (section) {
|
|
598
|
+
items = items.filter((d) => d.section?.toLowerCase() === section.toLowerCase());
|
|
599
|
+
}
|
|
600
|
+
return { ok: true, data: items, total: items.length };
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// src/server/routes/articles.ts
|
|
605
|
+
function registerArticlesRoute(app, data) {
|
|
606
|
+
app.get(
|
|
607
|
+
"/api/articles",
|
|
608
|
+
async (req) => {
|
|
609
|
+
let items = data.articles;
|
|
610
|
+
const tag = req.query.tag;
|
|
611
|
+
if (tag) {
|
|
612
|
+
items = items.filter((a) => a.tags?.some((t) => t.toLowerCase() === tag.toLowerCase()));
|
|
613
|
+
}
|
|
614
|
+
const page = Math.max(1, parseInt(req.query.page ?? "1", 10));
|
|
615
|
+
const limit = Math.min(50, Math.max(1, parseInt(req.query.limit ?? "10", 10)));
|
|
616
|
+
const total = items.length;
|
|
617
|
+
const start = (page - 1) * limit;
|
|
618
|
+
const paged = items.slice(start, start + limit);
|
|
619
|
+
return { ok: true, data: paged, total, page };
|
|
620
|
+
}
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// src/server/routes/pricing.ts
|
|
625
|
+
function registerPricingRoute(app, data) {
|
|
626
|
+
app.get("/api/pricing", async () => {
|
|
627
|
+
return { ok: true, data: data.pricing };
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// src/server/routes/changelog.ts
|
|
632
|
+
function registerChangelogRoute(app, data) {
|
|
633
|
+
app.get("/api/changelog", async () => {
|
|
634
|
+
return { ok: true, data: data.changelog };
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// src/server/app.ts
|
|
639
|
+
async function createApp(config, data) {
|
|
640
|
+
const app = Fastify({ logger: false });
|
|
641
|
+
await app.register(cors, { origin: true });
|
|
642
|
+
await app.register(rateLimit, {
|
|
643
|
+
max: config.server.rateLimit.max,
|
|
644
|
+
timeWindow: config.server.rateLimit.timeWindow
|
|
645
|
+
});
|
|
646
|
+
let accessLogger;
|
|
647
|
+
if (config.server.accessLog) {
|
|
648
|
+
accessLogger = new AccessLogger(config.output.dir);
|
|
649
|
+
accessLogger.register(app);
|
|
650
|
+
}
|
|
651
|
+
const filteredPages = data.pages.filter((page) => isPageAllowed(page, config));
|
|
652
|
+
const filteredData = { ...data, pages: filteredPages };
|
|
653
|
+
const searchIndex = buildSearchIndex(filteredData);
|
|
654
|
+
app.get("/api/health", async () => ({ ok: true }));
|
|
655
|
+
app.get("/api/access-log", async (req) => {
|
|
656
|
+
if (!accessLogger) return { entries: [], message: "Access logging is disabled" };
|
|
657
|
+
const query = req.query;
|
|
658
|
+
const limit = parseInt(query.limit, 10) || 100;
|
|
659
|
+
return { entries: accessLogger.getRecent(limit) };
|
|
660
|
+
});
|
|
661
|
+
app.get("/api/stats", async () => {
|
|
662
|
+
const typeCounts = {};
|
|
663
|
+
let totalWords = 0;
|
|
664
|
+
for (const p of filteredData.pages) {
|
|
665
|
+
typeCounts[p.type] = (typeCounts[p.type] ?? 0) + 1;
|
|
666
|
+
totalWords += p.wordCount;
|
|
667
|
+
}
|
|
668
|
+
return {
|
|
669
|
+
totalPages: filteredData.pages.length,
|
|
670
|
+
totalWords,
|
|
671
|
+
pageTypes: typeCounts,
|
|
672
|
+
dataFiles: {
|
|
673
|
+
docs: filteredData.docs.length,
|
|
674
|
+
faq: filteredData.faq.length,
|
|
675
|
+
products: filteredData.products.length,
|
|
676
|
+
articles: filteredData.articles.length,
|
|
677
|
+
pricing: filteredData.pricing.length,
|
|
678
|
+
changelog: filteredData.changelog.length
|
|
679
|
+
}
|
|
680
|
+
};
|
|
681
|
+
});
|
|
682
|
+
app.get("/api/config", async () => {
|
|
683
|
+
const sanitized = {
|
|
684
|
+
site: config.site,
|
|
685
|
+
scan: config.scan,
|
|
686
|
+
output: config.output,
|
|
687
|
+
server: { port: config.server.port, rateLimit: config.server.rateLimit, accessLog: config.server.accessLog },
|
|
688
|
+
access: config.access,
|
|
689
|
+
llm: config.llm ? { apiUrl: config.llm.apiUrl, model: config.llm.model, apiKey: "***" } : void 0
|
|
690
|
+
};
|
|
691
|
+
return sanitized;
|
|
692
|
+
});
|
|
693
|
+
app.get("/api/files", async () => {
|
|
694
|
+
const { readFileSync: readFileSync7, existsSync: existsSync9 } = await import("fs");
|
|
695
|
+
const outDir = config.output.dir;
|
|
696
|
+
const files = [];
|
|
697
|
+
const textFiles = ["llms.txt"];
|
|
698
|
+
const jsonFiles = ["agent-sitemap.json", "agent-index.json", "data/docs.json", "data/faq.json", "data/products.json", "data/articles.json", "data/pricing.json", "data/changelog.json"];
|
|
699
|
+
for (const f of textFiles) {
|
|
700
|
+
const p = `${outDir}/${f}`;
|
|
701
|
+
if (existsSync9(p)) {
|
|
702
|
+
const content = readFileSync7(p, "utf-8");
|
|
703
|
+
files.push({ name: f, size: content.length, content });
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
for (const f of jsonFiles) {
|
|
707
|
+
const p = `${outDir}/${f}`;
|
|
708
|
+
if (existsSync9(p)) {
|
|
709
|
+
const content = readFileSync7(p, "utf-8");
|
|
710
|
+
files.push({ name: f, size: content.length, content });
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return { files };
|
|
714
|
+
});
|
|
715
|
+
app.get("/", async (req, reply) => {
|
|
716
|
+
reply.type("text/html").send(dashboardHtml);
|
|
717
|
+
});
|
|
718
|
+
app.get("/api/pages-data", async () => {
|
|
719
|
+
return {
|
|
720
|
+
siteUrl: filteredData.pages[0]?.url.split("/").slice(0, 3).join("/") || "",
|
|
721
|
+
scannedAt: filteredData.pages[0]?.scannedAt || (/* @__PURE__ */ new Date()).toISOString(),
|
|
722
|
+
totalPages: filteredData.pages.length,
|
|
723
|
+
pages: filteredData.pages
|
|
724
|
+
};
|
|
725
|
+
});
|
|
726
|
+
if (config.access.allowSearch) {
|
|
727
|
+
registerSearchRoute(app, filteredData, searchIndex);
|
|
728
|
+
}
|
|
729
|
+
registerPagesRoute(app, filteredData, config);
|
|
730
|
+
registerFaqRoute(app, filteredData);
|
|
731
|
+
registerProductsRoute(app, filteredData);
|
|
732
|
+
registerDocsRoute(app, filteredData);
|
|
733
|
+
registerArticlesRoute(app, filteredData);
|
|
734
|
+
registerPricingRoute(app, filteredData);
|
|
735
|
+
registerChangelogRoute(app, filteredData);
|
|
736
|
+
app.post("/api/rescan", async () => {
|
|
737
|
+
try {
|
|
738
|
+
const { runScan } = await import("./scan-QWNB65DB.js");
|
|
739
|
+
const result = await runScan();
|
|
740
|
+
return { ok: true, message: `Rescan complete. ${result.totalPages} pages scanned.` };
|
|
741
|
+
} catch (err) {
|
|
742
|
+
return { ok: false, message: err.message };
|
|
743
|
+
}
|
|
744
|
+
});
|
|
745
|
+
app.post("/api/regenerate", async () => {
|
|
746
|
+
try {
|
|
747
|
+
const { runGenerate } = await import("./generate-V5JMMT4J.js");
|
|
748
|
+
await runGenerate();
|
|
749
|
+
return { ok: true, message: "Regeneration complete." };
|
|
750
|
+
} catch (err) {
|
|
751
|
+
return { ok: false, message: err.message };
|
|
752
|
+
}
|
|
753
|
+
});
|
|
754
|
+
return app;
|
|
755
|
+
}
|
|
756
|
+
function buildSearchIndex(data) {
|
|
757
|
+
const index = /* @__PURE__ */ new Map();
|
|
758
|
+
data.pages.forEach((page, i) => {
|
|
759
|
+
const text = `${page.title} ${page.summary} ${page.type}`.toLowerCase();
|
|
760
|
+
const words = text.split(/\W+/).filter((w) => w.length > 2);
|
|
761
|
+
for (const word of words) {
|
|
762
|
+
if (!index.has(word)) index.set(word, /* @__PURE__ */ new Set());
|
|
763
|
+
index.get(word).add(i);
|
|
764
|
+
}
|
|
765
|
+
});
|
|
766
|
+
return index;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
// src/commands/serve.ts
|
|
770
|
+
function loadJson(path) {
|
|
771
|
+
if (!existsSync4(path)) return [];
|
|
772
|
+
return JSON.parse(readFileSync3(path, "utf-8"));
|
|
773
|
+
}
|
|
774
|
+
function loadSiteData(outDir) {
|
|
775
|
+
const scanPath = `${outDir}/scan-result.json`;
|
|
776
|
+
if (!existsSync4(scanPath)) return null;
|
|
777
|
+
const scanResult = JSON.parse(readFileSync3(scanPath, "utf-8"));
|
|
778
|
+
const data = {
|
|
779
|
+
pages: scanResult.pages,
|
|
780
|
+
docs: loadJson(`${outDir}/data/docs.json`),
|
|
781
|
+
faq: loadJson(`${outDir}/data/faq.json`),
|
|
782
|
+
products: loadJson(`${outDir}/data/products.json`),
|
|
783
|
+
articles: loadJson(`${outDir}/data/articles.json`),
|
|
784
|
+
pricing: loadJson(`${outDir}/data/pricing.json`),
|
|
785
|
+
changelog: loadJson(`${outDir}/data/changelog.json`)
|
|
786
|
+
};
|
|
787
|
+
return { scanResult, data };
|
|
788
|
+
}
|
|
789
|
+
function registerServeCommand(program2) {
|
|
790
|
+
program2.command("serve").description("Start the Agent-friendly API server").option("-p, --port <port>", "Port number").action(async (opts) => {
|
|
791
|
+
const config = loadConfig();
|
|
792
|
+
const outDir = config.output.dir;
|
|
793
|
+
const primary = loadSiteData(outDir);
|
|
794
|
+
const data = primary?.data ?? {
|
|
795
|
+
pages: [],
|
|
796
|
+
docs: [],
|
|
797
|
+
faq: [],
|
|
798
|
+
products: [],
|
|
799
|
+
articles: [],
|
|
800
|
+
pricing: [],
|
|
801
|
+
changelog: []
|
|
802
|
+
};
|
|
803
|
+
if (!primary) {
|
|
804
|
+
log.warn("No scan data found. Server starting with empty data.");
|
|
805
|
+
log.warn("Use the Dashboard Operations panel or run `agentsite scan && agentsite generate` to populate.");
|
|
806
|
+
}
|
|
807
|
+
const port = parseInt(opts.port, 10) || config.server.port;
|
|
808
|
+
const app = await createApp(config, data);
|
|
809
|
+
if (config.resolvedSites.length > 1) {
|
|
810
|
+
for (const siteEntry of config.resolvedSites) {
|
|
811
|
+
const slug = toSlug(siteEntry.site.name);
|
|
812
|
+
const siteOutDir = siteEntry.output.dir;
|
|
813
|
+
const siteData = loadSiteData(siteOutDir);
|
|
814
|
+
if (siteData) {
|
|
815
|
+
app.get(`/api/${slug}/pages-data`, async () => ({
|
|
816
|
+
siteUrl: siteData.scanResult.siteUrl,
|
|
817
|
+
scannedAt: siteData.scanResult.scannedAt,
|
|
818
|
+
totalPages: siteData.scanResult.totalPages,
|
|
819
|
+
pages: siteData.data.pages
|
|
820
|
+
}));
|
|
821
|
+
app.get(`/api/${slug}/search`, async (req) => {
|
|
822
|
+
const query = req.query;
|
|
823
|
+
return { site: slug, query: query.q, results: [] };
|
|
824
|
+
});
|
|
825
|
+
app.get(`/api/${slug}/docs`, async () => siteData.data.docs);
|
|
826
|
+
app.get(`/api/${slug}/faq`, async () => siteData.data.faq);
|
|
827
|
+
app.get(`/api/${slug}/products`, async () => siteData.data.products);
|
|
828
|
+
app.get(`/api/${slug}/articles`, async () => siteData.data.articles);
|
|
829
|
+
app.get(`/api/${slug}/pricing`, async () => siteData.data.pricing);
|
|
830
|
+
app.get(`/api/${slug}/changelog`, async () => siteData.data.changelog);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
app.get("/api/sites", async () => {
|
|
835
|
+
return {
|
|
836
|
+
sites: config.resolvedSites.map((s) => ({
|
|
837
|
+
name: s.site.name,
|
|
838
|
+
slug: toSlug(s.site.name),
|
|
839
|
+
url: s.site.url,
|
|
840
|
+
description: s.site.description,
|
|
841
|
+
outDir: s.output.dir
|
|
842
|
+
}))
|
|
843
|
+
};
|
|
844
|
+
});
|
|
845
|
+
await app.listen({ port, host: "0.0.0.0" });
|
|
846
|
+
log.success(`API server running at http://localhost:${port}`);
|
|
847
|
+
log.info("Endpoints:");
|
|
848
|
+
console.log(" GET /api/health");
|
|
849
|
+
console.log(" GET /api/search?q=keyword");
|
|
850
|
+
console.log(" GET /api/pages/:id");
|
|
851
|
+
console.log(" GET /api/faq");
|
|
852
|
+
console.log(" GET /api/products");
|
|
853
|
+
console.log(" GET /api/docs");
|
|
854
|
+
console.log(" GET /api/articles");
|
|
855
|
+
console.log(" GET /api/pricing");
|
|
856
|
+
console.log(" GET /api/changelog");
|
|
857
|
+
console.log(" GET /api/stats");
|
|
858
|
+
console.log(" GET /api/config");
|
|
859
|
+
console.log(" GET /api/files");
|
|
860
|
+
console.log(" GET /api/access-log");
|
|
861
|
+
console.log(" GET /api/sites");
|
|
862
|
+
if (config.resolvedSites.length > 1) {
|
|
863
|
+
for (const s of config.resolvedSites) {
|
|
864
|
+
console.log(` GET /api/${toSlug(s.site.name)}/* (multi-site)`);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// src/commands/update.ts
|
|
871
|
+
import { writeFileSync as writeFileSync3, mkdirSync as mkdirSync2 } from "fs";
|
|
872
|
+
|
|
873
|
+
// src/change-detection/store.ts
|
|
874
|
+
import { readFileSync as readFileSync4, writeFileSync as writeFileSync2, existsSync as existsSync5 } from "fs";
|
|
875
|
+
function loadHashes(path) {
|
|
876
|
+
if (!existsSync5(path)) return {};
|
|
877
|
+
return JSON.parse(readFileSync4(path, "utf-8"));
|
|
878
|
+
}
|
|
879
|
+
function saveHashes(path, hashes) {
|
|
880
|
+
writeFileSync2(path, JSON.stringify(hashes, null, 2), "utf-8");
|
|
881
|
+
}
|
|
882
|
+
function findChangedUrls(oldHashes, newHashes) {
|
|
883
|
+
const changed = [];
|
|
884
|
+
for (const [url, hash] of Object.entries(newHashes)) {
|
|
885
|
+
if (oldHashes[url] !== hash) changed.push(url);
|
|
886
|
+
}
|
|
887
|
+
return changed;
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
// src/commands/update.ts
|
|
891
|
+
function registerUpdateCommand(program2) {
|
|
892
|
+
program2.command("update").description("Incrementally update: re-scan, detect changes, regenerate").action(async () => {
|
|
893
|
+
const config = loadConfig();
|
|
894
|
+
const outDir = config.output.dir;
|
|
895
|
+
const hashPath = `${outDir}/cache/hashes.json`;
|
|
896
|
+
mkdirSync2(`${outDir}/cache/pages`, { recursive: true });
|
|
897
|
+
mkdirSync2(`${outDir}/data`, { recursive: true });
|
|
898
|
+
const oldHashes = loadHashes(hashPath);
|
|
899
|
+
const sp = spinner("Re-scanning site...");
|
|
900
|
+
const sitemapUrls = await parseSitemap(config.site.url);
|
|
901
|
+
const crawled = await crawlSite(config, sitemapUrls, (url, i) => {
|
|
902
|
+
sp.text = `Scanning (${i})... ${url}`;
|
|
903
|
+
});
|
|
904
|
+
sp.succeed(`Scanned ${crawled.length} pages`);
|
|
905
|
+
const newHashes = {};
|
|
906
|
+
const pages = [];
|
|
907
|
+
for (const { url, html } of crawled) {
|
|
908
|
+
const content = extractContent(html, url);
|
|
909
|
+
const contentHash = sha256(content.bodyText);
|
|
910
|
+
newHashes[url] = contentHash;
|
|
911
|
+
const pageType = classifyPage({
|
|
912
|
+
url,
|
|
913
|
+
title: content.title,
|
|
914
|
+
metaOgType: content.metaOgType,
|
|
915
|
+
headings: content.headings,
|
|
916
|
+
bodyText: content.bodyText
|
|
917
|
+
});
|
|
918
|
+
pages.push({
|
|
919
|
+
url,
|
|
920
|
+
title: content.title,
|
|
921
|
+
type: pageType,
|
|
922
|
+
contentHash,
|
|
923
|
+
summary: content.summary,
|
|
924
|
+
headings: content.headings,
|
|
925
|
+
lastModified: content.lastModified,
|
|
926
|
+
scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
927
|
+
wordCount: content.wordCount,
|
|
928
|
+
tags: content.tags,
|
|
929
|
+
version: content.version,
|
|
930
|
+
author: content.author,
|
|
931
|
+
publishedAt: content.publishedAt,
|
|
932
|
+
updatedAt: content.lastModified
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
const changed = findChangedUrls(oldHashes, newHashes);
|
|
936
|
+
if (changed.length === 0) {
|
|
937
|
+
log.success("No changes detected. Everything is up to date.");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
log.info(`${changed.length} page(s) changed`);
|
|
941
|
+
const result = {
|
|
942
|
+
siteUrl: config.site.url,
|
|
943
|
+
scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
944
|
+
totalPages: pages.length,
|
|
945
|
+
pages
|
|
946
|
+
};
|
|
947
|
+
writeFileSync3(`${outDir}/scan-result.json`, JSON.stringify(result, null, 2), "utf-8");
|
|
948
|
+
const genSp = spinner("Regenerating files...");
|
|
949
|
+
writeFileSync3(`${outDir}/llms.txt`, generateLlmsTxt(result, config.site.name, config.site.description), "utf-8");
|
|
950
|
+
writeFileSync3(`${outDir}/agent-sitemap.json`, JSON.stringify(generateAgentSitemap(result), null, 2), "utf-8");
|
|
951
|
+
writeFileSync3(`${outDir}/agent-index.json`, JSON.stringify(generateAgentIndex(result, config.site.name, config.site.description), null, 2), "utf-8");
|
|
952
|
+
const { docs, faq, products, articles, pricing, changelog } = await generateStructuredExports(result, outDir, config);
|
|
953
|
+
writeFileSync3(`${outDir}/data/docs.json`, JSON.stringify(docs, null, 2), "utf-8");
|
|
954
|
+
writeFileSync3(`${outDir}/data/faq.json`, JSON.stringify(faq, null, 2), "utf-8");
|
|
955
|
+
writeFileSync3(`${outDir}/data/products.json`, JSON.stringify(products, null, 2), "utf-8");
|
|
956
|
+
writeFileSync3(`${outDir}/data/articles.json`, JSON.stringify(articles, null, 2), "utf-8");
|
|
957
|
+
writeFileSync3(`${outDir}/data/pricing.json`, JSON.stringify(pricing, null, 2), "utf-8");
|
|
958
|
+
writeFileSync3(`${outDir}/data/changelog.json`, JSON.stringify(changelog, null, 2), "utf-8");
|
|
959
|
+
saveHashes(hashPath, newHashes);
|
|
960
|
+
genSp.succeed("All files regenerated");
|
|
961
|
+
log.success(`Update complete. ${changed.length} page(s) updated.`);
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// src/commands/mcp.ts
|
|
966
|
+
import { existsSync as existsSync8 } from "fs";
|
|
967
|
+
|
|
968
|
+
// src/mcp/server.ts
|
|
969
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
970
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
971
|
+
import { z } from "zod";
|
|
972
|
+
import { readFileSync as readFileSync6, existsSync as existsSync7 } from "fs";
|
|
973
|
+
function loadData(outDir) {
|
|
974
|
+
const loadJson2 = (path) => {
|
|
975
|
+
if (!existsSync7(path)) return [];
|
|
976
|
+
return JSON.parse(readFileSync6(path, "utf-8"));
|
|
977
|
+
};
|
|
978
|
+
const scanResult = JSON.parse(readFileSync6(`${outDir}/scan-result.json`, "utf-8"));
|
|
979
|
+
return {
|
|
980
|
+
scanResult,
|
|
981
|
+
docs: loadJson2(`${outDir}/data/docs.json`),
|
|
982
|
+
faq: loadJson2(`${outDir}/data/faq.json`),
|
|
983
|
+
products: loadJson2(`${outDir}/data/products.json`),
|
|
984
|
+
articles: loadJson2(`${outDir}/data/articles.json`),
|
|
985
|
+
pricing: loadJson2(`${outDir}/data/pricing.json`),
|
|
986
|
+
changelog: loadJson2(`${outDir}/data/changelog.json`)
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
function buildSearchIndex2(pages) {
|
|
990
|
+
const index = /* @__PURE__ */ new Map();
|
|
991
|
+
pages.forEach((page, i) => {
|
|
992
|
+
const text = `${page.title} ${page.summary} ${page.type}`.toLowerCase();
|
|
993
|
+
const words = text.split(/\W+/).filter((w) => w.length > 2);
|
|
994
|
+
for (const word of words) {
|
|
995
|
+
if (!index.has(word)) index.set(word, /* @__PURE__ */ new Set());
|
|
996
|
+
index.get(word).add(i);
|
|
997
|
+
}
|
|
998
|
+
});
|
|
999
|
+
return index;
|
|
1000
|
+
}
|
|
1001
|
+
async function startMcpServer(outDir) {
|
|
1002
|
+
const data = loadData(outDir);
|
|
1003
|
+
const searchIndex = buildSearchIndex2(data.scanResult.pages);
|
|
1004
|
+
const server = new McpServer({
|
|
1005
|
+
name: "agentsite",
|
|
1006
|
+
version: "1.0.0"
|
|
1007
|
+
});
|
|
1008
|
+
server.tool(
|
|
1009
|
+
"search",
|
|
1010
|
+
"Search site content by keyword",
|
|
1011
|
+
{ query: z.string().describe("Search keyword"), limit: z.number().optional().describe("Max results (default 10)") },
|
|
1012
|
+
async ({ query, limit }) => {
|
|
1013
|
+
const max = limit ?? 10;
|
|
1014
|
+
const words = query.toLowerCase().split(/\W+/).filter((w) => w.length > 2);
|
|
1015
|
+
const scoreMap = /* @__PURE__ */ new Map();
|
|
1016
|
+
for (const word of words) {
|
|
1017
|
+
const matches = searchIndex.get(word);
|
|
1018
|
+
if (matches) {
|
|
1019
|
+
for (const idx of matches) {
|
|
1020
|
+
scoreMap.set(idx, (scoreMap.get(idx) ?? 0) + 1);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
const sorted = [...scoreMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, max).map(([idx]) => {
|
|
1025
|
+
const p = data.scanResult.pages[idx];
|
|
1026
|
+
return { url: p.url, title: p.title, type: p.type, summary: p.summary };
|
|
1027
|
+
});
|
|
1028
|
+
return { content: [{ type: "text", text: JSON.stringify(sorted, null, 2) }] };
|
|
1029
|
+
}
|
|
1030
|
+
);
|
|
1031
|
+
server.tool(
|
|
1032
|
+
"get_page",
|
|
1033
|
+
"Get detailed info about a specific page by URL",
|
|
1034
|
+
{ url: z.string().describe("Page URL") },
|
|
1035
|
+
async ({ url }) => {
|
|
1036
|
+
const page = data.scanResult.pages.find((p) => p.url === url);
|
|
1037
|
+
if (!page) {
|
|
1038
|
+
return { content: [{ type: "text", text: "Page not found" }] };
|
|
1039
|
+
}
|
|
1040
|
+
return { content: [{ type: "text", text: JSON.stringify(page, null, 2) }] };
|
|
1041
|
+
}
|
|
1042
|
+
);
|
|
1043
|
+
server.tool(
|
|
1044
|
+
"list_pages",
|
|
1045
|
+
"List all pages with optional type filter",
|
|
1046
|
+
{ type: z.string().optional().describe("Filter by page type: docs, faq, blog, product, pricing, about, contact, changelog") },
|
|
1047
|
+
async ({ type }) => {
|
|
1048
|
+
let pages = data.scanResult.pages;
|
|
1049
|
+
if (type) {
|
|
1050
|
+
pages = pages.filter((p) => p.type === type);
|
|
1051
|
+
}
|
|
1052
|
+
const summary = pages.map((p) => ({ url: p.url, title: p.title, type: p.type }));
|
|
1053
|
+
return { content: [{ type: "text", text: JSON.stringify(summary, null, 2) }] };
|
|
1054
|
+
}
|
|
1055
|
+
);
|
|
1056
|
+
server.tool(
|
|
1057
|
+
"list_faq",
|
|
1058
|
+
"List FAQ entries with optional category filter",
|
|
1059
|
+
{ category: z.string().optional().describe("Filter by category") },
|
|
1060
|
+
async ({ category }) => {
|
|
1061
|
+
let items = data.faq;
|
|
1062
|
+
if (category) {
|
|
1063
|
+
items = items.filter((f) => f.category.toLowerCase() === category.toLowerCase());
|
|
1064
|
+
}
|
|
1065
|
+
return { content: [{ type: "text", text: JSON.stringify(items, null, 2) }] };
|
|
1066
|
+
}
|
|
1067
|
+
);
|
|
1068
|
+
server.tool(
|
|
1069
|
+
"list_docs",
|
|
1070
|
+
"List documentation entries with optional section filter",
|
|
1071
|
+
{ section: z.string().optional().describe("Filter by section") },
|
|
1072
|
+
async ({ section }) => {
|
|
1073
|
+
let items = data.docs;
|
|
1074
|
+
if (section) {
|
|
1075
|
+
items = items.filter((d) => d.section.toLowerCase() === section.toLowerCase());
|
|
1076
|
+
}
|
|
1077
|
+
return { content: [{ type: "text", text: JSON.stringify(items, null, 2) }] };
|
|
1078
|
+
}
|
|
1079
|
+
);
|
|
1080
|
+
server.tool(
|
|
1081
|
+
"list_products",
|
|
1082
|
+
"List all product entries",
|
|
1083
|
+
{},
|
|
1084
|
+
async () => {
|
|
1085
|
+
return { content: [{ type: "text", text: JSON.stringify(data.products, null, 2) }] };
|
|
1086
|
+
}
|
|
1087
|
+
);
|
|
1088
|
+
server.tool(
|
|
1089
|
+
"list_articles",
|
|
1090
|
+
"List blog/article entries with optional tag filter",
|
|
1091
|
+
{ tag: z.string().optional().describe("Filter by tag") },
|
|
1092
|
+
async ({ tag }) => {
|
|
1093
|
+
let items = data.articles;
|
|
1094
|
+
if (tag) {
|
|
1095
|
+
items = items.filter((a) => a.tags.some((t) => t.toLowerCase() === tag.toLowerCase()));
|
|
1096
|
+
}
|
|
1097
|
+
return { content: [{ type: "text", text: JSON.stringify(items, null, 2) }] };
|
|
1098
|
+
}
|
|
1099
|
+
);
|
|
1100
|
+
server.tool(
|
|
1101
|
+
"list_pricing",
|
|
1102
|
+
"List pricing/plan entries",
|
|
1103
|
+
{},
|
|
1104
|
+
async () => {
|
|
1105
|
+
return { content: [{ type: "text", text: JSON.stringify(data.pricing, null, 2) }] };
|
|
1106
|
+
}
|
|
1107
|
+
);
|
|
1108
|
+
server.tool(
|
|
1109
|
+
"list_changelog",
|
|
1110
|
+
"List changelog/release entries",
|
|
1111
|
+
{ version: z.string().optional().describe("Filter by version") },
|
|
1112
|
+
async ({ version }) => {
|
|
1113
|
+
let items = data.changelog;
|
|
1114
|
+
if (version) {
|
|
1115
|
+
items = items.filter((c) => c.version.toLowerCase().includes(version.toLowerCase()));
|
|
1116
|
+
}
|
|
1117
|
+
return { content: [{ type: "text", text: JSON.stringify(items, null, 2) }] };
|
|
1118
|
+
}
|
|
1119
|
+
);
|
|
1120
|
+
server.tool(
|
|
1121
|
+
"get_config",
|
|
1122
|
+
"Get site configuration summary (API keys hidden)",
|
|
1123
|
+
{},
|
|
1124
|
+
async () => {
|
|
1125
|
+
const configPath = `${outDir}/../agentsite.config.yaml`;
|
|
1126
|
+
let configSummary = { note: "Config file not found at expected location" };
|
|
1127
|
+
if (existsSync7(configPath)) {
|
|
1128
|
+
const yaml3 = await import("js-yaml");
|
|
1129
|
+
const raw = readFileSync6(configPath, "utf-8");
|
|
1130
|
+
const parsed = yaml3.load(raw);
|
|
1131
|
+
if (parsed.llm && typeof parsed.llm === "object") {
|
|
1132
|
+
parsed.llm.apiKey = "***";
|
|
1133
|
+
}
|
|
1134
|
+
configSummary = parsed;
|
|
1135
|
+
}
|
|
1136
|
+
return { content: [{ type: "text", text: JSON.stringify(configSummary, null, 2) }] };
|
|
1137
|
+
}
|
|
1138
|
+
);
|
|
1139
|
+
server.tool(
|
|
1140
|
+
"site_overview",
|
|
1141
|
+
"Get a high-level overview of the site: name, description, page counts by type",
|
|
1142
|
+
{},
|
|
1143
|
+
async () => {
|
|
1144
|
+
const typeCounts = {};
|
|
1145
|
+
for (const p of data.scanResult.pages) {
|
|
1146
|
+
typeCounts[p.type] = (typeCounts[p.type] ?? 0) + 1;
|
|
1147
|
+
}
|
|
1148
|
+
const overview = {
|
|
1149
|
+
totalPages: data.scanResult.totalPages,
|
|
1150
|
+
scannedAt: data.scanResult.scannedAt,
|
|
1151
|
+
pageTypes: typeCounts,
|
|
1152
|
+
dataFiles: {
|
|
1153
|
+
docs: data.docs.length,
|
|
1154
|
+
faq: data.faq.length,
|
|
1155
|
+
products: data.products.length,
|
|
1156
|
+
articles: data.articles.length,
|
|
1157
|
+
pricing: data.pricing.length,
|
|
1158
|
+
changelog: data.changelog.length
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
1161
|
+
return { content: [{ type: "text", text: JSON.stringify(overview, null, 2) }] };
|
|
1162
|
+
}
|
|
1163
|
+
);
|
|
1164
|
+
const llmsPath = `${outDir}/llms.txt`;
|
|
1165
|
+
if (existsSync7(llmsPath)) {
|
|
1166
|
+
server.resource(
|
|
1167
|
+
"llms-txt",
|
|
1168
|
+
"file:///llms.txt",
|
|
1169
|
+
{ description: "LLM-friendly site overview in llms.txt format", mimeType: "text/plain" },
|
|
1170
|
+
async () => ({
|
|
1171
|
+
contents: [{ uri: "file:///llms.txt", text: readFileSync6(llmsPath, "utf-8"), mimeType: "text/plain" }]
|
|
1172
|
+
})
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
const indexPath = `${outDir}/agent-index.json`;
|
|
1176
|
+
if (existsSync7(indexPath)) {
|
|
1177
|
+
server.resource(
|
|
1178
|
+
"agent-index",
|
|
1179
|
+
"file:///agent-index.json",
|
|
1180
|
+
{ description: "Agent index with site metadata and API endpoints", mimeType: "application/json" },
|
|
1181
|
+
async () => ({
|
|
1182
|
+
contents: [{ uri: "file:///agent-index.json", text: readFileSync6(indexPath, "utf-8"), mimeType: "application/json" }]
|
|
1183
|
+
})
|
|
1184
|
+
);
|
|
1185
|
+
}
|
|
1186
|
+
const sitemapPath = `${outDir}/agent-sitemap.json`;
|
|
1187
|
+
if (existsSync7(sitemapPath)) {
|
|
1188
|
+
server.resource(
|
|
1189
|
+
"agent-sitemap",
|
|
1190
|
+
"file:///agent-sitemap.json",
|
|
1191
|
+
{ description: "Machine-readable sitemap with page metadata", mimeType: "application/json" },
|
|
1192
|
+
async () => ({
|
|
1193
|
+
contents: [{ uri: "file:///agent-sitemap.json", text: readFileSync6(sitemapPath, "utf-8"), mimeType: "application/json" }]
|
|
1194
|
+
})
|
|
1195
|
+
);
|
|
1196
|
+
}
|
|
1197
|
+
const dataFiles = ["docs", "faq", "products", "articles", "pricing", "changelog"];
|
|
1198
|
+
for (const name of dataFiles) {
|
|
1199
|
+
const dataPath = `${outDir}/data/${name}.json`;
|
|
1200
|
+
if (existsSync7(dataPath)) {
|
|
1201
|
+
server.resource(
|
|
1202
|
+
`data-${name}`,
|
|
1203
|
+
`file:///data/${name}.json`,
|
|
1204
|
+
{ description: `Structured ${name} data`, mimeType: "application/json" },
|
|
1205
|
+
async () => ({
|
|
1206
|
+
contents: [{ uri: `file:///data/${name}.json`, text: readFileSync6(dataPath, "utf-8"), mimeType: "application/json" }]
|
|
1207
|
+
})
|
|
1208
|
+
);
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
const transport = new StdioServerTransport();
|
|
1212
|
+
await server.connect(transport);
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
// src/commands/mcp.ts
|
|
1216
|
+
function registerMcpCommand(program2) {
|
|
1217
|
+
program2.command("mcp").description("Start MCP (Model Context Protocol) server over stdio").action(async () => {
|
|
1218
|
+
const config = loadConfig();
|
|
1219
|
+
const outDir = config.output.dir;
|
|
1220
|
+
if (!existsSync8(`${outDir}/scan-result.json`)) {
|
|
1221
|
+
log.error("No data found. Run `agentsite scan` and `agentsite generate` first.");
|
|
1222
|
+
process.exit(1);
|
|
1223
|
+
}
|
|
1224
|
+
await startMcpServer(outDir);
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
// src/index.ts
|
|
1229
|
+
var program = new Command();
|
|
1230
|
+
program.name("agentsite").description("Make any website Agent-friendly").version("1.0.0");
|
|
1231
|
+
registerInitCommand(program);
|
|
1232
|
+
registerScanCommand(program);
|
|
1233
|
+
registerGenerateCommand(program);
|
|
1234
|
+
registerServeCommand(program);
|
|
1235
|
+
registerUpdateCommand(program);
|
|
1236
|
+
registerMcpCommand(program);
|
|
1237
|
+
program.parse();
|
|
1238
|
+
//# sourceMappingURL=index.js.map
|