@amtp/protocol 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +386 -0
- package/USAGE_GUIDE.md +722 -0
- package/bin/amtp.ts +387 -0
- package/dist/client/amtp-client.d.ts +164 -0
- package/dist/client/amtp-client.js +460 -0
- package/dist/client/amtp-client.js.map +1 -0
- package/dist/client/examples/basic-client.d.ts +6 -0
- package/dist/client/examples/basic-client.js +35 -0
- package/dist/client/examples/basic-client.js.map +1 -0
- package/dist/crawler/amtp-crawler.d.ts +125 -0
- package/dist/crawler/amtp-crawler.js +359 -0
- package/dist/crawler/amtp-crawler.js.map +1 -0
- package/dist/crawler/examples/basic-crawler.d.ts +6 -0
- package/dist/crawler/examples/basic-crawler.js +28 -0
- package/dist/crawler/examples/basic-crawler.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +70 -0
- package/dist/index.js.map +1 -0
- package/dist/server/adapters/fastify-adapter.d.ts +86 -0
- package/dist/server/adapters/fastify-adapter.js +169 -0
- package/dist/server/adapters/fastify-adapter.js.map +1 -0
- package/dist/server/amtp-ql-executor.d.ts +24 -0
- package/dist/server/amtp-ql-executor.js +198 -0
- package/dist/server/amtp-ql-executor.js.map +1 -0
- package/dist/server/amtp-ql-parser.d.ts +30 -0
- package/dist/server/amtp-ql-parser.js +212 -0
- package/dist/server/amtp-ql-parser.js.map +1 -0
- package/dist/server/amtp-server.d.ts +183 -0
- package/dist/server/amtp-server.js +650 -0
- package/dist/server/amtp-server.js.map +1 -0
- package/dist/server/examples/basic-server.d.ts +6 -0
- package/dist/server/examples/basic-server.js +215 -0
- package/dist/server/examples/basic-server.js.map +1 -0
- package/dist/server/examples/saas-dashboard-server.d.ts +44 -0
- package/dist/server/examples/saas-dashboard-server.js +387 -0
- package/dist/server/examples/saas-dashboard-server.js.map +1 -0
- package/dist/server/markdown-parser.d.ts +31 -0
- package/dist/server/markdown-parser.js +463 -0
- package/dist/server/markdown-parser.js.map +1 -0
- package/dist/server/notifications.d.ts +40 -0
- package/dist/server/notifications.js +134 -0
- package/dist/server/notifications.js.map +1 -0
- package/dist/server/permissions.d.ts +40 -0
- package/dist/server/permissions.js +156 -0
- package/dist/server/permissions.js.map +1 -0
- package/dist/server/security.d.ts +127 -0
- package/dist/server/security.js +368 -0
- package/dist/server/security.js.map +1 -0
- package/dist/types/amtp.types.d.ts +720 -0
- package/dist/types/amtp.types.js +224 -0
- package/dist/types/amtp.types.js.map +1 -0
- package/package.json +89 -0
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* AMTP Crawler & Indexer
|
|
4
|
+
* Crawls AMTP-enabled websites and builds searchable index
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.SearchIndexer = exports.AMTPCrawler = void 0;
|
|
8
|
+
const amtp_client_1 = require("../client/amtp-client");
|
|
9
|
+
const amtp_types_1 = require("../types/amtp.types");
|
|
10
|
+
/**
|
|
11
|
+
* AMTP Crawler
|
|
12
|
+
*/
|
|
13
|
+
class AMTPCrawler {
|
|
14
|
+
constructor(config) {
|
|
15
|
+
this.visited = new Set();
|
|
16
|
+
this.queue = [];
|
|
17
|
+
this.index = new Map();
|
|
18
|
+
this.startTime = 0;
|
|
19
|
+
this.disallowRules = [];
|
|
20
|
+
this.crawlDelayMs = 0;
|
|
21
|
+
this.config = {
|
|
22
|
+
baseUrl: config.baseUrl,
|
|
23
|
+
maxPages: config.maxPages || 1000,
|
|
24
|
+
maxDepth: config.maxDepth || 5,
|
|
25
|
+
respectRobotsTxt: config.respectRobotsTxt !== false,
|
|
26
|
+
delays: {
|
|
27
|
+
betweenRequests: config.delays?.betweenRequests || 100,
|
|
28
|
+
betweenDomains: config.delays?.betweenDomains || 1000,
|
|
29
|
+
},
|
|
30
|
+
userAgent: config.userAgent || "AMTP-Crawler/1.0",
|
|
31
|
+
indexCallback: config.indexCallback,
|
|
32
|
+
};
|
|
33
|
+
this.client = new amtp_client_1.AMTPClient({
|
|
34
|
+
baseUrl: this.config.baseUrl,
|
|
35
|
+
capabilities: ["streaming", "pagination"],
|
|
36
|
+
});
|
|
37
|
+
this.stats = {
|
|
38
|
+
totalPages: 0,
|
|
39
|
+
totalTime: 0,
|
|
40
|
+
avgTimePerPage: 0,
|
|
41
|
+
totalLinksDiscovered: 0,
|
|
42
|
+
externalLinks: 0,
|
|
43
|
+
errors: 0,
|
|
44
|
+
startedAt: new Date().toISOString(),
|
|
45
|
+
completedAt: "",
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Start crawling
|
|
50
|
+
*/
|
|
51
|
+
async crawl() {
|
|
52
|
+
this.startTime = Date.now();
|
|
53
|
+
console.log(`🕷️ Starting AMTP crawl: ${this.config.baseUrl}`);
|
|
54
|
+
// Check robots.txt if enabled
|
|
55
|
+
if (this.config.respectRobotsTxt) {
|
|
56
|
+
await this.checkRobotsTxt();
|
|
57
|
+
}
|
|
58
|
+
// Initialize queue with root
|
|
59
|
+
this.queue.push({ url: this.config.baseUrl, depth: 0 });
|
|
60
|
+
// Process queue
|
|
61
|
+
while (this.queue.length > 0 &&
|
|
62
|
+
(this.visited.size < (this.config.maxPages ?? 1000))) {
|
|
63
|
+
const item = this.queue.shift();
|
|
64
|
+
if (!item)
|
|
65
|
+
break;
|
|
66
|
+
await this.crawlPage(item.url, item.depth);
|
|
67
|
+
const delayMs = Math.max(this.crawlDelayMs, this.config.delays?.betweenRequests ?? 100);
|
|
68
|
+
await this.delay(delayMs);
|
|
69
|
+
}
|
|
70
|
+
// Calculate stats
|
|
71
|
+
const endTime = Date.now();
|
|
72
|
+
this.stats.totalTime = endTime - this.startTime;
|
|
73
|
+
this.stats.avgTimePerPage = this.stats.totalPages > 0
|
|
74
|
+
? Math.round(this.stats.totalTime / this.stats.totalPages)
|
|
75
|
+
: 0;
|
|
76
|
+
this.stats.completedAt = new Date().toISOString();
|
|
77
|
+
console.log(`✅ Crawl complete. Pages: ${this.stats.totalPages}`);
|
|
78
|
+
this.printStats();
|
|
79
|
+
return Array.from(this.index.values());
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Crawl a single page
|
|
83
|
+
*/
|
|
84
|
+
async crawlPage(url, depth) {
|
|
85
|
+
// Skip if already visited
|
|
86
|
+
if (this.visited.has(url))
|
|
87
|
+
return;
|
|
88
|
+
this.visited.add(url);
|
|
89
|
+
// Skip if depth exceeded
|
|
90
|
+
if (depth > (this.config.maxDepth ?? 5))
|
|
91
|
+
return;
|
|
92
|
+
// SECURITY: Respect robots.txt disallow rules
|
|
93
|
+
if (this.config.respectRobotsTxt && this.disallowRules.length > 0 && !this.isAllowedByRobots(url)) {
|
|
94
|
+
console.log(`🚫 Skipping (robots.txt): ${url}`);
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
console.log(`📄 Crawling [${this.stats.totalPages + 1}] ${url}`);
|
|
99
|
+
const doc = await this.client.getPage(url);
|
|
100
|
+
// Index the page
|
|
101
|
+
const crawledPage = {
|
|
102
|
+
url,
|
|
103
|
+
title: doc.title,
|
|
104
|
+
content: doc.nodes.map((n) => n.content || "").join(" "),
|
|
105
|
+
structuredData: doc.structured_data || [],
|
|
106
|
+
links: doc.links,
|
|
107
|
+
indexedAt: new Date().toISOString(),
|
|
108
|
+
};
|
|
109
|
+
this.index.set(url, crawledPage);
|
|
110
|
+
this.stats.totalPages++;
|
|
111
|
+
// Call index callback if provided
|
|
112
|
+
if (this.config.indexCallback) {
|
|
113
|
+
await this.config.indexCallback(doc);
|
|
114
|
+
}
|
|
115
|
+
// Discover new links
|
|
116
|
+
for (const link of doc.links) {
|
|
117
|
+
if (link.type === amtp_types_1.LinkType.INTERNAL) {
|
|
118
|
+
const linkedUrl = new URL(link.url, this.config.baseUrl).href;
|
|
119
|
+
if (!this.visited.has(linkedUrl)) {
|
|
120
|
+
this.queue.push({ url: linkedUrl, depth: depth + 1 });
|
|
121
|
+
this.stats.totalLinksDiscovered++;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
else if (link.type === amtp_types_1.LinkType.EXTERNAL) {
|
|
125
|
+
this.stats.externalLinks++;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
console.error(`❌ Error crawling ${url}:`, error);
|
|
131
|
+
this.stats.errors++;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Check robots.txt — parse rules and enforce them during crawl.
|
|
136
|
+
*/
|
|
137
|
+
async checkRobotsTxt() {
|
|
138
|
+
try {
|
|
139
|
+
const robotsUrl = `${this.config.baseUrl}/robots.txt`;
|
|
140
|
+
const response = await fetch(robotsUrl);
|
|
141
|
+
if (!response.ok) {
|
|
142
|
+
console.log("📋 No robots.txt found (continuing)");
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
const content = await response.text();
|
|
146
|
+
console.log("📋 robots.txt found, parsing rules...");
|
|
147
|
+
const userAgent = this.config.userAgent || "AMTP-Crawler/1.0";
|
|
148
|
+
let relevantSection = false;
|
|
149
|
+
const disallows = [];
|
|
150
|
+
for (const line of content.split("\n")) {
|
|
151
|
+
const trimmed = line.trim();
|
|
152
|
+
if (/^User-agent:\s*/i.test(trimmed)) {
|
|
153
|
+
const agent = trimmed.replace(/^User-agent:\s*/i, "").trim();
|
|
154
|
+
relevantSection = agent === "*" || agent === userAgent;
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
if (relevantSection && /^Disallow:\s*/i.test(trimmed)) {
|
|
158
|
+
const path = trimmed.replace(/^Disallow:\s*/i, "").trim();
|
|
159
|
+
if (path) {
|
|
160
|
+
disallows.push(path);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
if (relevantSection && /^Crawl-delay:\s*/i.test(trimmed)) {
|
|
164
|
+
const delay = parseInt(trimmed.replace(/^Crawl-delay:\s*/i, "").trim(), 10);
|
|
165
|
+
if (!isNaN(delay) && delay > 0) {
|
|
166
|
+
this.crawlDelayMs = delay * 1000;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// Convert disallow paths to regex patterns for URL matching
|
|
171
|
+
this.disallowRules = disallows.map((d) => {
|
|
172
|
+
const escaped = d.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\*/g, ".*");
|
|
173
|
+
return new RegExp(`^${escaped}`);
|
|
174
|
+
});
|
|
175
|
+
if (this.disallowRules.length > 0) {
|
|
176
|
+
console.log(`📋 ${disallows.length} disallow rule(s) loaded`);
|
|
177
|
+
}
|
|
178
|
+
if (this.crawlDelayMs > 0) {
|
|
179
|
+
console.log(`📋 Crawl-delay: ${this.crawlDelayMs}ms`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
console.log("📋 Could not fetch robots.txt (continuing without restrictions)");
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
isAllowedByRobots(url) {
|
|
187
|
+
try {
|
|
188
|
+
const parsed = new URL(url);
|
|
189
|
+
const path = parsed.pathname + parsed.search;
|
|
190
|
+
for (const rule of this.disallowRules) {
|
|
191
|
+
if (rule.test(path))
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
// If URL is malformed, allow it (crawlPage will handle errors)
|
|
197
|
+
}
|
|
198
|
+
return true;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Search the index
|
|
202
|
+
*/
|
|
203
|
+
search(query) {
|
|
204
|
+
const lowerQuery = query.toLowerCase();
|
|
205
|
+
return Array.from(this.index.values()).filter((page) => page.title.toLowerCase().includes(lowerQuery) ||
|
|
206
|
+
page.content.toLowerCase().includes(lowerQuery));
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Get page by URL
|
|
210
|
+
*/
|
|
211
|
+
getPage(url) {
|
|
212
|
+
return this.index.get(url);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Export index as JSON
|
|
216
|
+
*/
|
|
217
|
+
exportIndex() {
|
|
218
|
+
return JSON.stringify({
|
|
219
|
+
crawledAt: new Date().toISOString(),
|
|
220
|
+
stats: this.stats,
|
|
221
|
+
pages: Array.from(this.index.values()),
|
|
222
|
+
}, null, 2);
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Print crawl statistics
|
|
226
|
+
*/
|
|
227
|
+
printStats() {
|
|
228
|
+
console.log("\n📊 Crawl Statistics:");
|
|
229
|
+
console.log(` Total Pages: ${this.stats.totalPages}`);
|
|
230
|
+
console.log(` Total Time: ${this.stats.totalTime}ms`);
|
|
231
|
+
console.log(` Avg Time/Page: ${this.stats.avgTimePerPage}ms`);
|
|
232
|
+
console.log(` Links Discovered: ${this.stats.totalLinksDiscovered}`);
|
|
233
|
+
console.log(` External Links: ${this.stats.externalLinks}`);
|
|
234
|
+
console.log(` Errors: ${this.stats.errors}`);
|
|
235
|
+
console.log();
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Delay execution
|
|
239
|
+
*/
|
|
240
|
+
delay(ms) {
|
|
241
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
exports.AMTPCrawler = AMTPCrawler;
|
|
245
|
+
/**
|
|
246
|
+
* Search Engine Indexer
|
|
247
|
+
* Example of building a search engine index from AMTP pages
|
|
248
|
+
*/
|
|
249
|
+
class SearchIndexer {
|
|
250
|
+
constructor() {
|
|
251
|
+
this.index = new Map();
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Add pages to index
|
|
255
|
+
*/
|
|
256
|
+
addPages(pages) {
|
|
257
|
+
for (const page of pages) {
|
|
258
|
+
// Extract keywords
|
|
259
|
+
const keywords = this.extractKeywords(page.content);
|
|
260
|
+
for (const keyword of keywords) {
|
|
261
|
+
if (!this.index.has(keyword)) {
|
|
262
|
+
this.index.set(keyword, []);
|
|
263
|
+
}
|
|
264
|
+
const entry = this.index.get(keyword);
|
|
265
|
+
if (entry)
|
|
266
|
+
entry.push(page);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Search index
|
|
272
|
+
*/
|
|
273
|
+
search(query, limit = 10) {
|
|
274
|
+
const keywords = query.toLowerCase().split(" ");
|
|
275
|
+
const results = new Map();
|
|
276
|
+
// Count keyword matches
|
|
277
|
+
for (const keyword of keywords) {
|
|
278
|
+
const pages = this.index.get(keyword) || [];
|
|
279
|
+
for (const page of pages) {
|
|
280
|
+
const url = page.url;
|
|
281
|
+
results.set(url, (results.get(url) || 0) + 1);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Sort by relevance
|
|
285
|
+
return Array.from(results.entries())
|
|
286
|
+
.sort((a, b) => b[1] - a[1])
|
|
287
|
+
.slice(0, limit)
|
|
288
|
+
.map(([url]) => {
|
|
289
|
+
// Get page from all pages
|
|
290
|
+
for (const pages of this.index.values()) {
|
|
291
|
+
const page = pages.find((p) => p.url === url);
|
|
292
|
+
if (page)
|
|
293
|
+
return page;
|
|
294
|
+
}
|
|
295
|
+
throw new Error(`Page ${url} not found`);
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Extract keywords from content
|
|
300
|
+
*/
|
|
301
|
+
extractKeywords(content) {
|
|
302
|
+
// Simple keyword extraction (production would use NLP)
|
|
303
|
+
return content
|
|
304
|
+
.toLowerCase()
|
|
305
|
+
.split(/\s+/)
|
|
306
|
+
.filter((word) => word.length > 3 && !this.isStopWord(word))
|
|
307
|
+
.slice(0, 50);
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Check if word is stop word
|
|
311
|
+
*/
|
|
312
|
+
isStopWord(word) {
|
|
313
|
+
const stopWords = new Set([
|
|
314
|
+
"the",
|
|
315
|
+
"a",
|
|
316
|
+
"an",
|
|
317
|
+
"and",
|
|
318
|
+
"or",
|
|
319
|
+
"but",
|
|
320
|
+
"is",
|
|
321
|
+
"are",
|
|
322
|
+
"was",
|
|
323
|
+
"were",
|
|
324
|
+
"been",
|
|
325
|
+
"be",
|
|
326
|
+
"have",
|
|
327
|
+
"has",
|
|
328
|
+
"had",
|
|
329
|
+
"do",
|
|
330
|
+
"does",
|
|
331
|
+
"did",
|
|
332
|
+
"will",
|
|
333
|
+
"would",
|
|
334
|
+
"could",
|
|
335
|
+
"should",
|
|
336
|
+
"may",
|
|
337
|
+
"might",
|
|
338
|
+
"must",
|
|
339
|
+
"can",
|
|
340
|
+
]);
|
|
341
|
+
return stopWords.has(word);
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Export index
|
|
345
|
+
*/
|
|
346
|
+
exportIndex() {
|
|
347
|
+
const data = {};
|
|
348
|
+
for (const [keyword, pages] of this.index) {
|
|
349
|
+
data[keyword] = pages.map((p) => p.url);
|
|
350
|
+
}
|
|
351
|
+
return JSON.stringify(data, null, 2);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
exports.SearchIndexer = SearchIndexer;
|
|
355
|
+
exports.default = {
|
|
356
|
+
AMTPCrawler,
|
|
357
|
+
SearchIndexer,
|
|
358
|
+
};
|
|
359
|
+
//# sourceMappingURL=amtp-crawler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"amtp-crawler.js","sourceRoot":"","sources":["../../src/crawler/amtp-crawler.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,uDAAmD;AACnD,oDAAiG;AA6CjG;;GAEG;AACH,MAAa,WAAW;IAStB,YAAY,MAAqB;QANzB,YAAO,GAAgB,IAAI,GAAG,EAAE,CAAC;QACjC,UAAK,GAAqC,EAAE,CAAC;QAC7C,UAAK,GAA6B,IAAI,GAAG,EAAE,CAAC;QAE5C,cAAS,GAAG,CAAC,CAAC;QAsId,kBAAa,GAAa,EAAE,CAAC;QAC7B,iBAAY,GAAG,CAAC,CAAC;QApIvB,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,CAAC;YAC9B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,KAAK,KAAK;YACnD,MAAM,EAAE;gBACN,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,IAAI,GAAG;gBACtD,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,IAAI,IAAI;aACtD;YACD,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,kBAAkB;YACjD,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,wBAAU,CAAC;YAC3B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,YAAY,EAAE,CAAC,WAAW,EAAE,YAAY,CAAC;SAC1C,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG;YACX,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,cAAc,EAAE,CAAC;YACjB,oBAAoB,EAAE,CAAC;YACvB,aAAa,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,WAAW,EAAE,EAAE;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAEhE,8BAA8B;QAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;YAChC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;SAC7B;QAED,6BAA6B;QAC7B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAExD,gBAAgB;QAChB,OACE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAK,CAAC,CAAC,EACrD;YACA,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI;gBAAE,MAAM;YAEjB,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,IAAI,GAAG,CAAC,CAAC;YACxF,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC3B;QAED,kBAAkB;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;QAChD,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC;YACnD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;YAC1D,CAAC,CAAC,CAAC,CAAC;QACN,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAElD,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,KAAa;QAChD,0BAA0B;QAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEtB,yBAAyB;QACzB,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YAAE,OAAO;QAEhD,8CAA8C;QAC9C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE;YACjG,OAAO,CAAC,GAAG,CAAC,6BAA6B,GAAG,EAAE,CAAC,CAAC;YAChD,OAAO;SACR;QAED,IAAI;YACF,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;YAEjE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAE3C,iBAAiB;YACjB,MAAM,WAAW,GAAgB;gBAC/B,GAAG;gBACH,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBACtE,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,EAAE;gBACzC,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YAExB,kCAAkC;YAClC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;gBAC7B,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;aACtC;YAED,qBAAqB;YACrB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE;gBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,qBAAQ,CAAC,QAAQ,EAAE;oBACnC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;oBAE9D,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;wBAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;wBACtD,IAAI,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC;qBACnC;iBACF;qBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,qBAAQ,CAAC,QAAQ,EAAE;oBAC1C,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;iBAC5B;aACF;SACF;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;SACrB;IACH,CAAC;IAKD;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,IAAI;YACF,MAAM,SAAS,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,aAAa,CAAC;YACtD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;YAExC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAChB,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;gBACnD,OAAO;aACR;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YAErD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;YAC9D,IAAI,eAAe,GAAG,KAAK,CAAC;YAC5B,MAAM,SAAS,GAAa,EAAE,CAAC;YAE/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBAE5B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;oBACpC,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC7D,eAAe,GAAG,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,SAAS,CAAC;oBACvD,SAAS;iBACV;gBAED,IAAI,eAAe,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;oBACrD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC1D,IAAI,IAAI,EAAE;wBACR,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBACtB;iBACF;gBAED,IAAI,eAAe,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;oBACxD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC5E,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE;wBAC9B,IAAI,CAAC,YAAY,GAAG,KAAK,GAAG,IAAI,CAAC;qBAClC;iBACF;aACF;YAED,4DAA4D;YAC5D,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACvC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAChF,OAAO,IAAI,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjC,OAAO,CAAC,GAAG,CAAC,MAAM,SAAS,CAAC,MAAM,0BAA0B,CAAC,CAAC;aAC/D;YACD,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE;gBACzB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;aACvD;SACF;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;SAChF;IACH,CAAC;IAEO,iBAAiB,CAAC,GAAW;QACnC,IAAI;YACF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE;gBACrC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,OAAO,KAAK,CAAC;aACnC;SACF;QAAC,MAAM;YACN,+DAA+D;SAChE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa;QAClB,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAC3C,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAClD,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAW;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,SAAS,CACnB;YACE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;SACvC,EACD,IAAI,EACJ,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,UAAU;QAChB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;CACF;AAhRD,kCAgRC;AAED;;;GAGG;AACH,MAAa,aAAa;IAA1B;QACU,UAAK,GAA+B,IAAI,GAAG,EAAE,CAAC;IA2GxD,CAAC;IAzGC;;OAEG;IACH,QAAQ,CAAC,KAAoB;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,mBAAmB;YACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEpD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;oBAC5B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;iBAC7B;gBACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtC,IAAI,KAAK;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC7B;SACF;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;QAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,OAAO,GAAwB,IAAI,GAAG,EAAE,CAAC;QAE/C,wBAAwB;QACxB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACxB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;aAC/C;SACF;QAED,oBAAoB;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;aACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE;YACb,0BAA0B;YAC1B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE;gBACvC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAC9C,IAAI,IAAI;oBAAE,OAAO,IAAI,CAAC;aACvB;YACD,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,OAAe;QACrC,uDAAuD;QACvD,OAAO,OAAO;aACX,WAAW,EAAE;aACb,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAC3D,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAY;QAC7B,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;YACxB,KAAK;YACL,GAAG;YACH,IAAI;YACJ,KAAK;YACL,IAAI;YACJ,KAAK;YACL,IAAI;YACJ,KAAK;YACL,KAAK;YACL,MAAM;YACN,MAAM;YACN,IAAI;YACJ,MAAM;YACN,KAAK;YACL,KAAK;YACL,IAAI;YACJ,MAAM;YACN,KAAK;YACL,MAAM;YACN,OAAO;YACP,OAAO;YACP,QAAQ;YACR,KAAK;YACL,OAAO;YACP,MAAM;YACN,KAAK;SACN,CAAC,CAAC;QACH,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,MAAM,IAAI,GAA6B,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;YACzC,IAAI,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SACzC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;CACF;AA5GD,sCA4GC;AAED,kBAAe;IACb,WAAW;IACX,aAAa;CACd,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Basic AMTP Crawler Example
|
|
4
|
+
*
|
|
5
|
+
* Run with: npm run crawler
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
const amtp_crawler_js_1 = require("../amtp-crawler.js");
|
|
9
|
+
async function main() {
|
|
10
|
+
const crawler = new amtp_crawler_js_1.AMTPCrawler({
|
|
11
|
+
baseUrl: process.env.AMTP_BASE_URL || "http://localhost:3000",
|
|
12
|
+
maxPages: 5,
|
|
13
|
+
maxDepth: 1,
|
|
14
|
+
respectRobotsTxt: false,
|
|
15
|
+
delays: { betweenRequests: 10, betweenDomains: 10 },
|
|
16
|
+
});
|
|
17
|
+
console.log("🕷️ AMTP Crawler Demo — (start server first for results)");
|
|
18
|
+
try {
|
|
19
|
+
const pages = await crawler.crawl();
|
|
20
|
+
console.log(`Crawled ${pages.length} pages`);
|
|
21
|
+
console.log("Crawl stats:", crawler.getStats?.() || "done");
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
console.log("Demo: server not running? Error:", err.message);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
main().catch(console.error);
|
|
28
|
+
//# sourceMappingURL=basic-crawler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basic-crawler.js","sourceRoot":"","sources":["../../../src/crawler/examples/basic-crawler.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;AAEH,wDAAiD;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,OAAO,GAAG,IAAI,6BAAW,CAAC;QAC9B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,uBAAuB;QAC7D,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,gBAAgB,EAAE,KAAK;QACvB,MAAM,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE;KACpD,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;IAEzE,IAAI;QACF,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,WAAW,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,EAAG,OAAe,CAAC,QAAQ,EAAE,EAAE,IAAI,MAAM,CAAC,CAAC;KACtE;IAAC,OAAO,GAAQ,EAAE;QACjB,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;KAC9D;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export { AMTPServer, AMTPMiddlewareFactory, SessionManager, ContentNegotiator, AMTPRequestParser, AMTPResponseBuilder } from "./server/amtp-server";
|
|
2
|
+
export type { AMTPServerConfig } from "./server/amtp-server";
|
|
3
|
+
export { AMTPMarkdownParser } from "./server/markdown-parser";
|
|
4
|
+
export { AMTPQLParser, AMTPQLSyntaxError, parseAMTPQL } from "./server/amtp-ql-parser";
|
|
5
|
+
export { AMTPQLExecutor, AMTPQLExecutionError, executeAMTPQL } from "./server/amtp-ql-executor";
|
|
6
|
+
export { NotificationBus, notificationBus } from "./server/notifications";
|
|
7
|
+
export { PermissionGuard } from "./server/permissions";
|
|
8
|
+
export type { PermissionCheckResult, PermissionGuardConfig } from "./server/permissions";
|
|
9
|
+
export { AMTPClient, AMTPMarkdownParser as ClientMarkdownParser, AutonomousAgent } from "./client/amtp-client";
|
|
10
|
+
export type { AMTPClientConfig } from "./client/amtp-client";
|
|
11
|
+
export { AMTPCrawler, SearchIndexer } from "./crawler/amtp-crawler";
|
|
12
|
+
export type { CrawlerConfig, CrawledPage, CrawlStats } from "./crawler/amtp-crawler";
|
|
13
|
+
export { secureRandomString, generateSecureSessionId, generateSecureRequestId, validateUrl, sanitizeHtml, validateTextField, InMemoryRateLimiter, isValidSessionId, csrfHeaderName, generateCsrfToken, isValidCsrfToken, sanitizeActionId, sanitizeEndpoint, sanitizeHttpMethod, sanitizeFreeText, SecurityError, readBodyWithLimit, DEFAULT_SESSION_TIMEOUT_MS, MAX_SESSION_LIFETIME_MS, DEFAULT_MAX_BODY_SIZE, AMTP_SECURITY_HEADERS, } from "./server/security";
|
|
14
|
+
export { AMTPVersion, MIMEType, HTTPMethod, StatusCode, AgentCapability, MarkdownNodeType, ParameterType, FormFieldType, LinkType, AuthMethod, StreamUpdateType, NotificationEventType, ErrorCode, AMTPError, } from "./types/amtp.types";
|
|
15
|
+
export type { AMTPRequestHeaders, AMTPPageRequest, AMTPActionRequest, AMTPBatchRequest, AMTPBatchResponse, AMTPQLQueryRequest, AMTPQLQueryResponse, AMTPResponseHeaders, AMTPSuccessResponse, AMTPErrorResponse, AMTPResponse, AMTPDocument, DocumentMetadata, MarkdownNode, Action, ActionParameter, ActionResult, RateLimit, BatchActionItem, Form, FormField, FormSubmission, FieldValidation, FormOption, Link, Pagination, Cursor, PageInfo, Session, LoginRequest, LoginResponse, ServerSentEvent, StreamUpdate, NotificationEvent, WebhookSubscription, RegisterWebhookRequest, RegisterWebhookResponse, StructuredData, StructuredDataType, ErrorDetail, Permission, Policy, PolicyCondition, Skill, PageRequestOptions, PageResponseOptions, AMTPContext, AMTPMiddleware, AMTPHandler, ContentNegotiation, AMTPQLQuery, AMTPQLResult, AMTPQLSelection, AMTPQLParsedQuery, } from "./types/amtp.types";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LinkType = exports.FormFieldType = exports.ParameterType = exports.MarkdownNodeType = exports.AgentCapability = exports.StatusCode = exports.HTTPMethod = exports.MIMEType = exports.AMTP_SECURITY_HEADERS = exports.DEFAULT_MAX_BODY_SIZE = exports.MAX_SESSION_LIFETIME_MS = exports.DEFAULT_SESSION_TIMEOUT_MS = exports.readBodyWithLimit = exports.SecurityError = exports.sanitizeFreeText = exports.sanitizeHttpMethod = exports.sanitizeEndpoint = exports.sanitizeActionId = exports.isValidCsrfToken = exports.generateCsrfToken = exports.csrfHeaderName = exports.isValidSessionId = exports.InMemoryRateLimiter = exports.validateTextField = exports.sanitizeHtml = exports.validateUrl = exports.generateSecureRequestId = exports.generateSecureSessionId = exports.secureRandomString = exports.SearchIndexer = exports.AMTPCrawler = exports.AutonomousAgent = exports.ClientMarkdownParser = exports.AMTPClient = exports.PermissionGuard = exports.notificationBus = exports.NotificationBus = exports.executeAMTPQL = exports.AMTPQLExecutionError = exports.AMTPQLExecutor = exports.parseAMTPQL = exports.AMTPQLSyntaxError = exports.AMTPQLParser = exports.AMTPMarkdownParser = exports.AMTPResponseBuilder = exports.AMTPRequestParser = exports.ContentNegotiator = exports.SessionManager = exports.AMTPMiddlewareFactory = exports.AMTPServer = void 0;
|
|
4
|
+
exports.AMTPError = exports.ErrorCode = exports.NotificationEventType = exports.StreamUpdateType = exports.AuthMethod = void 0;
|
|
5
|
+
var amtp_server_1 = require("./server/amtp-server");
|
|
6
|
+
Object.defineProperty(exports, "AMTPServer", { enumerable: true, get: function () { return amtp_server_1.AMTPServer; } });
|
|
7
|
+
Object.defineProperty(exports, "AMTPMiddlewareFactory", { enumerable: true, get: function () { return amtp_server_1.AMTPMiddlewareFactory; } });
|
|
8
|
+
Object.defineProperty(exports, "SessionManager", { enumerable: true, get: function () { return amtp_server_1.SessionManager; } });
|
|
9
|
+
Object.defineProperty(exports, "ContentNegotiator", { enumerable: true, get: function () { return amtp_server_1.ContentNegotiator; } });
|
|
10
|
+
Object.defineProperty(exports, "AMTPRequestParser", { enumerable: true, get: function () { return amtp_server_1.AMTPRequestParser; } });
|
|
11
|
+
Object.defineProperty(exports, "AMTPResponseBuilder", { enumerable: true, get: function () { return amtp_server_1.AMTPResponseBuilder; } });
|
|
12
|
+
var markdown_parser_1 = require("./server/markdown-parser");
|
|
13
|
+
Object.defineProperty(exports, "AMTPMarkdownParser", { enumerable: true, get: function () { return markdown_parser_1.AMTPMarkdownParser; } });
|
|
14
|
+
var amtp_ql_parser_1 = require("./server/amtp-ql-parser");
|
|
15
|
+
Object.defineProperty(exports, "AMTPQLParser", { enumerable: true, get: function () { return amtp_ql_parser_1.AMTPQLParser; } });
|
|
16
|
+
Object.defineProperty(exports, "AMTPQLSyntaxError", { enumerable: true, get: function () { return amtp_ql_parser_1.AMTPQLSyntaxError; } });
|
|
17
|
+
Object.defineProperty(exports, "parseAMTPQL", { enumerable: true, get: function () { return amtp_ql_parser_1.parseAMTPQL; } });
|
|
18
|
+
var amtp_ql_executor_1 = require("./server/amtp-ql-executor");
|
|
19
|
+
Object.defineProperty(exports, "AMTPQLExecutor", { enumerable: true, get: function () { return amtp_ql_executor_1.AMTPQLExecutor; } });
|
|
20
|
+
Object.defineProperty(exports, "AMTPQLExecutionError", { enumerable: true, get: function () { return amtp_ql_executor_1.AMTPQLExecutionError; } });
|
|
21
|
+
Object.defineProperty(exports, "executeAMTPQL", { enumerable: true, get: function () { return amtp_ql_executor_1.executeAMTPQL; } });
|
|
22
|
+
var notifications_1 = require("./server/notifications");
|
|
23
|
+
Object.defineProperty(exports, "NotificationBus", { enumerable: true, get: function () { return notifications_1.NotificationBus; } });
|
|
24
|
+
Object.defineProperty(exports, "notificationBus", { enumerable: true, get: function () { return notifications_1.notificationBus; } });
|
|
25
|
+
var permissions_1 = require("./server/permissions");
|
|
26
|
+
Object.defineProperty(exports, "PermissionGuard", { enumerable: true, get: function () { return permissions_1.PermissionGuard; } });
|
|
27
|
+
var amtp_client_1 = require("./client/amtp-client");
|
|
28
|
+
Object.defineProperty(exports, "AMTPClient", { enumerable: true, get: function () { return amtp_client_1.AMTPClient; } });
|
|
29
|
+
Object.defineProperty(exports, "ClientMarkdownParser", { enumerable: true, get: function () { return amtp_client_1.AMTPMarkdownParser; } });
|
|
30
|
+
Object.defineProperty(exports, "AutonomousAgent", { enumerable: true, get: function () { return amtp_client_1.AutonomousAgent; } });
|
|
31
|
+
var amtp_crawler_1 = require("./crawler/amtp-crawler");
|
|
32
|
+
Object.defineProperty(exports, "AMTPCrawler", { enumerable: true, get: function () { return amtp_crawler_1.AMTPCrawler; } });
|
|
33
|
+
Object.defineProperty(exports, "SearchIndexer", { enumerable: true, get: function () { return amtp_crawler_1.SearchIndexer; } });
|
|
34
|
+
var security_1 = require("./server/security");
|
|
35
|
+
Object.defineProperty(exports, "secureRandomString", { enumerable: true, get: function () { return security_1.secureRandomString; } });
|
|
36
|
+
Object.defineProperty(exports, "generateSecureSessionId", { enumerable: true, get: function () { return security_1.generateSecureSessionId; } });
|
|
37
|
+
Object.defineProperty(exports, "generateSecureRequestId", { enumerable: true, get: function () { return security_1.generateSecureRequestId; } });
|
|
38
|
+
Object.defineProperty(exports, "validateUrl", { enumerable: true, get: function () { return security_1.validateUrl; } });
|
|
39
|
+
Object.defineProperty(exports, "sanitizeHtml", { enumerable: true, get: function () { return security_1.sanitizeHtml; } });
|
|
40
|
+
Object.defineProperty(exports, "validateTextField", { enumerable: true, get: function () { return security_1.validateTextField; } });
|
|
41
|
+
Object.defineProperty(exports, "InMemoryRateLimiter", { enumerable: true, get: function () { return security_1.InMemoryRateLimiter; } });
|
|
42
|
+
Object.defineProperty(exports, "isValidSessionId", { enumerable: true, get: function () { return security_1.isValidSessionId; } });
|
|
43
|
+
Object.defineProperty(exports, "csrfHeaderName", { enumerable: true, get: function () { return security_1.csrfHeaderName; } });
|
|
44
|
+
Object.defineProperty(exports, "generateCsrfToken", { enumerable: true, get: function () { return security_1.generateCsrfToken; } });
|
|
45
|
+
Object.defineProperty(exports, "isValidCsrfToken", { enumerable: true, get: function () { return security_1.isValidCsrfToken; } });
|
|
46
|
+
Object.defineProperty(exports, "sanitizeActionId", { enumerable: true, get: function () { return security_1.sanitizeActionId; } });
|
|
47
|
+
Object.defineProperty(exports, "sanitizeEndpoint", { enumerable: true, get: function () { return security_1.sanitizeEndpoint; } });
|
|
48
|
+
Object.defineProperty(exports, "sanitizeHttpMethod", { enumerable: true, get: function () { return security_1.sanitizeHttpMethod; } });
|
|
49
|
+
Object.defineProperty(exports, "sanitizeFreeText", { enumerable: true, get: function () { return security_1.sanitizeFreeText; } });
|
|
50
|
+
Object.defineProperty(exports, "SecurityError", { enumerable: true, get: function () { return security_1.SecurityError; } });
|
|
51
|
+
Object.defineProperty(exports, "readBodyWithLimit", { enumerable: true, get: function () { return security_1.readBodyWithLimit; } });
|
|
52
|
+
Object.defineProperty(exports, "DEFAULT_SESSION_TIMEOUT_MS", { enumerable: true, get: function () { return security_1.DEFAULT_SESSION_TIMEOUT_MS; } });
|
|
53
|
+
Object.defineProperty(exports, "MAX_SESSION_LIFETIME_MS", { enumerable: true, get: function () { return security_1.MAX_SESSION_LIFETIME_MS; } });
|
|
54
|
+
Object.defineProperty(exports, "DEFAULT_MAX_BODY_SIZE", { enumerable: true, get: function () { return security_1.DEFAULT_MAX_BODY_SIZE; } });
|
|
55
|
+
Object.defineProperty(exports, "AMTP_SECURITY_HEADERS", { enumerable: true, get: function () { return security_1.AMTP_SECURITY_HEADERS; } });
|
|
56
|
+
var amtp_types_1 = require("./types/amtp.types");
|
|
57
|
+
Object.defineProperty(exports, "MIMEType", { enumerable: true, get: function () { return amtp_types_1.MIMEType; } });
|
|
58
|
+
Object.defineProperty(exports, "HTTPMethod", { enumerable: true, get: function () { return amtp_types_1.HTTPMethod; } });
|
|
59
|
+
Object.defineProperty(exports, "StatusCode", { enumerable: true, get: function () { return amtp_types_1.StatusCode; } });
|
|
60
|
+
Object.defineProperty(exports, "AgentCapability", { enumerable: true, get: function () { return amtp_types_1.AgentCapability; } });
|
|
61
|
+
Object.defineProperty(exports, "MarkdownNodeType", { enumerable: true, get: function () { return amtp_types_1.MarkdownNodeType; } });
|
|
62
|
+
Object.defineProperty(exports, "ParameterType", { enumerable: true, get: function () { return amtp_types_1.ParameterType; } });
|
|
63
|
+
Object.defineProperty(exports, "FormFieldType", { enumerable: true, get: function () { return amtp_types_1.FormFieldType; } });
|
|
64
|
+
Object.defineProperty(exports, "LinkType", { enumerable: true, get: function () { return amtp_types_1.LinkType; } });
|
|
65
|
+
Object.defineProperty(exports, "AuthMethod", { enumerable: true, get: function () { return amtp_types_1.AuthMethod; } });
|
|
66
|
+
Object.defineProperty(exports, "StreamUpdateType", { enumerable: true, get: function () { return amtp_types_1.StreamUpdateType; } });
|
|
67
|
+
Object.defineProperty(exports, "NotificationEventType", { enumerable: true, get: function () { return amtp_types_1.NotificationEventType; } });
|
|
68
|
+
Object.defineProperty(exports, "ErrorCode", { enumerable: true, get: function () { return amtp_types_1.ErrorCode; } });
|
|
69
|
+
Object.defineProperty(exports, "AMTPError", { enumerable: true, get: function () { return amtp_types_1.AMTPError; } });
|
|
70
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,oDAAoJ;AAA3I,yGAAA,UAAU,OAAA;AAAE,oHAAA,qBAAqB,OAAA;AAAE,6GAAA,cAAc,OAAA;AAAE,gHAAA,iBAAiB,OAAA;AAAE,gHAAA,iBAAiB,OAAA;AAAE,kHAAA,mBAAmB,OAAA;AAErH,4DAA8D;AAArD,qHAAA,kBAAkB,OAAA;AAC3B,0DAAuF;AAA9E,8GAAA,YAAY,OAAA;AAAE,mHAAA,iBAAiB,OAAA;AAAE,6GAAA,WAAW,OAAA;AACrD,8DAAgG;AAAvF,kHAAA,cAAc,OAAA;AAAE,wHAAA,oBAAoB,OAAA;AAAE,iHAAA,aAAa,OAAA;AAC5D,wDAA0E;AAAjE,gHAAA,eAAe,OAAA;AAAE,gHAAA,eAAe,OAAA;AACzC,oDAAuD;AAA9C,8GAAA,eAAe,OAAA;AAExB,oDAA+G;AAAtG,yGAAA,UAAU,OAAA;AAAE,mHAAA,kBAAkB,OAAwB;AAAE,8GAAA,eAAe,OAAA;AAEhF,uDAAoE;AAA3D,2GAAA,WAAW,OAAA;AAAE,6GAAA,aAAa,OAAA;AAEnC,8CAsB2B;AArBzB,8GAAA,kBAAkB,OAAA;AAClB,mHAAA,uBAAuB,OAAA;AACvB,mHAAA,uBAAuB,OAAA;AACvB,uGAAA,WAAW,OAAA;AACX,wGAAA,YAAY,OAAA;AACZ,6GAAA,iBAAiB,OAAA;AACjB,+GAAA,mBAAmB,OAAA;AACnB,4GAAA,gBAAgB,OAAA;AAChB,0GAAA,cAAc,OAAA;AACd,6GAAA,iBAAiB,OAAA;AACjB,4GAAA,gBAAgB,OAAA;AAChB,4GAAA,gBAAgB,OAAA;AAChB,4GAAA,gBAAgB,OAAA;AAChB,8GAAA,kBAAkB,OAAA;AAClB,4GAAA,gBAAgB,OAAA;AAChB,yGAAA,aAAa,OAAA;AACb,6GAAA,iBAAiB,OAAA;AACjB,sHAAA,0BAA0B,OAAA;AAC1B,mHAAA,uBAAuB,OAAA;AACvB,iHAAA,qBAAqB,OAAA;AACrB,iHAAA,qBAAqB,OAAA;AAEvB,iDAe4B;AAb1B,sGAAA,QAAQ,OAAA;AACR,wGAAA,UAAU,OAAA;AACV,wGAAA,UAAU,OAAA;AACV,6GAAA,eAAe,OAAA;AACf,8GAAA,gBAAgB,OAAA;AAChB,2GAAA,aAAa,OAAA;AACb,2GAAA,aAAa,OAAA;AACb,sGAAA,QAAQ,OAAA;AACR,wGAAA,UAAU,OAAA;AACV,8GAAA,gBAAgB,OAAA;AAChB,mHAAA,qBAAqB,OAAA;AACrB,uGAAA,SAAS,OAAA;AACT,uGAAA,SAAS,OAAA"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AMTP — Fastify Adapter
|
|
3
|
+
*
|
|
4
|
+
* Registers AMTP middleware on a Fastify instance: request parsing,
|
|
5
|
+
* session resolution, CORS, rate-limiting, body-size cap, security
|
|
6
|
+
* headers, and CSRF protection.
|
|
7
|
+
*
|
|
8
|
+
* Dependencies:
|
|
9
|
+
* npm install --save fastify
|
|
10
|
+
*
|
|
11
|
+
* Usage in your server entry point:
|
|
12
|
+
* import fastify from "fastify";
|
|
13
|
+
* import { fastifyAMTP, amtpReply, createAMTPFastifyApp } from "../src/server/adapters/fastify-adapter";
|
|
14
|
+
* import { AMTPResponseBuilder } from "../src/server/amtp-server";
|
|
15
|
+
*
|
|
16
|
+
* const app = fastifyAMTP.fastify({
|
|
17
|
+
* logger: { level: "info" },
|
|
18
|
+
* });
|
|
19
|
+
*
|
|
20
|
+
* await app.register(fastifyAMTP.plugin, {
|
|
21
|
+
* trustedOrigins: ["https://your-frontend.example.com"],
|
|
22
|
+
* });
|
|
23
|
+
*
|
|
24
|
+
* app.get("/hello", (req, reply) => {
|
|
25
|
+
* const doc = new AMTPResponseBuilder().build("# Hello\n\nWorld!");
|
|
26
|
+
* return reply.amtp(doc);
|
|
27
|
+
* });
|
|
28
|
+
*
|
|
29
|
+
* await app.listen({ port: 3000 });
|
|
30
|
+
*/
|
|
31
|
+
import { FastifyInstance, FastifyPluginAsync, FastifyPluginOptions } from "fastify";
|
|
32
|
+
import { AMTPResponseBuilder, SessionManager } from "../amtp-server.js";
|
|
33
|
+
import { AMTPMarkdownParser } from "../markdown-parser.js";
|
|
34
|
+
export { AMTPResponseBuilder, AMTPMarkdownParser, SessionManager };
|
|
35
|
+
/** Plugin options passed to fastifyAMTP.register(...) */
|
|
36
|
+
export interface FastifyAMTPOptions extends Partial<FastifyPluginOptions> {
|
|
37
|
+
/** List of allowed CORS origins. Empty = block all. Undefined = wildcard (dev only). */
|
|
38
|
+
trustedOrigins?: string[];
|
|
39
|
+
/** Max JSON body size in bytes. Default: 1 MB. */
|
|
40
|
+
maxRequestBodySize?: number;
|
|
41
|
+
/** Apply security headers (CSP, X-Frame-Options, etc.). Default: true. */
|
|
42
|
+
enableSecurityHeaders?: boolean;
|
|
43
|
+
/** Enable in-memory rate limiting. Default: true. */
|
|
44
|
+
enableRateLimit?: boolean;
|
|
45
|
+
/** Rate-limit window ms. Default: 60 000. */
|
|
46
|
+
rateLimitWindowMs?: number;
|
|
47
|
+
/** Max requests per window. Default: 60. */
|
|
48
|
+
rateLimitMaxRequests?: number;
|
|
49
|
+
/** Enabled CSRF protection. Default: true. */
|
|
50
|
+
csrfProtection?: boolean;
|
|
51
|
+
/** Shared SessionManager instance. */
|
|
52
|
+
sessionManager?: SessionManager;
|
|
53
|
+
}
|
|
54
|
+
declare module "fastify" {
|
|
55
|
+
interface FastifyRequest {
|
|
56
|
+
/** AMTP request headers extracted from the original HTTP request. */
|
|
57
|
+
amtpHeaders?: Record<string, string | string[] | null>;
|
|
58
|
+
/** AMTP session resolved from the incoming session id, if any. */
|
|
59
|
+
amtpSession?: any | null;
|
|
60
|
+
}
|
|
61
|
+
interface FastifyReply {
|
|
62
|
+
/**
|
|
63
|
+
* Short-hand to send an AMTP document.
|
|
64
|
+
* @param doc AMTP document string or `{ body, headers }` object
|
|
65
|
+
* @param sid Optional session id to set in the `X-Session-ID` response header
|
|
66
|
+
*/
|
|
67
|
+
amtp(doc: string | {
|
|
68
|
+
body: string;
|
|
69
|
+
headers: Record<string, string>;
|
|
70
|
+
}, sid?: string): FastifyReply;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export declare const plugin: FastifyPluginAsync;
|
|
74
|
+
/**
|
|
75
|
+
* NAMESPACE EXPORT — use for ergonomic named import.
|
|
76
|
+
*/
|
|
77
|
+
export declare const fastifyAMTP: {
|
|
78
|
+
plugin: FastifyPluginAsync;
|
|
79
|
+
createAMTPFastifyApp: typeof createAMTPFastifyApp;
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* Build and return a Fastify instance with AMTP and `reply.amtp()` pre-attached.
|
|
83
|
+
*
|
|
84
|
+
* @param opts AMTP plugin options
|
|
85
|
+
*/
|
|
86
|
+
export declare function createAMTPFastifyApp(opts?: FastifyAMTPOptions): FastifyInstance;
|