@knownagents/sdk 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/dist/KnownAgents.d.ts +1 -1
- package/dist/KnownAgents.d.ts.map +1 -1
- package/dist/KnownAgents.js +10 -9
- package/dist/headers.d.ts +2 -1
- package/dist/headers.d.ts.map +1 -1
- package/dist/headers.js +17 -2
- package/dist/paths.d.ts +1 -1
- package/dist/paths.d.ts.map +1 -1
- package/dist/paths.js +2 -2
- package/dist/types.d.ts +0 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/KnownAgents.ts +12 -11
- package/src/headers.ts +20 -1
- package/src/paths.ts +1 -1
- package/src/types.ts +0 -1
package/README.md
CHANGED
|
@@ -72,8 +72,7 @@ const mcpHandler = createMcpHandler(({ requestInfo }) => {
|
|
|
72
72
|
knownAgents.trackVisit(requestInfo, undefined, {
|
|
73
73
|
mcp: {
|
|
74
74
|
context: context,
|
|
75
|
-
result: result
|
|
76
|
-
toolName: "ping"
|
|
75
|
+
result: result
|
|
77
76
|
}
|
|
78
77
|
})
|
|
79
78
|
|
|
@@ -112,12 +111,12 @@ knownAgents.trackVisit(request, response, {
|
|
|
112
111
|
|
|
113
112
|
Generate a robots.txt that continuously updates with rules for new [crawlers, scrapers, AI agents, and other bots](https://knownagents.com/agents) as they're discovered.
|
|
114
113
|
|
|
115
|
-
Use the `
|
|
114
|
+
Use the `generateRobotsTxt` method. Select which `AgentType`s you want to block, and a string specifying which URLs are disallowed (e.g. `"/"` to disallow all paths).
|
|
116
115
|
|
|
117
116
|
```ts
|
|
118
117
|
import { AgentType } from "@knownagents/sdk"
|
|
119
118
|
|
|
120
|
-
const robotsTxt = await knownAgents.
|
|
119
|
+
const robotsTxt = await knownAgents.generateRobotsTxt([
|
|
121
120
|
AgentType.AIDataScraper,
|
|
122
121
|
AgentType.AIDataProvider,
|
|
123
122
|
AgentType.Scraper,
|
package/dist/KnownAgents.d.ts
CHANGED
|
@@ -44,7 +44,7 @@ export declare class KnownAgents {
|
|
|
44
44
|
* @returns A promise that resolves to the generated robots.txt content as a string.
|
|
45
45
|
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
46
46
|
*/
|
|
47
|
-
|
|
47
|
+
generateRobotsTxt(agentTypes: AgentType[], disallow?: string): Promise<string>;
|
|
48
48
|
/**
|
|
49
49
|
* Identifies and verifies a single agent from an HTTP request.
|
|
50
50
|
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"KnownAgents.d.ts","sourceRoot":"","sources":["../src/KnownAgents.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;AAI3D,OAAO,KAAK,EACR,SAAS,EACT,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EAEpB,MAAM,SAAS,CAAA;AAMhB;;GAEG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAQ;IACpD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,YAAY,CAA4B;IAChD,OAAO,CAAC,mBAAmB,CAAmC;IAE9D;;;;;OAKG;gBACS,WAAW,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAMjE;;;;;;;;OAQG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,QAAQ,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"KnownAgents.d.ts","sourceRoot":"","sources":["../src/KnownAgents.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;AAI3D,OAAO,KAAK,EACR,SAAS,EACT,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EAEpB,MAAM,SAAS,CAAA;AAMhB;;GAEG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAQ;IACpD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,YAAY,CAA4B;IAChD,OAAO,CAAC,mBAAmB,CAAmC;IAE9D;;;;;OAKG;gBACS,WAAW,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAMjE;;;;;;;;OAQG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,QAAQ,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,IAAI;IAsDvH;;;;;OAKG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBtB;;;;;;;;;OASG;IACG,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBzF;;;;;;;OAOG;IACG,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAsB5E;;;;;;;OAOG;IACG,cAAc,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAuBxF,OAAO,CAAC,YAAY;IAepB,OAAO,CAAC,aAAa;YAaP,UAAU;CAwB3B"}
|
package/dist/KnownAgents.js
CHANGED
|
@@ -6,7 +6,7 @@ const headers_1 = require("./headers");
|
|
|
6
6
|
const paths_1 = require("./paths");
|
|
7
7
|
const DEFAULT_FLUSH_INTERVAL_IN_MILLISECONDS = 10000;
|
|
8
8
|
const DEFAULT_FLUSH_QUEUE_SIZE = 1;
|
|
9
|
-
const NODE_PACKAGE_VERSION = "3.
|
|
9
|
+
const NODE_PACKAGE_VERSION = "3.1.0";
|
|
10
10
|
/**
|
|
11
11
|
* Server-side client for Known Agents analytics, robots.txt generation, and agent identification.
|
|
12
12
|
*/
|
|
@@ -35,10 +35,11 @@ class KnownAgents {
|
|
|
35
35
|
*/
|
|
36
36
|
trackVisit(request, response, options) {
|
|
37
37
|
const url = request instanceof Request ? new URL(request.url) : undefined;
|
|
38
|
-
const requestHeaders = request instanceof Request ? Object.fromEntries(request.headers
|
|
38
|
+
const requestHeaders = request instanceof Request ? Object.fromEntries(request.headers) : request.headers;
|
|
39
|
+
const mcpRequestName = (0, headers_1.getHeaderValue)(requestHeaders, "mcp-name");
|
|
39
40
|
const path = url ? `${url.pathname}${url.search}` : request.url;
|
|
40
41
|
const method = request.method;
|
|
41
|
-
if (!path || (0, paths_1.
|
|
42
|
+
if (!path || (0, paths_1.isPathExcluded)(path) || !method) {
|
|
42
43
|
return;
|
|
43
44
|
}
|
|
44
45
|
const created = new Date();
|
|
@@ -56,7 +57,7 @@ class KnownAgents {
|
|
|
56
57
|
response_headers: responseHeaders,
|
|
57
58
|
response_duration_in_milliseconds: responseDurationInMilliseconds,
|
|
58
59
|
mcp_request_method: mcp?.context?.mcpReq.method,
|
|
59
|
-
mcp_request_name:
|
|
60
|
+
mcp_request_name: mcpRequestName,
|
|
60
61
|
mcp_request_client_info_name: clientInfo?.name,
|
|
61
62
|
mcp_request_client_info_version: clientInfo?.version,
|
|
62
63
|
mcp_response_result_is_error: mcp?.result?.isError,
|
|
@@ -107,7 +108,7 @@ class KnownAgents {
|
|
|
107
108
|
* @returns A promise that resolves to the generated robots.txt content as a string.
|
|
108
109
|
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
109
110
|
*/
|
|
110
|
-
async
|
|
111
|
+
async generateRobotsTxt(agentTypes, disallow = "/") {
|
|
111
112
|
const response = await fetch("https://api.knownagents.com/robots-txts", {
|
|
112
113
|
method: "POST",
|
|
113
114
|
headers: {
|
|
@@ -143,7 +144,7 @@ class KnownAgents {
|
|
|
143
144
|
"Content-Type": "application/json"
|
|
144
145
|
},
|
|
145
146
|
body: JSON.stringify({
|
|
146
|
-
request_headers: (0, headers_1.
|
|
147
|
+
request_headers: (0, headers_1.filterHeaders)(request.headers),
|
|
147
148
|
request_ip: request.socket.remoteAddress,
|
|
148
149
|
request_path: request.url,
|
|
149
150
|
node_package_version: NODE_PACKAGE_VERSION
|
|
@@ -174,7 +175,7 @@ class KnownAgents {
|
|
|
174
175
|
body: JSON.stringify(requests.map((request) => {
|
|
175
176
|
return {
|
|
176
177
|
...request,
|
|
177
|
-
request_headers: (0, headers_1.
|
|
178
|
+
request_headers: (0, headers_1.filterHeaders)(request.request_headers),
|
|
178
179
|
node_package_version: NODE_PACKAGE_VERSION
|
|
179
180
|
};
|
|
180
181
|
}))
|
|
@@ -189,8 +190,8 @@ class KnownAgents {
|
|
|
189
190
|
enqueueVisit(request) {
|
|
190
191
|
this.visitQueue.push({
|
|
191
192
|
...request,
|
|
192
|
-
request_headers: (0, headers_1.
|
|
193
|
-
response_headers: request.response_headers ? (0, headers_1.
|
|
193
|
+
request_headers: (0, headers_1.filterHeaders)(request.request_headers),
|
|
194
|
+
response_headers: request.response_headers ? (0, headers_1.filterHeaders)(request.response_headers) : undefined,
|
|
194
195
|
});
|
|
195
196
|
if (this.visitQueue.length >= this.flushQueueSize) {
|
|
196
197
|
void this.flush();
|
package/dist/headers.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
import type { IncomingHttpHeaders, OutgoingHttpHeaders } from "http";
|
|
2
|
-
export declare function
|
|
2
|
+
export declare function getHeaderValue(headers: IncomingHttpHeaders | OutgoingHttpHeaders, headerName: string): string | undefined;
|
|
3
|
+
export declare function filterHeaders<Headers extends IncomingHttpHeaders | OutgoingHttpHeaders>(headers: Headers): Headers;
|
|
3
4
|
//# sourceMappingURL=headers.d.ts.map
|
package/dist/headers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../src/headers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,MAAM,CAAA;AA2BpE,wBAAgB,
|
|
1
|
+
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../src/headers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,MAAM,CAAA;AA2BpE,wBAAgB,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,mBAAmB,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAiBzH;AAED,wBAAgB,aAAa,CAAC,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAUlH"}
|
package/dist/headers.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.getHeaderValue = getHeaderValue;
|
|
4
|
+
exports.filterHeaders = filterHeaders;
|
|
4
5
|
const filteredHeaderNames = [
|
|
5
6
|
"authorization",
|
|
6
7
|
"proxy-authorization",
|
|
@@ -25,7 +26,21 @@ const filteredHeaderNames = [
|
|
|
25
26
|
"client-secret",
|
|
26
27
|
"private-token"
|
|
27
28
|
];
|
|
28
|
-
function
|
|
29
|
+
function getHeaderValue(headers, headerName) {
|
|
30
|
+
const normalizedHeaderName = headerName.toLowerCase();
|
|
31
|
+
const matchingHeaderName = Object.keys(headers).find((candidate) => {
|
|
32
|
+
return candidate.toLowerCase() === normalizedHeaderName;
|
|
33
|
+
});
|
|
34
|
+
if (!matchingHeaderName) {
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
const headerValue = headers[matchingHeaderName];
|
|
38
|
+
if (Array.isArray(headerValue)) {
|
|
39
|
+
return headerValue[0];
|
|
40
|
+
}
|
|
41
|
+
return headerValue?.toString();
|
|
42
|
+
}
|
|
43
|
+
function filterHeaders(headers) {
|
|
29
44
|
const filteredHeaders = { ...headers };
|
|
30
45
|
Object.keys(filteredHeaders).forEach((key) => {
|
|
31
46
|
if (filteredHeaderNames.includes(key.toLowerCase())) {
|
package/dist/paths.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare function
|
|
1
|
+
export declare function isPathExcluded(path: string): boolean;
|
|
2
2
|
//# sourceMappingURL=paths.d.ts.map
|
package/dist/paths.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../src/paths.ts"],"names":[],"mappings":"AAAA,wBAAgB,
|
|
1
|
+
{"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../src/paths.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAuEpD"}
|
package/dist/paths.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
function
|
|
3
|
+
exports.isPathExcluded = isPathExcluded;
|
|
4
|
+
function isPathExcluded(path) {
|
|
5
5
|
const pathWithoutQueryString = path.split(/[?#]/)[0];
|
|
6
6
|
const lowercasedPath = pathWithoutQueryString.toLowerCase();
|
|
7
7
|
const excludedPrefixes = [
|
package/dist/types.d.ts
CHANGED
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,MAAM,CAAA;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,yGAAyG;IACzG,2BAA2B,CAAC,EAAE,MAAM,CAAA;IACpC,sFAAsF;IACtF,cAAc,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B,8BAA8B,CAAC,EAAE,MAAM,CAAA;IACvC,GAAG,CAAC,EAAE;QACF,OAAO,CAAC,EAAE,aAAa,CAAA;QACvB,MAAM,CAAC,EAAE,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,MAAM,CAAA;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,yGAAyG;IACzG,2BAA2B,CAAC,EAAE,MAAM,CAAA;IACpC,sFAAsF;IACtF,cAAc,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B,8BAA8B,CAAC,EAAE,MAAM,CAAA;IACvC,GAAG,CAAC,EAAE;QACF,OAAO,CAAC,EAAE,aAAa,CAAA;QACvB,MAAM,CAAC,EAAE,cAAc,CAAA;KAC1B,CAAA;IACD,GAAG,CAAC,EAAE;QACF,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KACxC,CAAA;IACD,GAAG,CAAC,EAAE;QACF,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KACxC,CAAA;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAA;IACpB,2CAA2C;IAC3C,cAAc,EAAE,MAAM,CAAA;IACtB,8EAA8E;IAC9E,eAAe,EAAE,mBAAmB,CAAA;IACpC,oDAAoD;IACpD,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,+EAA+E;IAC/E,gBAAgB,CAAC,EAAE,mBAAmB,CAAA;IACtC,4CAA4C;IAC5C,iCAAiC,CAAC,EAAE,MAAM,CAAA;IAC1C,sCAAsC;IACtC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,gCAAgC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,kCAAkC;IAClC,4BAA4B,CAAC,EAAE,MAAM,CAAA;IACrC,qCAAqC;IACrC,+BAA+B,CAAC,EAAE,MAAM,CAAA;IACxC,kDAAkD;IAClD,4BAA4B,CAAC,EAAE,OAAO,CAAA;IACtC,4BAA4B;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC3C,4BAA4B;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC3C,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IAClC,4IAA4I;IAC5I,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,iQAAiQ;IACjQ,eAAe,EAAE,mBAAmB,CAAA;IACpC,0JAA0J;IAC1J,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;;OAMG;IACH,MAAM,EAAE,UAAU,GAAG,qBAAqB,GAAG,gBAAgB,GAAG,gBAAgB,CAAA;IAChF,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,iEAAiE;IACjE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kGAAkG;IAClG,2BAA2B,CAAC,EAAE,OAAO,CAAA;IACrC,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,iGAAiG;IACjG,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,CAAA;IACtE,sSAAsS;IACtS,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,KAAK,CACtB,gBAAgB,GACd,eAAe,GACf,aAAa,GACb,mBAAmB,GACnB,wBAAwB,CAC7B,CAAA;CACJ;AAED;;GAEG;AACH,oBAAY,SAAS;IACjB,OAAO,aAAa;IACpB,WAAW,iBAAiB;IAC5B,aAAa,oBAAoB;IACjC,cAAc,qBAAqB;IACnC,aAAa,oBAAoB;IACjC,eAAe,sBAAsB;IACrC,QAAQ,aAAa;IACrB,cAAc,oBAAoB;IAClC,eAAe,qBAAqB;IACpC,OAAO,YAAY;IACnB,oBAAoB,0BAA0B;IAC9C,OAAO,YAAY;IACnB,UAAU,gBAAgB;IAC1B,mBAAmB,0BAA0B;IAC7C,eAAe,qBAAqB;IACpC,mBAAmB,0BAA0B;CAChD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@knownagents/sdk",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.1.0",
|
|
4
4
|
"description": "The official Node.js SDK for Known Agents: AI agent analytics (bot traffic), LLM referral tracking, automatic robots.txt, agent identification",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
package/src/KnownAgents.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from "http"
|
|
2
2
|
import { CLIENT_INFO_META_KEY, type Implementation } from "@modelcontextprotocol/server"
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { filterHeaders, getHeaderValue } from "./headers"
|
|
4
|
+
import { isPathExcluded } from "./paths"
|
|
5
5
|
import type {
|
|
6
6
|
AgentType,
|
|
7
7
|
IdentificationRequest,
|
|
@@ -13,7 +13,7 @@ import type {
|
|
|
13
13
|
|
|
14
14
|
const DEFAULT_FLUSH_INTERVAL_IN_MILLISECONDS = 10000
|
|
15
15
|
const DEFAULT_FLUSH_QUEUE_SIZE = 1
|
|
16
|
-
const NODE_PACKAGE_VERSION = "3.
|
|
16
|
+
const NODE_PACKAGE_VERSION = "3.1.0"
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* Server-side client for Known Agents analytics, robots.txt generation, and agent identification.
|
|
@@ -49,11 +49,12 @@ export class KnownAgents {
|
|
|
49
49
|
*/
|
|
50
50
|
trackVisit(request: IncomingMessage | Request, response?: ServerResponse | Response, options?: TrackVisitOptions): void {
|
|
51
51
|
const url = request instanceof Request ? new URL(request.url) : undefined
|
|
52
|
-
const requestHeaders = request instanceof Request ? Object.fromEntries(request.headers
|
|
52
|
+
const requestHeaders = request instanceof Request ? Object.fromEntries(request.headers) : request.headers
|
|
53
|
+
const mcpRequestName = getHeaderValue(requestHeaders, "mcp-name")
|
|
53
54
|
const path = url ? `${url.pathname}${url.search}` : request.url
|
|
54
55
|
const method = request.method
|
|
55
56
|
|
|
56
|
-
if (!path ||
|
|
57
|
+
if (!path || isPathExcluded(path) || !method) {
|
|
57
58
|
return
|
|
58
59
|
}
|
|
59
60
|
|
|
@@ -76,7 +77,7 @@ export class KnownAgents {
|
|
|
76
77
|
response_headers: responseHeaders,
|
|
77
78
|
response_duration_in_milliseconds: responseDurationInMilliseconds,
|
|
78
79
|
mcp_request_method: mcp?.context?.mcpReq.method,
|
|
79
|
-
mcp_request_name:
|
|
80
|
+
mcp_request_name: mcpRequestName,
|
|
80
81
|
mcp_request_client_info_name: clientInfo?.name,
|
|
81
82
|
mcp_request_client_info_version: clientInfo?.version,
|
|
82
83
|
mcp_response_result_is_error: mcp?.result?.isError,
|
|
@@ -135,7 +136,7 @@ export class KnownAgents {
|
|
|
135
136
|
* @returns A promise that resolves to the generated robots.txt content as a string.
|
|
136
137
|
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
137
138
|
*/
|
|
138
|
-
async
|
|
139
|
+
async generateRobotsTxt(agentTypes: AgentType[], disallow: string = "/"): Promise<string> {
|
|
139
140
|
const response = await fetch("https://api.knownagents.com/robots-txts", {
|
|
140
141
|
method: "POST",
|
|
141
142
|
headers: {
|
|
@@ -172,7 +173,7 @@ export class KnownAgents {
|
|
|
172
173
|
"Content-Type": "application/json"
|
|
173
174
|
},
|
|
174
175
|
body: JSON.stringify({
|
|
175
|
-
request_headers:
|
|
176
|
+
request_headers: filterHeaders(request.headers),
|
|
176
177
|
request_ip: request.socket.remoteAddress,
|
|
177
178
|
request_path: request.url,
|
|
178
179
|
node_package_version: NODE_PACKAGE_VERSION
|
|
@@ -204,7 +205,7 @@ export class KnownAgents {
|
|
|
204
205
|
body: JSON.stringify(requests.map((request) => {
|
|
205
206
|
return {
|
|
206
207
|
...request,
|
|
207
|
-
request_headers:
|
|
208
|
+
request_headers: filterHeaders(request.request_headers),
|
|
208
209
|
node_package_version: NODE_PACKAGE_VERSION
|
|
209
210
|
}
|
|
210
211
|
}))
|
|
@@ -220,8 +221,8 @@ export class KnownAgents {
|
|
|
220
221
|
private enqueueVisit(request: VisitRequest): void {
|
|
221
222
|
this.visitQueue.push({
|
|
222
223
|
...request,
|
|
223
|
-
request_headers:
|
|
224
|
-
response_headers: request.response_headers ?
|
|
224
|
+
request_headers: filterHeaders(request.request_headers),
|
|
225
|
+
response_headers: request.response_headers ? filterHeaders(request.response_headers) : undefined,
|
|
225
226
|
})
|
|
226
227
|
|
|
227
228
|
if (this.visitQueue.length >= this.flushQueueSize) {
|
package/src/headers.ts
CHANGED
|
@@ -25,7 +25,26 @@ const filteredHeaderNames = [
|
|
|
25
25
|
"private-token"
|
|
26
26
|
]
|
|
27
27
|
|
|
28
|
-
export function
|
|
28
|
+
export function getHeaderValue(headers: IncomingHttpHeaders | OutgoingHttpHeaders, headerName: string): string | undefined {
|
|
29
|
+
const normalizedHeaderName = headerName.toLowerCase()
|
|
30
|
+
const matchingHeaderName = Object.keys(headers).find((candidate) => {
|
|
31
|
+
return candidate.toLowerCase() === normalizedHeaderName
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
if (!matchingHeaderName) {
|
|
35
|
+
return undefined
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const headerValue = headers[matchingHeaderName]
|
|
39
|
+
|
|
40
|
+
if (Array.isArray(headerValue)) {
|
|
41
|
+
return headerValue[0]
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return headerValue?.toString()
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function filterHeaders<Headers extends IncomingHttpHeaders | OutgoingHttpHeaders>(headers: Headers): Headers {
|
|
29
48
|
const filteredHeaders = { ...headers }
|
|
30
49
|
|
|
31
50
|
Object.keys(filteredHeaders).forEach((key) => {
|
package/src/paths.ts
CHANGED