@knownagents/sdk 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -9
- package/dist/index.d.ts +95 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +67 -24
- package/package.json +2 -2
- package/src/index.ts +129 -37
- package/tsconfig.json +2 -1
package/README.md
CHANGED
|
@@ -53,7 +53,6 @@ const knownAgents = new KnownAgents("YOUR_ACCESS_TOKEN")
|
|
|
53
53
|
|
|
54
54
|
app.use((req, res, next) => {
|
|
55
55
|
const start = Date.now()
|
|
56
|
-
|
|
57
56
|
res.on('finish', () => {
|
|
58
57
|
const duration = Date.now() - start
|
|
59
58
|
knownAgents.trackVisit(req, res, duration)
|
|
@@ -69,6 +68,33 @@ app.get("/", (req, res) => {
|
|
|
69
68
|
app.listen(3000, () => console.log("Server running on port 3000"))
|
|
70
69
|
```
|
|
71
70
|
|
|
71
|
+
### Batch Requests If Possible
|
|
72
|
+
|
|
73
|
+
For high-traffic websites, batch multiple visits together and send them periodically (e.g. every 30 seconds) using `trackVisits`:
|
|
74
|
+
|
|
75
|
+
```ts
|
|
76
|
+
import { VisitRequest } from "@knownagents/sdk"
|
|
77
|
+
|
|
78
|
+
const visits: VisitRequest[] = []
|
|
79
|
+
|
|
80
|
+
// Collect visits
|
|
81
|
+
visits.push({
|
|
82
|
+
request_path: req.url,
|
|
83
|
+
request_method: req.method,
|
|
84
|
+
request_headers: req.headers,
|
|
85
|
+
response_status_code: res.statusCode,
|
|
86
|
+
response_duration_in_milliseconds: duration,
|
|
87
|
+
created: new Date().toISOString()
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
// Send batch periodically
|
|
91
|
+
setInterval(() => {
|
|
92
|
+
if (visits.length > 0) {
|
|
93
|
+
knownAgents.trackVisits(visits.splice(0))
|
|
94
|
+
}
|
|
95
|
+
}, 30000)
|
|
96
|
+
```
|
|
97
|
+
|
|
72
98
|
### Test Your Integration
|
|
73
99
|
|
|
74
100
|
- Open your project's settings page
|
|
@@ -96,25 +122,45 @@ const robotsTxt = await knownAgents.generateRobotsTXT([
|
|
|
96
122
|
|
|
97
123
|
The return value is a plain text robots.txt string. Generate a `robotsTxt` periodically (e.g. once per day), then cache and serve it from your website's `/robots.txt` endpoint.
|
|
98
124
|
|
|
99
|
-
## How To Use Agent
|
|
125
|
+
## How To Use Agent Identification ([Full Docs](https://knownagents.com/docs/identification))
|
|
126
|
+
|
|
127
|
+
Use the `identifyAgent` and `identifyAgents` functions to [identify](https://knownagents.com/agents) and verify agents from network requests using Web Bot Auth (HTTP message signatures), IP matching, or other available methods. This can be useful for implementing access policies based on verified agent identity or enriching your own datasets.
|
|
100
128
|
|
|
101
|
-
|
|
129
|
+
### Identify a Single Request
|
|
102
130
|
|
|
103
|
-
Call `
|
|
131
|
+
Call `identifyAgent` with the incoming request.
|
|
104
132
|
|
|
105
133
|
```ts
|
|
106
|
-
const
|
|
134
|
+
const identification = await knownAgents.identifyAgent(request)
|
|
107
135
|
|
|
108
|
-
if (
|
|
136
|
+
if (identification.result === "verified") {
|
|
109
137
|
// Agent is legitimate
|
|
110
|
-
} else if (
|
|
138
|
+
} else if (identification.result === "verification_failed") {
|
|
111
139
|
// Agent is not legitimate
|
|
112
140
|
}
|
|
113
141
|
```
|
|
114
142
|
|
|
115
|
-
|
|
143
|
+
### Identify Multiple Requests
|
|
144
|
+
|
|
145
|
+
Use `identifyAgents` to identify multiple requests at once:
|
|
146
|
+
|
|
147
|
+
```ts
|
|
148
|
+
const identifications = await knownAgents.identifyAgents([
|
|
149
|
+
{
|
|
150
|
+
id: "request-1",
|
|
151
|
+
request_headers: request1.headers
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
id: "request-2",
|
|
155
|
+
request_headers: request2.headers
|
|
156
|
+
}
|
|
157
|
+
])
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
The functions return an object (or array of objects) with the following fields:
|
|
116
161
|
|
|
117
|
-
- `
|
|
162
|
+
- `id`: The identifier from the request (if provided)
|
|
163
|
+
- `result`: The identification result:
|
|
118
164
|
- `"verified"`: The agent is identified and verified
|
|
119
165
|
- `"verification_failed"`: The agent was identified but could not be verified
|
|
120
166
|
- `"unknown_agent"`: The agent is not in our database
|
package/dist/index.d.ts
CHANGED
|
@@ -1,31 +1,77 @@
|
|
|
1
1
|
import { IncomingMessage, ServerResponse } from "http";
|
|
2
|
+
/**
|
|
3
|
+
* A framework-agnostic HTTP request interface.
|
|
4
|
+
* Used as a convenience type for single-request methods.
|
|
5
|
+
*/
|
|
2
6
|
export interface Request {
|
|
3
|
-
/** The URL
|
|
7
|
+
/** The URL path and query string (e.g. "/about?foo=bar") */
|
|
4
8
|
path: string;
|
|
5
|
-
/** The
|
|
9
|
+
/** The HTTP method (e.g. "GET", "POST") */
|
|
6
10
|
method: string;
|
|
7
|
-
/** The request
|
|
11
|
+
/** The HTTP request headers */
|
|
8
12
|
headers: Record<string, string | string[] | undefined>;
|
|
9
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* A minimal HTTP response interface.
|
|
16
|
+
* Used as a convenience type for single-request methods.
|
|
17
|
+
*/
|
|
10
18
|
export interface Response {
|
|
11
|
-
/** The HTTP response status code */
|
|
19
|
+
/** The HTTP response status code (e.g. 200, 404) */
|
|
12
20
|
statusCode: number;
|
|
13
21
|
}
|
|
14
22
|
/**
|
|
15
|
-
*
|
|
23
|
+
* A visit tracking request for the Known Agents API.
|
|
24
|
+
* Used for batch visit tracking with `trackVisits()`.
|
|
25
|
+
*/
|
|
26
|
+
export interface VisitRequest {
|
|
27
|
+
/** The URL path of the request (e.g. "/about?foo=bar") */
|
|
28
|
+
request_path: string;
|
|
29
|
+
/** The HTTP method (e.g. "GET", "POST") */
|
|
30
|
+
request_method: string;
|
|
31
|
+
/** The HTTP request headers */
|
|
32
|
+
request_headers: Record<string, string | string[] | undefined>;
|
|
33
|
+
/** The HTTP response status code (e.g. 200, 404) */
|
|
34
|
+
response_status_code?: number;
|
|
35
|
+
/** The HTTP response headers */
|
|
36
|
+
response_headers?: Record<string, string | string[] | undefined>;
|
|
37
|
+
/** The response duration in milliseconds */
|
|
38
|
+
response_duration_in_milliseconds?: number;
|
|
39
|
+
/** The timestamp when the visit occurred (ISO 8601 format) */
|
|
40
|
+
created?: string;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* An agent identification request for the Known Agents API.
|
|
44
|
+
* Used for batch agent identification with `identifyAgents()`.
|
|
45
|
+
*/
|
|
46
|
+
export interface IdentificationRequest {
|
|
47
|
+
/** An identifier that will be echoed back in the response to match requests with results */
|
|
48
|
+
id?: string;
|
|
49
|
+
/** The HTTP request headers */
|
|
50
|
+
request_headers: Record<string, string | string[] | undefined>;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The result of an agent identification request.
|
|
16
54
|
*/
|
|
17
|
-
export interface
|
|
18
|
-
/** The
|
|
55
|
+
export interface IdentificationResult {
|
|
56
|
+
/** The identifier from the request (if provided) */
|
|
57
|
+
id?: string;
|
|
58
|
+
/**
|
|
59
|
+
* The identification result:
|
|
60
|
+
* - "verified": The agent was identified and verified
|
|
61
|
+
* - "verification_failed": The agent was identified but failed verification (spoofed)
|
|
62
|
+
* - "unknown_agent": The request is from an unrecognized agent or human
|
|
63
|
+
* - "not_verifiable": The agent was identified but cannot be verified
|
|
64
|
+
*/
|
|
19
65
|
result: "verified" | "verification_failed" | "unknown_agent" | "not_verifiable";
|
|
20
|
-
/** The unique ID of the agent
|
|
66
|
+
/** The unique ID of the identified agent */
|
|
21
67
|
agent_id?: string;
|
|
22
|
-
/** The name of the agent (e.g. "Googlebot")
|
|
68
|
+
/** The name of the identified agent (e.g. "Googlebot") */
|
|
23
69
|
agent_token?: string;
|
|
24
|
-
/** The documentation URL
|
|
70
|
+
/** The documentation URL for the identified agent */
|
|
25
71
|
agent_url?: string;
|
|
26
|
-
/** The type of agent (e.g. "AI Agent"
|
|
72
|
+
/** The type of the identified agent (e.g. "AI Agent", "Search Engine Crawler") */
|
|
27
73
|
agent_type_name?: string;
|
|
28
|
-
/** The company behind the agent (e.g. "Google")
|
|
74
|
+
/** The company behind the identified agent (e.g. "Google") */
|
|
29
75
|
operator_name?: string;
|
|
30
76
|
}
|
|
31
77
|
/**
|
|
@@ -34,6 +80,8 @@ export interface VerificationResult {
|
|
|
34
80
|
export declare enum AgentType {
|
|
35
81
|
AIAgent = "AI Agent",
|
|
36
82
|
AIAssistant = "AI Assistant",
|
|
83
|
+
AICodingAgent = "AI Coding Agent",
|
|
84
|
+
AIDataProvider = "AI Data Provider",
|
|
37
85
|
AIDataScraper = "AI Data Scraper",
|
|
38
86
|
AISearchCrawler = "AI Search Crawler",
|
|
39
87
|
Archiver = "Archiver",
|
|
@@ -56,31 +104,51 @@ export declare class KnownAgents {
|
|
|
56
104
|
*/
|
|
57
105
|
constructor(accessToken: string);
|
|
58
106
|
/**
|
|
59
|
-
* Tracks
|
|
107
|
+
* Tracks a single agent visit for analytics.
|
|
108
|
+
* This method is non-blocking and handles errors internally.
|
|
60
109
|
*
|
|
61
|
-
* @param request - The incoming request.
|
|
62
|
-
* @param response - The outgoing response (optional).
|
|
63
|
-
* @param responseDurationInMilliseconds - The
|
|
110
|
+
* @param request - The incoming HTTP request (Request or Node.js IncomingMessage).
|
|
111
|
+
* @param response - The outgoing HTTP response (optional).
|
|
112
|
+
* @param responseDurationInMilliseconds - The time taken to process the request in milliseconds (optional).
|
|
64
113
|
*/
|
|
65
114
|
trackVisit(request: Request | IncomingMessage, response?: Response | ServerResponse, responseDurationInMilliseconds?: number): void;
|
|
66
115
|
/**
|
|
67
|
-
*
|
|
68
|
-
*
|
|
116
|
+
* Tracks multiple agent visits in a batch for analytics.
|
|
117
|
+
* This method is non-blocking and handles errors internally.
|
|
118
|
+
* Recommended for high-traffic websites - batch visits and send periodically (e.g. every 30 seconds).
|
|
69
119
|
*
|
|
70
|
-
* @param
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
120
|
+
* @param visits - An array of visit tracking requests.
|
|
121
|
+
*/
|
|
122
|
+
trackVisits(visits: VisitRequest[]): void;
|
|
123
|
+
/**
|
|
124
|
+
* Generates a robots.txt file that disallows the specified agent types.
|
|
125
|
+
* The generated file stays up-to-date with all current and future agents in the specified categories.
|
|
126
|
+
* Cache and serve the result at your website's `/robots.txt` endpoint.
|
|
127
|
+
*
|
|
128
|
+
* @param agentTypes - An array of agent types to disallow.
|
|
129
|
+
* @param disallow - The path to disallow (default: "/" for all paths).
|
|
130
|
+
* @returns A promise that resolves to the generated robots.txt content as a string.
|
|
131
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
74
132
|
*/
|
|
75
133
|
generateRobotsTXT(agentTypes: AgentType[], disallow?: string): Promise<string>;
|
|
76
134
|
/**
|
|
77
|
-
*
|
|
135
|
+
* Identifies and verifies a single agent from an HTTP request.
|
|
136
|
+
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
137
|
+
*
|
|
138
|
+
* @param request - The incoming HTTP request (Request or Node.js IncomingMessage).
|
|
139
|
+
* @returns A promise that resolves to the identification result.
|
|
140
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
141
|
+
*/
|
|
142
|
+
identifyAgent(request: Request | IncomingMessage): Promise<IdentificationResult>;
|
|
143
|
+
/**
|
|
144
|
+
* Identifies and verifies multiple agents from HTTP requests in a batch.
|
|
145
|
+
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
78
146
|
*
|
|
79
|
-
* @param
|
|
80
|
-
* @returns A promise that resolves to
|
|
81
|
-
* @throws If the API call fails or returns a non-200 status.
|
|
147
|
+
* @param requests - An array of identification requests.
|
|
148
|
+
* @returns A promise that resolves to an array of identification results.
|
|
149
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
82
150
|
*/
|
|
83
|
-
|
|
151
|
+
identifyAgents(requests: IdentificationRequest[]): Promise<IdentificationResult[]>;
|
|
84
152
|
private filterHeaders;
|
|
85
153
|
}
|
|
86
154
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;AAEtD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACpB,4DAA4D;IAC5D,IAAI,EAAE,MAAM,CAAA;IACZ,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAA;IACd,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;CACzD;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACrB,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAA;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IACzB,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAA;IACpB,2CAA2C;IAC3C,cAAc,EAAE,MAAM,CAAA;IACtB,+BAA+B;IAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;IAC9D,oDAAoD;IACpD,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,gCAAgC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;IAChE,4CAA4C;IAC5C,iCAAiC,CAAC,EAAE,MAAM,CAAA;IAC1C,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IAClC,4FAA4F;IAC5F,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,+BAA+B;IAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;CACjE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,oDAAoD;IACpD,EAAE,CAAC,EAAE,MAAM,CAAA;IACX;;;;;;OAMG;IACH,MAAM,EAAE,UAAU,GAAG,qBAAqB,GAAG,eAAe,GAAG,gBAAgB,CAAA;IAC/E,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kFAAkF;IAClF,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,oBAAY,SAAS;IACjB,OAAO,aAAa;IACpB,WAAW,iBAAiB;IAC5B,aAAa,oBAAoB;IACjC,cAAc,qBAAqB;IACnC,aAAa,oBAAoB;IACjC,eAAe,sBAAsB;IACrC,QAAQ,aAAa;IACrB,eAAe,qBAAqB;IACpC,OAAO,YAAY;IACnB,aAAa,mBAAmB;IAChC,oBAAoB,0BAA0B;IAC9C,OAAO,YAAY;IACnB,UAAU,gBAAgB;IAC1B,mBAAmB,0BAA0B;IAC7C,eAAe,qBAAqB;IACpC,mBAAmB,0BAA0B;CAChD;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IAEpC;;;;OAIG;gBACS,WAAW,EAAE,MAAM;IAI/B;;;;;;;OAOG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,EAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,cAAc,EAAE,8BAA8B,CAAC,EAAE,MAAM,GAAG,IAAI;IAoBnI;;;;;;OAMG;IACH,WAAW,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI;IAazC;;;;;;;;;OASG;IACG,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBzF;;;;;;;OAOG;IACG,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAmBtF;;;;;;;OAOG;IACG,cAAc,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAmBxF,OAAO,CAAC,aAAa;CAcxB"}
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.KnownAgents = exports.AgentType = void 0;
|
|
4
|
-
const NODE_PACKAGE_VERSION = "1.8.0";
|
|
5
4
|
/**
|
|
6
5
|
* Agent types that can be controlled with robots.txt rules.
|
|
7
6
|
*/
|
|
@@ -9,6 +8,8 @@ var AgentType;
|
|
|
9
8
|
(function (AgentType) {
|
|
10
9
|
AgentType["AIAgent"] = "AI Agent";
|
|
11
10
|
AgentType["AIAssistant"] = "AI Assistant";
|
|
11
|
+
AgentType["AICodingAgent"] = "AI Coding Agent";
|
|
12
|
+
AgentType["AIDataProvider"] = "AI Data Provider";
|
|
12
13
|
AgentType["AIDataScraper"] = "AI Data Scraper";
|
|
13
14
|
AgentType["AISearchCrawler"] = "AI Search Crawler";
|
|
14
15
|
AgentType["Archiver"] = "Archiver";
|
|
@@ -32,11 +33,12 @@ class KnownAgents {
|
|
|
32
33
|
this.accessToken = accessToken;
|
|
33
34
|
}
|
|
34
35
|
/**
|
|
35
|
-
* Tracks
|
|
36
|
+
* Tracks a single agent visit for analytics.
|
|
37
|
+
* This method is non-blocking and handles errors internally.
|
|
36
38
|
*
|
|
37
|
-
* @param request - The incoming request.
|
|
38
|
-
* @param response - The outgoing response (optional).
|
|
39
|
-
* @param responseDurationInMilliseconds - The
|
|
39
|
+
* @param request - The incoming HTTP request (Request or Node.js IncomingMessage).
|
|
40
|
+
* @param response - The outgoing HTTP response (optional).
|
|
41
|
+
* @param responseDurationInMilliseconds - The time taken to process the request in milliseconds (optional).
|
|
40
42
|
*/
|
|
41
43
|
trackVisit(request, response, responseDurationInMilliseconds) {
|
|
42
44
|
fetch("https://api.knownagents.com/visits", {
|
|
@@ -51,7 +53,6 @@ class KnownAgents {
|
|
|
51
53
|
request_headers: this.filterHeaders(request.headers),
|
|
52
54
|
response_status_code: response?.statusCode,
|
|
53
55
|
response_duration_in_milliseconds: responseDurationInMilliseconds,
|
|
54
|
-
node_package_version: NODE_PACKAGE_VERSION,
|
|
55
56
|
created: new Date().toISOString()
|
|
56
57
|
})
|
|
57
58
|
}).catch(error => {
|
|
@@ -59,13 +60,33 @@ class KnownAgents {
|
|
|
59
60
|
});
|
|
60
61
|
}
|
|
61
62
|
/**
|
|
62
|
-
*
|
|
63
|
-
*
|
|
63
|
+
* Tracks multiple agent visits in a batch for analytics.
|
|
64
|
+
* This method is non-blocking and handles errors internally.
|
|
65
|
+
* Recommended for high-traffic websites - batch visits and send periodically (e.g. every 30 seconds).
|
|
64
66
|
*
|
|
65
|
-
* @param
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
* @param visits - An array of visit tracking requests.
|
|
68
|
+
*/
|
|
69
|
+
trackVisits(visits) {
|
|
70
|
+
fetch("https://api.knownagents.com/visits", {
|
|
71
|
+
method: "POST",
|
|
72
|
+
headers: {
|
|
73
|
+
"Authorization": `Bearer ${this.accessToken}`,
|
|
74
|
+
"Content-Type": "application/json"
|
|
75
|
+
},
|
|
76
|
+
body: JSON.stringify(visits)
|
|
77
|
+
}).catch(error => {
|
|
78
|
+
console.error(`Known Agents failed to track visits: ${error.message}`);
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Generates a robots.txt file that disallows the specified agent types.
|
|
83
|
+
* The generated file stays up-to-date with all current and future agents in the specified categories.
|
|
84
|
+
* Cache and serve the result at your website's `/robots.txt` endpoint.
|
|
85
|
+
*
|
|
86
|
+
* @param agentTypes - An array of agent types to disallow.
|
|
87
|
+
* @param disallow - The path to disallow (default: "/" for all paths).
|
|
88
|
+
* @returns A promise that resolves to the generated robots.txt content as a string.
|
|
89
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
69
90
|
*/
|
|
70
91
|
async generateRobotsTXT(agentTypes, disallow = "/") {
|
|
71
92
|
const response = await fetch("https://api.knownagents.com/robots-txts", {
|
|
@@ -76,8 +97,7 @@ class KnownAgents {
|
|
|
76
97
|
},
|
|
77
98
|
body: JSON.stringify({
|
|
78
99
|
agent_types: agentTypes,
|
|
79
|
-
disallow: disallow
|
|
80
|
-
node_package_version: NODE_PACKAGE_VERSION
|
|
100
|
+
disallow: disallow
|
|
81
101
|
})
|
|
82
102
|
});
|
|
83
103
|
if (response.ok) {
|
|
@@ -88,30 +108,53 @@ class KnownAgents {
|
|
|
88
108
|
}
|
|
89
109
|
}
|
|
90
110
|
/**
|
|
91
|
-
*
|
|
111
|
+
* Identifies and verifies a single agent from an HTTP request.
|
|
112
|
+
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
92
113
|
*
|
|
93
|
-
* @param request - The incoming request.
|
|
94
|
-
* @returns A promise that resolves to the
|
|
95
|
-
* @throws If the API call fails or returns a non-200 status.
|
|
114
|
+
* @param request - The incoming HTTP request (Request or Node.js IncomingMessage).
|
|
115
|
+
* @returns A promise that resolves to the identification result.
|
|
116
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
96
117
|
*/
|
|
97
|
-
async
|
|
98
|
-
const response = await fetch("https://api.knownagents.com/agent-
|
|
118
|
+
async identifyAgent(request) {
|
|
119
|
+
const response = await fetch("https://api.knownagents.com/agent-identifications", {
|
|
99
120
|
method: "POST",
|
|
100
121
|
headers: {
|
|
101
122
|
"Authorization": `Bearer ${this.accessToken}`,
|
|
102
123
|
"Content-Type": "application/json"
|
|
103
124
|
},
|
|
104
125
|
body: JSON.stringify({
|
|
105
|
-
|
|
106
|
-
request_headers: this.filterHeaders(request.headers),
|
|
107
|
-
node_package_version: NODE_PACKAGE_VERSION
|
|
126
|
+
request_headers: this.filterHeaders(request.headers)
|
|
108
127
|
})
|
|
109
128
|
});
|
|
110
129
|
if (response.ok) {
|
|
111
130
|
return await response.json();
|
|
112
131
|
}
|
|
113
132
|
else {
|
|
114
|
-
throw new Error(`Known Agents failed to
|
|
133
|
+
throw new Error(`Known Agents failed to identify agent: ${response.status} ${response.statusText}`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Identifies and verifies multiple agents from HTTP requests in a batch.
|
|
138
|
+
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
139
|
+
*
|
|
140
|
+
* @param requests - An array of identification requests.
|
|
141
|
+
* @returns A promise that resolves to an array of identification results.
|
|
142
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
143
|
+
*/
|
|
144
|
+
async identifyAgents(requests) {
|
|
145
|
+
const response = await fetch("https://api.knownagents.com/agent-identifications", {
|
|
146
|
+
method: "POST",
|
|
147
|
+
headers: {
|
|
148
|
+
"Authorization": `Bearer ${this.accessToken}`,
|
|
149
|
+
"Content-Type": "application/json"
|
|
150
|
+
},
|
|
151
|
+
body: JSON.stringify(requests)
|
|
152
|
+
});
|
|
153
|
+
if (response.ok) {
|
|
154
|
+
return await response.json();
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
throw new Error(`Known Agents failed to identify agents: ${response.status} ${response.statusText}`);
|
|
115
158
|
}
|
|
116
159
|
}
|
|
117
160
|
// Helpers
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@knownagents/sdk",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"description": "The official Node.js SDK for Known Agents: AI agent analytics (bot traffic), LLM referral tracking, automatic robots.txt, agent verification",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"node": ">=18"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
|
-
"@types/node": "^22.
|
|
55
|
+
"@types/node": "^22.19.17",
|
|
56
56
|
"typescript": "^5.8.2"
|
|
57
57
|
}
|
|
58
58
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,36 +1,82 @@
|
|
|
1
1
|
import { IncomingMessage, ServerResponse } from "http"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
/**
|
|
4
|
+
* A framework-agnostic HTTP request interface.
|
|
5
|
+
* Used as a convenience type for single-request methods.
|
|
6
|
+
*/
|
|
5
7
|
export interface Request {
|
|
6
|
-
/** The URL
|
|
8
|
+
/** The URL path and query string (e.g. "/about?foo=bar") */
|
|
7
9
|
path: string
|
|
8
|
-
/** The
|
|
10
|
+
/** The HTTP method (e.g. "GET", "POST") */
|
|
9
11
|
method: string
|
|
10
|
-
/** The request
|
|
12
|
+
/** The HTTP request headers */
|
|
11
13
|
headers: Record<string, string | string[] | undefined>
|
|
12
14
|
}
|
|
13
15
|
|
|
16
|
+
/**
|
|
17
|
+
* A minimal HTTP response interface.
|
|
18
|
+
* Used as a convenience type for single-request methods.
|
|
19
|
+
*/
|
|
14
20
|
export interface Response {
|
|
15
|
-
/** The HTTP response status code */
|
|
21
|
+
/** The HTTP response status code (e.g. 200, 404) */
|
|
16
22
|
statusCode: number
|
|
17
23
|
}
|
|
18
24
|
|
|
19
25
|
/**
|
|
20
|
-
*
|
|
26
|
+
* A visit tracking request for the Known Agents API.
|
|
27
|
+
* Used for batch visit tracking with `trackVisits()`.
|
|
28
|
+
*/
|
|
29
|
+
export interface VisitRequest {
|
|
30
|
+
/** The URL path of the request (e.g. "/about?foo=bar") */
|
|
31
|
+
request_path: string
|
|
32
|
+
/** The HTTP method (e.g. "GET", "POST") */
|
|
33
|
+
request_method: string
|
|
34
|
+
/** The HTTP request headers */
|
|
35
|
+
request_headers: Record<string, string | string[] | undefined>
|
|
36
|
+
/** The HTTP response status code (e.g. 200, 404) */
|
|
37
|
+
response_status_code?: number
|
|
38
|
+
/** The HTTP response headers */
|
|
39
|
+
response_headers?: Record<string, string | string[] | undefined>
|
|
40
|
+
/** The response duration in milliseconds */
|
|
41
|
+
response_duration_in_milliseconds?: number
|
|
42
|
+
/** The timestamp when the visit occurred (ISO 8601 format) */
|
|
43
|
+
created?: string
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* An agent identification request for the Known Agents API.
|
|
48
|
+
* Used for batch agent identification with `identifyAgents()`.
|
|
21
49
|
*/
|
|
22
|
-
export interface
|
|
23
|
-
/**
|
|
50
|
+
export interface IdentificationRequest {
|
|
51
|
+
/** An identifier that will be echoed back in the response to match requests with results */
|
|
52
|
+
id?: string
|
|
53
|
+
/** The HTTP request headers */
|
|
54
|
+
request_headers: Record<string, string | string[] | undefined>
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The result of an agent identification request.
|
|
59
|
+
*/
|
|
60
|
+
export interface IdentificationResult {
|
|
61
|
+
/** The identifier from the request (if provided) */
|
|
62
|
+
id?: string
|
|
63
|
+
/**
|
|
64
|
+
* The identification result:
|
|
65
|
+
* - "verified": The agent was identified and verified
|
|
66
|
+
* - "verification_failed": The agent was identified but failed verification (spoofed)
|
|
67
|
+
* - "unknown_agent": The request is from an unrecognized agent or human
|
|
68
|
+
* - "not_verifiable": The agent was identified but cannot be verified
|
|
69
|
+
*/
|
|
24
70
|
result: "verified" | "verification_failed" | "unknown_agent" | "not_verifiable"
|
|
25
|
-
/** The unique ID of the agent
|
|
71
|
+
/** The unique ID of the identified agent */
|
|
26
72
|
agent_id?: string
|
|
27
|
-
/** The name of the agent (e.g. "Googlebot")
|
|
73
|
+
/** The name of the identified agent (e.g. "Googlebot") */
|
|
28
74
|
agent_token?: string
|
|
29
|
-
/** The documentation URL
|
|
75
|
+
/** The documentation URL for the identified agent */
|
|
30
76
|
agent_url?: string
|
|
31
|
-
/** The type of agent (e.g. "AI Agent"
|
|
77
|
+
/** The type of the identified agent (e.g. "AI Agent", "Search Engine Crawler") */
|
|
32
78
|
agent_type_name?: string
|
|
33
|
-
/** The company behind the agent (e.g. "Google")
|
|
79
|
+
/** The company behind the identified agent (e.g. "Google") */
|
|
34
80
|
operator_name?: string
|
|
35
81
|
}
|
|
36
82
|
|
|
@@ -40,6 +86,8 @@ export interface VerificationResult {
|
|
|
40
86
|
export enum AgentType {
|
|
41
87
|
AIAgent = "AI Agent",
|
|
42
88
|
AIAssistant = "AI Assistant",
|
|
89
|
+
AICodingAgent = "AI Coding Agent",
|
|
90
|
+
AIDataProvider = "AI Data Provider",
|
|
43
91
|
AIDataScraper = "AI Data Scraper",
|
|
44
92
|
AISearchCrawler = "AI Search Crawler",
|
|
45
93
|
Archiver = "Archiver",
|
|
@@ -67,11 +115,12 @@ export class KnownAgents {
|
|
|
67
115
|
}
|
|
68
116
|
|
|
69
117
|
/**
|
|
70
|
-
* Tracks
|
|
118
|
+
* Tracks a single agent visit for analytics.
|
|
119
|
+
* This method is non-blocking and handles errors internally.
|
|
71
120
|
*
|
|
72
|
-
* @param request - The incoming request.
|
|
73
|
-
* @param response - The outgoing response (optional).
|
|
74
|
-
* @param responseDurationInMilliseconds - The
|
|
121
|
+
* @param request - The incoming HTTP request (Request or Node.js IncomingMessage).
|
|
122
|
+
* @param response - The outgoing HTTP response (optional).
|
|
123
|
+
* @param responseDurationInMilliseconds - The time taken to process the request in milliseconds (optional).
|
|
75
124
|
*/
|
|
76
125
|
trackVisit(request: Request | IncomingMessage, response?: Response | ServerResponse, responseDurationInMilliseconds?: number): void {
|
|
77
126
|
fetch("https://api.knownagents.com/visits", {
|
|
@@ -86,22 +135,42 @@ export class KnownAgents {
|
|
|
86
135
|
request_headers: this.filterHeaders(request.headers),
|
|
87
136
|
response_status_code: response?.statusCode,
|
|
88
137
|
response_duration_in_milliseconds: responseDurationInMilliseconds,
|
|
89
|
-
node_package_version: NODE_PACKAGE_VERSION,
|
|
90
138
|
created: new Date().toISOString()
|
|
91
139
|
})
|
|
92
140
|
}).catch(error => {
|
|
93
141
|
console.error(`Known Agents failed to track visit: ${error.message}`)
|
|
94
142
|
})
|
|
95
143
|
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Tracks multiple agent visits in a batch for analytics.
|
|
147
|
+
* This method is non-blocking and handles errors internally.
|
|
148
|
+
* Recommended for high-traffic websites - batch visits and send periodically (e.g. every 30 seconds).
|
|
149
|
+
*
|
|
150
|
+
* @param visits - An array of visit tracking requests.
|
|
151
|
+
*/
|
|
152
|
+
trackVisits(visits: VisitRequest[]): void {
|
|
153
|
+
fetch("https://api.knownagents.com/visits", {
|
|
154
|
+
method: "POST",
|
|
155
|
+
headers: {
|
|
156
|
+
"Authorization": `Bearer ${this.accessToken}`,
|
|
157
|
+
"Content-Type": "application/json"
|
|
158
|
+
},
|
|
159
|
+
body: JSON.stringify(visits)
|
|
160
|
+
}).catch(error => {
|
|
161
|
+
console.error(`Known Agents failed to track visits: ${error.message}`)
|
|
162
|
+
})
|
|
163
|
+
}
|
|
96
164
|
|
|
97
165
|
/**
|
|
98
|
-
* Generates a
|
|
99
|
-
*
|
|
166
|
+
* Generates a robots.txt file that disallows the specified agent types.
|
|
167
|
+
* The generated file stays up-to-date with all current and future agents in the specified categories.
|
|
168
|
+
* Cache and serve the result at your website's `/robots.txt` endpoint.
|
|
100
169
|
*
|
|
101
|
-
* @param agentTypes -
|
|
102
|
-
* @param disallow -
|
|
103
|
-
* @returns
|
|
104
|
-
* @throws If the API call fails or returns a non-200 status.
|
|
170
|
+
* @param agentTypes - An array of agent types to disallow.
|
|
171
|
+
* @param disallow - The path to disallow (default: "/" for all paths).
|
|
172
|
+
* @returns A promise that resolves to the generated robots.txt content as a string.
|
|
173
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
105
174
|
*/
|
|
106
175
|
async generateRobotsTXT(agentTypes: AgentType[], disallow: string = "/"): Promise<string> {
|
|
107
176
|
const response = await fetch("https://api.knownagents.com/robots-txts", {
|
|
@@ -112,8 +181,7 @@ export class KnownAgents {
|
|
|
112
181
|
},
|
|
113
182
|
body: JSON.stringify({
|
|
114
183
|
agent_types: agentTypes,
|
|
115
|
-
disallow: disallow
|
|
116
|
-
node_package_version: NODE_PACKAGE_VERSION
|
|
184
|
+
disallow: disallow
|
|
117
185
|
})
|
|
118
186
|
})
|
|
119
187
|
|
|
@@ -125,30 +193,54 @@ export class KnownAgents {
|
|
|
125
193
|
}
|
|
126
194
|
|
|
127
195
|
/**
|
|
128
|
-
*
|
|
196
|
+
* Identifies and verifies a single agent from an HTTP request.
|
|
197
|
+
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
129
198
|
*
|
|
130
|
-
* @param request - The incoming request.
|
|
131
|
-
* @returns A promise that resolves to the
|
|
132
|
-
* @throws If the API call fails or returns a non-200 status.
|
|
199
|
+
* @param request - The incoming HTTP request (Request or Node.js IncomingMessage).
|
|
200
|
+
* @returns A promise that resolves to the identification result.
|
|
201
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
133
202
|
*/
|
|
134
|
-
async
|
|
135
|
-
const response = await fetch("https://api.knownagents.com/agent-
|
|
203
|
+
async identifyAgent(request: Request | IncomingMessage): Promise<IdentificationResult> {
|
|
204
|
+
const response = await fetch("https://api.knownagents.com/agent-identifications", {
|
|
136
205
|
method: "POST",
|
|
137
206
|
headers: {
|
|
138
207
|
"Authorization": `Bearer ${this.accessToken}`,
|
|
139
208
|
"Content-Type": "application/json"
|
|
140
209
|
},
|
|
141
210
|
body: JSON.stringify({
|
|
142
|
-
|
|
143
|
-
request_headers: this.filterHeaders(request.headers),
|
|
144
|
-
node_package_version: NODE_PACKAGE_VERSION
|
|
211
|
+
request_headers: this.filterHeaders(request.headers)
|
|
145
212
|
})
|
|
146
213
|
})
|
|
147
214
|
|
|
148
215
|
if (response.ok) {
|
|
149
216
|
return await response.json()
|
|
150
217
|
} else {
|
|
151
|
-
throw new Error(`Known Agents failed to
|
|
218
|
+
throw new Error(`Known Agents failed to identify agent: ${response.status} ${response.statusText}`)
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Identifies and verifies multiple agents from HTTP requests in a batch.
|
|
224
|
+
* Uses Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
|
|
225
|
+
*
|
|
226
|
+
* @param requests - An array of identification requests.
|
|
227
|
+
* @returns A promise that resolves to an array of identification results.
|
|
228
|
+
* @throws {Error} If the API call fails or returns a non-200 status.
|
|
229
|
+
*/
|
|
230
|
+
async identifyAgents(requests: IdentificationRequest[]): Promise<IdentificationResult[]> {
|
|
231
|
+
const response = await fetch("https://api.knownagents.com/agent-identifications", {
|
|
232
|
+
method: "POST",
|
|
233
|
+
headers: {
|
|
234
|
+
"Authorization": `Bearer ${this.accessToken}`,
|
|
235
|
+
"Content-Type": "application/json"
|
|
236
|
+
},
|
|
237
|
+
body: JSON.stringify(requests)
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
if (response.ok) {
|
|
241
|
+
return await response.json()
|
|
242
|
+
} else {
|
|
243
|
+
throw new Error(`Known Agents failed to identify agents: ${response.status} ${response.statusText}`)
|
|
152
244
|
}
|
|
153
245
|
}
|
|
154
246
|
|
package/tsconfig.json
CHANGED
|
@@ -4,12 +4,13 @@
|
|
|
4
4
|
"target": "ES2020",
|
|
5
5
|
"module": "CommonJS",
|
|
6
6
|
"outDir": "dist",
|
|
7
|
+
"rootDir": "src",
|
|
7
8
|
"declaration": true,
|
|
8
9
|
"declarationMap": true,
|
|
9
10
|
"esModuleInterop": true,
|
|
10
11
|
"forceConsistentCasingInFileNames": true,
|
|
11
12
|
"strict": true,
|
|
12
13
|
"skipLibCheck": true,
|
|
13
|
-
"
|
|
14
|
+
"types": ["node"]
|
|
14
15
|
}
|
|
15
16
|
}
|