@knownagents/sdk 1.6.0 → 1.8.0

package/README.md

@@ -1,6 +1,6 @@
 # Known Agents SDK
 
-[![NPM version](https://img.shields.io/npm/v/@knownagents/sdk.svg)](https://npmjs.org/package/@knownagents/sdk) ![npm bundle size](https://img.shields.io/bundlephobia/minzip/@knownagents/sdk)
+[![NPM version](https://img.shields.io/npm/v/@knownagents/sdk.svg)](https://npmjs.org/package/@knownagents/sdk)
 
 This library provides convenient access to [Known Agents](https://knownagents.com/) from server-side TypeScript or JavaScript.
 
@@ -29,7 +29,13 @@ Get realtime insight into the hidden ecosystem of [crawlers, scrapers, AI agents
 To collect this data, call `trackVisit` for each incoming request in the endpoints where you serve your pages.
 
 ```ts
-knownAgents.trackVisit(incomingRequest)
+knownAgents.trackVisit(request)
+```
+
+For richer analytics, include the response and duration:
+
+```ts
+knownAgents.trackVisit(request, response, responseDurationInMilliseconds)
 ```
 
 ### Use Middleware if Possible
@@ -46,7 +52,13 @@ const app = express()
 const knownAgents = new KnownAgents("YOUR_ACCESS_TOKEN")
 
 app.use((req, res, next) => {
-    knownAgents.trackVisit(req)
+    const start = Date.now()
+    
+    res.on('finish', () => {
+        const duration = Date.now() - start
+        knownAgents.trackVisit(req, res, duration)
+    })
+    
     next()
 })
 
@@ -69,10 +81,10 @@ If your website is correctly connected, you should see visits from the Known Age
 
 Protect sensitive content from unwanted access and scraping. Generate a continuously updating robots.txt that stays up to date with [all current and future bots](https://knownagents.com/agents) in the specified categories automatically.
 
-Use the `generateRobotsTxt` function. Select which `AgentType`s you want to block, and a string specifying which URLs are disallowed (e.g. `"/"` to disallow all paths).
+Use the `generateRobotsTXT` function. Select which `AgentType`s you want to block, and a string specifying which URLs are disallowed (e.g. `"/"` to disallow all paths).
 
 ```ts
-const robotsTxt = await knownAgents.generateRobotsTxt([
+const robotsTxt = await knownAgents.generateRobotsTXT([
   AgentType.AIDataScraper,
   AgentType.Scraper,
   AgentType.IntelligenceGatherer,
@@ -82,7 +94,36 @@ const robotsTxt = await knownAgents.generateRobotsTxt([
 
 ```
 
-The return value is a plain text robots.txt string. Generate a `robotsTXT` periodically (e.g. once per day), then cache and serve it from your website's `/robots.txt` endpoint.
+The return value is a plain text robots.txt string. Generate a `robotsTxt` periodically (e.g. once per day), then cache and serve it from your website's `/robots.txt` endpoint.
+
+## How To Use Agent Verification ([Full Docs](https://knownagents.com/docs/verification))
+
+Use the `verifyAgent` function to [identify](https://knownagents.com/agents) and verify agents from network requests using Web Bot Auth (HTTP message signatures), IP matching, or other available methods.
+
+Call `verifyAgent` with the incoming request.
+
+```ts
+const verification = await knownAgents.verifyAgent(request)
+
+if (verification.result === "verified") {
+    // Agent is legitimate
+} else if (verification.result === "verification_failed") {
+    // Agent is not legitimate
+}
+```
+
+The function returns an object with the following fields:
+
+- `result`: The verification result:
+  - `"verified"`: The agent is identified and verified
+  - `"verification_failed"`: The agent was identified but could not be verified
+  - `"unknown_agent"`: The agent is not in our database
+  - `"not_verifiable"`: The agent cannot be verified (no verification method available)
+- `agent_id`: The unique ID of the agent (if identified)
+- `agent_token`: The name of the agent (e.g. `"Googlebot"`) (if identified)
+- `agent_url`: The documentation URL of the agent (if identified)
+- `agent_type_name`: The type of agent (e.g. `"AI Agent"`) (if identified)
+- `operator_name`: The company behind the agent (e.g. `"Google"`) (if identified)
 
 ## Requirements
 

package/dist/index.d.ts

@@ -1,12 +1,33 @@
-import { IncomingMessage } from "http";
+import { IncomingMessage, ServerResponse } from "http";
 export interface Request {
-    /** The path of the request's URL (e.g. "/about") */
+    /** The URL of the request, including the path and query string (e.g. "/about?foo=bar") */
     path: string;
     /** The request's HTTP method (e.g. "GET", "POST") */
     method: string;
     /** The request's HTTP headers */
     headers: Record<string, string | string[] | undefined>;
 }
+export interface Response {
+    /** The HTTP response status code */
+    statusCode: number;
+}
+/**
+ * The result of an agent verification check.
+ */
+export interface VerificationResult {
+    /** The verification result */
+    result: "verified" | "verification_failed" | "unknown_agent" | "not_verifiable";
+    /** The unique ID of the agent (if identified) */
+    agent_id?: string;
+    /** The name of the agent (e.g. "Googlebot") (if identified) */
+    agent_token?: string;
+    /** The documentation URL of the agent (if identified) */
+    agent_url?: string;
+    /** The type of agent (e.g. "AI Agent") (if identified) */
+    agent_type_name?: string;
+    /** The company behind the agent (e.g. "Google") (if identified) */
+    operator_name?: string;
+}
 /**
  * Agent types that can be controlled with robots.txt rules.
  */
@@ -37,10 +58,11 @@ export declare class KnownAgents {
     /**
      * Tracks an agent visit in Known Agents agent analytics.
      *
-     * @param visitRequest - The incoming visit request.
+     * @param request - The incoming request.
+     * @param response - The outgoing response (optional).
+     * @param responseDurationInMilliseconds - The response duration in milliseconds (optional).
      */
-    trackVisit(request: Request): void;
-    trackVisit(request: IncomingMessage): void;
+    trackVisit(request: Request | IncomingMessage, response?: Response | ServerResponse, responseDurationInMilliseconds?: number): void;
     /**
      * Generates a `robots.txt` file that disallows the specified agent types.
      * Cache and serve this string at your website’s `/robots.txt` endpoint.
@@ -50,7 +72,15 @@ export declare class KnownAgents {
      * @returns The generated `robots.txt` as a string.
      * @throws If the API call fails or returns a non-200 status.
      */
-    generateRobotsTxt(agentTypes: AgentType[], disallow?: string): Promise<string>;
+    generateRobotsTXT(agentTypes: AgentType[], disallow?: string): Promise<string>;
+    /**
+     * Verifies if an agent request is legitimate by checking it against Known Agents.
+     *
+     * @param request - The incoming request.
+     * @returns A promise that resolves to the verification result.
+     * @throws If the API call fails or returns a non-200 status.
+     */
+    verifyAgent(request: Request | IncomingMessage): Promise<VerificationResult>;
     private filterHeaders;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,MAAM,CAAA;AAItC,MAAM,WAAW,OAAO;IACpB,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAA;IACZ,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;CACzD;AAED;;GAEG;AACH,oBAAY,SAAS;IACjB,OAAO,aAAa;IACpB,WAAW,iBAAiB;IAC5B,aAAa,oBAAoB;IACjC,eAAe,sBAAsB;IACrC,QAAQ,aAAa;IACrB,eAAe,qBAAqB;IACpC,OAAO,YAAY;IACnB,aAAa,mBAAmB;IAChC,oBAAoB,0BAA0B;IAC9C,OAAO,YAAY;IACnB,UAAU,gBAAgB;IAC1B,mBAAmB,0BAA0B;IAC7C,eAAe,qBAAqB;IACpC,mBAAmB,0BAA0B;CAChD;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IAEpC;;;;OAIG;gBACS,WAAW,EAAE,MAAM;IAI/B;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAClC,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAyB1C;;;;;;;;OAQG;IACG,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBzF,OAAO,CAAC,aAAa;CAcxB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;AAItD,MAAM,WAAW,OAAO;IACpB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAA;IACZ,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;CACzD;AAED,MAAM,WAAW,QAAQ;IACrB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,8BAA8B;IAC9B,MAAM,EAAE,UAAU,GAAG,qBAAqB,GAAG,eAAe,GAAG,gBAAgB,CAAA;IAC/E,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,mEAAmE;IACnE,aAAa,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,oBAAY,SAAS;IACjB,OAAO,aAAa;IACpB,WAAW,iBAAiB;IAC5B,aAAa,oBAAoB;IACjC,eAAe,sBAAsB;IACrC,QAAQ,aAAa;IACrB,eAAe,qBAAqB;IACpC,OAAO,YAAY;IACnB,aAAa,mBAAmB;IAChC,oBAAoB,0BAA0B;IAC9C,OAAO,YAAY;IACnB,UAAU,gBAAgB;IAC1B,mBAAmB,0BAA0B;IAC7C,eAAe,qBAAqB;IACpC,mBAAmB,0BAA0B;CAChD;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IAEpC;;;;OAIG;gBACS,WAAW,EAAE,MAAM;IAI/B;;;;;;OAMG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,EAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,cAAc,EAAE,8BAA8B,CAAC,EAAE,MAAM,GAAG,IAAI;IAqBnI;;;;;;;;OAQG;IACG,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBzF;;;;;;OAMG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuBlF,OAAO,CAAC,aAAa;CAcxB"}

package/dist/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KnownAgents = exports.AgentType = void 0;
-const NODE_PACKAGE_VERSION = "1.3.0";
+const NODE_PACKAGE_VERSION = "1.8.0";
 /**
  * Agent types that can be controlled with robots.txt rules.
  */
@@ -31,12 +31,14 @@ class KnownAgents {
     constructor(accessToken) {
         this.accessToken = accessToken;
     }
-    trackVisit(visitRequest) {
-        const request = "path" in visitRequest ? visitRequest : {
-            path: visitRequest.url ? new URL(visitRequest.url, "https://example.org/").pathname : "/",
-            method: visitRequest.method ?? "GET",
-            headers: visitRequest.headers
-        };
+    /**
+     * Tracks an agent visit in Known Agents agent analytics.
+     *
+     * @param request - The incoming request.
+     * @param response - The outgoing response (optional).
+     * @param responseDurationInMilliseconds - The response duration in milliseconds (optional).
+     */
+    trackVisit(request, response, responseDurationInMilliseconds) {
         fetch("https://api.knownagents.com/visits", {
             method: "POST",
             headers: {
@@ -44,10 +46,13 @@ class KnownAgents {
                 "Content-Type": "application/json"
             },
             body: JSON.stringify({
-                request_path: request.path,
+                request_path: "path" in request ? request.path : request.url,
                 request_method: request.method,
                 request_headers: this.filterHeaders(request.headers),
-                node_package_version: NODE_PACKAGE_VERSION
+                response_status_code: response?.statusCode,
+                response_duration_in_milliseconds: responseDurationInMilliseconds,
+                node_package_version: NODE_PACKAGE_VERSION,
+                created: new Date().toISOString()
             })
         }).catch(error => {
             console.error(`Known Agents failed to track visit: ${error.message}`);
@@ -62,7 +67,7 @@ class KnownAgents {
      * @returns The generated `robots.txt` as a string.
      * @throws If the API call fails or returns a non-200 status.
      */
-    async generateRobotsTxt(agentTypes, disallow = "/") {
+    async generateRobotsTXT(agentTypes, disallow = "/") {
         const response = await fetch("https://api.knownagents.com/robots-txts", {
             method: "POST",
             headers: {
@@ -82,6 +87,33 @@ class KnownAgents {
             throw new Error(`Known Agents failed to generate robots.txt: ${response.status} ${response.statusText}`);
         }
     }
+    /**
+     * Verifies if an agent request is legitimate by checking it against Known Agents.
+     *
+     * @param request - The incoming request.
+     * @returns A promise that resolves to the verification result.
+     * @throws If the API call fails or returns a non-200 status.
+     */
+    async verifyAgent(request) {
+        const response = await fetch("https://api.knownagents.com/agent-verifications", {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${this.accessToken}`,
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                request_path: "path" in request ? request.path : request.url,
+                request_headers: this.filterHeaders(request.headers),
+                node_package_version: NODE_PACKAGE_VERSION
+            })
+        });
+        if (response.ok) {
+            return await response.json();
+        }
+        else {
+            throw new Error(`Known Agents failed to verify agent: ${response.status} ${response.statusText}`);
+        }
+    }
     // Helpers
     filterHeaders(headers) {
         const excludedHeaders = new Set([

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@knownagents/sdk",
-  "version": "1.6.0",
-  "description": "The official Node.js SDK for Known Agents: AI agent analytics (bot traffic), LLM referral tracking, automatic robots.txt",
+  "version": "1.8.0",
+  "description": "The official Node.js SDK for Known Agents: AI agent analytics (bot traffic), LLM referral tracking, automatic robots.txt, agent verification",
   "publishConfig": {
     "access": "public"
   },
@@ -13,13 +13,22 @@
     "ai scrapers",
     "ai crawlers",
     "ai bots",
+    "bot detection",
+    "crawler detection",
+    "llm analytics",
     "robots.txt",
     "automatic robots.txt",
     "robots.txt generator",
     "web scraping",
     "ai blocking",
     "bot blocking",
-    "block ai bots",
+    "agent verification",
+    "bot verification",
+    "verify ai agents",
+    "web bot auth",
+    "http signatures",
+    "security",
+    "typescript",
     "express middleware"
   ],
   "main": "dist/index.js",

package/src/index.ts

@@ -1,9 +1,9 @@
-import { IncomingMessage } from "http"
+import { IncomingMessage, ServerResponse } from "http"
 
-const NODE_PACKAGE_VERSION = "1.3.0"
+const NODE_PACKAGE_VERSION = "1.8.0"
 
 export interface Request {
-    /** The path of the request's URL (e.g. "/about") */
+    /** The URL of the request, including the path and query string (e.g. "/about?foo=bar") */
     path: string
     /** The request's HTTP method (e.g. "GET", "POST") */
     method: string
@@ -11,6 +11,29 @@ export interface Request {
     headers: Record<string, string | string[] | undefined>
 }
 
+export interface Response {
+    /** The HTTP response status code */
+    statusCode: number
+}
+
+/**
+ * The result of an agent verification check.
+ */
+export interface VerificationResult {
+    /** The verification result */
+    result: "verified" | "verification_failed" | "unknown_agent" | "not_verifiable"
+    /** The unique ID of the agent (if identified) */
+    agent_id?: string
+    /** The name of the agent (e.g. "Googlebot") (if identified) */
+    agent_token?: string
+    /** The documentation URL of the agent (if identified) */
+    agent_url?: string
+    /** The type of agent (e.g. "AI Agent") (if identified) */
+    agent_type_name?: string
+    /** The company behind the agent (e.g. "Google") (if identified) */
+    operator_name?: string
+}
+
 /**
  * Agent types that can be controlled with robots.txt rules.
  */
@@ -46,17 +69,11 @@ export class KnownAgents {
     /**
      * Tracks an agent visit in Known Agents agent analytics.
      *
-     * @param visitRequest - The incoming visit request.
+     * @param request - The incoming request.
+     * @param response - The outgoing response (optional).
+     * @param responseDurationInMilliseconds - The response duration in milliseconds (optional).
      */
-    trackVisit(request: Request): void
-    trackVisit(request: IncomingMessage): void
-    trackVisit(visitRequest: Request | IncomingMessage): void {
-        const request: Request = "path" in visitRequest ? visitRequest : {
-            path: visitRequest.url ? new URL(visitRequest.url, "https://example.org/").pathname : "/",
-            method: visitRequest.method ?? "GET",
-            headers: visitRequest.headers
-        }
-
+    trackVisit(request: Request | IncomingMessage, response?: Response | ServerResponse, responseDurationInMilliseconds?: number): void {
         fetch("https://api.knownagents.com/visits", {
             method: "POST",
             headers: {
@@ -64,16 +81,19 @@ export class KnownAgents {
                 "Content-Type": "application/json"
             },
             body: JSON.stringify({
-                request_path: request.path,
+                request_path: "path" in request ? request.path : request.url,
                 request_method: request.method,
                 request_headers: this.filterHeaders(request.headers),
-                node_package_version: NODE_PACKAGE_VERSION
+                response_status_code: response?.statusCode,
+                response_duration_in_milliseconds: responseDurationInMilliseconds,
+                node_package_version: NODE_PACKAGE_VERSION,
+                created: new Date().toISOString()
             })
         }).catch(error => {
             console.error(`Known Agents failed to track visit: ${error.message}`)
         })
     }
-
+    
     /**
      * Generates a `robots.txt` file that disallows the specified agent types.
      * Cache and serve this string at your website’s `/robots.txt` endpoint.
@@ -83,7 +103,7 @@ export class KnownAgents {
      * @returns The generated `robots.txt` as a string.
      * @throws If the API call fails or returns a non-200 status.
      */
-    async generateRobotsTxt(agentTypes: AgentType[], disallow: string = "/"): Promise<string> {
+    async generateRobotsTXT(agentTypes: AgentType[], disallow: string = "/"): Promise<string> {
         const response = await fetch("https://api.knownagents.com/robots-txts", {
             method: "POST",
             headers: {
@@ -103,6 +123,34 @@ export class KnownAgents {
             throw new Error(`Known Agents failed to generate robots.txt: ${response.status} ${response.statusText}`)
         }
     }
+    
+    /**
+     * Verifies if an agent request is legitimate by checking it against Known Agents.
+     *
+     * @param request - The incoming request.
+     * @returns A promise that resolves to the verification result.
+     * @throws If the API call fails or returns a non-200 status.
+     */
+    async verifyAgent(request: Request | IncomingMessage): Promise<VerificationResult> {
+        const response = await fetch("https://api.knownagents.com/agent-verifications", {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${this.accessToken}`,
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                request_path: "path" in request ? request.path : request.url,
+                request_headers: this.filterHeaders(request.headers),
+                node_package_version: NODE_PACKAGE_VERSION
+            })
+        })
+
+        if (response.ok) {
+            return await response.json()
+        } else {
+            throw new Error(`Known Agents failed to verify agent: ${response.status} ${response.statusText}`)
+        }
+    }
 
     // Helpers