@knownagents/sdk 1.7.0 → 1.9.0

package/README.md

@@ -1,6 +1,6 @@
 # Known Agents SDK
 
-[![NPM version](https://img.shields.io/npm/v/@knownagents/sdk.svg)](https://npmjs.org/package/@knownagents/sdk) ![npm bundle size](https://img.shields.io/bundlephobia/minzip/@knownagents/sdk)
+[![NPM version](https://img.shields.io/npm/v/@knownagents/sdk.svg)](https://npmjs.org/package/@knownagents/sdk)
 
 This library provides convenient access to [Known Agents](https://knownagents.com/) from server-side TypeScript or JavaScript.
 
@@ -53,7 +53,6 @@ const knownAgents = new KnownAgents("YOUR_ACCESS_TOKEN")
 
 app.use((req, res, next) => {
     const start = Date.now()
-    
     res.on('finish', () => {
         const duration = Date.now() - start
         knownAgents.trackVisit(req, res, duration)
@@ -81,10 +80,10 @@ If your website is correctly connected, you should see visits from the Known Age
 
 Protect sensitive content from unwanted access and scraping. Generate a continuously updating robots.txt that stays up to date with [all current and future bots](https://knownagents.com/agents) in the specified categories automatically.
 
-Use the `generateRobotsTxt` function. Select which `AgentType`s you want to block, and a string specifying which URLs are disallowed (e.g. `"/"` to disallow all paths).
+Use the `generateRobotsTXT` function. Select which `AgentType`s you want to block, and a string specifying which URLs are disallowed (e.g. `"/"` to disallow all paths).
 
 ```ts
-const robotsTxt = await knownAgents.generateRobotsTxt([
+const robotsTxt = await knownAgents.generateRobotsTXT([
   AgentType.AIDataScraper,
   AgentType.Scraper,
   AgentType.IntelligenceGatherer,
@@ -96,6 +95,35 @@ const robotsTxt = await knownAgents.generateRobotsTxt([
 
 The return value is a plain text robots.txt string. Generate a `robotsTxt` periodically (e.g. once per day), then cache and serve it from your website's `/robots.txt` endpoint.
 
+## How To Use Agent Verification ([Full Docs](https://knownagents.com/docs/verification))
+
+Use the `verifyAgent` function to [identify](https://knownagents.com/agents) and verify agents from network requests using Web Bot Auth (HTTP message signatures), IP matching, or other available methods. This can be useful for implementing access policies based on verified agent identity.
+
+Call `verifyAgent` with the incoming request.
+
+```ts
+const verification = await knownAgents.verifyAgent(request)
+
+if (verification.result === "verified") {
+    // Agent is legitimate
+} else if (verification.result === "verification_failed") {
+    // Agent is not legitimate
+}
+```
+
+The function returns an object with the following fields:
+
+- `result`: The verification result:
+  - `"verified"`: The agent is identified and verified
+  - `"verification_failed"`: The agent was identified but could not be verified
+  - `"unknown_agent"`: The agent is not in our database
+  - `"not_verifiable"`: The agent cannot be verified (no verification method available)
+- `agent_id`: The unique ID of the agent (if identified)
+- `agent_token`: The name of the agent (e.g. `"Googlebot"`) (if identified)
+- `agent_url`: The documentation URL of the agent (if identified)
+- `agent_type_name`: The type of agent (e.g. `"AI Agent"`) (if identified)
+- `operator_name`: The company behind the agent (e.g. `"Google"`) (if identified)
+
 ## Requirements
 
 TypeScript >= 4.7 is supported.

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { IncomingMessage, ServerResponse } from "http";
 export interface Request {
-    /** The path of the request's URL (e.g. "/about") */
+    /** The URL of the request, including the path and query string (e.g. "/about?foo=bar") */
     path: string;
     /** The request's HTTP method (e.g. "GET", "POST") */
     method: string;
@@ -11,6 +11,23 @@ export interface Response {
     /** The HTTP response status code */
     statusCode: number;
 }
+/**
+ * The result of an agent verification check.
+ */
+export interface VerificationResult {
+    /** The verification result */
+    result: "verified" | "verification_failed" | "unknown_agent" | "not_verifiable";
+    /** The unique ID of the agent (if identified) */
+    agent_id?: string;
+    /** The name of the agent (e.g. "Googlebot") (if identified) */
+    agent_token?: string;
+    /** The documentation URL of the agent (if identified) */
+    agent_url?: string;
+    /** The type of agent (e.g. "AI Agent") (if identified) */
+    agent_type_name?: string;
+    /** The company behind the agent (e.g. "Google") (if identified) */
+    operator_name?: string;
+}
 /**
  * Agent types that can be controlled with robots.txt rules.
  */
@@ -41,12 +58,11 @@ export declare class KnownAgents {
     /**
      * Tracks an agent visit in Known Agents agent analytics.
      *
-     * @param visitRequest - The incoming visit request.
-     * @param visitResponse - The outgoing visit response (optional).
+     * @param request - The incoming request.
+     * @param response - The outgoing response (optional).
      * @param responseDurationInMilliseconds - The response duration in milliseconds (optional).
      */
-    trackVisit(request: Request, response?: Response, responseDurationInMilliseconds?: number): void;
-    trackVisit(request: IncomingMessage, response?: ServerResponse, responseDurationInMilliseconds?: number): void;
+    trackVisit(request: Request | IncomingMessage, response?: Response | ServerResponse, responseDurationInMilliseconds?: number): void;
     /**
      * Generates a `robots.txt` file that disallows the specified agent types.
      * Cache and serve this string at your website’s `/robots.txt` endpoint.
@@ -56,7 +72,15 @@ export declare class KnownAgents {
      * @returns The generated `robots.txt` as a string.
      * @throws If the API call fails or returns a non-200 status.
      */
-    generateRobotsTxt(agentTypes: AgentType[], disallow?: string): Promise<string>;
+    generateRobotsTXT(agentTypes: AgentType[], disallow?: string): Promise<string>;
+    /**
+     * Verifies if an agent request is legitimate by checking it against Known Agents.
+     *
+     * @param request - The incoming request.
+     * @returns A promise that resolves to the verification result.
+     * @throws If the API call fails or returns a non-200 status.
+     */
+    verifyAgent(request: Request | IncomingMessage): Promise<VerificationResult>;
     private filterHeaders;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;AAItD,MAAM,WAAW,OAAO;IACpB,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAA;IACZ,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;CACzD;AAED,MAAM,WAAW,QAAQ;IACrB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,oBAAY,SAAS;IACjB,OAAO,aAAa;IACpB,WAAW,iBAAiB;IAC5B,aAAa,oBAAoB;IACjC,eAAe,sBAAsB;IACrC,QAAQ,aAAa;IACrB,eAAe,qBAAqB;IACpC,OAAO,YAAY;IACnB,aAAa,mBAAmB;IAChC,oBAAoB,0BAA0B;IAC9C,OAAO,YAAY;IACnB,UAAU,gBAAgB;IAC1B,mBAAmB,0BAA0B;IAC7C,eAAe,qBAAqB;IACpC,mBAAmB,0BAA0B;CAChD;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IAEpC;;;;OAIG;gBACS,WAAW,EAAE,MAAM;IAI/B;;;;;;OAMG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,8BAA8B,CAAC,EAAE,MAAM,GAAG,IAAI;IAChG,UAAU,CAAC,OAAO,EAAE,eAAe,EAAE,QAAQ,CAAC,EAAE,cAAc,EAAE,8BAA8B,CAAC,EAAE,MAAM,GAAG,IAAI;IA4B9G;;;;;;;;OAQG;IACG,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBzF,OAAO,CAAC,aAAa;CAcxB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAA;AAItD,MAAM,WAAW,OAAO;IACpB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAA;IACZ,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;CACzD;AAED,MAAM,WAAW,QAAQ;IACrB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,8BAA8B;IAC9B,MAAM,EAAE,UAAU,GAAG,qBAAqB,GAAG,eAAe,GAAG,gBAAgB,CAAA;IAC/E,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,mEAAmE;IACnE,aAAa,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,oBAAY,SAAS;IACjB,OAAO,aAAa;IACpB,WAAW,iBAAiB;IAC5B,aAAa,oBAAoB;IACjC,eAAe,sBAAsB;IACrC,QAAQ,aAAa;IACrB,eAAe,qBAAqB;IACpC,OAAO,YAAY;IACnB,aAAa,mBAAmB;IAChC,oBAAoB,0BAA0B;IAC9C,OAAO,YAAY;IACnB,UAAU,gBAAgB;IAC1B,mBAAmB,0BAA0B;IAC7C,eAAe,qBAAqB;IACpC,mBAAmB,0BAA0B;CAChD;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IAEpC;;;;OAIG;gBACS,WAAW,EAAE,MAAM;IAI/B;;;;;;OAMG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,EAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,cAAc,EAAE,8BAA8B,CAAC,EAAE,MAAM,GAAG,IAAI;IAqBnI;;;;;;;;OAQG;IACG,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBzF;;;;;;OAMG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuBlF,OAAO,CAAC,aAAa;CAcxB"}

package/dist/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KnownAgents = exports.AgentType = void 0;
-const NODE_PACKAGE_VERSION = "1.7.0";
+const NODE_PACKAGE_VERSION = "1.9.0";
 /**
  * Agent types that can be controlled with robots.txt rules.
  */
@@ -31,12 +31,14 @@ class KnownAgents {
     constructor(accessToken) {
         this.accessToken = accessToken;
     }
-    trackVisit(visitRequest, visitResponse, responseDurationInMilliseconds) {
-        const request = "path" in visitRequest ? visitRequest : {
-            path: visitRequest.url ? new URL(visitRequest.url, "https://example.org/").pathname : "/",
-            method: visitRequest.method ?? "GET",
-            headers: visitRequest.headers
-        };
+    /**
+     * Tracks an agent visit in Known Agents agent analytics.
+     *
+     * @param request - The incoming request.
+     * @param response - The outgoing response (optional).
+     * @param responseDurationInMilliseconds - The response duration in milliseconds (optional).
+     */
+    trackVisit(request, response, responseDurationInMilliseconds) {
         fetch("https://api.knownagents.com/visits", {
             method: "POST",
             headers: {
@@ -44,10 +46,10 @@ class KnownAgents {
                 "Content-Type": "application/json"
             },
             body: JSON.stringify({
-                request_path: request.path,
+                request_path: "path" in request ? request.path : request.url,
                 request_method: request.method,
                 request_headers: this.filterHeaders(request.headers),
-                response_status_code: visitResponse?.statusCode,
+                response_status_code: response?.statusCode,
                 response_duration_in_milliseconds: responseDurationInMilliseconds,
                 node_package_version: NODE_PACKAGE_VERSION,
                 created: new Date().toISOString()
@@ -65,7 +67,7 @@ class KnownAgents {
      * @returns The generated `robots.txt` as a string.
      * @throws If the API call fails or returns a non-200 status.
      */
-    async generateRobotsTxt(agentTypes, disallow = "/") {
+    async generateRobotsTXT(agentTypes, disallow = "/") {
         const response = await fetch("https://api.knownagents.com/robots-txts", {
             method: "POST",
             headers: {
@@ -85,6 +87,33 @@ class KnownAgents {
             throw new Error(`Known Agents failed to generate robots.txt: ${response.status} ${response.statusText}`);
         }
     }
+    /**
+     * Verifies if an agent request is legitimate by checking it against Known Agents.
+     *
+     * @param request - The incoming request.
+     * @returns A promise that resolves to the verification result.
+     * @throws If the API call fails or returns a non-200 status.
+     */
+    async verifyAgent(request) {
+        const response = await fetch("https://api.knownagents.com/agent-verifications", {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${this.accessToken}`,
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                request_path: "path" in request ? request.path : request.url,
+                request_headers: this.filterHeaders(request.headers),
+                node_package_version: NODE_PACKAGE_VERSION
+            })
+        });
+        if (response.ok) {
+            return await response.json();
+        }
+        else {
+            throw new Error(`Known Agents failed to verify agent: ${response.status} ${response.statusText}`);
+        }
+    }
     // Helpers
     filterHeaders(headers) {
         const excludedHeaders = new Set([

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@knownagents/sdk",
-  "version": "1.7.0",
-  "description": "The official Node.js SDK for Known Agents: AI agent analytics (bot traffic), LLM referral tracking, automatic robots.txt",
+  "version": "1.9.0",
+  "description": "The official Node.js SDK for Known Agents: AI agent analytics (bot traffic), LLM referral tracking, automatic robots.txt, agent verification",
   "publishConfig": {
     "access": "public"
   },
@@ -13,13 +13,22 @@
     "ai scrapers",
     "ai crawlers",
     "ai bots",
+    "bot detection",
+    "crawler detection",
+    "llm analytics",
     "robots.txt",
     "automatic robots.txt",
     "robots.txt generator",
     "web scraping",
     "ai blocking",
     "bot blocking",
-    "block ai bots",
+    "agent verification",
+    "bot verification",
+    "verify ai agents",
+    "web bot auth",
+    "http signatures",
+    "security",
+    "typescript",
     "express middleware"
   ],
   "main": "dist/index.js",

package/src/index.ts

@@ -1,9 +1,9 @@
 import { IncomingMessage, ServerResponse } from "http"
 
-const NODE_PACKAGE_VERSION = "1.7.0"
+const NODE_PACKAGE_VERSION = "1.9.0"
 
 export interface Request {
-    /** The path of the request's URL (e.g. "/about") */
+    /** The URL of the request, including the path and query string (e.g. "/about?foo=bar") */
     path: string
     /** The request's HTTP method (e.g. "GET", "POST") */
     method: string
@@ -16,6 +16,24 @@ export interface Response {
     statusCode: number
 }
 
+/**
+ * The result of an agent verification check.
+ */
+export interface VerificationResult {
+    /** The verification result */
+    result: "verified" | "verification_failed" | "unknown_agent" | "not_verifiable"
+    /** The unique ID of the agent (if identified) */
+    agent_id?: string
+    /** The name of the agent (e.g. "Googlebot") (if identified) */
+    agent_token?: string
+    /** The documentation URL of the agent (if identified) */
+    agent_url?: string
+    /** The type of agent (e.g. "AI Agent") (if identified) */
+    agent_type_name?: string
+    /** The company behind the agent (e.g. "Google") (if identified) */
+    operator_name?: string
+}
+
 /**
  * Agent types that can be controlled with robots.txt rules.
  */
@@ -51,19 +69,11 @@ export class KnownAgents {
     /**
      * Tracks an agent visit in Known Agents agent analytics.
      *
-     * @param visitRequest - The incoming visit request.
-     * @param visitResponse - The outgoing visit response (optional).
+     * @param request - The incoming request.
+     * @param response - The outgoing response (optional).
      * @param responseDurationInMilliseconds - The response duration in milliseconds (optional).
      */
-    trackVisit(request: Request, response?: Response, responseDurationInMilliseconds?: number): void
-    trackVisit(request: IncomingMessage, response?: ServerResponse, responseDurationInMilliseconds?: number): void
-    trackVisit(visitRequest: Request | IncomingMessage, visitResponse?: Response | ServerResponse, responseDurationInMilliseconds?: number): void {
-        const request: Request = "path" in visitRequest ? visitRequest : {
-            path: visitRequest.url ? new URL(visitRequest.url, "https://example.org/").pathname : "/",
-            method: visitRequest.method ?? "GET",
-            headers: visitRequest.headers
-        }
-
+    trackVisit(request: Request | IncomingMessage, response?: Response | ServerResponse, responseDurationInMilliseconds?: number): void {
         fetch("https://api.knownagents.com/visits", {
             method: "POST",
             headers: {
@@ -71,10 +81,10 @@ export class KnownAgents {
                 "Content-Type": "application/json"
             },
             body: JSON.stringify({
-                request_path: request.path,
+                request_path: "path" in request ? request.path : request.url,
                 request_method: request.method,
                 request_headers: this.filterHeaders(request.headers),
-                response_status_code: visitResponse?.statusCode,
+                response_status_code: response?.statusCode,
                 response_duration_in_milliseconds: responseDurationInMilliseconds,
                 node_package_version: NODE_PACKAGE_VERSION,
                 created: new Date().toISOString()
@@ -83,7 +93,7 @@ export class KnownAgents {
             console.error(`Known Agents failed to track visit: ${error.message}`)
         })
     }
-
+    
     /**
      * Generates a `robots.txt` file that disallows the specified agent types.
      * Cache and serve this string at your website’s `/robots.txt` endpoint.
@@ -93,7 +103,7 @@ export class KnownAgents {
      * @returns The generated `robots.txt` as a string.
      * @throws If the API call fails or returns a non-200 status.
      */
-    async generateRobotsTxt(agentTypes: AgentType[], disallow: string = "/"): Promise<string> {
+    async generateRobotsTXT(agentTypes: AgentType[], disallow: string = "/"): Promise<string> {
         const response = await fetch("https://api.knownagents.com/robots-txts", {
             method: "POST",
             headers: {
@@ -113,6 +123,34 @@ export class KnownAgents {
             throw new Error(`Known Agents failed to generate robots.txt: ${response.status} ${response.statusText}`)
         }
     }
+    
+    /**
+     * Verifies if an agent request is legitimate by checking it against Known Agents.
+     *
+     * @param request - The incoming request.
+     * @returns A promise that resolves to the verification result.
+     * @throws If the API call fails or returns a non-200 status.
+     */
+    async verifyAgent(request: Request | IncomingMessage): Promise<VerificationResult> {
+        const response = await fetch("https://api.knownagents.com/agent-verifications", {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${this.accessToken}`,
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                request_path: "path" in request ? request.path : request.url,
+                request_headers: this.filterHeaders(request.headers),
+                node_package_version: NODE_PACKAGE_VERSION
+            })
+        })
+
+        if (response.ok) {
+            return await response.json()
+        } else {
+            throw new Error(`Known Agents failed to verify agent: ${response.status} ${response.statusText}`)
+        }
+    }
 
     // Helpers