@pickrate/collector 0.1.0

package/README.md

@@ -0,0 +1,65 @@
+# @pickrate/collector
+
+See which AI agents visit your site. One file, no data pipeline.
+
+The analytics you already run (Google Analytics, Amplitude) can't see AI agents, because agents don't run JavaScript. This collector sits in your server middleware, where it *can* see them, classifies each request (which agent, what it read), and sends it to [Pickrate](https://pickrate.io).
+
+## Install (Next.js)
+
+```bash
+npm install @pickrate/collector
+```
+
+```ts
+// middleware.ts
+export { middleware } from "@pickrate/collector/next";
+export const config = { matcher: ["/((?!_next|favicon).*)"] };
+```
+
+```bash
+# .env
+PICKRATE_KEY=sk_live_xxx   # your Pickrate secret key
+```
+
+That's it. Agent hits start showing up in your Pickrate dashboard. Human traffic classifies to nothing and pays zero cost.
+
+## What it captures
+
+Every request from a known agent (by user-agent) or to a machine-readable surface (`/llms.txt`, `/openapi.json`, `/.well-known/*`, `*.md`, `sitemap.xml`, `robots.txt`). For each it sends an `agent_interaction`:
+
+- **actor** — OpenAI GPTBot, Anthropic ClaudeBot, Perplexity, ...
+- **actorType** — assistant (a person is waiting) vs crawler (background)
+- **surface** — llms_txt, openapi_spec, well_known_endpoint, content, ...
+- **interaction** — llms_txt_requested, page_requested, ...
+- **object / path** — what was read
+
+It never reports app routes, assets, or `/api`. Human page loads are ignored.
+
+## Custom instrumentation
+
+Compose your own middleware, or report events the request path can't see (e.g. an MCP `tool_invoked`):
+
+```ts
+import { createCollector } from "@pickrate/collector";
+const pickrate = createCollector({ key: process.env.PICKRATE_KEY! });
+
+// From an MCP tool handler:
+await pickrate.send({
+  type: "agent_interaction",
+  actor: "chatgpt",
+  actorType: "assistant",
+  protocol: "http",
+  surface: "remote_mcp_server",
+  interaction: "tool_invoked",
+  object: "create_automation",
+  path: "/mcp",
+});
+```
+
+## Notes
+
+- **Secret key.** `agent_interaction` is server-detected, so the ingest requires a secret (`sk_`) key. The collector runs server-side, so the key never reaches a browser.
+- **Fail-safe.** Every send is fire-and-forget and timeout-bounded. A logging failure never surfaces to a visitor.
+- **Runtime.** The core (`@pickrate/collector`) is framework-agnostic (Edge, Node, Cloudflare). The Next adapter (`@pickrate/collector/next`) uses `waitUntil` when available so nothing blocks the response.
+
+MIT © Pickrate

package/index.d.ts

@@ -0,0 +1,52 @@
+// Type declarations for @pickrate/collector (framework-agnostic core).
+
+export interface RequestInfo {
+  method: string;
+  /** pathname only, no query string */
+  path: string;
+  userAgent: string;
+}
+
+export interface InteractionPayload {
+  type: "agent_interaction";
+  /** clean display name, or "unknown" */
+  actor: string;
+  /** assistant | crawler | unknown_machine */
+  actorType: string;
+  protocol: "http";
+  /** llms_txt | well_known_endpoint | content | ... */
+  surface: string;
+  /** llms_txt_requested | page_requested | ... */
+  interaction: string;
+  /** the path acted on */
+  object: string;
+  path: string;
+  format?: string;
+}
+
+export interface CollectorConfig {
+  /** A Pickrate SECRET key (sk_). agent_interaction is server-detected, so the ingest requires it. */
+  key: string;
+  /** Ingest endpoint. Default https://pickrate.io/api/collect */
+  endpoint?: string;
+  /** Per-request send timeout, ms. Default 1500. */
+  timeoutMs?: number;
+}
+
+export interface Collector {
+  /** Classify a request and report it if it's an agent hit. Resolves true when something was sent. */
+  report(req: RequestInfo): Promise<boolean>;
+  /** Send a pre-built payload (e.g. an MCP tool_invoked the request path can't see). */
+  send(payload: InteractionPayload): Promise<void>;
+}
+
+export interface SurfaceInfo {
+  surface: string;
+  interaction: string;
+  format?: string;
+  dedicated: boolean;
+}
+
+export function surfaceFor(path: string): SurfaceInfo;
+export function classifyRequest(req: RequestInfo): InteractionPayload | null;
+export function createCollector(config: CollectorConfig): Collector;

package/index.js

@@ -0,0 +1,132 @@
+// @pickrate/collector — see which AI agents visit your site. Framework-agnostic core.
+// Hand-authored ESM to match packages/attribution (no build step). Logic mirrors the tested source
+// at app/src/collector; keep them in sync.
+
+// UA substring -> clean name. Order matters where one signature contains another.
+const BOTS = [
+  ["gptbot", "OpenAI GPTBot"],
+  ["oai-searchbot", "OpenAI SearchBot"],
+  ["chatgpt-user", "OpenAI ChatGPT-User"],
+  ["claudebot", "Anthropic ClaudeBot"],
+  ["claude-user", "Anthropic Claude-User"],
+  ["claude-web", "Anthropic Claude-Web"],
+  ["anthropic-ai", "Anthropic"],
+  ["perplexitybot", "Perplexity"],
+  ["perplexity-user", "Perplexity-User"],
+  ["google-extended", "Google-Extended"],
+  ["googlebot", "Googlebot"],
+  ["bingbot", "Bingbot"],
+  ["applebot-extended", "Applebot-Extended"],
+  ["applebot", "Applebot"],
+  ["ccbot", "Common Crawl"],
+  ["bytespider", "ByteDance"],
+  ["amazonbot", "Amazon"],
+  ["meta-externalagent", "Meta"],
+  ["cohere-ai", "Cohere"],
+  ["perplexity", "Perplexity"],
+];
+
+const RETRIEVAL = new Set([
+  "OpenAI SearchBot",
+  "OpenAI ChatGPT-User",
+  "Anthropic Claude-Web",
+  "Anthropic Claude-User",
+  "Perplexity",
+  "Perplexity-User",
+]);
+
+const SEARCH = new Set(["Googlebot", "Bingbot", "Applebot"]);
+
+function botFor(ua) {
+  const l = String(ua || "").toLowerCase();
+  for (const [needle, name] of BOTS) if (l.includes(needle)) return name;
+  return null;
+}
+
+function actorTypeFor(name) {
+  if (RETRIEVAL.has(name)) return "assistant";
+  if (SEARCH.has(name)) return "crawler";
+  return "crawler";
+}
+
+function isExcluded(path) {
+  return (
+    path.startsWith("/api") ||
+    path.startsWith("/_next") ||
+    path.startsWith("/dashboard") ||
+    path.startsWith("/admin") ||
+    path.startsWith("/auth") ||
+    path.startsWith("/login") ||
+    path === "/favicon.ico" ||
+    /\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff2?|ttf)$/i.test(path)
+  );
+}
+
+export function surfaceFor(path) {
+  if (path === "/llms.txt") return { surface: "llms_txt", interaction: "llms_txt_requested", format: "plain_text", dedicated: true };
+  if (path === "/robots.txt") return { surface: "robots_txt", interaction: "robots_txt_requested", format: "plain_text", dedicated: true };
+  if (path === "/sitemap.xml") return { surface: "sitemap_xml", interaction: "sitemap_requested", format: "xml", dedicated: true };
+  if (path === "/openapi.json") return { surface: "openapi_spec", interaction: "openapi_spec_requested", format: "json", dedicated: true };
+  if (path.endsWith(".md")) return { surface: "markdown_document", interaction: "markdown_retrieved", format: "markdown", dedicated: true };
+  if (path.startsWith("/.well-known/")) return { surface: "well_known_endpoint", interaction: "manifest_requested", format: "json", dedicated: true };
+  return { surface: "content", interaction: "page_requested", dedicated: false };
+}
+
+// Classify a request. Reports when it's a KNOWN agent (by UA) OR targets an agent surface. Returns
+// null for human, asset, and excluded traffic.
+export function classifyRequest(req) {
+  if (req.method !== "GET" && req.method !== "HEAD") return null;
+  const path = String(req.path || "/").split(/[?#]/)[0];
+  if (isExcluded(path)) return null;
+
+  const agent = botFor(req.userAgent);
+  const sf = surfaceFor(path);
+  if (!agent && !sf.dedicated) return null;
+
+  const payload = {
+    type: "agent_interaction",
+    actor: agent || "unknown",
+    actorType: agent ? actorTypeFor(agent) : "unknown_machine",
+    protocol: "http",
+    surface: sf.surface,
+    interaction: sf.interaction,
+    object: path,
+    path,
+  };
+  if (sf.format) payload.format = sf.format;
+  return payload;
+}
+
+const DEFAULT_ENDPOINT = "https://pickrate.io/api/collect";
+
+// Create a collector bound to a secret key. report(req) classifies + POSTs an agent hit; send()
+// posts a pre-built payload. Fire-and-forget, timeout-bounded: a logging failure never surfaces.
+export function createCollector(config) {
+  const endpoint = (config && config.endpoint) || DEFAULT_ENDPOINT;
+  const timeoutMs = config && typeof config.timeoutMs === "number" ? config.timeoutMs : 1500;
+  const key = (config && config.key) || "";
+
+  async function send(payload) {
+    if (!key) return;
+    try {
+      await fetch(endpoint, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
+        body: JSON.stringify({ events: [payload] }),
+        cache: "no-store",
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+    } catch {
+      // swallow: a visitor (or bot) must never see a logging error
+    }
+  }
+
+  async function report(req) {
+    const payload = classifyRequest(req);
+    if (!payload) return false;
+    await send(payload);
+    return true;
+  }
+
+  return { report, send };
+}

package/next.d.ts

@@ -0,0 +1,9 @@
+// Type declarations for @pickrate/collector/next.
+import type { NextRequest, NextFetchEvent, NextResponse } from "next/server";
+import type { CollectorConfig } from "./index.js";
+
+export function createMiddleware(
+  config: CollectorConfig,
+): (req: NextRequest, ev?: NextFetchEvent) => Promise<NextResponse>;
+
+export const middleware: (req: NextRequest, ev?: NextFetchEvent) => Promise<NextResponse>;

package/next.js

@@ -0,0 +1,33 @@
+// @pickrate/collector/next — the Next.js middleware adapter. The one-file install:
+//
+//   // middleware.ts
+//   export { middleware } from "@pickrate/collector/next";
+//   export const config = { matcher: ["/((?!_next|favicon).*)"] };
+//
+// Set PICKRATE_KEY (a secret sk_ key) in the environment.
+
+import { NextResponse } from "next/server";
+import { createCollector } from "./index.js";
+
+// Build a middleware bound to a specific config (for composing your own middleware).
+export function createMiddleware(config) {
+  const collector = createCollector(config);
+  return async function middleware(req, ev) {
+    const info = {
+      method: req.method,
+      path: req.nextUrl.pathname,
+      userAgent: req.headers.get("user-agent") || "",
+    };
+    // Prefer waitUntil (Vercel/Edge) so nothing blocks the response. Where it's unavailable, await:
+    // only agent hits actually send, so a human page load still returns instantly.
+    if (ev && typeof ev.waitUntil === "function") ev.waitUntil(collector.report(info));
+    else await collector.report(info);
+    return NextResponse.next();
+  };
+}
+
+// Zero-config default for the one-file install. Reads PICKRATE_KEY (and optional PICKRATE_ENDPOINT).
+export const middleware = createMiddleware({
+  key: process.env.PICKRATE_KEY || "",
+  endpoint: process.env.PICKRATE_ENDPOINT,
+});

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@pickrate/collector",
+  "version": "0.1.0",
+  "description": "See which AI agents visit your site. One-file server-side collector for Pickrate Agent Analytics.",
+  "type": "module",
+  "main": "index.js",
+  "module": "index.js",
+  "types": "index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    },
+    "./next": {
+      "types": "./next.d.ts",
+      "default": "./next.js"
+    }
+  },
+  "files": ["index.js", "index.d.ts", "next.js", "next.d.ts", "README.md"],
+  "engines": { "node": ">=18" },
+  "license": "MIT",
+  "homepage": "https://pickrate.io",
+  "repository": { "type": "git", "url": "git+https://github.com/TimHey/pickrate.git", "directory": "packages/collector" },
+  "keywords": ["pickrate", "agent", "analytics", "ai", "crawler", "mcp", "llms.txt", "attribution", "middleware", "next"],
+  "peerDependencies": { "next": ">=13" },
+  "peerDependenciesMeta": { "next": { "optional": true } },
+  "sideEffects": false
+}