analytics-platform 0.0.0-stage → 0.2.2

package/lib/analytics.js

@@ -0,0 +1,105 @@
+// Analytics module: what only the server sees for sure (payments, sign-ups,
+// plan changes), next to the browser tracker's visits. Not pageviews: the
+// tracker counts those, and counting them here too would count them twice.
+// - track(): events, batched to POST /events/server-events. Each gets an
+//   event_uid, so a retried batch can be told apart from new events.
+// - identify(): links the browser's visit to your user ID (/events/identify).
+// - lead(): adds or updates a lead (/events/lead).
+// Pass `req` (or session_id) to link an event to the visit: the page sends
+// window.analyticsSession.getId() in the x-analytics-session header.
+"use strict";
+
+const { randomUUID } = require("node:crypto");
+const { sessionOf } = require("./core.js");
+
+const BATCH = 100;
+const QUEUE_MAX = 1000;
+const FLUSH_MS = 2000;
+const RETRY_MS = 15000;
+
+function createAnalytics(core) {
+  const log = (...args) => core.log("analytics", ...args);
+  let queue = [];
+  let timer = null;
+  let sending = Promise.resolve();
+
+  function schedule(ms = FLUSH_MS) {
+    if (timer || !queue.length) return;
+    timer = setTimeout(() => {
+      timer = null;
+      flush();
+    }, ms);
+    timer.unref?.();
+  }
+
+  async function sendBatch(events) {
+    const res = await core.send("analytics", "/events/server-events", { body: { events } });
+    // Not taken: back to the front of the queue (as room allows), tried again later.
+    if (!res || res.status === 429 || res.status >= 500) {
+      queue = [...events, ...queue].slice(0, QUEUE_MAX);
+      schedule(RETRY_MS);
+      return;
+    }
+    if (!res.ok) log("events refused:", await res.text().catch(() => ""));
+  }
+
+  function flush() {
+    clearTimeout(timer);
+    timer = null;
+    const batches = [];
+    while (queue.length) batches.push(queue.splice(0, BATCH));
+    sending = sending.then(() => Promise.all(batches.map(sendBatch)));
+    return sending;
+  }
+
+  // name: the event's name; fields: value, numeric_value, currency, url,
+  // event_label, variant, id, session_id, timestamp, event_uid (see the
+  // server events docs). Returns the event_uid, or "" when not queued.
+  function track(name, fields = {}, { req } = {}) {
+    try {
+      if (!name || typeof name !== "string" || !core.isOn("analytics")) return "";
+      const event = {
+        timestamp: new Date().toISOString(),
+        session_id: sessionOf(req),
+        ...fields,
+        event_name: name.slice(0, 100),
+        event_uid: fields.event_uid || randomUUID(),
+      };
+      if (typeof event.numeric_value === "number") event.numeric_value = String(event.numeric_value);
+      if (queue.length >= QUEUE_MAX) {
+        log("queue full, event dropped");
+        return "";
+      }
+      queue.push(event);
+      if (queue.length >= BATCH) flush();
+      else schedule();
+      return event.event_uid;
+    } catch (error) {
+      log("could not queue event:", error?.message);
+      return "";
+    }
+  }
+
+  // Links the visit (req or session_id) to your user ID. Resolves true when stored.
+  async function identify(userId, { req, session_id } = {}) {
+    const session = session_id || sessionOf(req);
+    if (!userId || !session || !core.isOn("analytics")) return false;
+    const res = await core.send("analytics", "/events/identify", { body: { user_id: String(userId), session_id: session } });
+    return Boolean(res?.ok);
+  }
+
+  // fields: email or phone, and name, status, tags, note, deal fields...
+  // (the leads docs). Resolves to { status, lead_id }, or null.
+  async function lead(fields = {}, { req } = {}) {
+    if (!core.isOn("analytics")) return null;
+    const res = await core.send("analytics", "/events/lead", { body: { session_id: sessionOf(req), ...fields } });
+    if (!res) return null;
+    const data = await res.json().catch(() => null);
+    if (!res.ok) log("lead refused:", data?.error || res.status);
+    return res.ok ? data : null;
+  }
+
+  return { track, identify, lead, flush, close: flush };
+}
+
+module.exports = { createAnalytics };

package/lib/audit.js

@@ -0,0 +1,57 @@
+// Audit log module: who did what on your site, sent to POST /events/audit
+// in small batches (Security → Audit log). audit("user.role_changed", {
+// actor, target, details, req }); req adds the actor's address and visit.
+// What goes in (names, emails) is up to you. Sending never throws.
+"use strict";
+
+const { sessionOf } = require("./core.js");
+
+const BATCH = 100;
+const QUEUE_MAX = 1000;
+const FLUSH_MS = 5000;
+
+// clientIp(req): the firewall's way of finding the visitor's address.
+function createAudit(core, { clientIp } = {}) {
+  let queue = [];
+  let timer = null;
+  let sending = Promise.resolve();
+
+  function flush() {
+    clearTimeout(timer);
+    timer = null;
+    const batches = [];
+    while (queue.length) batches.push(queue.splice(0, BATCH));
+    sending = sending.then(() => Promise.all(batches.map((entries) => core.send("audit", "/events/audit", { body: { entries } }))));
+    return sending;
+  }
+
+  // action: what happened, e.g. "user.login", "settings.changed" (letters,
+  // numbers and . _ : -). Options: actor (who), target (what it happened
+  // to), details (any JSON), req.
+  function audit(action, { actor = "", target = "", details, req } = {}) {
+    try {
+      if (!core.isOn("audit") || typeof action !== "string" || !action) return;
+      if (queue.length >= QUEUE_MAX) return core.log("audit", "queue full, entry dropped");
+      queue.push({
+        at: new Date().toISOString(),
+        action: action.slice(0, 64),
+        actor: String(actor).slice(0, 200),
+        target: String(target).slice(0, 300),
+        details,
+        actor_ip: req ? (clientIp ? clientIp(req) : req.ip || req.socket?.remoteAddress || "") : "",
+        session_id: sessionOf(req),
+      });
+      if (queue.length >= BATCH) flush();
+      else if (!timer) {
+        timer = setTimeout(flush, FLUSH_MS);
+        timer.unref?.();
+      }
+    } catch (error) {
+      core.log("audit", "could not record:", error?.message);
+    }
+  }
+
+  return { audit, flush, close: flush };
+}
+
+module.exports = { createAudit };

package/lib/bans.js

@@ -0,0 +1,78 @@
+// Bot protection on the server itself: an address that floods (over the rate
+// limit) or fails too many logins is banned here straight away, before the
+// next rules arrive; the report that goes back bans it on the site's other
+// servers too (the rules' "banned" list). Also the rules' `probes`: addresses
+// of files no visitor asks for (.env, .git/...). No Node built-ins (edge-safe).
+"use strict";
+
+const { parseIp } = require("./ip.js");
+
+const MAX = 50000;
+
+// One key per address however it's spelled: "1.2.3.4" and "::ffff:1.2.3.4"
+// (and IPv6 written two ways) are the same visitor.
+function keyOf(ip) {
+  const address = parseIp(ip);
+  return address ? `${address.family}:${address.n}` : String(ip);
+}
+
+function createLocalBans() {
+  const bans = new Map(); // address -> when the ban ends (ms)
+  const logins = new Map(); // address -> { start, count }
+
+  function banned(address) {
+    const ip = keyOf(address);
+    const until = bans.get(ip);
+    if (!until) return false;
+    if (until > Date.now()) return true;
+    bans.delete(ip);
+    return false;
+  }
+
+  function ban(address, seconds) {
+    if (!(seconds > 0)) return;
+    const ip = keyOf(address);
+    if (bans.size >= MAX) bans.clear();
+    bans.set(ip, Math.max(bans.get(ip) || 0, Date.now() + seconds * 1000));
+  }
+
+  // One failed login; login: the rules' { limit, window, ban } (seconds).
+  // True when it got the address banned.
+  function loginFailed(address, login) {
+    if (!login) return false;
+    const ip = keyOf(address);
+    const now = Date.now();
+    let entry = logins.get(ip);
+    if (!entry || now - entry.start >= login.window * 1000) {
+      if (!entry && logins.size >= MAX) logins.clear();
+      entry = { start: now, count: 0 };
+      logins.set(ip, entry);
+    }
+    entry.count += 1;
+    if (entry.count < login.limit) return false;
+    ban(address, login.ban);
+    logins.delete(ip);
+    return true;
+  }
+
+  return { banned, ban, loginFailed };
+}
+
+// The rules' probes -> { exact: Set, prefix: [...], ban (seconds) }, or null.
+function compileProbes(probes) {
+  if (!probes || typeof probes !== "object") return null;
+  const paths = (list) => (Array.isArray(list) ? list.filter((p) => typeof p === "string" && p).map((p) => p.toLowerCase()) : []);
+  const exact = new Set(paths(probes.exact));
+  const prefix = paths(probes.prefix);
+  if (!exact.size && !prefix.length) return null;
+  return { exact, prefix, ban: Number(probes.ban) > 0 ? Number(probes.ban) : 0 };
+}
+
+// True when the request's path (no query, lower-cased, "//" as "/") is a probe.
+function isProbe(path, probes) {
+  if (!probes) return false;
+  const clean = String(path || "").split(/[?#]/)[0].toLowerCase().replace(/\/{2,}/g, "/");
+  return probes.exact.has(clean) || probes.prefix.some((p) => clean.startsWith(p));
+}
+
+module.exports = { createLocalBans, compileProbes, isProbe };

package/lib/core.js

@@ -0,0 +1,122 @@
+// What every module shares: the site's credentials, sending to the
+// collector, logging, and the dashboard's module switches (Site settings →
+// Server, GET /events/platform/config about once a minute). A module the
+// dashboard turns off stays loaded but does nothing; until the switches
+// arrive, or when they can't be fetched, every loaded module stays on.
+// Sending never throws into the app.
+"use strict";
+
+const VERSION = "0.2.2";
+const TIMEOUT_MS = 5000;
+const MIN_SYNC_S = 30;
+const RETRY_S = 60;
+
+function createCore(options) {
+  const siteId = String(options.siteId || "");
+  const secret = String(options.secret || "");
+  const endpoint = String(options.endpoint || "").replace(/\/+$/, "");
+  const valid = /^[a-f0-9]{24}$/i.test(siteId) && Boolean(secret) && Boolean(endpoint);
+  const debug = Boolean(options.debug);
+  // options.client names the integration for the dashboard (default "node").
+  const userAgent = `analytics-platform-${options.client || "node"}/${VERSION}`;
+
+  const log = (module, ...args) => {
+    if (debug) console.warn(`[analytics-platform:${module}]`, ...args);
+  };
+
+  // JSON to the collector. auth: "body" puts site_id and sitesecret in the
+  // body (the event endpoints); "bearer" sends the secret as a header (the
+  // firewall). Resolves to the Response, or null when it couldn't be sent.
+  async function send(module, path, { method = "POST", body, auth = "body", headers = {} } = {}) {
+    if (!valid) return null;
+    const payload = body === undefined ? undefined
+      : JSON.stringify(auth === "body" ? { site_id: siteId, sitesecret: secret, ...body } : body);
+    try {
+      const res = await fetch(`${endpoint}${path}`, {
+        method,
+        headers: {
+          ...(payload === undefined ? {} : { "content-type": "application/json" }),
+          ...(auth === "bearer" ? { authorization: `Bearer ${secret}` } : {}),
+          "user-agent": userAgent,
+          ...headers,
+        },
+        body: payload,
+        signal: typeof AbortSignal.timeout === "function" ? AbortSignal.timeout(TIMEOUT_MS) : undefined,
+      });
+      if (!res.ok && res.status !== 304) log(module, `collector answered ${res.status} to ${path}`);
+      return res;
+    } catch (error) {
+      log(module, `could not reach the collector (${path}):`, error?.message);
+      return null;
+    }
+  }
+
+  // ---- Module switches ----
+  let switches = {};
+  let securityHeaders = [];
+  let flags = {};
+  let maintenance = { on: false };
+  let etag = "";
+  let timer = null;
+  let loaded = [];
+
+  // False only when the dashboard turned the module off.
+  const isOn = (name) => switches[name] !== false;
+
+  async function syncSwitches() {
+    let wait = RETRY_S;
+    try {
+      const res = await send("core", `/events/platform/config?site_id=${encodeURIComponent(siteId)}`, {
+        method: "GET", auth: "bearer", headers: { "x-platform-modules": loaded.join(","), ...(etag ? { "if-none-match": etag } : {}) },
+      });
+      if (res?.status === 200) {
+        const data = await res.json();
+        if (data?.v === 1 && data.modules && typeof data.modules === "object") {
+          switches = data.modules;
+          // Only well-formed headers (no line breaks can get through).
+          securityHeaders = (Array.isArray(data.headers) ? data.headers : []).filter((pair) => Array.isArray(pair)
+            && /^[A-Za-z][A-Za-z0-9-]{0,63}$/.test(pair[0]) && typeof pair[1] === "string" && !/[\r\n]/.test(pair[1]) && pair[1].length <= 4000);
+          flags = data.flags && typeof data.flags === "object" ? data.flags : {};
+          maintenance = data.maintenance && typeof data.maintenance === "object" ? data.maintenance : { on: false };
+          etag = res.headers.get("etag") || "";
+        }
+        wait = Math.max(MIN_SYNC_S, Number(data?.sync) || RETRY_S);
+      } else if (res?.status === 304) wait = RETRY_S;
+    } catch (error) {
+      log("core", "could not read module switches:", error?.message);
+    }
+    if (timer !== false) {
+      timer = setTimeout(syncSwitches, wait * 1000);
+      timer.unref?.();
+    }
+  }
+
+  // Starts fetching the switches; `names` are the modules the code loaded.
+  function start(names) {
+    loaded = names;
+    return valid ? syncSwitches() : Promise.resolve();
+  }
+
+  function stop() {
+    clearTimeout(timer);
+    timer = false;
+  }
+
+  // The security headers picked on the dashboard, as [name, value] pairs.
+  const headers = () => securityHeaders;
+
+  return {
+    siteId, secret, endpoint, valid, debug, log, send, userAgent, version: VERSION, isOn, headers, start, stop,
+    flags: () => flags,
+    maintenance: () => maintenance,
+  };
+}
+
+// The browser's visit, when the page passed it on (window.analyticsSession.getId()).
+const SESSION_HEADER = "x-analytics-session";
+function sessionOf(req) {
+  const value = req?.headers?.[SESSION_HEADER];
+  return typeof value === "string" ? value : "";
+}
+
+module.exports = { createCore, sessionOf, VERSION };

package/lib/crawlers.js

@@ -0,0 +1,216 @@
+// Crawlers module: counts the search engines and AI bots that fetch the site
+// and sends one row a minute per bot, path and status to POST /events/crawlers
+// (the protocol is sdk/protocols/crawlers.md). This traffic is invisible to
+// the browser tracker, because crawlers don't run JavaScript: the server is
+// the only place it can be counted.
+//
+// Only requests whose user agent names a known crawler are counted, so
+// visitors are never counted twice. Requests the firewall turned away count
+// too, with the status the crawler got. Nothing about a visitor is sent; bot
+// addresses go with the batch only so the collector can check a crawler is
+// itself (reverse DNS), and are not stored.
+"use strict";
+
+const os = require("node:os");
+
+const FLUSH_MS = 60 * 1000;
+const MAX_KEYS = 300;
+const MAX_ROWS = 500;
+const MAX_IPS_PER_BOT = 5;
+const MAX_PATH = 400;
+const OTHER = "(other)";
+
+// The catalogue of the protocol, most specific first: "ChatGPT-User" must be
+// tried before "GPTBot", "Claude-User" before "ClaudeBot". Keep in step with
+// the collector (expresscollect/Analytics/crawlers.js CATALOGUE).
+const BOTS = [
+  // Fetched because a person asked an AI about the page.
+  ["ChatGPT-User", /ChatGPT-User/i],
+  ["OAI-SearchBot", /OAI-SearchBot/i],
+  ["Perplexity-User", /Perplexity-User/i],
+  ["Claude-User", /Claude-User/i],
+  ["Claude-SearchBot", /Claude-SearchBot/i],
+  ["MistralAI-User", /MistralAI-User/i],
+  ["MistralAI-SearchBot", /MistralAI-SearchBot/i],
+  ["Meta-ExternalFetcher", /meta-externalfetcher/i],
+  // Bulk collection, for training or a dataset.
+  ["GPTBot", /GPTBot/i],
+  ["ClaudeBot", /ClaudeBot|Claude-Web/i],
+  ["anthropic-ai", /anthropic-ai/i],
+  ["CCBot", /CCBot/i],
+  ["Bytespider", /Bytespider/i],
+  ["Meta-ExternalAgent", /meta-externalagent|facebookbot/i],
+  ["Amazonbot", /Amazonbot/i],
+  ["Diffbot", /Diffbot/i],
+  ["omgili", /omgili/i],
+  ["AI2Bot", /AI2Bot/i],
+  ["Timpibot", /Timpibot/i],
+  ["Webzio-Extended", /Webzio-Extended/i],
+  ["PerplexityBot", /PerplexityBot/i],
+  ["DuckAssistBot", /DuckAssistBot/i],
+  ["YouBot", /YouBot/i],
+  // Search engines. Google's tools before Googlebot itself.
+  ["Google-InspectionTool", /Google-InspectionTool/i],
+  ["Storebot-Google", /Storebot-Google/i],
+  ["AdsBot-Google", /AdsBot-Google/i],
+  ["Googlebot", /Googlebot|Google-Site-Verification/i],
+  ["bingbot", /bingbot|BingPreview|adidxbot/i],
+  ["YandexBot", /Yandex(Bot|Mobile|Images|Webmaster)?/i],
+  ["Baiduspider", /Baiduspider/i],
+  ["DuckDuckBot", /DuckDuckBot|DuckDuckGo-Favicons/i],
+  ["Applebot", /Applebot/i],
+  ["PetalBot", /PetalBot|AspiegelBot/i],
+  ["SeznamBot", /SeznamBot/i],
+  ["Yeti", /Yeti\//i],
+  ["MojeekBot", /MojeekBot/i],
+  ["Qwantbot", /Qwantbot|Qwantify/i],
+  // SEO and backlink tools.
+  ["AhrefsBot", /AhrefsBot|AhrefsSiteAudit/i],
+  ["SemrushBot", /SemrushBot|SiteAuditBot/i],
+  ["DotBot", /DotBot/i],
+  ["MJ12bot", /MJ12bot/i],
+  ["BLEXBot", /BLEXBot/i],
+  ["DataForSeoBot", /DataForSeoBot/i],
+  ["Barkrowler", /Barkrowler/i],
+  ["ScreamingFrog", /Screaming\s?Frog/i],
+  ["ZoominfoBot", /ZoominfoBot/i],
+  // Link previews.
+  ["facebookexternalhit", /facebookexternalhit/i],
+  ["Twitterbot", /Twitterbot/i],
+  ["LinkedInBot", /LinkedInBot/i],
+  ["Slackbot", /Slackbot|Slack-ImgProxy/i],
+  ["Discordbot", /Discordbot/i],
+  ["TelegramBot", /TelegramBot/i],
+  ["WhatsApp", /WhatsApp/i],
+  ["Pinterestbot", /Pinterest(bot)?\//i],
+  ["redditbot", /redditbot/i],
+  // Uptime and speed checks.
+  ["UptimeRobot", /UptimeRobot/i],
+  ["Pingdom", /Pingdom/i],
+  ["BetterUptime", /Better\s?Uptime|betteruptime/i],
+  ["Lighthouse", /Chrome-Lighthouse|Google Page Speed/i],
+  ["GTmetrix", /GTmetrix/i],
+  // Archives.
+  ["ia_archiver", /ia_archiver/i],
+  ["archive.org_bot", /archive\.org_bot|Wayback/i],
+];
+
+// Anything else that names itself a bot: counted as "Other bot", so a new
+// crawler shows up as traffic instead of disappearing.
+const GENERIC = /bot\b|crawler|crawling|spider|scrapy|curl\/|wget\/|python-requests|headlesschrome|http-client|okhttp|java\/|go-http-client/i;
+
+// The crawler's catalogue name, or "" when the user agent isn't one.
+function detect(userAgent) {
+  const ua = String(userAgent || "");
+  if (!ua) return "";
+  for (const [name, pattern] of BOTS) if (pattern.test(ua)) return name;
+  return GENERIC.test(ua) ? "Other bot" : "";
+}
+
+// Options: serverName (default the host name), clientIp(req) (the firewall's,
+// so the address is found the same way behind a proxy).
+function createCrawlers(core, options = {}) {
+  const log = (...args) => core.log("crawlers", ...args);
+  const serverName = String(options.serverName ?? os.hostname()).slice(0, 100);
+  const runtime = `Node ${process.versions.node}`;
+  const addressOf = typeof options.clientIp === "function"
+    ? options.clientIp
+    : (req) => req.ip || req.socket?.remoteAddress || "";
+  let rows = new Map();
+  const keysIn = new Map(); // minute -> rows it has
+  let agents = new Map();
+  let sending = Promise.resolve();
+
+  // Records one finished request from a crawler.
+  function record({ bot, path, status, bytes = 0, ms = 0, ip = "" }) {
+    if (!bot || !core.isOn("crawlers")) return;
+    const minute = Math.floor(Date.now() / 60000) * 60000;
+    const clean = String(path || "/").split(/[?#]/)[0].slice(0, MAX_PATH) || "/";
+    let key = `${minute}|${bot}|${clean}|${status}`;
+    // Past MAX_KEYS a crawler asking for thousands of URLs becomes one row.
+    // The cap is per minute: rows of a minute not yet sent don't count.
+    const used = keysIn.get(minute) || 0;
+    if (!rows.has(key) && used >= MAX_KEYS) key = `${minute}|${bot}|${OTHER}|${status}`;
+    let row = rows.get(key);
+    if (!row) {
+      row = { minute, bot, path: key.split("|")[2], status, count: 0, bytes: 0, sum_ms: 0 };
+      rows.set(key, row);
+      keysIn.set(minute, used + 1);
+    }
+    row.count += 1;
+    row.bytes += Math.max(0, Number(bytes) || 0);
+    row.sum_ms += Math.max(0, Number(ms) || 0);
+    if (ip) {
+      let seen = agents.get(bot);
+      if (!seen) agents.set(bot, (seen = new Set()));
+      if (seen.size < MAX_IPS_PER_BOT) seen.add(ip);
+    }
+  }
+
+  // Express / Connect / node:http middleware. Runs before the firewall, so a
+  // crawler that was turned away is still counted, with the status it got.
+  function middleware(req, res, next = () => {}) {
+    const bot = detect(req.headers?.["user-agent"]);
+    if (!bot) return next();
+    const start = process.hrtime.bigint();
+    res.once("finish", () => {
+      try {
+        const length = Number(res.getHeader?.("content-length")) || 0;
+        record({
+          bot,
+          path: req.originalUrl || req.url || "/",
+          status: res.statusCode || 0,
+          bytes: length,
+          ms: Number(process.hrtime.bigint() - start) / 1e6,
+          ip: addressOf(req),
+        });
+      } catch (error) {
+        log("could not count a crawler request:", error?.message);
+      }
+    });
+    return next();
+  }
+
+  // Sends the rows of finished minutes (all of them with all = true).
+  function flush(all = false) {
+    const current = Math.floor(Date.now() / 60000) * 60000;
+    const ready = [];
+    for (const [key, row] of rows) {
+      if (all || row.minute < current) {
+        ready.push({ ...row, minute: new Date(row.minute).toISOString(), sum_ms: Math.round(row.sum_ms) });
+        rows.delete(key);
+      }
+    }
+    for (const minute of keysIn.keys()) if (all || minute < current) keysIn.delete(minute);
+    if (!ready.length) return sending;
+    const sent = new Set(ready.map((row) => row.bot));
+    const batchAgents = [...agents].filter(([bot]) => sent.has(bot)).map(([bot, ips]) => ({ bot, ips: [...ips] }));
+    // A bot with rows still waiting (this minute isn't over) keeps its
+    // addresses, so the next batch can be checked as well.
+    const waiting = new Set([...rows.values()].map((row) => row.bot));
+    agents = new Map([...agents].filter(([bot]) => waiting.has(bot)));
+    for (let i = 0; i < ready.length; i += MAX_ROWS) {
+      const batch = ready.slice(i, i + MAX_ROWS);
+      sending = sending.then(() => core.send("crawlers", "/events/crawlers", {
+        body: { server_name: serverName, runtime, rows: batch, agents: i === 0 ? batchAgents : [] },
+      }));
+    }
+    return sending;
+  }
+
+  const timer = core.valid ? setInterval(() => flush(), FLUSH_MS) : null;
+  timer?.unref?.();
+
+  return {
+    middleware,
+    record,
+    detect,
+    flush: () => flush(true),
+    close() {
+      clearInterval(timer);
+      return flush(true);
+    },
+  };
+}
+
+module.exports = { createCrawlers, detect, BOTS };