@litemetrics/node 0.1.0 → 0.1.1

package/README.md

@@ -0,0 +1,108 @@
+# @litemetrics/node
+
+Self-hosted analytics server for Litemetrics. Includes event collection, query API, site management, GeoIP enrichment, and bot filtering.
+
+## Installation
+
+```bash
+npm install @litemetrics/node
+```
+
+## Quick Start
+
+```ts
+import express from 'express';
+import { createCollector } from '@litemetrics/node';
+
+const app = express();
+app.use(express.json());
+
+const collector = await createCollector({
+  db: { url: 'http://localhost:8123' }, // ClickHouse (default)
+  adminSecret: 'your-admin-secret',
+  geoip: true,
+  cors: { origins: [] }, // Allow all origins
+});
+
+// Event collection endpoint (receives tracker data)
+app.post('/api/collect', collector.handler());
+
+// Query API (stats, time series, retention)
+app.get('/api/stats', collector.queryHandler());
+
+// Event & user listing
+app.all('/api/events', collector.eventsHandler());
+app.all('/api/users/*', collector.usersHandler());
+
+// Site management (CRUD)
+app.all('/api/sites/*', collector.sitesHandler());
+
+// Serve tracker script
+app.use(express.static('node_modules/@litemetrics/tracker/dist'));
+
+app.listen(3000);
+```
+
+## Database Adapters
+
+### ClickHouse (Default)
+
+```ts
+const collector = await createCollector({
+  db: { url: 'http://localhost:8123' },
+});
+```
+
+Uses `MergeTree` for events and `ReplacingMergeTree` for sites. Tables are auto-created on init.
+
+### MongoDB
+
+```ts
+const collector = await createCollector({
+  db: { adapter: 'mongodb', url: 'mongodb://localhost:27017/litemetrics' },
+});
+```
+
+## Features
+
+- **Event Collection** - Receives batched events from the browser tracker
+- **Bot Filtering** - Automatically drops events from known bots and crawlers
+- **GeoIP Enrichment** - Resolves country/city from IP using MaxMind GeoLite2
+- **User-Agent Parsing** - Extracts browser, OS, and device type
+- **Hostname Filtering** - Only count events from allowed hostnames per site
+- **Query API** - 12 built-in metrics (pageviews, visitors, sessions, top pages, etc.)
+- **Time Series** - Hourly/daily/weekly/monthly breakdowns
+- **Retention Analysis** - Weekly cohort retention
+- **Site Management** - Multi-tenant CRUD with secret key auth
+- **Server-Side Tracking** - Track events and identify users from your backend
+
+## Server-Side Tracking
+
+```ts
+// Track events from your backend
+await collector.track('site-id', 'Purchase', { amount: 99 }, { userId: 'user-123' });
+
+// Identify users
+await collector.identify('site-id', 'user-123', { plan: 'pro' });
+```
+
+## Metrics
+
+| Metric | Description |
+|--------|-------------|
+| `pageviews` | Total page views |
+| `visitors` | Unique visitors |
+| `sessions` | Unique sessions |
+| `events` | Custom events count |
+| `top_pages` | Most visited pages |
+| `top_referrers` | Top traffic sources |
+| `top_countries` | Visitors by country |
+| `top_cities` | Visitors by city |
+| `top_events` | Most common custom events |
+| `top_browsers` | Browser breakdown |
+| `top_os` | OS breakdown |
+| `top_devices` | Device type breakdown |
+
+## License
+
+MIT

package/dist/index.cjs

@@ -32,7 +32,8 @@ var index_exports = {};
 __export(index_exports, {
   ClickHouseAdapter: () => ClickHouseAdapter,
   MongoDBAdapter: () => MongoDBAdapter,
-  createCollector: () => createCollector
+  createCollector: () => createCollector,
+  isBot: () => isBot
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -230,6 +231,10 @@ CREATE TABLE IF NOT EXISTS ${SITES_TABLE} (
   ORDER BY (site_id)
   SETTINGS index_granularity = 8192
 `;
+function toCHDateTime(d) {
+  const iso = typeof d === "string" ? d : d.toISOString();
+  return iso.replace("T", " ").replace("Z", "");
+}
 var ClickHouseAdapter = class {
   client;
   constructor(url) {
@@ -253,7 +258,7 @@ var ClickHouseAdapter = class {
     const rows = events.map((e) => ({
       site_id: e.siteId,
       type: e.type,
-      timestamp: new Date(e.timestamp).toISOString(),
+      timestamp: toCHDateTime(new Date(e.timestamp)),
       session_id: e.sessionId,
       visitor_id: e.visitorId,
       url: e.url ?? null,
@@ -293,8 +298,8 @@ var ClickHouseAdapter = class {
     const limit = q.limit ?? 10;
     const params = {
       siteId,
-      from: dateRange.from,
-      to: dateRange.to,
+      from: toCHDateTime(dateRange.from),
+      to: toCHDateTime(dateRange.to),
       limit
     };
     let data = [];
@@ -542,8 +547,8 @@ var ClickHouseAdapter = class {
     }
     const rows = await this.queryRows(sql, {
       siteId: params.siteId,
-      from: dateRange.from,
-      to: dateRange.to
+      from: toCHDateTime(dateRange.from),
+      to: toCHDateTime(dateRange.to)
     });
     const mappedRows = rows.map((r) => ({
       _id: this.convertClickHouseBucket(r.bucket, granularity),
@@ -609,7 +614,7 @@ var ClickHouseAdapter = class {
       GROUP BY visitor_id`,
       {
         siteId: params.siteId,
-        since: startDate.toISOString()
+        since: toCHDateTime(startDate)
       }
     );
     const cohortMap = /* @__PURE__ */ new Map();
@@ -676,8 +681,8 @@ var ClickHouseAdapter = class {
         dateTo: params.dateTo
       });
       conditions.push(`timestamp >= {from:String} AND timestamp <= {to:String}`);
-      queryParams.from = dateRange.from;
-      queryParams.to = dateRange.to;
+      queryParams.from = toCHDateTime(dateRange.from);
+      queryParams.to = toCHDateTime(dateRange.to);
     }
     const where = conditions.join(" AND ");
     const [events, countRows] = await Promise.all([
@@ -783,15 +788,17 @@ var ClickHouseAdapter = class {
   }
   // ─── Site Management ──────────────────────────────────────
   async createSite(data) {
-    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const now = /* @__PURE__ */ new Date();
+    const nowISO = now.toISOString();
+    const nowCH = toCHDateTime(now);
     const site = {
       siteId: generateSiteId(),
       secretKey: generateSecretKey(),
       name: data.name,
       domain: data.domain,
       allowedOrigins: data.allowedOrigins,
-      createdAt: now,
-      updatedAt: now
+      createdAt: nowISO,
+      updatedAt: nowISO
     };
     await this.client.insert({
       table: SITES_TABLE,
@@ -801,8 +808,8 @@ var ClickHouseAdapter = class {
         name: site.name,
         domain: site.domain ?? null,
         allowed_origins: site.allowedOrigins ? JSON.stringify(site.allowedOrigins) : null,
-        created_at: now,
-        updated_at: now,
+        created_at: nowCH,
+        updated_at: nowCH,
         version: 1,
         is_deleted: 0
       }],
@@ -847,7 +854,9 @@ var ClickHouseAdapter = class {
     );
     if (currentRows.length === 0) return null;
     const current = currentRows[0];
-    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const now = /* @__PURE__ */ new Date();
+    const nowISO = now.toISOString();
+    const nowCH = toCHDateTime(now);
     const newVersion = Number(current.version) + 1;
     const newName = data.name !== void 0 ? data.name : String(current.name);
     const newDomain = data.domain !== void 0 ? data.domain || null : current.domain ? String(current.domain) : null;
@@ -860,8 +869,8 @@ var ClickHouseAdapter = class {
         name: newName,
         domain: newDomain,
         allowed_origins: newOrigins,
-        created_at: String(current.created_at),
-        updated_at: now,
+        created_at: toCHDateTime(String(current.created_at)),
+        updated_at: nowCH,
         version: newVersion,
         is_deleted: 0
       }],
@@ -874,7 +883,7 @@ var ClickHouseAdapter = class {
       domain: newDomain ?? void 0,
       allowedOrigins: newOrigins ? JSON.parse(newOrigins) : void 0,
       createdAt: String(current.created_at),
-      updatedAt: now
+      updatedAt: nowISO
     };
   }
   async deleteSite(siteId) {
@@ -886,7 +895,7 @@ var ClickHouseAdapter = class {
     );
     if (currentRows.length === 0) return false;
     const current = currentRows[0];
-    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const nowCH = toCHDateTime(/* @__PURE__ */ new Date());
     await this.client.insert({
       table: SITES_TABLE,
       values: [{
@@ -895,8 +904,8 @@ var ClickHouseAdapter = class {
         name: String(current.name),
         domain: current.domain ? String(current.domain) : null,
         allowed_origins: current.allowed_origins ? String(current.allowed_origins) : null,
-        created_at: String(current.created_at),
-        updated_at: now,
+        created_at: toCHDateTime(String(current.created_at)),
+        updated_at: nowCH,
         version: Number(current.version) + 1,
         is_deleted: 1
       }],
@@ -913,7 +922,9 @@ var ClickHouseAdapter = class {
     );
     if (currentRows.length === 0) return null;
     const current = currentRows[0];
-    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const now = /* @__PURE__ */ new Date();
+    const nowISO = now.toISOString();
+    const nowCH = toCHDateTime(now);
     const newSecret = generateSecretKey();
     await this.client.insert({
       table: SITES_TABLE,
@@ -923,8 +934,8 @@ var ClickHouseAdapter = class {
         name: String(current.name),
         domain: current.domain ? String(current.domain) : null,
         allowed_origins: current.allowed_origins ? String(current.allowed_origins) : null,
-        created_at: String(current.created_at),
-        updated_at: now,
+        created_at: toCHDateTime(String(current.created_at)),
+        updated_at: nowCH,
         version: Number(current.version) + 1,
         is_deleted: 0
       }],
@@ -937,7 +948,7 @@ var ClickHouseAdapter = class {
       domain: current.domain ? String(current.domain) : void 0,
       allowedOrigins: current.allowed_origins ? JSON.parse(String(current.allowed_origins)) : void 0,
       createdAt: String(current.created_at),
-      updatedAt: now
+      updatedAt: nowISO
     };
   }
   // ─── Helpers ─────────────────────────────────────────────
@@ -1700,6 +1711,63 @@ function resolveDeviceType(type) {
   return "desktop";
 }
 
+// src/botfilter.ts
+var BOT_PATTERNS = [
+  // Headless browsers
+  /HeadlessChrome/i,
+  /PhantomJS/i,
+  /Selenium/i,
+  /Puppeteer/i,
+  /Playwright/i,
+  // Common bots
+  /bot\b/i,
+  /spider/i,
+  /crawl/i,
+  /slurp/i,
+  /mediapartners/i,
+  /facebookexternalhit/i,
+  /Twitterbot/i,
+  /LinkedInBot/i,
+  /WhatsApp/i,
+  /Discordbot/i,
+  /TelegramBot/i,
+  /Applebot/i,
+  /Baiduspider/i,
+  /YandexBot/i,
+  /DuckDuckBot/i,
+  /Sogou/i,
+  /Exabot/i,
+  /ia_archiver/i,
+  // HTTP libraries & API tools
+  /PostmanRuntime/i,
+  /axios/i,
+  /node-fetch/i,
+  /python-requests/i,
+  /Go-http-client/i,
+  /Java\//i,
+  /libwww-perl/i,
+  /wget/i,
+  /curl/i,
+  /httpie/i,
+  // Monitoring / uptime
+  /UptimeRobot/i,
+  /Pingdom/i,
+  /StatusCake/i,
+  /Site24x7/i,
+  /NewRelic/i,
+  /Datadog/i,
+  // Preview/embed
+  /Slackbot/i,
+  /Embedly/i,
+  /Quora Link Preview/i,
+  /redditbot/i,
+  /Pinterestbot/i
+];
+function isBot(ua) {
+  if (!ua || ua.length === 0) return true;
+  return BOT_PATTERNS.some((re) => re.test(ua));
+}
+
 // src/collector.ts
 async function createCollector(config) {
   const db = createAdapter(config.db);
@@ -1772,9 +1840,36 @@ async function createCollector(config) {
           sendJson(res, 400, { ok: false, error: "Too many events (max 100)" });
           return;
         }
-        const ip = extractIp(req);
         const userAgent = req.headers?.["user-agent"] || "";
+        if (isBot(userAgent)) {
+          sendJson(res, 200, { ok: true });
+          return;
+        }
+        const ip = extractIp(req);
         const enriched = enrichEvents(payload.events, ip, userAgent);
+        const siteId = enriched[0]?.siteId;
+        if (siteId) {
+          const site = await db.getSite(siteId);
+          if (site?.allowedOrigins && site.allowedOrigins.length > 0) {
+            const allowed = new Set(site.allowedOrigins.map((h) => h.toLowerCase()));
+            const filtered = enriched.filter((event) => {
+              if (!event.url) return true;
+              try {
+                const hostname = new URL(event.url).hostname.toLowerCase();
+                return allowed.has(hostname);
+              } catch {
+                return true;
+              }
+            });
+            if (filtered.length === 0) {
+              sendJson(res, 200, { ok: true });
+              return;
+            }
+            await db.insertEvents(filtered);
+            sendJson(res, 200, { ok: true });
+            return;
+          }
+        }
         await db.insertEvents(enriched);
         sendJson(res, 200, { ok: true });
       } catch (err) {
@@ -2115,6 +2210,7 @@ function sendJson(res, status, body) {
 0 && (module.exports = {
   ClickHouseAdapter,
   MongoDBAdapter,
-  createCollector
+  createCollector,
+  isBot
 });
 //# sourceMappingURL=index.cjs.map