@rawdash/connector-clerk 0.24.0

package/dist/index.js

@@ -0,0 +1,638 @@
+// ../../connector-shared/dist/index.js
+var HTTP_CLIENT_VERSION = "0.0.0";
+var DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
+function connectorUserAgent(connectorId) {
+  return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
+}
+function standardRateLimitPolicy(config) {
+  const { remainingHeader, resetHeader, resetUnit, resetFallbackMs } = config;
+  const multiplier = resetUnit === "s" ? 1e3 : 1;
+  return {
+    parse(h) {
+      const remainingRaw = h.get(remainingHeader);
+      if (remainingRaw === null || remainingRaw.trim() === "") {
+        return null;
+      }
+      const remaining = Number(remainingRaw);
+      if (!Number.isFinite(remaining)) {
+        return null;
+      }
+      const resetRaw = h.get(resetHeader);
+      if (resetRaw === null) {
+        if (resetFallbackMs === void 0) {
+          return null;
+        }
+        return {
+          remaining,
+          resetAt: new Date(Date.now() + resetFallbackMs)
+        };
+      }
+      if (resetRaw.trim() === "") {
+        return null;
+      }
+      const reset = Number(resetRaw);
+      if (!Number.isFinite(reset) || reset < 0) {
+        return null;
+      }
+      const resetMs = reset * multiplier;
+      if (!Number.isFinite(resetMs)) {
+        return null;
+      }
+      return { remaining, resetAt: new Date(resetMs) };
+    }
+  };
+}
+function parseEpoch(value, unit) {
+  if (value === null || value === void 0) {
+    return null;
+  }
+  if (unit === "iso") {
+    if (typeof value !== "string") {
+      return null;
+    }
+    const ms = new Date(value).getTime();
+    return Number.isFinite(ms) ? ms : null;
+  }
+  if (typeof value === "string" && value.trim() === "") {
+    return null;
+  }
+  const n = typeof value === "number" ? value : Number(value);
+  if (!Number.isFinite(n)) {
+    return null;
+  }
+  const result = unit === "s" ? n * 1e3 : n;
+  return Number.isFinite(result) ? result : null;
+}
+
+// src/clerk.ts
+import {
+  BaseConnector,
+  defineConfigFields,
+  defineConnectorDoc,
+  defineResources,
+  makeChunkedCursorGuard,
+  paginateChunked,
+  schemasFromResources,
+  selectActivePhases
+} from "@rawdash/core";
+import { z } from "zod";
+var configFields = defineConfigFields(
+  z.object({
+    secretKey: z.object({ $secret: z.string().min(1) }).meta({
+      label: "Secret key",
+      description: "Clerk Backend API secret key (starts with `sk_test_` or `sk_live_`). Create one at Clerk Dashboard -> API Keys.",
+      placeholder: "CLERK_SECRET_KEY",
+      secret: true
+    }),
+    apiUrl: z.string().trim().url('Must be a full URL, e.g. "https://api.clerk.com".').default("https://api.clerk.com").meta({
+      label: "API base URL",
+      description: "Clerk Backend API base URL. Defaults to https://api.clerk.com; override only if you are pinned to the legacy https://api.clerk.dev host.",
+      placeholder: "https://api.clerk.com"
+    }),
+    resources: z.array(
+      z.enum(["users", "organizations", "sessions", "daily_active_users"])
+    ).nonempty().optional().meta({
+      label: "Resources",
+      description: "Which Clerk resources to sync. Omit to sync all of them. The secret key has read access to every resource by default; the allowlist exists to skip phases your dashboards do not query."
+    }),
+    dauLookbackDays: z.number().int().positive().max(90).optional().meta({
+      label: "DAU lookback (days)",
+      description: "How many days back to bucket users by last_active_at when computing the daily_active_users metric. Defaults to 30; the cap is 90.",
+      placeholder: "30"
+    })
+  })
+);
+var doc = defineConnectorDoc({
+  displayName: "Clerk",
+  category: "security",
+  brandColor: "#6C47FF",
+  tagline: "Sync users, organizations, sessions, and a derived daily-active-users metric from a Clerk application for sign-up, DAU, and active-session dashboards.",
+  vendor: {
+    name: "Clerk",
+    domain: "clerk.com",
+    apiDocs: "https://clerk.com/docs/reference/backend-api",
+    website: "https://clerk.com"
+  },
+  auth: {
+    summary: "A Clerk Backend API secret key (Bearer token). Anyone with the key has read access to every resource the connector syncs.",
+    setup: [
+      "Open the Clerk Dashboard for the application you want to sync and navigate to API Keys.",
+      "Copy the Secret key (it starts with `sk_test_` for development instances or `sk_live_` for production).",
+      'Store it as a rawdash secret and reference it from the connector config as `secretKey: secret("CLERK_SECRET_KEY")`.',
+      "Treat the secret key like a root credential - rotate it from the dashboard if it leaks."
+    ]
+  },
+  rateLimit: "Clerk Backend API throttles per instance (~20 req/s for production, lower for dev). Responses publish X-RateLimit-Remaining / X-RateLimit-Reset (Unix seconds) headers and the shared HTTP client backs off on 429 using the standard rate-limit policy.",
+  limitations: [
+    "Each phase paginates via limit / offset and is capped at 50 pages per sync (~25,000 rows). Instances larger than that should run more frequent incremental syncs so each window fits under the cap.",
+    "The daily_active_users metric is derived by bucketing users by the day of their last_active_at timestamp - it counts users whose most recent activity fell on each day, not unique users active across overlapping days.",
+    "Webhooks, JWT templates, instance settings, and impersonation tokens are out of scope."
+  ]
+});
+var clerkCredentials = {
+  secretKey: {
+    description: "Clerk Backend API secret key",
+    auth: "required"
+  }
+};
+var clerkRateLimit = standardRateLimitPolicy({
+  remainingHeader: "x-ratelimit-remaining",
+  resetHeader: "x-ratelimit-reset",
+  resetUnit: "s"
+});
+var PHASE_ORDER = [
+  "users",
+  "organizations",
+  "sessions",
+  "daily_active_users"
+];
+var isClerkSyncCursor = makeChunkedCursorGuard(PHASE_ORDER);
+var USER_ENTITY = "clerk_user";
+var ORG_ENTITY = "clerk_organization";
+var SESSION_EVENT = "clerk_session";
+var DAU_METRIC = "clerk_daily_active_users";
+var PAGE_SIZE = 500;
+var MAX_PAGES = 50;
+var DEFAULT_DAU_LOOKBACK_DAYS = 30;
+var DEFAULT_API_URL = "https://api.clerk.com";
+var DAY_MS = 24 * 60 * 60 * 1e3;
+var SESSION_STATUSES = [
+  "abandoned",
+  "active",
+  "ended",
+  "expired",
+  "removed",
+  "replaced",
+  "revoked"
+];
+var idString = z.string().min(1);
+var emailAddressSchema = z.object({
+  id: z.string().optional(),
+  email_address: z.string().nullish(),
+  verification: z.object({ status: z.string().nullish() }).nullish()
+});
+var userSchema = z.object({
+  id: idString,
+  primary_email_address_id: z.string().nullish(),
+  email_addresses: z.array(emailAddressSchema).nullish(),
+  first_name: z.string().nullish(),
+  last_name: z.string().nullish(),
+  username: z.string().nullish(),
+  last_sign_in_at: z.number().nullish(),
+  last_active_at: z.number().nullish(),
+  created_at: z.number().nullish(),
+  updated_at: z.number().nullish(),
+  banned: z.boolean().nullish(),
+  locked: z.boolean().nullish()
+});
+var usersResponseSchema = z.array(userSchema);
+var organizationSchema = z.object({
+  id: idString,
+  name: z.string().nullish(),
+  slug: z.string().nullish(),
+  members_count: z.number().nullish(),
+  created_at: z.number().nullish(),
+  updated_at: z.number().nullish()
+});
+var organizationsResponseSchema = z.union([
+  z.object({
+    data: z.array(organizationSchema),
+    total_count: z.number().optional()
+  }),
+  z.array(organizationSchema)
+]);
+var sessionSchema = z.object({
+  id: idString,
+  user_id: z.string().nullish(),
+  client_id: z.string().nullish(),
+  status: z.string(),
+  last_active_at: z.number().nullish(),
+  expire_at: z.number().nullish(),
+  abandon_at: z.number().nullish(),
+  created_at: z.number().nullish(),
+  updated_at: z.number().nullish()
+});
+var sessionsResponseSchema = z.array(sessionSchema);
+var clerkResources = defineResources({
+  [USER_ENTITY]: {
+    shape: "entity",
+    filterable: [
+      { field: "banned", ops: ["eq"], values: ["true", "false"] },
+      { field: "locked", ops: ["eq"], values: ["true", "false"] }
+    ],
+    description: "Clerk users keyed by user id, with primary email, sign-in / activity timestamps, and banned / locked flags.",
+    endpoint: "GET /v1/users",
+    notes: "Uses offset pagination (limit / offset) capped at 50 pages (~25,000 users) per sync. Incremental syncs pass options.since through as the last_active_at_since filter.",
+    fields: [
+      { name: "email", description: "Primary email address (when present)." },
+      {
+        name: "emailVerified",
+        description: "Whether the primary email address is verified (null if no email is set)."
+      },
+      {
+        name: "lastSignInAt",
+        description: "Most recent sign-in timestamp (Unix ms)."
+      },
+      {
+        name: "lastActiveAt",
+        description: "Most recent activity timestamp (Unix ms). Clerk updates this on every successful client request."
+      },
+      {
+        name: "banned",
+        description: "Whether the user has been banned."
+      },
+      {
+        name: "locked",
+        description: "Whether the user is locked from signing in."
+      },
+      {
+        name: "createdAt",
+        description: "When the user account was created (Unix ms)."
+      }
+    ],
+    responses: { users: usersResponseSchema }
+  },
+  [ORG_ENTITY]: {
+    shape: "entity",
+    filterable: [],
+    description: "Clerk organizations keyed by organization id, with display name, slug, and members count.",
+    endpoint: "GET /v1/organizations",
+    notes: "Uses offset pagination (limit / offset) capped at 50 pages. Clerk has no created_at / updated_at filter for organizations, so each sync re-scans the full list and short-circuits once a page is entirely older than options.since.",
+    fields: [
+      { name: "name", description: "Organization display name." },
+      { name: "slug", description: "Organization URL slug." },
+      {
+        name: "membersCount",
+        description: "Number of users in the organization at sync time."
+      },
+      {
+        name: "createdAt",
+        description: "When the organization was created (Unix ms)."
+      }
+    ],
+    responses: { organizations: organizationsResponseSchema }
+  },
+  [SESSION_EVENT]: {
+    shape: "event",
+    filterable: [
+      {
+        field: "status",
+        ops: ["eq"],
+        values: SESSION_STATUSES
+      }
+    ],
+    description: "Clerk session events. One event per session row with start_ts set to created_at and attributes carrying user id, status, and last activity.",
+    endpoint: "GET /v1/sessions",
+    notes: "Uses offset pagination (limit / offset) capped at 50 pages. Clerk has no since filter on /v1/sessions, so the sync walks newest-first and stops once a page is entirely older than options.since.",
+    fields: [
+      { name: "sessionId", description: "Clerk session id." },
+      { name: "userId", description: "User the session belongs to." },
+      {
+        name: "status",
+        description: "Session status (active | ended | expired | abandoned | removed | replaced | revoked)."
+      },
+      {
+        name: "lastActiveAt",
+        description: "Most recent activity timestamp on the session (Unix ms)."
+      }
+    ],
+    responses: { sessions: sessionsResponseSchema }
+  },
+  [DAU_METRIC]: {
+    shape: "metric",
+    description: "Daily active users derived from the Clerk users endpoint: one sample per UTC day in the configured lookback window, counting users whose last_active_at fell on that day.",
+    endpoint: "GET /v1/users",
+    unit: "count",
+    granularity: "1d",
+    dimensions: [],
+    responses: { dau_users: usersResponseSchema }
+  }
+});
+var id = "clerk";
+function primaryEmail(user) {
+  const list = user.email_addresses ?? [];
+  if (list.length === 0) {
+    return { email: null, verified: null };
+  }
+  const primaryId = user.primary_email_address_id ?? null;
+  const primary = (primaryId !== null ? list.find((e) => e.id === primaryId) : void 0) ?? list[0];
+  const verified = primary.verification?.status === "verified" ? true : primary.verification?.status ? false : null;
+  return { email: primary.email_address ?? null, verified };
+}
+function isSessionStatus(value) {
+  return SESSION_STATUSES.includes(value);
+}
+function dayBucket(tsMs) {
+  return Math.floor(tsMs / DAY_MS) * DAY_MS;
+}
+function unwrapOrganizations(body) {
+  if (Array.isArray(body)) {
+    return { items: body, totalCount: null };
+  }
+  return { items: body.data, totalCount: body.total_count ?? null };
+}
+var ClerkConnector = class _ClerkConnector extends BaseConnector {
+  static id = id;
+  static resources = clerkResources;
+  static schemas = schemasFromResources(clerkResources);
+  static create(input, ctx) {
+    const parsed = configFields.parse(input);
+    return new _ClerkConnector(
+      {
+        apiUrl: parsed.apiUrl,
+        resources: parsed.resources,
+        dauLookbackDays: parsed.dauLookbackDays
+      },
+      {
+        secretKey: parsed.secretKey
+      },
+      ctx
+    );
+  }
+  id = id;
+  credentials = clerkCredentials;
+  dauBuckets = /* @__PURE__ */ new Map();
+  baseUrl() {
+    const raw = this.settings.apiUrl ?? DEFAULT_API_URL;
+    return raw.replace(/\/+$/, "");
+  }
+  dauLookbackDays() {
+    return this.settings.dauLookbackDays ?? DEFAULT_DAU_LOOKBACK_DAYS;
+  }
+  dauCutoffMs() {
+    return Date.now() - this.dauLookbackDays() * DAY_MS;
+  }
+  parsePageCursor(page) {
+    if (!page) {
+      return 0;
+    }
+    const n = Number.parseInt(page, 10);
+    if (!Number.isFinite(n) || n < 0) {
+      return 0;
+    }
+    return n;
+  }
+  async apiGet(url, resource, signal) {
+    return this.get(url, {
+      resource,
+      headers: {
+        Authorization: `Bearer ${this.creds.secretKey}`,
+        Accept: "application/json",
+        "User-Agent": connectorUserAgent("clerk")
+      },
+      rateLimit: clerkRateLimit,
+      signal
+    });
+  }
+  buildUsersUrl(offset, options) {
+    const u = new URL(`${this.baseUrl()}/v1/users`);
+    u.searchParams.set("limit", String(PAGE_SIZE));
+    u.searchParams.set("offset", String(offset));
+    u.searchParams.set("order_by", "-last_active_at");
+    if (options.since) {
+      const sinceMs = Date.parse(options.since);
+      if (Number.isFinite(sinceMs)) {
+        u.searchParams.set("last_active_at_since", String(sinceMs));
+      }
+    }
+    return u.toString();
+  }
+  buildOrganizationsUrl(offset) {
+    const u = new URL(`${this.baseUrl()}/v1/organizations`);
+    u.searchParams.set("limit", String(PAGE_SIZE));
+    u.searchParams.set("offset", String(offset));
+    u.searchParams.set("order_by", "-created_at");
+    return u.toString();
+  }
+  buildSessionsUrl(offset) {
+    const u = new URL(`${this.baseUrl()}/v1/sessions`);
+    u.searchParams.set("limit", String(PAGE_SIZE));
+    u.searchParams.set("offset", String(offset));
+    return u.toString();
+  }
+  buildDauUsersUrl(offset) {
+    const u = new URL(`${this.baseUrl()}/v1/users`);
+    u.searchParams.set("limit", String(PAGE_SIZE));
+    u.searchParams.set("offset", String(offset));
+    u.searchParams.set("order_by", "-last_active_at");
+    u.searchParams.set("last_active_at_since", String(this.dauCutoffMs()));
+    return u.toString();
+  }
+  async fetchUsersPage(page, options, signal) {
+    const offset = this.parsePageCursor(page);
+    const url = this.buildUsersUrl(offset, options);
+    const res = await this.apiGet(url, "users", signal);
+    const items = res.body;
+    const nextOffset = offset + PAGE_SIZE;
+    const pageIndex = Math.floor(offset / PAGE_SIZE);
+    const hasMore = items.length >= PAGE_SIZE && pageIndex + 1 < MAX_PAGES;
+    return { items, next: hasMore ? String(nextOffset) : null };
+  }
+  async fetchOrganizationsPage(page, options, signal) {
+    const offset = this.parsePageCursor(page);
+    const url = this.buildOrganizationsUrl(offset);
+    const res = await this.apiGet(
+      url,
+      "organizations",
+      signal
+    );
+    const { items } = unwrapOrganizations(res.body);
+    const sinceMs = options.since ? Date.parse(options.since) : NaN;
+    const allOlder = Number.isFinite(sinceMs) && items.length > 0 && items.every((o) => (o.created_at ?? 0) < sinceMs);
+    const nextOffset = offset + PAGE_SIZE;
+    const pageIndex = Math.floor(offset / PAGE_SIZE);
+    const hasMore = items.length >= PAGE_SIZE && !allOlder && pageIndex + 1 < MAX_PAGES;
+    return { items, next: hasMore ? String(nextOffset) : null };
+  }
+  async fetchSessionsPage(page, options, signal) {
+    const offset = this.parsePageCursor(page);
+    const url = this.buildSessionsUrl(offset);
+    const res = await this.apiGet(url, "sessions", signal);
+    const items = res.body;
+    const sinceMs = options.since ? Date.parse(options.since) : NaN;
+    const allOlder = Number.isFinite(sinceMs) && items.length > 0 && items.every((s) => (s.created_at ?? 0) < sinceMs);
+    const nextOffset = offset + PAGE_SIZE;
+    const pageIndex = Math.floor(offset / PAGE_SIZE);
+    const hasMore = items.length >= PAGE_SIZE && !allOlder && pageIndex + 1 < MAX_PAGES;
+    return { items, next: hasMore ? String(nextOffset) : null };
+  }
+  async fetchDauUsersPage(page, signal) {
+    const offset = this.parsePageCursor(page);
+    const url = this.buildDauUsersUrl(offset);
+    const res = await this.apiGet(url, "dau_users", signal);
+    const items = res.body;
+    const nextOffset = offset + PAGE_SIZE;
+    const pageIndex = Math.floor(offset / PAGE_SIZE);
+    const hasMore = items.length >= PAGE_SIZE && pageIndex + 1 < MAX_PAGES;
+    return { items, next: hasMore ? String(nextOffset) : null };
+  }
+  async writeUsers(storage, items) {
+    for (const u of items) {
+      const { email, verified } = primaryEmail(u);
+      const lastSignIn = parseEpoch(u.last_sign_in_at ?? null, "ms");
+      const lastActive = parseEpoch(u.last_active_at ?? null, "ms");
+      const createdAt = parseEpoch(u.created_at ?? null, "ms");
+      const updatedAt = parseEpoch(u.updated_at ?? null, "ms");
+      await storage.entity({
+        type: USER_ENTITY,
+        id: u.id,
+        attributes: {
+          email,
+          emailVerified: verified,
+          lastSignInAt: lastSignIn,
+          lastActiveAt: lastActive,
+          banned: u.banned ?? false,
+          locked: u.locked ?? false,
+          createdAt
+        },
+        updated_at: updatedAt ?? createdAt ?? 0
+      });
+    }
+  }
+  async writeOrganizations(storage, items) {
+    for (const o of items) {
+      const createdAt = parseEpoch(o.created_at ?? null, "ms");
+      const updatedAt = parseEpoch(o.updated_at ?? null, "ms");
+      await storage.entity({
+        type: ORG_ENTITY,
+        id: o.id,
+        attributes: {
+          name: o.name ?? null,
+          slug: o.slug ?? null,
+          membersCount: o.members_count ?? null,
+          createdAt
+        },
+        updated_at: updatedAt ?? createdAt ?? 0
+      });
+    }
+  }
+  async writeSessions(storage, items) {
+    for (const s of items) {
+      const startTs = parseEpoch(s.created_at ?? null, "ms");
+      if (startTs === null) {
+        continue;
+      }
+      const status = isSessionStatus(s.status) ? s.status : "active";
+      const lastActive = parseEpoch(s.last_active_at ?? null, "ms");
+      await storage.event({
+        name: SESSION_EVENT,
+        start_ts: startTs,
+        end_ts: null,
+        attributes: {
+          sessionId: s.id,
+          userId: s.user_id ?? null,
+          status,
+          lastActiveAt: lastActive
+        }
+      });
+    }
+  }
+  accumulateDau(items) {
+    const cutoff = this.dauCutoffMs();
+    for (const u of items) {
+      const ts = u.last_active_at;
+      if (typeof ts !== "number" || !Number.isFinite(ts) || ts < cutoff) {
+        continue;
+      }
+      const bucket = dayBucket(ts);
+      const set = this.dauBuckets.get(bucket) ?? /* @__PURE__ */ new Set();
+      set.add(u.id);
+      this.dauBuckets.set(bucket, set);
+    }
+  }
+  async writeDauSamples(storage) {
+    const samples = Array.from(this.dauBuckets.entries()).sort(([a], [b]) => a - b).map(([ts, set]) => ({
+      name: DAU_METRIC,
+      ts,
+      value: set.size,
+      attributes: {}
+    }));
+    await storage.metrics(samples, { names: [DAU_METRIC] });
+  }
+  async clearScopeOnFirstPage(storage, phase, isFull) {
+    if (phase === "daily_active_users") {
+      this.dauBuckets.clear();
+      await storage.metrics([], { names: [DAU_METRIC] });
+      return;
+    }
+    if (!isFull) {
+      return;
+    }
+    switch (phase) {
+      case "users":
+        await storage.entities([], { types: [USER_ENTITY] });
+        return;
+      case "organizations":
+        await storage.entities([], { types: [ORG_ENTITY] });
+        return;
+      case "sessions":
+        await storage.events([], { names: [SESSION_EVENT] });
+        return;
+    }
+  }
+  resolveCursor(cursor) {
+    return isClerkSyncCursor(cursor) ? cursor : void 0;
+  }
+  async sync(options, storage, signal) {
+    const cursor = this.resolveCursor(options.cursor);
+    const isFull = options.mode === "full";
+    const phases = selectActivePhases(
+      (r) => r,
+      PHASE_ORDER,
+      this.settings.resources
+    );
+    return paginateChunked({
+      phases,
+      cursor,
+      signal,
+      logger: this.logger,
+      fetchPage: async (phase, page, sig) => {
+        switch (phase) {
+          case "users":
+            return this.fetchUsersPage(page, options, sig);
+          case "organizations":
+            return this.fetchOrganizationsPage(page, options, sig);
+          case "sessions":
+            return this.fetchSessionsPage(page, options, sig);
+          case "daily_active_users":
+            return this.fetchDauUsersPage(page, sig);
+        }
+      },
+      writeBatch: async (phase, items, page) => {
+        if (page === null) {
+          await this.clearScopeOnFirstPage(storage, phase, isFull);
+        }
+        switch (phase) {
+          case "users":
+            await this.writeUsers(storage, items);
+            return;
+          case "organizations":
+            await this.writeOrganizations(
+              storage,
+              items
+            );
+            return;
+          case "sessions":
+            await this.writeSessions(storage, items);
+            return;
+          case "daily_active_users":
+            this.accumulateDau(items);
+            await this.writeDauSamples(storage);
+            return;
+        }
+      }
+    });
+  }
+};
+
+// src/index.ts
+var index_default = ClerkConnector;
+export {
+  ClerkConnector,
+  configFields,
+  index_default as default,
+  doc,
+  id,
+  clerkResources as resources
+};
+//# sourceMappingURL=index.js.map