lytx 0.3.0

package/db/adapter.ts

@@ -0,0 +1,166 @@
+import { getDashboardDataFromDurableObject, getSiteInfo } from "@db/durable/durableObjectClient";
+import { processDirectSiteEvents, batchProcessSiteEvents } from "@/api/queueWorker";
+
+import type { DashboardOptions, DBAdapter } from "@db/types";
+import type { SiteEventInput } from "@/session/siteSchema";
+import { IS_DEV } from "rwsdk/constants";
+
+/**
+ * Updated Site Adapter - All dashboard reads now go to durable objects for performance
+ */
+const SiteAdapter = {
+  //NOTE: The sites are from durable objects, but all account admin is d1
+  "sqlite": getDashboardDataFromDurableObject,
+  "postgres": getDashboardDataFromDurableObject, // Fast reads from durable object
+  "singlestore": getDashboardDataFromDurableObject,
+  "analytics_engine": getDashboardDataFromDurableObject
+} as const;
+
+/**
+ * Get dashboard data from site-specific durable objects
+ * All adapters now use durable objects for fast dashboard access
+ */
+export async function getDashboardData<T extends DBAdapter>(
+  adapter: T,
+  options: DashboardOptions,
+) {
+  if (IS_DEV) {
+    console.log(`Adapter set to : ${adapter} with options:`);
+    console.dir(options);
+  }
+  return SiteAdapter[adapter](options);
+}
+
+/**
+ * Insert site events with intelligent routing based on site adapter
+ * - sqlite sites: Direct write to durable object
+ * - postgres/singlestore sites: Queue for dual-write (original DB + durable object)
+ */
+export async function insertSiteEvents(
+  site_id: number,
+  site_uuid: string,
+  events: SiteEventInput[],
+  db_adapter: DBAdapter
+): Promise<{ success: boolean; inserted?: number; error?: string }> {
+  try {
+    // Get site information to determine routing strategy
+    if (db_adapter === 'sqlite') {
+      // Direct write to durable object for sqlite sites
+      return await processDirectSiteEvents(site_id, site_uuid, events);
+
+    } else if (db_adapter === 'postgres' || db_adapter === 'singlestore') {
+      // Queue for dual-write (postgres/singlestore + durable object)
+      const eventsBySite = new Map();
+      eventsBySite.set(site_id, {
+        siteUuid: site_uuid,
+        teamId: 1, // TODO: Get actual team_id from site info
+        adapter: db_adapter,
+        events
+      });
+
+      await batchProcessSiteEvents(eventsBySite);
+
+      return {
+        success: true,
+        inserted: events.length
+      };
+
+    } else {
+      return {
+        success: false,
+        error: `Unsupported adapter: ${db_adapter}`
+      };
+    }
+
+  } catch (error) {
+    console.error(`Error inserting site events for site ${site_uuid}:`, error);
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error'
+    };
+  }
+}
+
+/**
+ * Batch insert events for multiple sites
+ */
+export async function batchInsertSiteEvents(
+  eventsBySite: Map<number, SiteEventInput[]>,
+  env: Env
+): Promise<Map<number, { success: boolean; inserted?: number; error?: string }>> {
+  const results = new Map();
+
+  // Group sites by adapter type for efficient processing
+  const sqliteSites = new Map<number, SiteEventInput[]>();
+  const queueSites = new Map<number, { teamId: number; adapter: string; events: SiteEventInput[] }>();
+
+  // Categorize sites by adapter
+  for (const [siteId, events] of eventsBySite) {
+    const siteInfo = await getSiteInfo(siteId, env);
+
+    if (!siteInfo) {
+      results.set(siteId, { success: false, error: `Site ${siteId} not found` });
+      continue;
+    }
+
+    if (siteInfo.site_db_adapter === 'sqlite') {
+      sqliteSites.set(siteId, events);
+    } else {
+      queueSites.set(siteId, {
+        teamId: 1, // TODO: Get actual team_id
+        adapter: siteInfo.site_db_adapter,
+        events
+      });
+    }
+  }
+
+  // Process sqlite sites directly (parallel)
+  const sqlitePromises = Array.from(sqliteSites.entries()).map(async ([siteId, events]) => {
+    const siteInfo = await getSiteInfo(siteId, env);
+    if (!siteInfo) {
+      return [siteId, { success: false, error: `Site ${siteId} not found` }] as const;
+    }
+
+    const result = await processDirectSiteEvents(siteId, siteInfo.tag_id, events);
+    return [siteId, result] as const;
+  });
+
+  const sqliteResults = await Promise.allSettled(sqlitePromises);
+  sqliteResults.forEach((result, index) => {
+    const siteId = Array.from(sqliteSites.keys())[index];
+    if (result.status === 'fulfilled') {
+      results.set(result.value[0], result.value[1]);
+    } else {
+      results.set(siteId, { success: false, error: `Batch processing failed: ${result.reason}` });
+    }
+  });
+
+  // Process queue sites in batch
+  if (queueSites.size > 0) {
+    try {
+      await batchProcessSiteEvents(queueSites);
+      // Mark all queue sites as successful
+      for (const [siteId, { events }] of queueSites) {
+        results.set(siteId, { success: true, inserted: events.length });
+      }
+    } catch (error) {
+      // Mark all queue sites as failed
+      for (const siteId of queueSites.keys()) {
+        results.set(siteId, {
+          success: false,
+          error: error instanceof Error ? error.message : 'Queue processing failed'
+        });
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Get site adapter type for routing decisions
+ */
+export async function getSiteAdapter(siteId: number, env: Env): Promise<DBAdapter | null> {
+  const siteInfo = await getSiteInfo(siteId, env);
+  return siteInfo ? (siteInfo.site_db_adapter as DBAdapter) : null;
+}

package/db/client.ts

@@ -0,0 +1,16 @@
+import { d1_client } from "@db/d1/client";
+import { pg_client } from "@db/postgres/client";
+import type { AdapterToClient, DBAdapter } from "@db/types";
+
+export function getClient<T extends DBAdapter>(adapter: T, connectionString?: string): AdapterToClient[T] {
+  switch (adapter) {
+    case "sqlite":
+      return d1_client as AdapterToClient[T]
+    case "postgres":
+      if (!connectionString) throw new Error("connectionString is required for postgres");
+      return pg_client(connectionString) as AdapterToClient[T];
+    default:
+      return d1_client as AdapterToClient[T];
+  }
+}
+

package/db/d1/client.ts

@@ -0,0 +1,8 @@
+import { drizzle } from 'drizzle-orm/d1';
+import { env } from 'cloudflare:workers';
+
+export const d1_client = drizzle(env.lytx_core_db);
+
+
+
+

package/db/d1/drizzle.config.ts

@@ -0,0 +1,35 @@
+// import * as dotenv from 'dotenv';
+import { defineConfig, type Config } from 'drizzle-kit';
+// dotenv.config();
+// const mode = process.env.MODE;
+
+
+export default defineConfig({
+	schema: './db/d1/schema.ts',
+	out: './db/d1/migrations',
+	dialect: 'sqlite',
+});
+
+// const config: Config = {
+// 	out: './db/d1/migrations',
+// 	schema: './db/d1/schema.ts',
+// 	dialect: 'sqlite',
+//
+//
+// }
+// if (mode == "DEV") {
+// 	//@ts-ignore
+// 	config.dbCredentials = {
+// 		url: process.env.LOCAL_D1!
+// 	}
+// } else {
+// 	//@ts-ignore
+// 	config.driver = "d1-http";
+// 	//@ts-ignore
+// 	config.dbCredentials = {
+// 		accountId: process.env.CLOUDFLARE_ACCOUNT_ID!,
+// 		databaseId: process.env.CLOUDFLARE_DATABASE_ID!,
+// 		token: process.env.CLOUDFLARE_D1_TOKEN!,
+// 	}
+// }
+// export default defineConfig(config);

package/db/d1/migrations/0000_true_maelstrom.sql

@@ -0,0 +1,165 @@
+CREATE TABLE `account` (
+	`id` text PRIMARY KEY NOT NULL,
+	`account_id` text NOT NULL,
+	`provider_id` text NOT NULL,
+	`user_id` text NOT NULL,
+	`access_token` text,
+	`refresh_token` text,
+	`id_token` text,
+	`access_token_expires_at` integer,
+	`refresh_token_expires_at` integer,
+	`scope` text,
+	`password` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL,
+	FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+CREATE INDEX `account_user_id_idx` ON `account` (`user_id`);--> statement-breakpoint
+CREATE TABLE `api_key` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`key` text,
+	`team_id` integer NOT NULL,
+	`enabled` integer DEFAULT true,
+	`permissions` text DEFAULT '{"read":true,"write":true}' NOT NULL,
+	`allowed_team_members` text DEFAULT '[]',
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `api_key_team_id_idx` ON `api_key` (`team_id`);--> statement-breakpoint
+CREATE TABLE `invited_user` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`team_id` integer NOT NULL,
+	`accepted` integer NOT NULL,
+	`email` text NOT NULL,
+	`role` text DEFAULT 'editor' NOT NULL,
+	`name` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `invited_user_email_unique` ON `invited_user` (`email`);--> statement-breakpoint
+CREATE INDEX `invited_user_team_id_idx` ON `invited_user` (`team_id`);--> statement-breakpoint
+CREATE INDEX `invited_user_email_idx` ON `invited_user` (`email`);--> statement-breakpoint
+CREATE TABLE `session` (
+	`id` text PRIMARY KEY NOT NULL,
+	`expires_at` integer NOT NULL,
+	`token` text NOT NULL,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL,
+	`ip_address` text,
+	`user_agent` text,
+	`user_id` text NOT NULL,
+	FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint
+CREATE INDEX `session_user_id_idx` ON `session` (`user_id`);--> statement-breakpoint
+CREATE INDEX `session_token_idx` ON `session` (`token`);--> statement-breakpoint
+CREATE TABLE `site_events` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`team_id` integer,
+	`bot_data` text,
+	`browser` text,
+	`city` text,
+	`client_page_url` text,
+	`country` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL,
+	`custom_data` text,
+	`device_type` text,
+	`event` text NOT NULL,
+	`operating_system` text,
+	`page_url` text,
+	`postal` text,
+	`query_params` text,
+	`referer` text,
+	`region` text,
+	`rid` text,
+	`screen_height` integer,
+	`screen_width` integer,
+	`site_id` integer NOT NULL,
+	`tag_id` text NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `site_events_team_id_idx` ON `site_events` (`team_id`);--> statement-breakpoint
+CREATE INDEX `site_events_site_id_idx` ON `site_events` (`site_id`);--> statement-breakpoint
+CREATE INDEX `site_events_tag_id_idx` ON `site_events` (`tag_id`);--> statement-breakpoint
+CREATE INDEX `site_events_created_at_idx` ON `site_events` (`created_at`);--> statement-breakpoint
+CREATE INDEX `site_events_team_site_idx` ON `site_events` (`team_id`,`site_id`);--> statement-breakpoint
+CREATE INDEX `site_events_team_tag_idx` ON `site_events` (`team_id`,`tag_id`);--> statement-breakpoint
+CREATE INDEX `site_events_site_created_idx` ON `site_events` (`site_id`,`created_at`);--> statement-breakpoint
+CREATE INDEX `site_events_team_created_idx` ON `site_events` (`team_id`,`created_at`);--> statement-breakpoint
+CREATE INDEX `site_events_country_idx` ON `site_events` (`country`);--> statement-breakpoint
+CREATE INDEX `site_events_device_type_idx` ON `site_events` (`device_type`);--> statement-breakpoint
+CREATE INDEX `site_events_event_idx` ON `site_events` (`event`);--> statement-breakpoint
+CREATE INDEX `site_events_referer_idx` ON `site_events` (`referer`);--> statement-breakpoint
+CREATE TABLE `sites` (
+	`site_id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`uuid` text NOT NULL,
+	`tag_id` text NOT NULL,
+	`track_web_events` integer DEFAULT false NOT NULL,
+	`gdpr` integer DEFAULT false,
+	`event_load_strategy` text DEFAULT 'sdk' NOT NULL,
+	`external_id` integer DEFAULT 0 NOT NULL,
+	`site_db_adapter` text DEFAULT 'sqlite' NOT NULL,
+	`team_id` integer NOT NULL,
+	`name` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL,
+	`domain` text,
+	`tag_manager` integer DEFAULT false,
+	`autocapture` integer DEFAULT true,
+	`rid_salt` text,
+	`rid_salt_expire` integer,
+	`tag_id_override` text
+);
+--> statement-breakpoint
+CREATE INDEX `sites_team_id_idx` ON `sites` (`team_id`);--> statement-breakpoint
+CREATE INDEX `sites_tag_id_idx` ON `sites` (`tag_id`);--> statement-breakpoint
+CREATE INDEX `sites_domain_idx` ON `sites` (`domain`);--> statement-breakpoint
+CREATE TABLE `team` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`external_id` integer DEFAULT 0 NOT NULL,
+	`created_at` integer,
+	`updated_at` integer NOT NULL,
+	`created_by` text NOT NULL,
+	`name` text,
+	`uuid` text,
+	`db_adapter` text DEFAULT 'sqlite' NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE `team_member` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`team_id` integer NOT NULL,
+	`role` text DEFAULT 'editor' NOT NULL,
+	`user_id` text NOT NULL,
+	`allowed_site_ids` text DEFAULT '["all"]',
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `team_member_team_id_idx` ON `team_member` (`team_id`);--> statement-breakpoint
+CREATE INDEX `team_member_user_id_idx` ON `team_member` (`user_id`);--> statement-breakpoint
+CREATE INDEX `team_member_team_user_idx` ON `team_member` (`team_id`,`user_id`);--> statement-breakpoint
+CREATE TABLE `user` (
+	`id` text PRIMARY KEY NOT NULL,
+	`name` text NOT NULL,
+	`email` text NOT NULL,
+	`email_verified` integer NOT NULL,
+	`image` text,
+	`timezone` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint
+CREATE TABLE `verification` (
+	`id` text PRIMARY KEY NOT NULL,
+	`identifier` text NOT NULL,
+	`value` text NOT NULL,
+	`expires_at` integer NOT NULL,
+	`created_at` integer,
+	`updated_at` integer NOT NULL
+);

package/db/d1/migrations/0001_wonderful_bloodaxe.sql

@@ -0,0 +1,12 @@
+CREATE TABLE `event_labels` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`site_id` integer NOT NULL,
+	`event_name` text NOT NULL,
+	`label` text NOT NULL,
+	`description` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `event_labels_site_id_idx` ON `event_labels` (`site_id`);--> statement-breakpoint
+CREATE INDEX `event_labels_site_event_idx` ON `event_labels` (`site_id`,`event_name`);

package/db/d1/migrations/0002_late_frightful_four.sql

@@ -0,0 +1 @@
+ALTER TABLE `user` ADD `last_site_id` integer;

package/db/d1/migrations/0003_cuddly_obadiah_stane.sql

@@ -0,0 +1,16 @@
+CREATE TABLE `team_billing` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`team_id` integer NOT NULL,
+	`stripe_customer_id` text,
+	`stripe_subscription_id` text,
+	`stripe_price_id` text,
+	`plan_name` text,
+	`status` text,
+	`current_period_end` integer,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `team_billing_team_id_idx` ON `team_billing` (`team_id`);--> statement-breakpoint
+CREATE INDEX `team_billing_customer_idx` ON `team_billing` (`stripe_customer_id`);--> statement-breakpoint
+CREATE INDEX `team_billing_subscription_idx` ON `team_billing` (`stripe_subscription_id`);

package/db/d1/migrations/0004_mute_stardust.sql

@@ -0,0 +1 @@
+ALTER TABLE `user` ADD `last_team_id` integer;

package/db/d1/migrations/0005_awesome_silvermane.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `team_billing` ADD `current_period_start` integer;--> statement-breakpoint
+ALTER TABLE `team_billing` ADD `usage_mode` text DEFAULT 'metered' NOT NULL;--> statement-breakpoint
+ALTER TABLE `team_billing` ADD `usage_cap` integer;

package/db/d1/migrations/0006_volatile_shriek.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `api_key` ADD `site_id` integer;--> statement-breakpoint
+CREATE INDEX `api_key_site_id_idx` ON `api_key` (`site_id`);

package/db/d1/migrations/0007_superb_lila_cheney.sql

@@ -0,0 +1 @@
+ALTER TABLE `team_billing` ADD `billing_method` text DEFAULT 'stripe_subscription' NOT NULL;

package/db/d1/migrations/0008_bitter_longshot.sql

@@ -0,0 +1,17 @@
+CREATE TABLE `custom_reports` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`uuid` text NOT NULL,
+	`team_id` integer NOT NULL,
+	`site_id` integer NOT NULL,
+	`name` text NOT NULL,
+	`description` text,
+	`config` text NOT NULL,
+	`created_by` text NOT NULL,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `custom_reports_uuid_unique` ON `custom_reports` (`uuid`);--> statement-breakpoint
+CREATE INDEX `custom_reports_team_id_idx` ON `custom_reports` (`team_id`);--> statement-breakpoint
+CREATE INDEX `custom_reports_site_id_idx` ON `custom_reports` (`site_id`);--> statement-breakpoint
+CREATE INDEX `custom_reports_team_site_idx` ON `custom_reports` (`team_id`,`site_id`);

package/db/d1/migrations/0009_wonderful_madame_masque.sql

@@ -0,0 +1,28 @@
+CREATE TABLE `team_ai_usage` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`team_id` integer NOT NULL,
+	`user_id` text,
+	`site_id` integer,
+	`request_id` text,
+	`request_type` text NOT NULL,
+	`provider` text,
+	`model` text,
+	`status` text DEFAULT 'success' NOT NULL,
+	`error_code` text,
+	`error_message` text,
+	`input_tokens` integer,
+	`output_tokens` integer,
+	`total_tokens` integer,
+	`tool_calls` integer,
+	`message_count` integer,
+	`prompt_chars` integer,
+	`completion_chars` integer,
+	`duration_ms` integer,
+	`created_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `team_ai_usage_team_id_idx` ON `team_ai_usage` (`team_id`);--> statement-breakpoint
+CREATE INDEX `team_ai_usage_team_created_idx` ON `team_ai_usage` (`team_id`,`created_at`);--> statement-breakpoint
+CREATE INDEX `team_ai_usage_team_type_created_idx` ON `team_ai_usage` (`team_id`,`request_type`,`created_at`);--> statement-breakpoint
+CREATE INDEX `team_ai_usage_request_id_idx` ON `team_ai_usage` (`request_id`);--> statement-breakpoint
+CREATE INDEX `team_ai_usage_user_created_idx` ON `team_ai_usage` (`user_id`,`created_at`);