@upstash/redis-analytics 0.1.0

package/src/services/search-index.ts

@@ -0,0 +1,335 @@
+import type { Redis } from "@upstash/redis";
+import { s } from "@upstash/redis";
+import type { AnalyticsConfig, EventSchema, SearchIndexInfo } from "../types";
+import { LoggingService } from "./logging";
+import { SchemaRegistry } from "./schema-registry";
+import { FeatureFlagService } from "./feature-flags";
+
+export class SearchIndexService {
+  constructor(
+    private redis: Redis,
+    private config: AnalyticsConfig,
+    private schemaRegistry: SchemaRegistry,
+    private featureFlags: FeatureFlagService,
+    private logger: LoggingService
+  ) {}
+
+  private get indexName() {
+    return this.config.search.indexName;
+  }
+
+  private get indexNameB() {
+    return `${this.indexName}-b`;
+  }
+
+  private buildSchema(
+    eventSchemas: Record<string, EventSchema>,
+    featureFlagNames: string[]
+  ) {
+    // Build the schema object for Redis Search
+    // Using `any` for mixed field types since s.object accepts heterogeneous schemas
+    const schemaFields: Record<string, any> = {};
+
+    // Fixed fields
+    schemaFields["eventName"] = s.keyword();
+    schemaFields["eventTime"] = s.date().fast();
+    schemaFields["eventTimeMs"] = s.number("F64");
+    schemaFields["sessionId"] = s.string().noTokenize();
+    schemaFields["sessionCreationTimestamp"] = s.date().fast();
+
+    // Feature flag fields from definitions stored in Redis
+    for (const flagName of featureFlagNames) {
+      schemaFields[`featureFlags.${flagName}`] = s.string().noTokenize();
+    }
+
+    // Dynamic fields from event schemas
+    for (const [eventName, eventSchema] of Object.entries(eventSchemas)) {
+      // Session metadata schema uses sessionMetadata.* prefix
+      const prefix = eventName === "__session_metadata__" ? "sessionMetadata" : "properties";
+
+      for (const [propName, propDef] of Object.entries(
+        eventSchema.properties
+      )) {
+        const fieldName = `${prefix}.${propName}`;
+        if (schemaFields[fieldName]) continue; // Already defined
+
+        switch (propDef.type) {
+          case "string":
+            schemaFields[fieldName] = s.string();
+            break;
+          case "number":
+            schemaFields[fieldName] = s.number("F64");
+            break;
+          case "boolean":
+            schemaFields[fieldName] = s.boolean();
+            break;
+        }
+      }
+    }
+
+    return s.object(schemaFields);
+  }
+
+  private async findAliasTarget(): Promise<string | null> {
+    try {
+      const aliases = await this.redis.search.alias.list();
+      // aliases is a record of alias -> index name
+      const target = aliases[this.indexName];
+      return target ?? null;
+    } catch {
+      return null;
+    }
+  }
+
+  private async indexExists(name: string): Promise<boolean> {
+    try {
+      const index = this.redis.search.index({ name });
+      await index.describe();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async rebuild(): Promise<void> {
+    const [eventSchemas, flagDefinitions] = await Promise.all([
+      this.schemaRegistry.getAllSchemas(),
+      this.featureFlags.getDefinitions(),
+    ]);
+    const schema = this.buildSchema(eventSchemas, Object.keys(flagDefinitions));
+
+    const aliasTarget = await this.findAliasTarget();
+    const indexAExists = await this.indexExists(this.indexName);
+    const indexBExists = await this.indexExists(this.indexNameB);
+
+    // Case 1: No index exists at all — create index A
+    if (!indexAExists && !indexBExists && !aliasTarget) {
+      const newIndex = await this.redis.search.createIndex({
+        name: this.indexName,
+        schema,
+        dataType: "json",
+        prefix: "event-",
+        existsOk: true,
+      });
+      await newIndex.waitIndexing();
+
+      await this.logger.log({
+        logType: "index_update",
+        metadata: { action: "create", indexName: this.indexName },
+        changes: {
+          before: null,
+          after: { activeIndex: this.indexName },
+        },
+      });
+      return;
+    }
+
+    // Case 2: Alias exists pointing to B — swap back to A
+    if (aliasTarget === this.indexNameB) {
+      const newIndex = await this.redis.search.createIndex({
+        name: this.indexName,
+        schema,
+        dataType: "json",
+        prefix: "event-",
+        existsOk: true,
+      });
+      await newIndex.waitIndexing();
+
+      // Delete alias and old index B
+      await this.redis.search.alias.delete({ alias: this.indexName });
+      try {
+        await this.redis.search.index({ name: this.indexNameB }).drop();
+      } catch {
+        // Ignore if already gone
+      }
+
+      await this.logger.log({
+        logType: "index_update",
+        metadata: { action: "update", indexName: this.indexName },
+        changes: {
+          before: { activeIndex: this.indexNameB },
+          after: { activeIndex: this.indexName },
+        },
+      });
+      return;
+    }
+
+    // Case 3: No alias, index A exists — create B, alias to B, delete A
+    if (indexAExists && !aliasTarget) {
+      const newIndex = await this.redis.search.createIndex({
+        name: this.indexNameB,
+        schema,
+        dataType: "json",
+        prefix: "event-",
+        existsOk: true,
+      });
+      await newIndex.waitIndexing();
+
+      // Create alias pointing to B, then delete old index A
+      await this.redis.search.index({ name: this.indexName }).drop();
+      await this.redis.search.alias.add({
+        alias: this.indexName,
+        indexName: this.indexNameB,
+      });
+
+      await this.logger.log({
+        logType: "index_update",
+        metadata: { action: "update", indexName: this.indexNameB },
+        changes: {
+          before: { activeIndex: this.indexName },
+          after: { activeIndex: this.indexNameB },
+        },
+      });
+      return;
+    }
+
+    // Fallback: unexpected state — recreate from scratch
+    try {
+      await this.redis.search.alias.delete({ alias: this.indexName });
+    } catch { /* no alias */ }
+    try {
+      await this.redis.search.index({ name: this.indexName }).drop();
+    } catch { /* doesn't exist */ }
+    try {
+      await this.redis.search.index({ name: this.indexNameB }).drop();
+    } catch { /* doesn't exist */ }
+
+    const newIndex = await this.redis.search.createIndex({
+      name: this.indexName,
+      schema,
+      dataType: "json",
+      prefix: "event-",
+      existsOk: true,
+    });
+    await newIndex.waitIndexing();
+
+    await this.logger.log({
+      logType: "index_update",
+      metadata: { action: "create", indexName: this.indexName },
+      changes: {
+        before: null,
+        after: { activeIndex: this.indexName },
+      },
+    });
+  }
+
+  async getInfo(): Promise<SearchIndexInfo> {
+    // Try main index first, then B
+    for (const name of [this.indexName, this.indexNameB]) {
+      try {
+        const index = this.redis.search.index({ name });
+        const info = await index.describe();
+        if (!info) continue;
+        const countResult = await index.count({ filter: {} });
+
+        return {
+          currentIndex: name,
+          alias: this.indexName,
+          schema: info.schema as Record<string, unknown>,
+          documentCount: countResult.count,
+        };
+      } catch {
+        continue;
+      }
+    }
+
+    return {
+      currentIndex: "none",
+      alias: this.indexName,
+      schema: {},
+      documentCount: 0,
+    };
+  }
+
+  getActiveIndex() {
+    // Return an index reference for querying
+    return this.redis.search.index({ name: this.indexName });
+  }
+
+  getActiveIndexWithFallback() {
+    // Try main index, fallback to B
+    return {
+      primary: this.redis.search.index({ name: this.indexName }),
+      fallback: this.redis.search.index({ name: this.indexNameB }),
+    };
+  }
+
+  /**
+   * Compares the expected schema (built from config + schema registry)
+   * against the actual index schema. Returns drift info.
+   */
+  async checkSchemaDrift(): Promise<{
+    hasDrift: boolean;
+    missingFields: string[];
+    extraFields: string[];
+    indexExists: boolean;
+  }> {
+    const info = await this.getInfo();
+
+    if (info.currentIndex === "none") {
+      return {
+        hasDrift: false,
+        missingFields: [],
+        extraFields: [],
+        indexExists: false,
+      };
+    }
+
+    // Build expected schema from registries
+    const [eventSchemas, flagDefinitions] = await Promise.all([
+      this.schemaRegistry.getAllSchemas(),
+      this.featureFlags.getDefinitions(),
+    ]);
+    const expectedFields = new Set<string>();
+
+    // Fixed fields
+    expectedFields.add("eventName");
+    expectedFields.add("eventTime");
+    expectedFields.add("eventTimeMs");
+    expectedFields.add("sessionId");
+    expectedFields.add("sessionCreationTimestamp");
+
+    // Feature flag fields
+    for (const flagName of Object.keys(flagDefinitions)) {
+      expectedFields.add(`featureFlags.${flagName}`);
+    }
+
+    // Dynamic fields from event schemas
+    for (const [eventName, eventSchema] of Object.entries(eventSchemas)) {
+      const prefix = eventName === "__session_metadata__" ? "sessionMetadata" : "properties";
+      for (const propName of Object.keys(eventSchema.properties)) {
+        expectedFields.add(`${prefix}.${propName}`);
+      }
+    }
+
+    // Get actual indexed fields from the schema info
+    // The schema from describe() uses $ prefix for JSON paths
+    const actualFields = new Set<string>();
+    for (const key of Object.keys(info.schema)) {
+      // Normalize: remove leading "$." prefix if present
+      const normalized = key.startsWith("$.") ? key.slice(2) : key;
+      actualFields.add(normalized);
+    }
+
+    const missingFields: string[] = [];
+    for (const field of expectedFields) {
+      if (!actualFields.has(field)) {
+        missingFields.push(field);
+      }
+    }
+
+    const extraFields: string[] = [];
+    for (const field of actualFields) {
+      if (!expectedFields.has(field)) {
+        extraFields.push(field);
+      }
+    }
+
+    return {
+      hasDrift: missingFields.length > 0 || extraFields.length > 0,
+      missingFields,
+      extraFields,
+      indexExists: true,
+    };
+  }
+}

package/src/services/sessions.ts

@@ -0,0 +1,86 @@
+import type { Redis } from "@upstash/redis";
+import type {
+  AnalyticsConfig,
+  FeatureFlagAssignment,
+  Session,
+  SessionMetadata,
+  SessionResult,
+} from "../types";
+import { FeatureFlagService } from "./feature-flags";
+import { LoggingService } from "./logging";
+import { generateSessionId } from "../utils";
+
+const SESSION_PREFIX = "analytics:session:";
+
+export class SessionService {
+  constructor(
+    private redis: Redis,
+    private config: AnalyticsConfig,
+    private featureFlags: FeatureFlagService,
+    private logger: LoggingService
+  ) {}
+
+  async createSession(options?: {
+    featureFlags?: Record<string, FeatureFlagAssignment>;
+    metadata?: Record<string, unknown>;
+  }): Promise<Session> {
+    // Resolve feature flags
+    const resolvedFlags = await this.featureFlags.resolveFlags(
+      options?.featureFlags
+    );
+
+    const sessionId = generateSessionId();
+    const createdAt = Date.now();
+
+    const sessionData: SessionMetadata = {
+      sessionId,
+      createdAt,
+      featureFlags: resolvedFlags,
+      ...(options?.metadata ? { metadata: options.metadata } : {}),
+    };
+
+    // Store session with expiration
+    const key = `${SESSION_PREFIX}${sessionId}`;
+    await this.redis.set(key, JSON.stringify(sessionData), {
+      px: this.config.session.expirationMs,
+    });
+
+    return {
+      id: sessionId,
+      createdAt,
+      featureFlags: resolvedFlags,
+      ...(options?.metadata ? { metadata: options.metadata } : {}),
+    };
+  }
+
+  async getSession(sessionId: string): Promise<SessionResult> {
+    const key = `${SESSION_PREFIX}${sessionId}`;
+    const data = await this.redis.get<SessionMetadata | string>(key);
+
+    if (!data) {
+      return { exists: false };
+    }
+
+    const session: SessionMetadata =
+      typeof data === "string" ? JSON.parse(data) : data;
+
+    return { exists: true, session };
+  }
+
+  async getFeatureFlag(
+    sessionId: string,
+    flagName: string
+  ): Promise<string | null> {
+    const result = await this.getSession(sessionId);
+    if (!result.exists) return null;
+    return result.session.featureFlags[flagName] ?? null;
+  }
+
+  async getAllFeatureFlags(
+    sessionId: string
+  ): Promise<Record<string, string> | null> {
+    const result = await this.getSession(sessionId);
+    if (!result.exists) return null;
+    return result.session.featureFlags;
+  }
+}

package/src/types.ts

@@ -0,0 +1,194 @@
+import type { Redis } from "@upstash/redis";
+
+// ─── Utility Types ──────────────────────────────────────────────
+
+export type DeepPartial<T> = {
+  [P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
+};
+
+// ─── Feature Flags ───────────────────────────────────────────────
+
+export type FeatureFlagDefinition = {
+  possibleValues: string[];
+  defaultValue: string;
+  description?: string;
+};
+
+export type FeatureFlagConfig = FeatureFlagDefinition & {
+  name: string;
+};
+
+export type FeatureFlagDefinitions = Record<string, FeatureFlagDefinition>;
+
+// Feature flag assignment: either explicit value or distribution weights
+export type FeatureFlagAssignment =
+  | string
+  | Record<string, number>;
+
+// ─── Sessions ────────────────────────────────────────────────────
+
+export type SessionMetadata<
+  TFeatureFlags extends Record<string, string> = Record<string, string>,
+> = {
+  sessionId: string;
+  createdAt: number;
+  featureFlags: TFeatureFlags;
+  metadata?: Record<string, unknown>;
+};
+
+export type SessionResult<
+  TFeatureFlags extends Record<string, string> = Record<string, string>,
+> =
+  | { exists: true; session: SessionMetadata<TFeatureFlags> }
+  | { exists: false };
+
+export type Session<
+  TFeatureFlags extends Record<string, string> = Record<string, string>,
+> = {
+  id: string;
+  createdAt: number;
+  featureFlags: TFeatureFlags;
+  metadata?: Record<string, unknown>;
+};
+
+// ─── Events ──────────────────────────────────────────────────────
+
+export type StandardEventMap = {
+  pageview: { path: string };
+  click: { element: string };
+  error: { message: string };
+  warning: { message: string };
+  info: { message: string };
+};
+
+export type StandardEventName = keyof StandardEventMap;
+
+export type EventData = {
+  sessionCreationTimestamp: number;
+  sessionId: string;
+  /** ISO 8601 date string for date filtering */
+  eventTime: string;
+  /** Millisecond epoch for numeric aggregation (histogram) */
+  eventTimeMs: number;
+  eventName: string;
+  featureFlags: Record<string, string>;
+  sessionMetadata?: Record<string, unknown>;
+  properties?: Record<string, unknown>;
+};
+
+export type CaptureEventInput<
+  TCustomEvents extends Record<string, Record<string, unknown>> = Record<
+    string,
+    Record<string, unknown>
+  >,
+> =
+  | {
+      [K in keyof TCustomEvents]: {
+        sessionId: string;
+        eventName: K & string;
+        properties: TCustomEvents[K];
+      };
+    }[keyof TCustomEvents]
+  | {
+      [K in StandardEventName]: {
+        sessionId: string;
+        eventName: K;
+        properties: StandardEventMap[K];
+      };
+    }[StandardEventName];
+
+// ─── Schema Registry ────────────────────────────────────────────
+
+export type SchemaPropertyType = "string" | "number" | "boolean";
+
+export type SchemaProperty = {
+  type: SchemaPropertyType;
+};
+
+export type EventSchema = {
+  properties: Record<string, SchemaProperty>;
+  lastUpdated: number;
+  version: number;
+};
+
+// ─── System Logging ─────────────────────────────────────────────
+
+export type LogType =
+  | "schema_update"
+  | "feature_flag_update"
+  | "validation_error"
+  | "system_error"
+  | "index_update";
+
+export type SystemLog = {
+  logType: LogType;
+  timestamp: number;
+  eventName?: string;
+  changes?: {
+    before: unknown;
+    after: unknown;
+  };
+  metadata?: Record<string, unknown>;
+};
+
+// ─── Search Index ───────────────────────────────────────────────
+
+export type SearchIndexInfo = {
+  currentIndex: string;
+  alias: string;
+  schema: Record<string, unknown>;
+  documentCount: number;
+};
+
+// ─── Configuration ──────────────────────────────────────────────
+
+export type AnalyticsConfig = {
+  session: {
+    expirationMs: number;
+  };
+  featureFlags: FeatureFlagDefinitions;
+  events: {
+    customEventRetentionDays: number;
+    /** Maximum number of events allowed in a single batch request. Default: 20 */
+    maxBatchSize: number;
+  };
+  schemaValidation: {
+    /** Probability of validating schema on each event, between 0 and 1. Default: 1 (every event). Set lower (e.g. 0.01) to reduce Redis costs at the expense of delayed schema detection. */
+    checkFrequency: number;
+  };
+  logging: {
+    retentionDays: number;
+    enabledTypes: LogType[];
+  };
+  search: {
+    indexName: string;
+    autoUpdate: boolean;
+    rebuildOnStartup: boolean;
+  };
+};
+
+export type SDKOptions = {
+  redis: Redis;
+  config: AnalyticsConfig;
+};
+
+// ─── Server Configuration ───────────────────────────────────────
+
+export type ServerConfig = {
+  redis: {
+    url: string;
+    token: string;
+  };
+  config?: DeepPartial<AnalyticsConfig>;
+};
+
+// ─── Client Configuration ───────────────────────────────────────
+
+export type ClientConfig = {
+  /** Base URL for the analytics endpoint. Defaults to "/api/analytics" */
+  endpoint?: string;
+  /** Debounce time in ms for batching event captures. 0 = no batching. Defaults to 0. */
+  flushInterval?: number;
+  /** Maximum number of events to send in a single batch request. Default: 20. */
+  maxBatchSize?: number;
+};

package/src/utils.ts

@@ -0,0 +1,45 @@
+export function generateShortId(): string {
+  return Math.random().toString(36).substring(2, 10);
+}
+
+export function generateSessionId(): string {
+  const timestamp = Date.now();
+  const shortId = generateShortId();
+  return `session-${timestamp}-${shortId}`;
+}
+
+export function generateEventKey(): string {
+  return `event-${generateShortId()}-${Date.now()}-${generateShortId()}`;
+}
+
+export function generateLogKey(): string {
+  const timestamp = Date.now();
+  const shortId = generateShortId();
+  return `system-log-${timestamp}-${shortId}`;
+}
+
+export function selectByDistribution(
+  distribution: Record<string, number>
+): string {
+  const entries = Object.entries(distribution);
+  const totalWeight = entries.reduce((sum, [, weight]) => sum + weight, 0);
+  let random = Math.random() * totalWeight;
+
+  for (const [value, weight] of entries) {
+    random -= weight;
+    if (random <= 0) {
+      return value;
+    }
+  }
+
+  // Fallback to last value
+  return entries[entries.length - 1][0];
+}
+
+export function inferPropertyType(
+  value: unknown
+): "string" | "number" | "boolean" {
+  if (typeof value === "boolean") return "boolean";
+  if (typeof value === "number") return "number";
+  return "string";
+}