@reachy/audience-module 1.0.18 → 1.0.20

package/src/engine/V2AudienceEngine.ts

@@ -1,5 +1,6 @@
 import { CriteriaParser } from '../builders/CriteriaParser'
 import { AudienceCriteria } from '../types'
+import { ClickHouseEventQueryExecutor } from '../executors/ClickHouseEventQueryExecutor'
 
 type Logger = {
   log: (...args: any[]) => void
@@ -281,23 +282,27 @@ export class V2AudienceEngine {
   private supabase: any
   private debug: boolean
   private logger: Logger
+  private chExecutor: ClickHouseEventQueryExecutor | null = null
 
-  constructor(params: { supabaseClient: any; debug?: boolean; logger?: Logger }) {
+  private static _tzCache = new Map<string, { tz: string; fetchedAt: number }>()
+  private static TZ_CACHE_TTL = 5 * 60 * 1000 // 5 minutes
+
+  constructor(params: { supabaseClient: any; clickhouseClient?: any; debug?: boolean; logger?: Logger }) {
     this.supabase = params.supabaseClient
     this.debug = !!params.debug
     this.logger = params.logger || console
+    if (params.clickhouseClient) {
+      this.chExecutor = new ClickHouseEventQueryExecutor(params.clickhouseClient, params.logger || console)
+      if (this.debug) this.logger.log('[V2AudienceEngine] ClickHouse executor initialized')
+    }
   }
 
-  async getContactIdsByAudienceCriteriaV2(
-    organizationId: string,
-    projectId: string,
-    criteriaRaw: any
-  ): Promise<Set<string>> {
-    let criteria: AudienceCriteria = CriteriaParser.parse(criteriaRaw as any) as any
-
-    // Timezone do projeto: usado em filtros de tempo (time_of_day/day_of_week/day_of_month)
-    // Fonte: projects.settings.timezone (mesma tela de Settings > Project no frontend)
-    let projectTimezone = 'UTC'
+  private async getProjectTimezone(organizationId: string, projectId: string): Promise<string> {
+    const cacheKey = `${organizationId}:${projectId}`
+    const cached = V2AudienceEngine._tzCache.get(cacheKey)
+    if (cached && Date.now() - cached.fetchedAt < V2AudienceEngine.TZ_CACHE_TTL) {
+      return cached.tz
+    }
     try {
       const { data } = await this.supabase
         .from('projects')
@@ -306,12 +311,25 @@ export class V2AudienceEngine {
         .eq('id', projectId)
         .single()
       const tzCandidate = (data as any)?.settings?.timezone
-      if (typeof tzCandidate === 'string' && isValidIanaTimezone(tzCandidate)) {
-        projectTimezone = tzCandidate.trim()
-      }
+      const tz = (typeof tzCandidate === 'string' && isValidIanaTimezone(tzCandidate))
+        ? tzCandidate.trim()
+        : 'UTC'
+      V2AudienceEngine._tzCache.set(cacheKey, { tz, fetchedAt: Date.now() })
+      return tz
     } catch {
-      // fallback silencioso
+      V2AudienceEngine._tzCache.set(cacheKey, { tz: 'UTC', fetchedAt: Date.now() })
+      return 'UTC'
     }
+  }
+
+  async getContactIdsByAudienceCriteriaV2(
+    organizationId: string,
+    projectId: string,
+    criteriaRaw: any
+  ): Promise<Set<string>> {
+    let criteria: AudienceCriteria = CriteriaParser.parse(criteriaRaw as any) as any
+
+    const projectTimezone = await this.getProjectTimezone(organizationId, projectId)
 
     // Compat: permitir critérios legados (filters/conditions) sem groups,
     // convertendo para groups V2 para que o engine avalie sozinho.
@@ -336,7 +354,8 @@ export class V2AudienceEngine {
 
     const typeId = String((criteria as any)?.type || '')
 
-    const fetchAll = async (baseQuery: any, pageSize: number): Promise<any[]> => {
+    const MAX_FETCH_ROWS = 500_000
+    const fetchAll = async (baseQuery: any, pageSize: number, maxRows: number = MAX_FETCH_ROWS): Promise<any[]> => {
       let offset = 0
       let out: any[] = []
       while (true) {
@@ -344,41 +363,72 @@ export class V2AudienceEngine {
         if (error) throw error
         if (!data || data.length === 0) break
         out = out.concat(data)
+        if (out.length >= maxRows) {
+          this.dlog('fetchAll: max rows reached', { maxRows, fetched: out.length })
+          break
+        }
         if (data.length < pageSize) break
         offset += pageSize
       }
       return out
     }
 
-    // Carregar contatos do projeto (necessário para negate e mapeamento de identidade)
-    const allContacts = await fetchAll(
-      this.supabase
-        .from('contacts')
-        .select('id, reachy_id, email')
-        .eq('organization_id', organizationId)
-        .eq('project_id', projectId),
-      1000
-    )
-
-    const allContactIds = new Set<string>((allContacts || []).map((c: any) => c.id as string))
-    const reachyIdToContactId = new Map<string, string>()
-    const contactIdToReachyId = new Map<string, string>()
-    const emailToContactId = new Map<string, string>()
-
-    for (const c of allContacts || []) {
-      const rid = c?.reachy_id ? String(c.reachy_id).trim() : ''
-      const cid = c?.id ? String(c.id).trim() : ''
-      if (rid && cid) {
-        reachyIdToContactId.set(rid, cid)
-        contactIdToReachyId.set(cid, rid)
+    // Lazy loading: só carrega contatos quando necessário (negate ou event rules que precisam de identity maps)
+    const groups = (criteria as any).groups as any[]
+    const hasNegate = groups.some((g: any) => (g.rules || []).some((r: any) => r.negate))
+    const hasEventRules = groups.some((g: any) => (g.rules || []).some((r: any) => r.kind === 'event'))
+
+    let allContactIds = new Set<string>()
+    let reachyIdToContactId = new Map<string, string>()
+    let contactIdToReachyId = new Map<string, string>()
+    let emailToContactId = new Map<string, string>()
+    let _contactsLoaded = false
+
+    const ensureContactsLoaded = async () => {
+      if (_contactsLoaded) return
+      _contactsLoaded = true
+      const allContacts = await fetchAll(
+        this.supabase
+          .from('contacts')
+          .select('id, reachy_id, email')
+          .eq('organization_id', organizationId)
+          .eq('project_id', projectId),
+        1000
+      )
+      allContactIds = new Set<string>((allContacts || []).map((c: any) => c.id as string))
+      for (const c of allContacts || []) {
+        const rid = c?.reachy_id ? String(c.reachy_id).trim() : ''
+        const cid = c?.id ? String(c.id).trim() : ''
+        if (rid && cid) {
+          reachyIdToContactId.set(rid, cid)
+          contactIdToReachyId.set(cid, rid)
+        }
+        const email = c?.email ? String(c.email).trim().toLowerCase() : ''
+        if (email && cid) emailToContactId.set(email, cid)
       }
-      const email = c?.email ? String(c.email).trim().toLowerCase() : ''
-      if (email && cid) emailToContactId.set(email, cid)
     }
 
-    const union = (a: Set<string>, b: Set<string>) => new Set([...a, ...b])
-    const intersect = (a: Set<string>, b: Set<string>) => new Set([...a].filter(x => b.has(x)))
-    const diff = (a: Set<string>, b: Set<string>) => new Set([...a].filter(x => !b.has(x)))
+    // Preload contatos se sabemos que será necessário
+    if (hasNegate || hasEventRules) {
+      await ensureContactsLoaded()
+    }
+
+    const union = (a: Set<string>, b: Set<string>): Set<string> => {
+      const result = new Set(a)
+      for (const x of b) result.add(x)
+      return result
+    }
+    const intersect = (a: Set<string>, b: Set<string>): Set<string> => {
+      const [smaller, larger] = a.size <= b.size ? [a, b] : [b, a]
+      const result = new Set<string>()
+      for (const x of smaller) { if (larger.has(x)) result.add(x) }
+      return result
+    }
+    const diff = (a: Set<string>, b: Set<string>): Set<string> => {
+      const result = new Set<string>()
+      for (const x of a) { if (!b.has(x)) result.add(x) }
+      return result
+    }
 
     const resolveEventContactId = (row: any): string | null => {
       const rawReachyId = row?.reachy_id ? String(row.reachy_id).trim() : ''
@@ -405,14 +455,45 @@ export class V2AudienceEngine {
 
     const evalEventRule = async (rule: any): Promise<Set<string>> => {
       const cfg = (criteria as any)?.config || {}
+
+      // Route to ClickHouse when available (replaces N paginated Supabase queries + JS aggregation)
+      if (this.chExecutor) {
+        try {
+          return await this.chExecutor.execute(
+            rule, criteria, organizationId, projectId, projectTimezone,
+            reachyIdToContactId, contactIdToReachyId, allContactIds
+          )
+        } catch (chErr) {
+          this.logger.warn('[V2AudienceEngine] ClickHouse evalEventRule failed, falling back to Supabase:', chErr)
+          // Fall through to Supabase path
+        }
+      }
+
       const effectiveEventName =
         cfg.eventType && String(cfg.eventType).trim() !== ''
           ? String(cfg.eventType)
           : String(rule.eventName)
 
+      // Determine minimal SELECT fields based on what's needed
+      const ruleFiltersPrecheck = Array.isArray((rule as any).filters) ? (rule as any).filters : []
+      const needsEventData = !!(
+        (rule.interest && rule.interest.key) ||
+        ruleFiltersPrecheck.some((f: any) => ['event_property', 'time_of_day', 'day_of_week', 'day_of_month'].includes(String(f?.type || '').trim())) ||
+        (rule.frequency && rule.frequency.type && rule.frequency.type !== 'count') ||
+        typeId === 'live-page-count'
+      )
+      const needsTimestamp = !!(
+        ruleFiltersPrecheck.some((f: any) => ['first_time', 'last_time', 'time_of_day', 'day_of_week', 'day_of_month'].includes(String(f?.type || '').trim()))
+      )
+
+      let selectFields = 'contact_id, reachy_id'
+      if (needsEventData) selectFields += ', event_data'
+      if (needsTimestamp) selectFields += ', event_timestamp'
+      if (needsEventData || needsTimestamp) selectFields += ', event_name'
+
       let query = this.supabase
         .from('contact_events')
-        .select('contact_id, reachy_id, event_data, event_timestamp, event_name')
+        .select(selectFields)
         .eq('organization_id', organizationId)
         .eq('project_id', projectId)
         .eq('event_name', effectiveEventName)
@@ -597,7 +678,8 @@ export class V2AudienceEngine {
         if (this.debug) this.logger.warn('[AUDIENCE_MODULE_ENGINE] event attributes filter error')
       }
 
-      const data = await fetchAll(query, 500)
+      const MAX_EVENT_ROWS = 200_000
+      const data = await fetchAll(query, 1000, MAX_EVENT_ROWS)
       if (!data || data.length === 0) return new Set<string>()
 
       // Event advanced filters (ex.: event_property dentro do DID)
@@ -1173,35 +1255,81 @@ export class V2AudienceEngine {
       return new Set<string>(data.map((r: any) => r.id as string))
     }
 
+    const evalRule = (rule: any) => rule.kind === 'event' ? evalEventRule(rule) : evalPropertyRule(rule)
+
     const evalGroup = async (group: any): Promise<Set<string>> => {
+      const rules = group.rules || []
+      if (rules.length === 0) return new Set<string>()
+
+      const op = group.operator
+
+      // OR: run all rules in parallel then union
+      if (op === 'OR') {
+        const results = await Promise.all(rules.map(async (rule: any) => {
+          const set = await evalRule(rule)
+          return rule.negate ? diff(allContactIds, set) : set
+        }))
+        let acc = results[0]!
+        for (let i = 1; i < results.length; i++) {
+          acc = union(acc, results[i]!)
+        }
+        return acc
+      }
+
+      // AND / NOT: sequential with early exit on empty set
       let acc: Set<string> | null = null
-      for (const rule of group.rules || []) {
-        const set = rule.kind === 'event' ? await evalEventRule(rule) : await evalPropertyRule(rule)
-        const shouldNegate = rule.negate
-        const s = shouldNegate ? diff(allContactIds, set) : set
+      for (const rule of rules) {
+        const set = await evalRule(rule)
+        const s = rule.negate ? diff(allContactIds, set) : set
 
         if (acc == null) {
           acc = s
         } else {
-          if (group.operator === 'AND') acc = intersect(acc, s)
-          else if (group.operator === 'OR') acc = union(acc, s)
-          else if (group.operator === 'NOT') acc = diff(acc, s)
+          if (op === 'AND') acc = intersect(acc, s)
+          else if (op === 'NOT') acc = diff(acc, s)
         }
+
+        // Early exit: AND/NOT with empty set can't recover
+        if (acc.size === 0) return acc
       }
       return acc || new Set<string>()
     }
 
+    // Evaluate groups — detect if we can parallelize
+    const allGroups = (criteria as any).groups as any[]
     let result: Set<string> | null = null
-    for (let i = 0; i < (criteria as any).groups.length; i++) {
-      const group = (criteria as any).groups[i]
-      const gset = await evalGroup(group)
-      if (result == null) {
-        result = gset
-      } else {
-        const betweenOp = String((group as any).combineOperator || group.operator || 'AND').toUpperCase()
-        if (betweenOp === 'AND') result = intersect(result, gset)
-        else if (betweenOp === 'OR') result = union(result, gset)
-        else if (betweenOp === 'NOT') result = diff(result, gset)
+
+    // Check if all groups combine with OR → can run in parallel
+    const allCombineWithOr = allGroups.length > 1 && allGroups.every((g: any, i: number) => {
+      if (i === 0) return true
+      return String((g as any).combineOperator || g.operator || 'AND').toUpperCase() === 'OR'
+    })
+
+    if (allCombineWithOr) {
+      const groupResults = await Promise.all(allGroups.map((g: any) => evalGroup(g)))
+      result = groupResults[0]!
+      for (let i = 1; i < groupResults.length; i++) {
+        result = union(result, groupResults[i]!)
+      }
+    } else {
+      for (let i = 0; i < allGroups.length; i++) {
+        const group = allGroups[i]
+        const gset = await evalGroup(group)
+        if (result == null) {
+          result = gset
+        } else {
+          const betweenOp = String((group as any).combineOperator || group.operator || 'AND').toUpperCase()
+          if (betweenOp === 'AND') result = intersect(result, gset)
+          else if (betweenOp === 'OR') result = union(result, gset)
+          else if (betweenOp === 'NOT') result = diff(result, gset)
+        }
+        // Early exit for AND between groups
+        if (result.size === 0) {
+          const nextOp = i + 1 < allGroups.length
+            ? String(((allGroups[i + 1] as any).combineOperator || (allGroups[i + 1] as any).operator || 'AND')).toUpperCase()
+            : ''
+          if (nextOp === 'AND' || nextOp === 'NOT') break
+        }
       }
     }
 
@@ -1235,22 +1363,7 @@ export class V2AudienceEngine {
 
     const typeId = String((criteria as any)?.type || '')
 
-    // Timezone do projeto (para filtros de tempo em eventos)
-    let projectTimezone = 'UTC'
-    try {
-      const { data } = await this.supabase
-        .from('projects')
-        .select('settings')
-        .eq('organization_id', organizationId)
-        .eq('id', projectId)
-        .single()
-      const tzCandidate = (data as any)?.settings?.timezone
-      if (typeof tzCandidate === 'string' && isValidIanaTimezone(tzCandidate)) {
-        projectTimezone = tzCandidate.trim()
-      }
-    } catch {
-      // fallback silencioso
-    }
+    const projectTimezone = await this.getProjectTimezone(organizationId, projectId)
 
     // Carregar identidade do contato (para reachy_id)
     const { data: contactRow } = await this.supabase
@@ -1265,6 +1378,29 @@ export class V2AudienceEngine {
 
     const evalEventRuleForContact = async (rule: any): Promise<boolean> => {
       const cfg = (criteria as any)?.config || {}
+
+      // Route to ClickHouse when available — use batch engine scoped to 1 contact
+      if (this.chExecutor) {
+        try {
+          // Build single-contact identity maps for the executor
+          const singleContactIdToReachyId = new Map<string, string>()
+          const singleReachyIdToContactId = new Map<string, string>()
+          const singleAllContactIds = new Set<string>([contactId])
+          if (reachyId) {
+            singleReachyIdToContactId.set(reachyId, contactId)
+            singleContactIdToReachyId.set(contactId, reachyId)
+          }
+
+          const matchSet = await this.chExecutor.execute(
+            rule, criteria, organizationId, projectId, projectTimezone,
+            singleReachyIdToContactId, singleContactIdToReachyId, singleAllContactIds
+          )
+          return matchSet.has(contactId)
+        } catch (chErr) {
+          this.logger.warn('[V2AudienceEngine] ClickHouse evalEventRuleForContact failed, falling back to Supabase:', chErr)
+        }
+      }
+
       const effectiveEventName =
         cfg.eventType && String(cfg.eventType).trim() !== ''
           ? String(cfg.eventType)
@@ -1871,15 +2007,23 @@ export class V2AudienceEngine {
     }
 
     const evalGroupForContact = async (group: any): Promise<boolean> => {
+      const rules = group.rules || []
+      if (rules.length === 0) return false
+      const op = group.operator
+
       let acc: boolean | null = null
-      for (const rule of group.rules || []) {
+      for (const rule of rules) {
         const r = await evalRuleForContact(rule)
-        if (acc == null) acc = r
-        else {
-          if (group.operator === 'AND') acc = acc && r
-          else if (group.operator === 'OR') acc = acc || r
-          else if (group.operator === 'NOT') acc = acc && !r
+        if (acc == null) {
+          acc = r
+        } else {
+          if (op === 'AND') acc = acc && r
+          else if (op === 'OR') acc = acc || r
+          else if (op === 'NOT') acc = acc && !r
         }
+        // Early exit: AND false can't become true; OR true can't become false
+        if (op === 'AND' && acc === false) return false
+        if (op === 'OR' && acc === true) return true
       }
       return !!acc
     }
@@ -1887,13 +2031,24 @@ export class V2AudienceEngine {
     let result: boolean | null = null
     for (const group of (criteria as any).groups) {
       const g = await evalGroupForContact(group)
-      if (result == null) result = g
-      else {
+      if (result == null) {
+        result = g
+      } else {
         const betweenOp = String((group as any).combineOperator || group.operator || 'AND').toUpperCase()
         if (betweenOp === 'AND') result = result && g
         else if (betweenOp === 'OR') result = result || g
         else if (betweenOp === 'NOT') result = result && !g
       }
+      // Early exit between groups
+      if (result === false) {
+        // If all remaining groups are AND/NOT, result stays false
+        const remaining = (criteria as any).groups.slice((criteria as any).groups.indexOf(group) + 1)
+        const allAnd = remaining.every((g2: any) => {
+          const op2 = String(g2.combineOperator || g2.operator || 'AND').toUpperCase()
+          return op2 === 'AND' || op2 === 'NOT'
+        })
+        if (allAnd) return false
+      }
     }
 
     return !!result