cdp-edge 2.0.2 → 2.0.4

package/server-edge-tracker/modules/ml/segmentation.js

@@ -0,0 +1,316 @@
+/**
+ * CDP Edge — ML Clustering (Fase 1)
+ * Handlers das rotas /api/segmentation/*
+ */
+
+import { tryParseJson } from '../utils.js';
+
+// ── POST /api/segmentation/cluster ────────────────────────────────────────────
+export async function handleSegmentationCluster(env, request, headers) {
+  if (!env.DB) return new Response(JSON.stringify({ error: 'DB não configurado' }), { status: 503, headers });
+  if (!env.AI) return new Response(JSON.stringify({ error: 'Workers AI não configurado (verifique binding AI no wrangler.toml)' }), { status: 503, headers });
+
+  const url            = new URL(request.url);
+  const algorithm      = url.searchParams.get('algorithm') || 'kmeans';
+  const nClusters      = Math.min(10, Math.max(3, parseInt(url.searchParams.get('n_clusters') || '5')));
+  const clientVertical = url.searchParams.get('vertical') || 'general';
+  const forceRecluster = url.searchParams.get('force') === 'true';
+
+  if (!['kmeans', 'dbscan', 'hierarchical'].includes(algorithm)) {
+    return new Response(JSON.stringify({ error: 'algorithm deve ser: kmeans, dbscan ou hierarchical', received: algorithm }), { status: 400, headers });
+  }
+
+  try {
+    // 1. Cluster recente? Evitar re-clustering desnecessário (< 7 dias)
+    if (!forceRecluster) {
+      const existing = await env.DB.prepare(`
+        SELECT id, created_at, cluster_name FROM ml_segments
+        WHERE clustering_algorithm = ? AND is_active = 1 AND client_vertical = ?
+        ORDER BY created_at DESC LIMIT 1
+      `).bind(algorithm, clientVertical).first();
+
+      if (existing) {
+        const ageDays = (Date.now() - new Date(existing.created_at).getTime()) / (1000 * 60 * 60 * 24);
+        if (ageDays < 7) {
+          return new Response(JSON.stringify({
+            success: true, message: 'Cluster existente ainda válido (< 7 dias). Use ?force=true para re-clustering.',
+            cluster_id: existing.id, cluster_name: existing.cluster_name,
+            age_days: Math.round(ageDays * 10) / 10, use_existing: true,
+          }), { status: 200, headers });
+        }
+      }
+    }
+
+    // 2. Extrair leads históricos do D1 (últimos 6 meses, excluindo bots confirmados)
+    const leadsRes = await env.DB.prepare(`
+      SELECT id, predicted_ltv_class, engagement_score, intention_level,
+             country, state, utm_source, utm_medium, bot_score,
+             CAST(strftime('%H', created_at) AS INTEGER) AS hour_of_day,
+             CAST(julianday('now') - julianday(created_at) AS INTEGER) AS days_since_lead,
+             CASE WHEN strftime('%w', created_at) IN ('0','6') THEN 1 ELSE 0 END AS is_weekend
+      FROM leads
+      WHERE created_at >= datetime('now', '-6 months')
+        AND (bot_score IS NULL OR bot_score < 2)
+      ORDER BY RANDOM()
+      LIMIT 2000
+    `).all();
+
+    const leads = leadsRes.results || [];
+
+    if (leads.length < 50) {
+      return new Response(JSON.stringify({
+        error: 'Dados insuficientes para clustering. Mínimo: 50 leads nos últimos 6 meses.',
+        leads_found: leads.length, required: 50,
+      }), { status: 400, headers });
+    }
+
+    // 3. Feature Engineering — normalização 0–1
+    const features = leads.map(l => ({
+      id:         l.id,
+      ltv:        l.predicted_ltv_class === 'High' ? 1 : (l.predicted_ltv_class === 'Medium' ? 0.5 : 0),
+      engagement: Math.min((l.engagement_score || 0) / 100, 1),
+      intention:  l.intention_level === 'comprador' || l.intention_level === 'high_intent' ? 1
+                : l.intention_level === 'interessado' ? 0.6
+                : l.intention_level === 'curioso'     ? 0.3 : 0,
+      recency:    Math.max(0, 1 - (l.days_since_lead || 0) / 180),
+      hour:       (l.hour_of_day || 12) / 23,
+      is_weekend: l.is_weekend || 0,
+      is_br:      l.country === 'BR' ? 1 : 0,
+      is_paid:    ['facebook','google','tiktok','instagram','youtube'].includes((l.utm_source || '').toLowerCase()) ? 1 : 0,
+    }));
+
+    // 4. Prompt para Workers AI
+    const sampleSize = Math.min(features.length, 100);
+    const sample     = features.slice(0, sampleSize);
+
+    const clusteringPrompt =
+`You are a customer segmentation ML expert. Perform ${algorithm} clustering on ${sampleSize} customers into ${nClusters} segments.
+
+Customer features (all normalized 0-1):
+- ltv: predicted lifetime value (0=Low, 0.5=Medium, 1=High)
+- engagement: browser engagement score
+- intention: purchase intention (0=none, 0.3=curious, 0.6=interested, 1=buyer)
+- recency: lead recency (1=today, 0=6 months ago)
+- hour: conversion hour of day
+- is_weekend: converted on weekend (0/1)
+- is_br: lead from Brazil (0/1)
+- is_paid: from paid traffic channel (0/1)
+
+Data (${sampleSize} customers): ${JSON.stringify(sample.slice(0, 50))}
+
+Return ONLY valid JSON, zero explanation:
+{
+  "clusters": [
+    {
+      "cluster_id": 0,
+      "name": "[Nome Descritivo em Português]",
+      "size": ${Math.round(sampleSize / nClusters)},
+      "percentage": ${Math.round(100 / nClusters)},
+      "characteristics": {
+        "avg_ltv_class": 0.5,
+        "avg_behavior_score": 0.5,
+        "avg_engagement_score": 0.5,
+        "avg_intention_level": 0.5,
+        "avg_days_since_lead": 30,
+        "dominant_countries": ["BR"],
+        "dominant_states": ["SP", "RJ"],
+        "dominant_utm_sources": ["facebook"],
+        "top_features": ["ltv", "engagement"]
+      },
+      "centroid": { "ltv": 0.5, "engagement": 0.5, "intention": 0.5 },
+      "action_recommendation": "[Recomendação de campanha específica para este segmento]"
+    }
+  ],
+  "silhouette_score": 0.65,
+  "total_processed": ${sampleSize}
+}`;
+
+    // 5. Workers AI
+    const startTime = Date.now();
+    const aiRes = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', {
+      messages:   [{ role: 'user', content: clusteringPrompt }],
+      max_tokens: 2000,
+    });
+    const duration = Date.now() - startTime;
+
+    if (!aiRes?.response) throw new Error('Workers AI não retornou resposta');
+
+    const jsonMatch = aiRes.response.trim().match(/\{[\s\S]*\}/);
+    if (!jsonMatch) throw new Error('Resposta do Workers AI não contém JSON válido');
+    const mlResult = JSON.parse(jsonMatch[0]);
+
+    if (!Array.isArray(mlResult.clusters) || mlResult.clusters.length === 0) {
+      throw new Error('Workers AI não retornou clusters válidos');
+    }
+
+    // 6. Inativar clusters anteriores
+    await env.DB.prepare(`UPDATE ml_segments SET is_active = 0 WHERE clustering_algorithm = ? AND client_vertical = ? AND is_active = 1`).bind(algorithm, clientVertical).run();
+
+    // 7. Persistir novos clusters
+    const now = new Date().toISOString();
+    for (const cluster of mlResult.clusters) {
+      const ch = cluster.characteristics || {};
+      await env.DB.prepare(`
+        INSERT INTO ml_segments (
+          cluster_id, cluster_name, clustering_algorithm, client_vertical,
+          size, percentage, avg_ltv_class, avg_behavior_score, avg_engagement_score,
+          avg_intention_level, avg_days_since_lead,
+          dominant_countries, dominant_states, dominant_utm_sources, dominant_features,
+          silhouette_score, action_recommendations, bid_recommendations, campaign_recommendations,
+          is_active, created_at, updated_at
+        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?)
+      `).bind(
+        cluster.cluster_id || 0, cluster.name || `Segmento ${cluster.cluster_id}`, algorithm, clientVertical,
+        cluster.size || 0, cluster.percentage || 0,
+        ch.avg_ltv_class || 0, ch.avg_behavior_score || 0, ch.avg_engagement_score || 0,
+        ch.avg_intention_level || 0, ch.avg_days_since_lead || 0,
+        JSON.stringify(ch.dominant_countries || ['BR']), JSON.stringify(ch.dominant_states || []),
+        JSON.stringify(ch.dominant_utm_sources || []), JSON.stringify(ch.top_features || []),
+        mlResult.silhouette_score || 0,
+        JSON.stringify([cluster.action_recommendation || '']), JSON.stringify([]), JSON.stringify([]),
+        now, now,
+      ).run();
+    }
+
+    // 8. Log no histórico
+    try {
+      await env.DB.prepare(`
+        INSERT INTO ml_clustering_history (
+          clustering_id, started_at, completed_at, algorithm,
+          n_leads_processed, n_clusters_created, total_duration_ms,
+          workers_ai_neurons_used, status, parameters, results_summary
+        ) VALUES (0, ?, datetime('now'), ?, ?, ?, ?, ?, 'completed', ?, ?)
+      `).bind(
+        new Date(startTime).toISOString(), algorithm, leads.length, mlResult.clusters.length,
+        duration, Math.ceil(duration * 0.01),
+        JSON.stringify({ algorithm, n_clusters: nClusters, vertical: clientVertical }),
+        JSON.stringify({ clusters: mlResult.clusters.length, silhouette: mlResult.silhouette_score }),
+      ).run();
+    } catch (e) { console.error('[Segmentation] history log error:', e.message); }
+
+    return new Response(JSON.stringify({
+      success: true, algorithm, n_clusters: mlResult.clusters.length, client_vertical: clientVertical,
+      leads_analyzed: leads.length, duration_ms: duration, silhouette_score: mlResult.silhouette_score || null,
+      clusters: mlResult.clusters, generated_at: now,
+    }), { status: 200, headers });
+
+  } catch (err) {
+    console.error('[Segmentation] cluster error:', err.message);
+    try {
+      if (env.DB) {
+        await env.DB.prepare(`
+          INSERT INTO ml_clustering_history (clustering_id, started_at, algorithm, n_leads_processed, n_clusters_created, total_duration_ms, workers_ai_neurons_used, status, error_message, parameters, results_summary)
+          VALUES (0, datetime('now'), ?, 0, 0, 0, 0, 'failed', ?, ?, '{}')
+        `).bind(algorithm, err.message, JSON.stringify({ algorithm, n_clusters: nClusters })).run();
+      }
+    } catch { /* não bloquear a resposta de erro */ }
+
+    return new Response(JSON.stringify({ error: 'Erro ao executar clustering', message: err.message }), { status: 500, headers });
+  }
+}
+
+// ── GET /api/segmentation/list ────────────────────────────────────────────────
+export async function handleSegmentationList(env, request, headers) {
+  if (!env.DB) return new Response(JSON.stringify({ error: 'DB não configurado' }), { status: 503, headers });
+
+  const url       = new URL(request.url);
+  const algorithm = url.searchParams.get('algorithm') || null;
+  const vertical  = url.searchParams.get('vertical')  || null;
+
+  try {
+    const conditions = ['is_active = 1'];
+    const bindings   = [];
+    if (algorithm) { conditions.push('clustering_algorithm = ?'); bindings.push(algorithm); }
+    if (vertical)  { conditions.push('client_vertical = ?');      bindings.push(vertical); }
+
+    const result   = await env.DB.prepare(`
+      SELECT id, cluster_id, cluster_name, clustering_algorithm, client_vertical,
+             size, percentage, avg_ltv_class, avg_behavior_score, avg_engagement_score,
+             avg_intention_level, avg_days_since_lead, silhouette_score,
+             dominant_countries, dominant_states, dominant_utm_sources, dominant_features,
+             action_recommendations, bid_recommendations, campaign_recommendations,
+             is_active, created_at, updated_at
+      FROM ml_segments
+      WHERE ${conditions.join(' AND ')}
+      ORDER BY created_at DESC
+      LIMIT 50
+    `).bind(...bindings).all();
+
+    const segments = (result.results || []).map(s => ({
+      ...s,
+      dominant_countries:       tryParseJson(s.dominant_countries, []),
+      dominant_states:          tryParseJson(s.dominant_states, []),
+      dominant_utm_sources:     tryParseJson(s.dominant_utm_sources, []),
+      dominant_features:        tryParseJson(s.dominant_features, []),
+      action_recommendations:   tryParseJson(s.action_recommendations, []),
+      bid_recommendations:      tryParseJson(s.bid_recommendations, []),
+      campaign_recommendations: tryParseJson(s.campaign_recommendations, []),
+    }));
+
+    return new Response(JSON.stringify({ success: true, total: segments.length, segments }), { status: 200, headers });
+  } catch (err) {
+    console.error('[Segmentation] list error:', err.message);
+    return new Response(JSON.stringify({ error: err.message }), { status: 500, headers });
+  }
+}
+
+// ── GET /api/segmentation/outliers ───────────────────────────────────────────
+export async function handleSegmentationOutliers(env, request, headers) {
+  if (!env.DB) return new Response(JSON.stringify({ error: 'DB não configurado' }), { status: 503, headers });
+
+  const url   = new URL(request.url);
+  const limit = Math.min(parseInt(url.searchParams.get('limit') || '50'), 200);
+  const days  = parseInt(url.searchParams.get('days') || '30');
+
+  try {
+    const result = await env.DB.prepare(`
+      SELECT msm.lead_id, msm.cluster_id, msm.confidence, msm.is_outlier, msm.outlier_reason, msm.assigned_at,
+             l.email, l.phone, l.country, l.state, l.city, l.utm_source, l.bot_score, l.engagement_score, l.intention_level, l.created_at AS lead_created_at
+      FROM ml_segment_members msm
+      LEFT JOIN leads l ON CAST(msm.lead_id AS INTEGER) = l.id
+      WHERE msm.is_outlier = 1 AND msm.assigned_at >= datetime('now', '-' || ? || ' days')
+      ORDER BY msm.assigned_at DESC
+      LIMIT ?
+    `).bind(days, limit).all();
+
+    return new Response(JSON.stringify({ success: true, total: (result.results || []).length, period_days: days, outliers: result.results || [] }), { status: 200, headers });
+  } catch (err) {
+    console.error('[Segmentation] outliers error:', err.message);
+    return new Response(JSON.stringify({ error: err.message }), { status: 500, headers });
+  }
+}
+
+// ── PUT /api/segmentation/update ─────────────────────────────────────────────
+export async function handleSegmentationUpdate(env, request, headers) {
+  if (!env.DB) return new Response(JSON.stringify({ error: 'DB não configurado' }), { status: 503, headers });
+
+  let body;
+  try { body = await request.json(); }
+  catch { return new Response(JSON.stringify({ error: 'JSON inválido no body da requisição' }), { status: 400, headers }); }
+
+  const { cluster_id, action_recommendations, bid_recommendations, campaign_recommendations } = body;
+  if (cluster_id === undefined || cluster_id === null) {
+    return new Response(JSON.stringify({ error: 'cluster_id é obrigatório' }), { status: 400, headers });
+  }
+
+  try {
+    const sets     = [];
+    const bindings = [];
+    if (action_recommendations   !== undefined) { sets.push('action_recommendations = ?');   bindings.push(JSON.stringify(action_recommendations)); }
+    if (bid_recommendations      !== undefined) { sets.push('bid_recommendations = ?');      bindings.push(JSON.stringify(bid_recommendations)); }
+    if (campaign_recommendations !== undefined) { sets.push('campaign_recommendations = ?'); bindings.push(JSON.stringify(campaign_recommendations)); }
+
+    if (sets.length === 0) {
+      return new Response(JSON.stringify({ error: 'Nenhum campo válido para atualizar (action_recommendations, bid_recommendations, campaign_recommendations)' }), { status: 400, headers });
+    }
+
+    sets.push("updated_at = datetime('now')");
+    bindings.push(cluster_id);
+
+    await env.DB.prepare(`UPDATE ml_segments SET ${sets.join(', ')} WHERE id = ?`).bind(...bindings).run();
+    return new Response(JSON.stringify({ success: true, cluster_id, fields_updated: sets.length - 1 }), { status: 200, headers });
+  } catch (err) {
+    console.error('[Segmentation] update error:', err.message);
+    return new Response(JSON.stringify({ error: err.message }), { status: 500, headers });
+  }
+}

package/server-edge-tracker/modules/utils.js

@@ -0,0 +1,89 @@
+/**
+ * CDP Edge — Utilities
+ * Funções puras sem dependências externas.
+ * Importadas por todos os outros módulos.
+ */
+
+// ── CORS ──────────────────────────────────────────────────────────────────────
+export function isAllowedOrigin(origin, siteDomain) {
+  if (!origin || !siteDomain) return false;
+  return origin === `https://${siteDomain}`
+    || origin.endsWith(`.${siteDomain}`)
+    || origin === 'http://localhost:3000'
+    || origin === 'http://localhost:5173';
+}
+
+export function corsHeaders(origin, siteDomain) {
+  const allowed = isAllowedOrigin(origin, siteDomain) ? origin : `https://${siteDomain}`;
+  return {
+    'Access-Control-Allow-Origin':  allowed,
+    'Access-Control-Allow-Methods': 'POST, GET, OPTIONS',
+    'Access-Control-Allow-Headers': 'Content-Type, X-Requested-With',
+    'Access-Control-Max-Age':       '86400',
+  };
+}
+
+// ── SHA-256 via WebCrypto (obrigatório no Cloudflare Workers) ─────────────────
+export async function sha256(value) {
+  if (!value) return undefined;
+  const clean = String(value).toLowerCase().trim();
+  if (!clean) return undefined;
+  const buf = await crypto.subtle.digest(
+    'SHA-256',
+    new TextEncoder().encode(clean)
+  );
+  return Array.from(new Uint8Array(buf))
+    .map(b => b.toString(16).padStart(2, '0'))
+    .join('');
+}
+
+// ── Normalização de telefone → somente dígitos + DDI 55 ──────────────────────
+export function normalizePhone(phone) {
+  if (!phone) return undefined;
+  let digits = String(phone).replace(/\D/g, '');
+  if (digits.length === 11 && !digits.startsWith('55')) digits = '55' + digits;
+  if (digits.length === 10 && !digits.startsWith('55')) digits = '55' + digits;
+  return digits.length >= 10 ? digits : undefined;
+}
+
+// ── Normalização de cidade → lowercase sem acentos ────────────────────────────
+export function normalizeCity(city) {
+  if (!city) return undefined;
+  return String(city)
+    .toLowerCase()
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .replace(/[^a-z0-9]/g, '');
+}
+
+// ── Parse seguro de JSON armazenado como TEXT no D1 ───────────────────────────
+export function tryParseJson(str, fallback) {
+  if (!str) return fallback !== undefined ? fallback : null;
+  try { return JSON.parse(str); } catch { return fallback !== undefined ? fallback : null; }
+}
+
+// ── Mapa Meta → GA4 event names ───────────────────────────────────────────────
+export const META_TO_GA4 = {
+  PageView:             'page_view',
+  ViewContent:          'view_item',
+  Lead:                 'generate_lead',
+  Contact:              'generate_lead',
+  Schedule:             'generate_lead',
+  InitiateCheckout:     'begin_checkout',
+  AddToCart:            'add_to_cart',
+  AddPaymentInfo:       'add_payment_info',
+  Purchase:             'purchase',
+  CompleteRegistration: 'sign_up',
+  Subscribe:            'subscribe',
+  StartTrial:           'start_trial',
+  Search:               'search',
+  AddToWishlist:        'add_to_wishlist',
+};
+
+// ── Lista canônica de eventos válidos (19 eventos) ────────────────────────────
+export const VALID_EVENT_NAMES = new Set([
+  'PageView','ViewContent','Lead','Purchase','InitiateCheckout',
+  'AddToCart','CompleteRegistration','Contact','Schedule',
+  'StartTrial','Subscribe','SubmitApplication','Search',
+  'video_start','video_25','video_50','video_75','video_complete',
+]);

package/server-edge-tracker/schema-indexes.sql

@@ -0,0 +1,67 @@
+-- CDP Edge — Composite Indexes para Performance
+-- Executar após todos os outros schemas:
+--   wrangler d1 execute cdp-edge-db --file=schema-indexes.sql --remote
+--
+-- Todos os índices usam IF NOT EXISTS — idempotente, seguro para re-executar.
+
+-- ── leads: queries mais comuns ────────────────────────────────────────────────
+
+-- Deduplicação por email + janela temporal (upsertProfile, saveLead)
+CREATE INDEX IF NOT EXISTS idx_leads_email_created
+  ON leads(email, created_at DESC);
+
+-- Relatórios por plataforma + período (Intelligence Agent, Dashboard)
+CREATE INDEX IF NOT EXISTS idx_leads_platform_created
+  ON leads(platform, created_at DESC);
+
+-- Lookup de conversão por evento + período (auditErrorRates, generateDailyReport)
+CREATE INDEX IF NOT EXISTS idx_leads_event_created
+  ON leads(event_name, created_at DESC);
+
+-- Identity resolution: user_id + evento para evitar duplicatas
+CREATE INDEX IF NOT EXISTS idx_leads_userid_event
+  ON leads(user_id, event_name);
+
+-- Exportação Customer Match: event_name + email filtrado por data
+CREATE INDEX IF NOT EXISTS idx_leads_event_email
+  ON leads(event_name, email);
+
+-- CRM Dashboard: ordenação por status + criação
+CREATE INDEX IF NOT EXISTS idx_leads_crm_created
+  ON leads(crm_status, created_at DESC);
+
+-- ── user_profiles: enrichment e Advanced Matching ────────────────────────────
+
+-- Busca por LTV class + atualização (Bidding ML consome isso)
+CREATE INDEX IF NOT EXISTS idx_profiles_ltv_updated
+  ON user_profiles(predicted_ltv_class, updated_at DESC);
+
+-- Score + cohort para segmentação ML (handleSegmentationCluster)
+CREATE INDEX IF NOT EXISTS idx_profiles_score_cohort
+  ON user_profiles(score DESC, cohort_label);
+
+-- Lookup email + updated_at (resolveDeviceGraph, getProfileByEmail)
+CREATE INDEX IF NOT EXISTS idx_profiles_email_updated
+  ON user_profiles(email, updated_at DESC);
+
+-- ── fraud_signals: dashboard e alertas ───────────────────────────────────────
+
+-- handleFraudAlerts: filtra por ip + período
+CREATE INDEX IF NOT EXISTS idx_fraud_ip_created
+  ON fraud_signals(ip_address, created_at DESC);
+
+-- handleFraudStats: fraud_score >= threshold ordenado por data
+CREATE INDEX IF NOT EXISTS idx_fraud_score_created
+  ON fraud_signals(fraud_score DESC, created_at DESC);
+
+-- ── ltv_ab_assignments: resultados de A/B test ───────────────────────────────
+
+-- handleLtvAbTestResults: test_id + predicted_class para análise
+CREATE INDEX IF NOT EXISTS idx_ab_testid_class
+  ON ltv_ab_assignments(test_id, predicted_class);
+
+-- ── ml_segment_members: join com leads para bidding ─────────────────────────
+
+-- handleBiddingRecommend: segment_id lookup
+CREATE INDEX IF NOT EXISTS idx_seg_members_segid
+  ON ml_segment_members(segment_id, joined_at DESC);

package/server-edge-tracker/wrangler.toml

@@ -1,4 +1,6 @@
 name = "server-edge-tracker"
+# Entry point: worker.js (monólito original, 100% compatível)
+# Para usar a versão modular ES Modules: altere para main = "index.js"
 main = "worker.js"
 compatibility_date = "2025-01-01"
 compatibility_flags = ["nodejs_compat"]