@kaikybrofc/omnizap-system 2.2.4 → 2.2.6

package/app/services/featureFlagService.js

@@ -0,0 +1,137 @@
+import { createHash } from 'node:crypto';
+
+import logger from '../utils/logger/loggerModule.js';
+import { executeQuery, TABLES } from '../../database/index.js';
+
+const FEATURE_FLAG_CACHE_TTL_MS = Math.max(
+  5_000,
+  Number(process.env.FEATURE_FLAG_CACHE_TTL_MS) || 30_000,
+);
+
+let cacheState = {
+  loadedAt: 0,
+  byName: new Map(),
+  tableAvailable: true,
+};
+
+const normalizeFlagName = (value) =>
+  String(value || '')
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9_:-]/g, '')
+    .slice(0, 120);
+
+const toPercent = (value) => {
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric)) return 0;
+  return Math.max(0, Math.min(100, Math.floor(numeric)));
+};
+
+const toBool = (value, fallback = false) => {
+  if (value === true || value === 1) return true;
+  if (value === false || value === 0) return false;
+  if (value === undefined || value === null || value === '') return fallback;
+  const normalized = String(value).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false;
+  return fallback;
+};
+
+const normalizeRow = (row) => ({
+  flag_name: normalizeFlagName(row?.flag_name),
+  is_enabled: toBool(row?.is_enabled, false),
+  rollout_percent: toPercent(row?.rollout_percent ?? 100),
+});
+
+const resolveCohortBucket = (subjectKey) => {
+  const normalized = String(subjectKey || '').trim();
+  if (!normalized) return 0;
+  const digest = createHash('sha1').update(normalized).digest();
+  const value = digest.readUInt32BE(0);
+  return value % 100;
+};
+
+const loadFlagsFromDatabase = async () => {
+  const rows = await executeQuery(
+    `SELECT flag_name, is_enabled, rollout_percent
+     FROM ${TABLES.FEATURE_FLAG}`,
+    [],
+  );
+
+  const byName = new Map();
+  (Array.isArray(rows) ? rows : []).forEach((row) => {
+    const normalized = normalizeRow(row);
+    if (!normalized.flag_name) return;
+    byName.set(normalized.flag_name, normalized);
+  });
+  return byName;
+};
+
+export const refreshFeatureFlags = async ({ force = false } = {}) => {
+  const now = Date.now();
+  const isFresh = now - cacheState.loadedAt < FEATURE_FLAG_CACHE_TTL_MS;
+  if (!force && isFresh) return cacheState.byName;
+
+  if (!cacheState.tableAvailable) return cacheState.byName;
+
+  try {
+    const byName = await loadFlagsFromDatabase();
+    cacheState = {
+      loadedAt: now,
+      byName,
+      tableAvailable: true,
+    };
+  } catch (error) {
+    if (error?.code === 'ER_NO_SUCH_TABLE') {
+      cacheState = {
+        ...cacheState,
+        tableAvailable: false,
+      };
+      logger.warn('Tabela de feature flags indisponível. Usando fallback por env/default.', {
+        action: 'feature_flag_table_unavailable',
+      });
+      return cacheState.byName;
+    }
+    logger.warn('Falha ao carregar feature flags. Mantendo cache anterior.', {
+      action: 'feature_flag_refresh_failed',
+      error: error?.message,
+    });
+  }
+
+  return cacheState.byName;
+};
+
+const resolveEnvFallback = (flagName, fallback) => {
+  const envKey = `FEATURE_${flagName.toUpperCase()}`;
+  return toBool(process.env[envKey], fallback);
+};
+
+export const isFeatureEnabled = async (
+  flagName,
+  { fallback = false, subjectKey = '' } = {},
+) => {
+  const normalizedFlagName = normalizeFlagName(flagName);
+  if (!normalizedFlagName) return Boolean(fallback);
+
+  const byName = await refreshFeatureFlags();
+  const entry = byName.get(normalizedFlagName);
+  if (!entry) {
+    return resolveEnvFallback(normalizedFlagName, fallback);
+  }
+
+  if (!entry.is_enabled) return false;
+  if (entry.rollout_percent >= 100) return true;
+  if (entry.rollout_percent <= 0) return false;
+
+  const bucket = resolveCohortBucket(subjectKey || normalizedFlagName);
+  return bucket < entry.rollout_percent;
+};
+
+export const getFeatureFlagsSnapshot = async () => {
+  const byName = await refreshFeatureFlags();
+  return Array.from(byName.values()).map((entry) => ({
+    flag_name: entry.flag_name,
+    is_enabled: Boolean(entry.is_enabled),
+    rollout_percent: Number(entry.rollout_percent || 0),
+  }));
+};

package/database/index.js

@@ -104,8 +104,13 @@ export const TABLES = {
   SEMANTIC_THEME_SUGGESTION_CACHE: 'semantic_theme_suggestion_cache',
   STICKER_PACK_ENGAGEMENT: 'sticker_pack_engagement',
   STICKER_PACK_INTERACTION_EVENT: 'sticker_pack_interaction_event',
+  STICKER_PACK_SCORE_SNAPSHOT: 'sticker_pack_score_snapshot',
   STICKER_ASSET_REPROCESS_QUEUE: 'sticker_asset_reprocess_queue',
   STICKER_WORKER_TASK_QUEUE: 'sticker_worker_task_queue',
+  STICKER_WORKER_TASK_DLQ: 'sticker_worker_task_dlq',
+  DOMAIN_EVENT_OUTBOX: 'domain_event_outbox',
+  DOMAIN_EVENT_OUTBOX_DLQ: 'domain_event_outbox_dlq',
+  FEATURE_FLAG: 'feature_flag',
   STICKER_WEB_GOOGLE_USER: 'sticker_web_google_user',
   STICKER_WEB_GOOGLE_SESSION: 'sticker_web_google_session',
   STICKER_WEB_ADMIN_BAN: 'sticker_web_admin_ban',

package/database/migrations/20260228_0022_sticker_scale_indexes.sql

@@ -0,0 +1,16 @@
+ALTER TABLE sticker_asset
+  ADD INDEX idx_sticker_asset_created_at (created_at);
+
+ALTER TABLE sticker_asset_classification
+  ADD INDEX idx_sticker_asset_classification_confidence (confidence),
+  ADD INDEX idx_sticker_asset_classification_updated_at (updated_at),
+  ADD INDEX idx_sticker_asset_classification_version_updated (classification_version, updated_at);
+
+ALTER TABLE sticker_pack_item
+  ADD INDEX idx_sticker_pack_item_sticker_id (sticker_id);
+
+ALTER TABLE sticker_pack
+  ADD INDEX idx_sticker_pack_catalog_lookup (deleted_at, status, pack_status, visibility, updated_at);
+
+ALTER TABLE sticker_pack_interaction_event
+  ADD INDEX idx_sticker_pack_interaction_created_at (created_at);

package/database/migrations/20260228_0023_sticker_pack_score_snapshot.sql

@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS sticker_pack_score_snapshot (
+  pack_id CHAR(36) PRIMARY KEY,
+  ranking_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  pack_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  trend_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  quality_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  engagement_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  diversity_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  cohesion_score DECIMAL(10,6) NOT NULL DEFAULT 0,
+  sensitive_content TINYINT(1) NOT NULL DEFAULT 0,
+  nsfw_level ENUM('safe', 'suggestive', 'explicit') NOT NULL DEFAULT 'safe',
+  sticker_count INT UNSIGNED NOT NULL DEFAULT 0,
+  tags JSON NULL,
+  scores_json JSON NULL,
+  source_version VARCHAR(32) NOT NULL DEFAULT 'v1',
+  refreshed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  CONSTRAINT fk_sticker_pack_score_snapshot_pack
+    FOREIGN KEY (pack_id) REFERENCES sticker_pack(id)
+    ON DELETE CASCADE ON UPDATE CASCADE,
+  INDEX idx_sticker_pack_score_snapshot_ranking (ranking_score),
+  INDEX idx_sticker_pack_score_snapshot_trend (trend_score),
+  INDEX idx_sticker_pack_score_snapshot_refresh (refreshed_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

package/database/migrations/20260228_0024_domain_event_outbox.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS domain_event_outbox (
+  id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT,
+  event_type VARCHAR(96) NOT NULL,
+  aggregate_type VARCHAR(96) NOT NULL,
+  aggregate_id VARCHAR(128) NOT NULL,
+  payload JSON NULL,
+  status ENUM('pending', 'processing', 'completed', 'failed') NOT NULL DEFAULT 'pending',
+  priority TINYINT UNSIGNED NOT NULL DEFAULT 50,
+  idempotency_key VARCHAR(180) NULL,
+  available_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
+  max_attempts TINYINT UNSIGNED NOT NULL DEFAULT 10,
+  worker_token CHAR(36) NULL,
+  last_error VARCHAR(255) NULL,
+  locked_at TIMESTAMP NULL DEFAULT NULL,
+  processed_at TIMESTAMP NULL DEFAULT NULL,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  UNIQUE KEY uq_domain_event_outbox_idempotency_key (idempotency_key),
+  INDEX idx_domain_event_outbox_status_sched (status, available_at, priority),
+  INDEX idx_domain_event_outbox_event_type (event_type, status, available_at),
+  INDEX idx_domain_event_outbox_aggregate (aggregate_type, aggregate_id, created_at),
+  INDEX idx_domain_event_outbox_worker_token (worker_token)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS domain_event_outbox_dlq (
+  id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT,
+  outbox_event_id BIGINT UNSIGNED NULL,
+  event_type VARCHAR(96) NOT NULL,
+  aggregate_type VARCHAR(96) NOT NULL,
+  aggregate_id VARCHAR(128) NOT NULL,
+  payload JSON NULL,
+  attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
+  max_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
+  last_error VARCHAR(255) NULL,
+  failed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  UNIQUE KEY uq_domain_event_outbox_dlq_outbox_event (outbox_event_id),
+  INDEX idx_domain_event_outbox_dlq_event (event_type, failed_at),
+  INDEX idx_domain_event_outbox_dlq_aggregate (aggregate_type, aggregate_id, failed_at),
+  INDEX idx_domain_event_outbox_dlq_outbox_event_id (outbox_event_id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

package/database/migrations/20260228_0025_sticker_worker_task_idempotency_dlq.sql

@@ -0,0 +1,23 @@
+ALTER TABLE sticker_worker_task_queue
+  ADD COLUMN IF NOT EXISTS idempotency_key VARCHAR(180) NULL AFTER task_type;
+
+CREATE UNIQUE INDEX uq_sticker_worker_task_idempotency_key
+  ON sticker_worker_task_queue (idempotency_key);
+
+CREATE TABLE IF NOT EXISTS sticker_worker_task_dlq (
+  id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT,
+  task_id BIGINT UNSIGNED NULL,
+  task_type ENUM('classification_cycle', 'curation_cycle', 'rebuild_cycle') NOT NULL,
+  payload JSON NULL,
+  idempotency_key VARCHAR(180) NULL,
+  attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
+  max_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
+  priority TINYINT UNSIGNED NOT NULL DEFAULT 0,
+  last_error VARCHAR(255) NULL,
+  failed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  UNIQUE KEY uq_sticker_worker_task_dlq_task_id (task_id),
+  INDEX idx_sticker_worker_task_dlq_type_failed_at (task_type, failed_at),
+  INDEX idx_sticker_worker_task_dlq_task_id (task_id),
+  INDEX idx_sticker_worker_task_dlq_idempotency_key (idempotency_key)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

package/database/migrations/20260228_0026_feature_flags.sql

@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS feature_flag (
+  id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT,
+  flag_name VARCHAR(120) NOT NULL,
+  is_enabled TINYINT(1) NOT NULL DEFAULT 0,
+  rollout_percent TINYINT UNSIGNED NOT NULL DEFAULT 100,
+  description VARCHAR(255) NULL,
+  updated_by VARCHAR(120) NULL,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  UNIQUE KEY uq_feature_flag_name (flag_name),
+  INDEX idx_feature_flag_enabled (is_enabled, rollout_percent)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+INSERT INTO feature_flag (flag_name, is_enabled, rollout_percent, description)
+VALUES
+  ('enable_ranking_snapshot_read', 1, 100, 'Leitura HTTP do ranking/sinais a partir de snapshot'),
+  ('enable_domain_event_outbox', 1, 100, 'Publicacao e consumo de eventos de dominio via outbox interno'),
+  ('enable_worker_dedicated_processes', 0, 100, 'Ativa workers dedicados por tipo de task'),
+  ('enable_object_storage_delivery', 0, 100, 'Entrega de assets via object storage/CDN com URL segura')
+ON DUPLICATE KEY UPDATE
+  description = VALUES(description);

package/ecosystem.prod.config.cjs

@@ -2,6 +2,21 @@ require('dotenv').config();
 
 const appName = process.env.PM2_APP_NAME || 'omnizap-system';
 
+const baseEnv = {
+  NODE_ENV: 'production',
+  COMMAND_PREFIX: '/',
+  LOG_LEVEL: 'info',
+  DB_LOG_EVERY_QUERY: 'false',
+  DB_MONITOR_ENABLED: 'false',
+  LID_BACKFILL_ON_START: 'false',
+  STICKER_CLASSIFICATION_BACKGROUND_ENABLED: 'true',
+  STICKER_REPROCESS_QUEUE_ENABLED: 'true',
+  STICKER_AUTO_PACK_BY_TAGS_ENABLED: 'true',
+  STICKER_WORKER_PIPELINE_ENABLED: 'true',
+  STICKER_WORKER_PIPELINE_INLINE_POLLER_ENABLED: 'true',
+  STICKER_DEDICATED_WORKERS_ENABLED: 'true',
+};
+
 module.exports = {
   apps: [
     {
@@ -17,19 +32,65 @@ module.exports = {
       out_file: `logs/${appName}-out.log`,
       error_file: `logs/${appName}-error.log`,
       env: {
-        NODE_ENV: 'production',
-        COMMAND_PREFIX: '/',
-        LOG_LEVEL: 'info',
-        DB_LOG_EVERY_QUERY: 'false',
-        DB_MONITOR_ENABLED: 'false',
-        LID_BACKFILL_ON_START: 'false',
-        STICKER_CLASSIFICATION_BACKGROUND_ENABLED: 'true',
-        STICKER_REPROCESS_QUEUE_ENABLED: 'true',
-        STICKER_AUTO_PACK_BY_TAGS_ENABLED: 'true',
+        ...baseEnv,
       },
       wait_ready: true,
       listen_timeout: 10000,
       kill_timeout: 5000,
     },
+    {
+      name: `${appName}-worker-classification`,
+      script: './scripts/sticker-worker-task.mjs',
+      args: '--task-type classification_cycle',
+      cwd: __dirname,
+      exec_mode: 'fork',
+      instances: 1,
+      autorestart: true,
+      watch: false,
+      max_memory_restart: '2G',
+      log_date_format: 'YYYY-MM-DD HH:mm:ss',
+      out_file: `logs/${appName}-worker-classification-out.log`,
+      error_file: `logs/${appName}-worker-classification-error.log`,
+      env: {
+        ...baseEnv,
+      },
+      kill_timeout: 5000,
+    },
+    {
+      name: `${appName}-worker-curation`,
+      script: './scripts/sticker-worker-task.mjs',
+      args: '--task-type curation_cycle',
+      cwd: __dirname,
+      exec_mode: 'fork',
+      instances: 1,
+      autorestart: true,
+      watch: false,
+      max_memory_restart: '2G',
+      log_date_format: 'YYYY-MM-DD HH:mm:ss',
+      out_file: `logs/${appName}-worker-curation-out.log`,
+      error_file: `logs/${appName}-worker-curation-error.log`,
+      env: {
+        ...baseEnv,
+      },
+      kill_timeout: 5000,
+    },
+    {
+      name: `${appName}-worker-rebuild`,
+      script: './scripts/sticker-worker-task.mjs',
+      args: '--task-type rebuild_cycle',
+      cwd: __dirname,
+      exec_mode: 'fork',
+      instances: 1,
+      autorestart: true,
+      watch: false,
+      max_memory_restart: '2G',
+      log_date_format: 'YYYY-MM-DD HH:mm:ss',
+      out_file: `logs/${appName}-worker-rebuild-out.log`,
+      error_file: `logs/${appName}-worker-rebuild-error.log`,
+      env: {
+        ...baseEnv,
+      },
+      kill_timeout: 5000,
+    },
   ],
 };

package/index.js

@@ -39,6 +39,14 @@ import {
   startStickerWorkerPipeline,
   stopStickerWorkerPipeline,
 } from './app/modules/stickerPackModule/stickerWorkerPipelineRuntime.js';
+import {
+  startStickerPackScoreSnapshotRuntime,
+  stopStickerPackScoreSnapshotRuntime,
+} from './app/modules/stickerPackModule/stickerPackScoreSnapshotRuntime.js';
+import {
+  startStickerDomainEventConsumer,
+  stopStickerDomainEventConsumer,
+} from './app/modules/stickerPackModule/stickerDomainEventConsumerRuntime.js';
 
 /**
  * Timeout máximo para inicialização do banco (criar/verificar DB + tabelas).
@@ -216,6 +224,8 @@ async function startApp() {
       startStickerClassificationBackground();
       startStickerAutoPackByTagsBackground();
     }
+    startStickerPackScoreSnapshotRuntime();
+    startStickerDomainEventConsumer();
 
     // Backfill é opcional, rodando em background.
     const shouldBackfill = process.env.LID_BACKFILL_ON_START !== 'false';
@@ -363,6 +373,22 @@ async function shutdown(signal, error) {
       });
     }
 
+    try {
+      stopStickerPackScoreSnapshotRuntime();
+    } catch (snapshotError) {
+      logger.warn('Falha ao encerrar runtime de snapshot de score dos packs.', {
+        error: snapshotError?.message,
+      });
+    }
+
+    try {
+      stopStickerDomainEventConsumer();
+    } catch (consumerError) {
+      logger.warn('Falha ao encerrar consumidor interno de eventos de domínio.', {
+        error: consumerError?.message,
+      });
+    }
+
     // 5) Encerrar MySQL pool
     await closeDatabasePool();
 

package/observability/sticker-catalog-slo.md

@@ -0,0 +1,83 @@
+# Sticker Catalog 10x Baseline e SLOs
+
+Este documento define a baseline operacional da camada HTTP + pipeline de classificação para o módulo de stickers.
+
+## 1. Metas SLO (fase inicial)
+
+### HTTP catálogo (`/api/sticker-packs*`, `/stickers*`, `/api/marketplace/stats`)
+
+- **Latência p95**: `<= 750ms`
+- **Latência p99**: `<= 1500ms`
+- **Taxa de erro (5xx + timeout)**: `<= 2%` por janela de 5 minutos
+- **Throughput alvo**: escalar linearmente com workers/processos sem aumento abrupto do p95
+
+### Classificação de stickers
+
+- **Duração média do ciclo**: `<= 10s`
+- **Throughput mínimo (assets classificados/min)**: `>= 300` (ajustar por hardware)
+- **Backlog de fila (`sticker_reprocess_pending`)**: tendência de queda após picos; alerta se cresce por mais de 15 min
+
+## 2. Métricas instrumentadas
+
+### HTTP
+
+- `omnizap_http_requests_total{route_group,method,status_class}`
+- `omnizap_http_request_duration_ms{route_group,method,status_class}`
+- `omnizap_http_slo_violation_total{route_group,method}`
+
+`route_group` segmenta tráfego em:
+
+- `catalog_api_public`
+- `catalog_api_auth`
+- `catalog_api_admin`
+- `catalog_api_upload`
+- `catalog_web`
+- `catalog_data_asset`
+- `catalog_user_profile`
+- `marketplace_stats`
+- `metrics`
+- `other`
+
+### Classificação
+
+- `omnizap_sticker_classification_cycle_duration_ms{status}`
+- `omnizap_sticker_classification_cycle_total{status}`
+- `omnizap_sticker_classification_assets_total{outcome}`
+- `omnizap_queue_depth{queue}`
+
+## 3. Tracing mínimo
+
+- Cada request HTTP agora recebe/propaga `X-Request-Id`.
+- Se o cliente enviar `X-Request-Id`, o valor é reaproveitado.
+- Sem header, o servidor gera UUID.
+
+## 4. Baseline de carga (script local)
+
+Script: `scripts/sticker-catalog-loadtest.mjs`
+
+Exemplo:
+
+```bash
+node scripts/sticker-catalog-loadtest.mjs \
+  --base-url http://127.0.0.1:9102 \
+  --duration-seconds 60 \
+  --concurrency 40 \
+  --paths "/api/sticker-packs?limit=24&sort=popular,/api/sticker-packs/stats,/api/sticker-packs/creators?limit=25" \
+  --out /tmp/sticker-loadtest-report.json
+```
+
+Interpretação rápida:
+
+- `latency_ms.p95 <= 750` = SLO de latência cumprido
+- `error_rate <= 0.02` = estabilidade aceitável
+- `throughput_rps` = referência para comparar antes/depois de otimizações
+
+## 5. Gate de rollout sugerido
+
+1. Capturar baseline com carga atual.
+2. Aplicar mudança de arquitetura/índice/cache.
+3. Reexecutar carga com mesmos parâmetros.
+4. Aprovar rollout apenas se:
+   - p95 não piorar mais de 10%
+   - erro não subir acima de 2%
+   - backlog voltar ao patamar normal em até 15 min

package/observability/sticker-scale-hardening-rollout.md

@@ -0,0 +1,128 @@
+# Sticker 10x Hardening And Rollout
+
+## Scope
+
+This runbook covers phases 4-8 of the sticker-pack scale plan:
+
+1. ranking snapshot read path
+2. internal outbox/event consumer
+3. dedicated workers (classification/curation/rebuild)
+4. object storage delivery with secure URLs
+5. canary rollout, rollback, and final tuning
+
+## Feature Flags
+
+Flags are stored in `feature_flag`:
+
+- `enable_ranking_snapshot_read`
+- `enable_domain_event_outbox`
+- `enable_worker_dedicated_processes`
+- `enable_object_storage_delivery`
+
+### Query Current Status
+
+```sql
+SELECT flag_name, is_enabled, rollout_percent, updated_at
+FROM feature_flag
+WHERE flag_name IN (
+  'enable_ranking_snapshot_read',
+  'enable_domain_event_outbox',
+  'enable_worker_dedicated_processes',
+  'enable_object_storage_delivery'
+)
+ORDER BY flag_name;
+```
+
+### Update Rollout Percent
+
+```sql
+UPDATE feature_flag
+SET is_enabled = 1, rollout_percent = 25, updated_by = 'ops'
+WHERE flag_name = 'enable_worker_dedicated_processes';
+```
+
+### Emergency Disable
+
+```sql
+UPDATE feature_flag
+SET is_enabled = 0, rollout_percent = 0, updated_by = 'ops'
+WHERE flag_name IN (
+  'enable_worker_dedicated_processes',
+  'enable_object_storage_delivery',
+  'enable_domain_event_outbox'
+);
+```
+
+## Canary Sequence
+
+1. `enable_ranking_snapshot_read`: 10% -> 50% -> 100%
+2. `enable_domain_event_outbox`: 10% -> 50% -> 100%
+3. start dedicated worker processes and set `enable_worker_dedicated_processes`: 10% -> 50% -> 100%
+4. `enable_object_storage_delivery`: 5% -> 25% -> 100%
+
+Promotion gate for each step:
+
+- HTTP p95 within target
+- queue backlog stable (`pending`, `failed`)
+- outbox DLQ not growing unexpectedly
+- no sustained error-rate increase
+
+## Dedicated Workers
+
+Run workers as isolated processes:
+
+```bash
+npm run worker:sticker:classification
+npm run worker:sticker:curation
+npm run worker:sticker:rebuild
+```
+
+PM2 production profile includes these workers in `ecosystem.prod.config.cjs`.
+
+## 10x Validation
+
+### HTTP Stress
+
+```bash
+npm run loadtest:stickers -- --base-url http://127.0.0.1:9102 --duration-seconds 120 --concurrency 200 --slo-ms 750
+```
+
+### Queue/Worker Validation
+
+Monitor:
+
+- `sticker_worker_tasks_pending`
+- `sticker_worker_tasks_processing`
+- `sticker_worker_tasks_failed`
+- `domain_event_outbox_pending`
+- `domain_event_outbox_failed`
+
+Acceptance:
+
+- failed queues remain near zero (transient spikes allowed)
+- pending queues recover after load burst
+- no monotonic growth in DLQ tables
+
+## Rollback Plan
+
+1. Disable `enable_object_storage_delivery`.
+2. Disable `enable_worker_dedicated_processes` (inline poller resumes).
+3. Disable `enable_domain_event_outbox` if event flow is unstable.
+4. Keep `enable_ranking_snapshot_read` enabled only if snapshot freshness is healthy.
+
+Data safety notes:
+
+- tasks/events are persisted in SQL queues
+- failed terminal tasks/events are preserved in DLQ tables
+- local disk read path remains fallback for sticker asset serving
+
+## Post-Rollout Tuning
+
+Tune these env vars after baseline:
+
+- `STICKER_WORKER_CLASSIFICATION_CADENCE_MS`
+- `STICKER_WORKER_CURATION_CADENCE_MS`
+- `STICKER_WORKER_REBUILD_CADENCE_MS`
+- `STICKER_DEDICATED_WORKER_POLL_INTERVAL_MS`
+- `STICKER_SCORE_SNAPSHOT_REFRESH_INTERVAL_MS`
+- `STICKER_OBJECT_STORAGE_SIGNED_URL_TTL_SECONDS`