@kaikybrofc/omnizap-system 2.2.4 → 2.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +5 -0
- package/README.md +13 -13
- package/app/modules/stickerPackModule/catalogHandlers/catalogAdminHttp.js +68 -0
- package/app/modules/stickerPackModule/catalogHandlers/catalogAuthHttp.js +34 -0
- package/app/modules/stickerPackModule/catalogHandlers/catalogPublicHttp.js +179 -0
- package/app/modules/stickerPackModule/catalogHandlers/catalogUploadHttp.js +92 -0
- package/app/modules/stickerPackModule/catalogRouter.js +79 -0
- package/app/modules/stickerPackModule/domainEventOutboxRepository.js +243 -0
- package/app/modules/stickerPackModule/domainEvents.js +61 -0
- package/app/modules/stickerPackModule/stickerAssetClassificationRepository.js +21 -0
- package/app/modules/stickerPackModule/stickerAssetRepository.js +19 -0
- package/app/modules/stickerPackModule/stickerClassificationBackgroundRuntime.js +55 -15
- package/app/modules/stickerPackModule/stickerDedicatedTaskWorkerRuntime.js +238 -0
- package/app/modules/stickerPackModule/stickerDomainEventBus.js +71 -0
- package/app/modules/stickerPackModule/stickerDomainEventConsumerRuntime.js +198 -0
- package/app/modules/stickerPackModule/stickerObjectStorageService.js +285 -0
- package/app/modules/stickerPackModule/stickerPackCatalogHttp.js +537 -529
- package/app/modules/stickerPackModule/stickerPackEngagementRepository.js +44 -0
- package/app/modules/stickerPackModule/stickerPackItemRepository.js +18 -0
- package/app/modules/stickerPackModule/stickerPackRepository.js +51 -0
- package/app/modules/stickerPackModule/stickerPackScoreSnapshotRepository.js +191 -0
- package/app/modules/stickerPackModule/stickerPackScoreSnapshotRuntime.js +301 -0
- package/app/modules/stickerPackModule/stickerStorageService.js +111 -10
- package/app/modules/stickerPackModule/stickerWorkerPipelineRuntime.js +21 -0
- package/app/modules/stickerPackModule/stickerWorkerTaskQueueRepository.js +59 -7
- package/app/observability/metrics.js +169 -0
- package/app/services/featureFlagService.js +137 -0
- package/database/index.js +5 -0
- package/database/migrations/20260228_0022_sticker_scale_indexes.sql +16 -0
- package/database/migrations/20260228_0023_sticker_pack_score_snapshot.sql +25 -0
- package/database/migrations/20260228_0024_domain_event_outbox.sql +42 -0
- package/database/migrations/20260228_0025_sticker_worker_task_idempotency_dlq.sql +23 -0
- package/database/migrations/20260228_0026_feature_flags.sql +21 -0
- package/ecosystem.prod.config.cjs +70 -9
- package/index.js +26 -0
- package/package.json +5 -1
- package/public/index.html +30 -3
- package/scripts/sticker-catalog-loadtest.mjs +208 -0
- package/scripts/sticker-worker-task.mjs +122 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
import { executeQuery, TABLES } from '../../../database/index.js';
|
|
4
|
+
import { normalizeDomainEventPayload } from './domainEvents.js';
|
|
5
|
+
|
|
6
|
+
const normalizeStatus = (value) => {
|
|
7
|
+
const normalized = String(value || '').trim().toLowerCase();
|
|
8
|
+
if (['pending', 'processing', 'completed', 'failed'].includes(normalized)) return normalized;
|
|
9
|
+
return null;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const parseJson = (value, fallback = null) => {
|
|
13
|
+
if (value === null || value === undefined) return fallback;
|
|
14
|
+
if (typeof value === 'object') return value;
|
|
15
|
+
if (Buffer.isBuffer(value)) {
|
|
16
|
+
try {
|
|
17
|
+
return JSON.parse(value.toString('utf8'));
|
|
18
|
+
} catch {
|
|
19
|
+
return fallback;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
if (typeof value === 'string') {
|
|
23
|
+
try {
|
|
24
|
+
return JSON.parse(value);
|
|
25
|
+
} catch {
|
|
26
|
+
return fallback;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return fallback;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const clampInt = (value, fallback, min, max) => {
|
|
33
|
+
const numeric = Number(value);
|
|
34
|
+
if (!Number.isFinite(numeric)) return fallback;
|
|
35
|
+
return Math.max(min, Math.min(max, Math.floor(numeric)));
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const CLAIM_LOCK_TIMEOUT_SECONDS = clampInt(
|
|
39
|
+
process.env.DOMAIN_EVENT_OUTBOX_LOCK_TIMEOUT_SECONDS,
|
|
40
|
+
15 * 60,
|
|
41
|
+
30,
|
|
42
|
+
24 * 60 * 60,
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
const normalizeRow = (row) => {
|
|
46
|
+
if (!row) return null;
|
|
47
|
+
return {
|
|
48
|
+
id: Number(row.id),
|
|
49
|
+
event_type: row.event_type,
|
|
50
|
+
aggregate_type: row.aggregate_type,
|
|
51
|
+
aggregate_id: row.aggregate_id,
|
|
52
|
+
payload: parseJson(row.payload, {}),
|
|
53
|
+
status: row.status,
|
|
54
|
+
priority: Number(row.priority || 0),
|
|
55
|
+
idempotency_key: row.idempotency_key || null,
|
|
56
|
+
available_at: row.available_at || null,
|
|
57
|
+
attempts: Number(row.attempts || 0),
|
|
58
|
+
max_attempts: Number(row.max_attempts || 0),
|
|
59
|
+
worker_token: row.worker_token || null,
|
|
60
|
+
last_error: row.last_error || null,
|
|
61
|
+
locked_at: row.locked_at || null,
|
|
62
|
+
processed_at: row.processed_at || null,
|
|
63
|
+
created_at: row.created_at || null,
|
|
64
|
+
updated_at: row.updated_at || null,
|
|
65
|
+
};
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
export async function enqueueDomainEvent(eventPayload, connection = null) {
|
|
69
|
+
const normalized = normalizeDomainEventPayload(eventPayload);
|
|
70
|
+
if (!normalized) return false;
|
|
71
|
+
|
|
72
|
+
await executeQuery(
|
|
73
|
+
`INSERT INTO ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
74
|
+
(
|
|
75
|
+
event_type,
|
|
76
|
+
aggregate_type,
|
|
77
|
+
aggregate_id,
|
|
78
|
+
payload,
|
|
79
|
+
status,
|
|
80
|
+
priority,
|
|
81
|
+
idempotency_key,
|
|
82
|
+
available_at,
|
|
83
|
+
attempts,
|
|
84
|
+
max_attempts
|
|
85
|
+
)
|
|
86
|
+
VALUES (?, ?, ?, ?, 'pending', ?, ?, COALESCE(?, UTC_TIMESTAMP()), 0, ?)
|
|
87
|
+
ON DUPLICATE KEY UPDATE
|
|
88
|
+
priority = GREATEST(priority, VALUES(priority)),
|
|
89
|
+
available_at = LEAST(available_at, VALUES(available_at)),
|
|
90
|
+
updated_at = CURRENT_TIMESTAMP`,
|
|
91
|
+
[
|
|
92
|
+
normalized.event_type,
|
|
93
|
+
normalized.aggregate_type,
|
|
94
|
+
normalized.aggregate_id,
|
|
95
|
+
JSON.stringify(normalized.payload ?? {}),
|
|
96
|
+
normalized.priority,
|
|
97
|
+
normalized.idempotency_key,
|
|
98
|
+
normalized.available_at,
|
|
99
|
+
normalized.max_attempts,
|
|
100
|
+
],
|
|
101
|
+
connection,
|
|
102
|
+
);
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export async function claimDomainEvent(
|
|
107
|
+
{
|
|
108
|
+
eventTypes = [],
|
|
109
|
+
allowRetryFailed = true,
|
|
110
|
+
} = {},
|
|
111
|
+
connection = null,
|
|
112
|
+
) {
|
|
113
|
+
const workerToken = randomUUID();
|
|
114
|
+
const statusClause = allowRetryFailed
|
|
115
|
+
? `(status = 'pending'
|
|
116
|
+
OR (status = 'failed' AND attempts < max_attempts)
|
|
117
|
+
OR (status = 'processing' AND locked_at <= (UTC_TIMESTAMP() - INTERVAL ${CLAIM_LOCK_TIMEOUT_SECONDS} SECOND)))`
|
|
118
|
+
: `(status = 'pending'
|
|
119
|
+
OR (status = 'processing' AND locked_at <= (UTC_TIMESTAMP() - INTERVAL ${CLAIM_LOCK_TIMEOUT_SECONDS} SECOND)))`;
|
|
120
|
+
|
|
121
|
+
const normalizedTypes = Array.from(
|
|
122
|
+
new Set((Array.isArray(eventTypes) ? eventTypes : [])
|
|
123
|
+
.map((type) => String(type || '').trim().toUpperCase())
|
|
124
|
+
.filter(Boolean)),
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
let eventTypeClause = '';
|
|
128
|
+
let params = [workerToken];
|
|
129
|
+
if (normalizedTypes.length) {
|
|
130
|
+
eventTypeClause = `AND event_type IN (${normalizedTypes.map(() => '?').join(', ')})`;
|
|
131
|
+
params = [workerToken, ...normalizedTypes];
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
await executeQuery(
|
|
135
|
+
`UPDATE ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
136
|
+
SET status = 'processing',
|
|
137
|
+
worker_token = ?,
|
|
138
|
+
locked_at = UTC_TIMESTAMP(),
|
|
139
|
+
attempts = attempts + 1,
|
|
140
|
+
updated_at = UTC_TIMESTAMP()
|
|
141
|
+
WHERE id = (
|
|
142
|
+
SELECT id FROM (
|
|
143
|
+
SELECT id
|
|
144
|
+
FROM ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
145
|
+
WHERE ${statusClause}
|
|
146
|
+
${eventTypeClause}
|
|
147
|
+
AND available_at <= UTC_TIMESTAMP()
|
|
148
|
+
ORDER BY priority DESC, available_at ASC, id ASC
|
|
149
|
+
LIMIT 1
|
|
150
|
+
) picked
|
|
151
|
+
)`,
|
|
152
|
+
params,
|
|
153
|
+
connection,
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
const rows = await executeQuery(
|
|
157
|
+
`SELECT *
|
|
158
|
+
FROM ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
159
|
+
WHERE worker_token = ?
|
|
160
|
+
AND status = 'processing'
|
|
161
|
+
ORDER BY id DESC
|
|
162
|
+
LIMIT 1`,
|
|
163
|
+
[workerToken],
|
|
164
|
+
connection,
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
return normalizeRow(rows?.[0] || null);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export async function completeDomainEvent(eventId, connection = null) {
|
|
171
|
+
if (!eventId) return false;
|
|
172
|
+
await executeQuery(
|
|
173
|
+
`UPDATE ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
174
|
+
SET status = 'completed',
|
|
175
|
+
processed_at = UTC_TIMESTAMP(),
|
|
176
|
+
worker_token = NULL,
|
|
177
|
+
locked_at = NULL,
|
|
178
|
+
last_error = NULL,
|
|
179
|
+
updated_at = CURRENT_TIMESTAMP
|
|
180
|
+
WHERE id = ?`,
|
|
181
|
+
[eventId],
|
|
182
|
+
connection,
|
|
183
|
+
);
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export async function failDomainEvent(
|
|
188
|
+
eventId,
|
|
189
|
+
{
|
|
190
|
+
error = null,
|
|
191
|
+
retryDelaySeconds = 0,
|
|
192
|
+
} = {},
|
|
193
|
+
connection = null,
|
|
194
|
+
) {
|
|
195
|
+
if (!eventId) return false;
|
|
196
|
+
|
|
197
|
+
const safeDelay = clampInt(retryDelaySeconds, 0, 0, 86400 * 7);
|
|
198
|
+
const message = String(error || '').trim().slice(0, 255) || null;
|
|
199
|
+
|
|
200
|
+
await executeQuery(
|
|
201
|
+
`UPDATE ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
202
|
+
SET status = IF(attempts >= max_attempts, 'failed', 'pending'),
|
|
203
|
+
worker_token = NULL,
|
|
204
|
+
locked_at = NULL,
|
|
205
|
+
last_error = ?,
|
|
206
|
+
available_at = IF(attempts >= max_attempts, available_at, UTC_TIMESTAMP() + INTERVAL ${safeDelay} SECOND),
|
|
207
|
+
updated_at = CURRENT_TIMESTAMP,
|
|
208
|
+
processed_at = IF(attempts >= max_attempts, UTC_TIMESTAMP(), processed_at)
|
|
209
|
+
WHERE id = ?`,
|
|
210
|
+
[message, eventId],
|
|
211
|
+
connection,
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
await executeQuery(
|
|
215
|
+
`INSERT INTO ${TABLES.DOMAIN_EVENT_OUTBOX_DLQ}
|
|
216
|
+
(outbox_event_id, event_type, aggregate_type, aggregate_id, payload, attempts, max_attempts, last_error)
|
|
217
|
+
SELECT id, event_type, aggregate_type, aggregate_id, payload, attempts, max_attempts, last_error
|
|
218
|
+
FROM ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
219
|
+
WHERE id = ?
|
|
220
|
+
AND status = 'failed'
|
|
221
|
+
ON DUPLICATE KEY UPDATE
|
|
222
|
+
last_error = VALUES(last_error),
|
|
223
|
+
attempts = VALUES(attempts),
|
|
224
|
+
max_attempts = VALUES(max_attempts),
|
|
225
|
+
failed_at = CURRENT_TIMESTAMP`,
|
|
226
|
+
[eventId],
|
|
227
|
+
connection,
|
|
228
|
+
).catch(() => null);
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export async function countDomainEventsByStatus(status = 'pending', connection = null) {
|
|
233
|
+
const normalized = normalizeStatus(status);
|
|
234
|
+
if (!normalized) return 0;
|
|
235
|
+
const rows = await executeQuery(
|
|
236
|
+
`SELECT COUNT(*) AS total
|
|
237
|
+
FROM ${TABLES.DOMAIN_EVENT_OUTBOX}
|
|
238
|
+
WHERE status = ?`,
|
|
239
|
+
[normalized],
|
|
240
|
+
connection,
|
|
241
|
+
);
|
|
242
|
+
return Number(rows?.[0]?.total || 0);
|
|
243
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export const STICKER_DOMAIN_EVENTS = Object.freeze({
|
|
2
|
+
STICKER_ASSET_CREATED: 'STICKER_ASSET_CREATED',
|
|
3
|
+
STICKER_CLASSIFIED: 'STICKER_CLASSIFIED',
|
|
4
|
+
PACK_UPDATED: 'PACK_UPDATED',
|
|
5
|
+
ENGAGEMENT_RECORDED: 'ENGAGEMENT_RECORDED',
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
export const STICKER_DOMAIN_EVENT_TYPES = new Set(Object.values(STICKER_DOMAIN_EVENTS));
|
|
9
|
+
|
|
10
|
+
const normalizeType = (value) =>
|
|
11
|
+
String(value || '')
|
|
12
|
+
.trim()
|
|
13
|
+
.toUpperCase()
|
|
14
|
+
.replace(/[^A-Z0-9_]/g, '')
|
|
15
|
+
.slice(0, 96);
|
|
16
|
+
|
|
17
|
+
const normalizeAggregateType = (value) =>
|
|
18
|
+
String(value || '')
|
|
19
|
+
.trim()
|
|
20
|
+
.toLowerCase()
|
|
21
|
+
.replace(/[^a-z0-9_:-]/g, '')
|
|
22
|
+
.slice(0, 96);
|
|
23
|
+
|
|
24
|
+
const normalizeAggregateId = (value) =>
|
|
25
|
+
String(value || '')
|
|
26
|
+
.trim()
|
|
27
|
+
.slice(0, 128);
|
|
28
|
+
|
|
29
|
+
const normalizeIdempotencyKey = (value) =>
|
|
30
|
+
String(value || '')
|
|
31
|
+
.trim()
|
|
32
|
+
.replace(/[^a-zA-Z0-9_:-]/g, '')
|
|
33
|
+
.slice(0, 180);
|
|
34
|
+
|
|
35
|
+
export const normalizeDomainEventPayload = ({
|
|
36
|
+
eventType,
|
|
37
|
+
aggregateType,
|
|
38
|
+
aggregateId,
|
|
39
|
+
payload = null,
|
|
40
|
+
priority = 50,
|
|
41
|
+
availableAt = null,
|
|
42
|
+
idempotencyKey = '',
|
|
43
|
+
maxAttempts = 10,
|
|
44
|
+
} = {}) => {
|
|
45
|
+
const normalizedType = normalizeType(eventType);
|
|
46
|
+
if (!normalizedType) return null;
|
|
47
|
+
const normalizedAggregateType = normalizeAggregateType(aggregateType);
|
|
48
|
+
const normalizedAggregateId = normalizeAggregateId(aggregateId);
|
|
49
|
+
if (!normalizedAggregateType || !normalizedAggregateId) return null;
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
event_type: normalizedType,
|
|
53
|
+
aggregate_type: normalizedAggregateType,
|
|
54
|
+
aggregate_id: normalizedAggregateId,
|
|
55
|
+
payload: payload && typeof payload === 'object' ? payload : payload ?? null,
|
|
56
|
+
priority: Math.max(1, Math.min(100, Number(priority) || 50)),
|
|
57
|
+
available_at: availableAt ? new Date(availableAt) : null,
|
|
58
|
+
idempotency_key: normalizeIdempotencyKey(idempotencyKey) || null,
|
|
59
|
+
max_attempts: Math.max(1, Math.min(30, Number(maxAttempts) || 10)),
|
|
60
|
+
};
|
|
61
|
+
};
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { executeQuery, TABLES } from '../../../database/index.js';
|
|
2
|
+
import { STICKER_DOMAIN_EVENTS } from './domainEvents.js';
|
|
3
|
+
import { publishStickerDomainEvent } from './stickerDomainEventBus.js';
|
|
2
4
|
|
|
3
5
|
const parseJson = (value, fallback = null) => {
|
|
4
6
|
if (value === null || value === undefined) return fallback;
|
|
@@ -163,6 +165,25 @@ export async function upsertStickerAssetClassification(payload, connection = nul
|
|
|
163
165
|
connection,
|
|
164
166
|
);
|
|
165
167
|
|
|
168
|
+
await publishStickerDomainEvent(
|
|
169
|
+
{
|
|
170
|
+
eventType: STICKER_DOMAIN_EVENTS.STICKER_CLASSIFIED,
|
|
171
|
+
aggregateType: 'sticker_asset',
|
|
172
|
+
aggregateId: payload.asset_id,
|
|
173
|
+
payload: {
|
|
174
|
+
asset_id: payload.asset_id,
|
|
175
|
+
category: payload.category || null,
|
|
176
|
+
confidence: payload.confidence ?? null,
|
|
177
|
+
nsfw_score: payload.nsfw_score ?? null,
|
|
178
|
+
is_nsfw: Boolean(payload.is_nsfw),
|
|
179
|
+
classification_version: payload.classification_version || 'v1',
|
|
180
|
+
},
|
|
181
|
+
priority: 80,
|
|
182
|
+
idempotencyKey: `sticker_classified:${payload.asset_id}:${payload.classification_version || 'v1'}`,
|
|
183
|
+
},
|
|
184
|
+
{ connection },
|
|
185
|
+
);
|
|
186
|
+
|
|
166
187
|
return findStickerClassificationByAssetId(payload.asset_id, connection);
|
|
167
188
|
}
|
|
168
189
|
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { executeQuery, TABLES } from '../../../database/index.js';
|
|
2
|
+
import { STICKER_DOMAIN_EVENTS } from './domainEvents.js';
|
|
3
|
+
import { publishStickerDomainEvent } from './stickerDomainEventBus.js';
|
|
2
4
|
|
|
3
5
|
/**
|
|
4
6
|
* Converte valores numéricos/booleanos vindos do banco para booleano.
|
|
@@ -358,6 +360,23 @@ export async function createStickerAsset(asset, connection = null) {
|
|
|
358
360
|
connection,
|
|
359
361
|
);
|
|
360
362
|
|
|
363
|
+
await publishStickerDomainEvent(
|
|
364
|
+
{
|
|
365
|
+
eventType: STICKER_DOMAIN_EVENTS.STICKER_ASSET_CREATED,
|
|
366
|
+
aggregateType: 'sticker_asset',
|
|
367
|
+
aggregateId: asset.id,
|
|
368
|
+
payload: {
|
|
369
|
+
asset_id: asset.id,
|
|
370
|
+
owner_jid: asset.owner_jid,
|
|
371
|
+
sha256: asset.sha256,
|
|
372
|
+
mimetype: asset.mimetype,
|
|
373
|
+
},
|
|
374
|
+
priority: 85,
|
|
375
|
+
idempotencyKey: `sticker_asset_created:${asset.id}`,
|
|
376
|
+
},
|
|
377
|
+
{ connection },
|
|
378
|
+
);
|
|
379
|
+
|
|
361
380
|
return findStickerAssetById(asset.id, connection);
|
|
362
381
|
}
|
|
363
382
|
|
|
@@ -2,7 +2,7 @@ import fs from 'node:fs/promises';
|
|
|
2
2
|
import os from 'node:os';
|
|
3
3
|
|
|
4
4
|
import logger from '../../utils/logger/loggerModule.js';
|
|
5
|
-
import { setQueueDepth } from '../../observability/metrics.js';
|
|
5
|
+
import { recordStickerClassificationCycle, setQueueDepth } from '../../observability/metrics.js';
|
|
6
6
|
import { listStickerAssetsPendingClassification, findStickerAssetById } from './stickerAssetRepository.js';
|
|
7
7
|
import { classifierConfig, ensureStickerAssetClassified } from './stickerClassificationService.js';
|
|
8
8
|
import {
|
|
@@ -351,30 +351,70 @@ export const runStickerClassificationCycle = async ({
|
|
|
351
351
|
processReprocess = true,
|
|
352
352
|
processDeterministic = true,
|
|
353
353
|
} = {}) => {
|
|
354
|
+
const startedAt = Date.now();
|
|
354
355
|
const shouldProcessClassifier = classifierConfig.enabled;
|
|
355
356
|
const shouldProcessDeterministic = deterministicReclassificationConfig.enabled;
|
|
356
357
|
|
|
357
358
|
if (!BACKGROUND_ENABLED || (!shouldProcessClassifier && !shouldProcessDeterministic)) {
|
|
359
|
+
recordStickerClassificationCycle({
|
|
360
|
+
status: 'skipped',
|
|
361
|
+
durationMs: Date.now() - startedAt,
|
|
362
|
+
processed: 0,
|
|
363
|
+
classified: 0,
|
|
364
|
+
failed: 0,
|
|
365
|
+
});
|
|
358
366
|
return {
|
|
359
367
|
skipped: true,
|
|
360
368
|
reason: !BACKGROUND_ENABLED ? 'background_disabled' : 'no_active_processors',
|
|
361
369
|
};
|
|
362
370
|
}
|
|
363
371
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
372
|
+
try {
|
|
373
|
+
const reprocessStats = processReprocess && shouldProcessClassifier ? await processReprocessQueue() : null;
|
|
374
|
+
const pendingStats = processPending && shouldProcessClassifier ? await processPendingAssets() : null;
|
|
375
|
+
const deterministicStats = processDeterministic
|
|
376
|
+
? await processDeterministicReclassification()
|
|
377
|
+
: null;
|
|
378
|
+
|
|
379
|
+
const processed =
|
|
380
|
+
Number(pendingStats?.processed || 0)
|
|
381
|
+
+ Number(reprocessStats?.processed || 0)
|
|
382
|
+
+ Number(deterministicStats?.processed || 0);
|
|
383
|
+
const classified =
|
|
384
|
+
Number(pendingStats?.classified || 0)
|
|
385
|
+
+ Number(reprocessStats?.classified || 0)
|
|
386
|
+
+ Number(deterministicStats?.updated || 0);
|
|
387
|
+
const failed =
|
|
388
|
+
Number(pendingStats?.failed || 0)
|
|
389
|
+
+ Number(reprocessStats?.failed || 0)
|
|
390
|
+
+ Number(deterministicStats?.failed || 0);
|
|
391
|
+
const durationMs = Date.now() - startedAt;
|
|
392
|
+
|
|
393
|
+
recordStickerClassificationCycle({
|
|
394
|
+
status: 'ok',
|
|
395
|
+
durationMs,
|
|
396
|
+
processed,
|
|
397
|
+
classified,
|
|
398
|
+
failed,
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
return {
|
|
402
|
+
skipped: false,
|
|
403
|
+
duration_ms: durationMs,
|
|
404
|
+
pending: pendingStats,
|
|
405
|
+
reprocess: reprocessStats,
|
|
406
|
+
deterministic_reclassification: deterministicStats,
|
|
407
|
+
};
|
|
408
|
+
} catch (error) {
|
|
409
|
+
recordStickerClassificationCycle({
|
|
410
|
+
status: 'failed',
|
|
411
|
+
durationMs: Date.now() - startedAt,
|
|
412
|
+
processed: 0,
|
|
413
|
+
classified: 0,
|
|
414
|
+
failed: 1,
|
|
415
|
+
});
|
|
416
|
+
throw error;
|
|
417
|
+
}
|
|
378
418
|
};
|
|
379
419
|
|
|
380
420
|
const clearCycleHandle = () => {
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import logger from '../../utils/logger/loggerModule.js';
|
|
2
|
+
import { setQueueDepth } from '../../observability/metrics.js';
|
|
3
|
+
import { isFeatureEnabled } from '../../services/featureFlagService.js';
|
|
4
|
+
import { runStickerClassificationCycle } from './stickerClassificationBackgroundRuntime.js';
|
|
5
|
+
import { runStickerAutoPackByTagsCycle } from './stickerAutoPackByTagsRuntime.js';
|
|
6
|
+
import {
|
|
7
|
+
claimWorkerTask,
|
|
8
|
+
completeWorkerTask,
|
|
9
|
+
countWorkerTasksByStatus,
|
|
10
|
+
failWorkerTask,
|
|
11
|
+
} from './stickerWorkerTaskQueueRepository.js';
|
|
12
|
+
|
|
13
|
+
const parseEnvBool = (value, fallback) => {
|
|
14
|
+
if (value === undefined || value === null || value === '') return fallback;
|
|
15
|
+
const normalized = String(value).trim().toLowerCase();
|
|
16
|
+
if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true;
|
|
17
|
+
if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false;
|
|
18
|
+
return fallback;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const clampInt = (value, fallback, min, max) => {
|
|
22
|
+
const numeric = Number(value);
|
|
23
|
+
if (!Number.isFinite(numeric)) return fallback;
|
|
24
|
+
return Math.max(min, Math.min(max, Math.floor(numeric)));
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const DEDICATED_WORKERS_ENABLED = parseEnvBool(process.env.STICKER_DEDICATED_WORKERS_ENABLED, true);
|
|
28
|
+
const DEDICATED_WORKERS_FORCE_ENABLED = parseEnvBool(process.env.STICKER_DEDICATED_WORKERS_FORCE_ENABLED, false);
|
|
29
|
+
const DEDICATED_WORKER_RETRY_DELAY_SECONDS = clampInt(
|
|
30
|
+
process.env.STICKER_DEDICATED_WORKER_RETRY_DELAY_SECONDS,
|
|
31
|
+
60,
|
|
32
|
+
5,
|
|
33
|
+
3600,
|
|
34
|
+
);
|
|
35
|
+
const DEDICATED_WORKER_POLL_INTERVAL_MS = clampInt(
|
|
36
|
+
process.env.STICKER_DEDICATED_WORKER_POLL_INTERVAL_MS,
|
|
37
|
+
2500,
|
|
38
|
+
250,
|
|
39
|
+
60_000,
|
|
40
|
+
);
|
|
41
|
+
const DEDICATED_WORKER_MAX_TASKS_PER_TICK = clampInt(
|
|
42
|
+
process.env.STICKER_DEDICATED_WORKER_MAX_TASKS_PER_TICK,
|
|
43
|
+
1,
|
|
44
|
+
1,
|
|
45
|
+
25,
|
|
46
|
+
);
|
|
47
|
+
const DEDICATED_WORKER_COHORT_KEY =
|
|
48
|
+
String(process.env.STICKER_DEDICATED_WORKER_COHORT_KEY || process.env.HOSTNAME || process.pid).trim()
|
|
49
|
+
|| 'worker';
|
|
50
|
+
|
|
51
|
+
const SUPPORTED_TASK_TYPES = new Set([
|
|
52
|
+
'classification_cycle',
|
|
53
|
+
'curation_cycle',
|
|
54
|
+
'rebuild_cycle',
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
const normalizeTaskType = (value) => {
|
|
58
|
+
const normalized = String(value || '').trim().toLowerCase();
|
|
59
|
+
return SUPPORTED_TASK_TYPES.has(normalized) ? normalized : null;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const runTaskHandler = async (taskType, payload = {}) => {
|
|
63
|
+
if (taskType === 'classification_cycle') {
|
|
64
|
+
return runStickerClassificationCycle({
|
|
65
|
+
processPending: true,
|
|
66
|
+
processReprocess: true,
|
|
67
|
+
processDeterministic: true,
|
|
68
|
+
...payload,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
if (taskType === 'curation_cycle') {
|
|
72
|
+
return runStickerAutoPackByTagsCycle({
|
|
73
|
+
enableAdditions: true,
|
|
74
|
+
enableRebuild: false,
|
|
75
|
+
...payload,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
if (taskType === 'rebuild_cycle') {
|
|
79
|
+
return runStickerAutoPackByTagsCycle({
|
|
80
|
+
enableAdditions: false,
|
|
81
|
+
enableRebuild: true,
|
|
82
|
+
...payload,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
throw new Error(`unsupported_task_type:${taskType}`);
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const refreshQueueDepthMetrics = async () => {
|
|
89
|
+
const [pending, processing, failed] = await Promise.all([
|
|
90
|
+
countWorkerTasksByStatus('pending'),
|
|
91
|
+
countWorkerTasksByStatus('processing'),
|
|
92
|
+
countWorkerTasksByStatus('failed'),
|
|
93
|
+
]);
|
|
94
|
+
setQueueDepth('sticker_worker_tasks_pending', pending);
|
|
95
|
+
setQueueDepth('sticker_worker_tasks_processing', processing);
|
|
96
|
+
setQueueDepth('sticker_worker_tasks_failed', failed);
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
const canRunDedicatedWorkers = async (taskType) => {
|
|
100
|
+
if (!DEDICATED_WORKERS_ENABLED) return false;
|
|
101
|
+
if (DEDICATED_WORKERS_FORCE_ENABLED) return true;
|
|
102
|
+
return isFeatureEnabled('enable_worker_dedicated_processes', {
|
|
103
|
+
fallback: false,
|
|
104
|
+
subjectKey: `worker:${taskType}:${DEDICATED_WORKER_COHORT_KEY}`,
|
|
105
|
+
});
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
export const isSupportedStickerWorkerTaskType = (taskType) => Boolean(normalizeTaskType(taskType));
|
|
109
|
+
|
|
110
|
+
export const runDedicatedStickerWorkerTick = async (
|
|
111
|
+
{
|
|
112
|
+
taskType,
|
|
113
|
+
maxTasks = DEDICATED_WORKER_MAX_TASKS_PER_TICK,
|
|
114
|
+
retryDelaySeconds = DEDICATED_WORKER_RETRY_DELAY_SECONDS,
|
|
115
|
+
} = {},
|
|
116
|
+
) => {
|
|
117
|
+
const normalizedTaskType = normalizeTaskType(taskType);
|
|
118
|
+
if (!normalizedTaskType) {
|
|
119
|
+
return { executed: false, reason: 'invalid_task_type', task_type: taskType || null };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const enabled = await canRunDedicatedWorkers(normalizedTaskType);
|
|
123
|
+
if (!enabled) {
|
|
124
|
+
return { executed: false, reason: 'feature_disabled', task_type: normalizedTaskType };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const safeMaxTasks = clampInt(maxTasks, DEDICATED_WORKER_MAX_TASKS_PER_TICK, 1, 25);
|
|
128
|
+
const stats = {
|
|
129
|
+
executed: true,
|
|
130
|
+
task_type: normalizedTaskType,
|
|
131
|
+
claimed: 0,
|
|
132
|
+
completed: 0,
|
|
133
|
+
failed: 0,
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
for (let i = 0; i < safeMaxTasks; i += 1) {
|
|
137
|
+
const task = await claimWorkerTask({ taskType: normalizedTaskType });
|
|
138
|
+
if (!task) break;
|
|
139
|
+
|
|
140
|
+
stats.claimed += 1;
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
await runTaskHandler(normalizedTaskType, task.payload || {});
|
|
144
|
+
await completeWorkerTask(task.id);
|
|
145
|
+
stats.completed += 1;
|
|
146
|
+
} catch (error) {
|
|
147
|
+
stats.failed += 1;
|
|
148
|
+
await failWorkerTask(task.id, {
|
|
149
|
+
error: error?.message || 'dedicated_worker_task_failed',
|
|
150
|
+
retryDelaySeconds,
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
logger.warn('Task falhou no worker dedicado.', {
|
|
154
|
+
action: 'sticker_dedicated_worker_task_failed',
|
|
155
|
+
task_type: normalizedTaskType,
|
|
156
|
+
task_id: task.id,
|
|
157
|
+
attempts: task.attempts,
|
|
158
|
+
error: error?.message,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (stats.claimed > 0) {
|
|
164
|
+
await refreshQueueDepthMetrics().catch(() => null);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return stats;
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
export const startDedicatedStickerWorker = ({
|
|
171
|
+
taskType,
|
|
172
|
+
pollIntervalMs = DEDICATED_WORKER_POLL_INTERVAL_MS,
|
|
173
|
+
maxTasksPerTick = DEDICATED_WORKER_MAX_TASKS_PER_TICK,
|
|
174
|
+
retryDelaySeconds = DEDICATED_WORKER_RETRY_DELAY_SECONDS,
|
|
175
|
+
label = '',
|
|
176
|
+
} = {}) => {
|
|
177
|
+
const normalizedTaskType = normalizeTaskType(taskType);
|
|
178
|
+
if (!normalizedTaskType) {
|
|
179
|
+
throw new Error(`invalid_task_type:${taskType}`);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const safePollIntervalMs = clampInt(pollIntervalMs, DEDICATED_WORKER_POLL_INTERVAL_MS, 250, 60_000);
|
|
183
|
+
let tickInFlight = false;
|
|
184
|
+
let stopped = false;
|
|
185
|
+
|
|
186
|
+
const runTick = async () => {
|
|
187
|
+
if (stopped || tickInFlight) return;
|
|
188
|
+
tickInFlight = true;
|
|
189
|
+
try {
|
|
190
|
+
await runDedicatedStickerWorkerTick({
|
|
191
|
+
taskType: normalizedTaskType,
|
|
192
|
+
maxTasks: maxTasksPerTick,
|
|
193
|
+
retryDelaySeconds,
|
|
194
|
+
});
|
|
195
|
+
} catch (error) {
|
|
196
|
+
if (error?.code !== 'ER_NO_SUCH_TABLE') {
|
|
197
|
+
logger.error('Falha no worker dedicado de sticker.', {
|
|
198
|
+
action: 'sticker_dedicated_worker_tick_failed',
|
|
199
|
+
task_type: normalizedTaskType,
|
|
200
|
+
error: error?.message,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
} finally {
|
|
204
|
+
tickInFlight = false;
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const intervalHandle = setInterval(() => {
|
|
209
|
+
void runTick();
|
|
210
|
+
}, safePollIntervalMs);
|
|
211
|
+
if (typeof intervalHandle?.unref === 'function') {
|
|
212
|
+
intervalHandle.unref();
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
void runTick();
|
|
216
|
+
|
|
217
|
+
logger.info('Worker dedicado de sticker iniciado.', {
|
|
218
|
+
action: 'sticker_dedicated_worker_started',
|
|
219
|
+
task_type: normalizedTaskType,
|
|
220
|
+
poll_interval_ms: safePollIntervalMs,
|
|
221
|
+
max_tasks_per_tick: maxTasksPerTick,
|
|
222
|
+
label: label || null,
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
return {
|
|
226
|
+
taskType: normalizedTaskType,
|
|
227
|
+
stop: () => {
|
|
228
|
+
if (stopped) return;
|
|
229
|
+
stopped = true;
|
|
230
|
+
clearInterval(intervalHandle);
|
|
231
|
+
logger.info('Worker dedicado de sticker encerrado.', {
|
|
232
|
+
action: 'sticker_dedicated_worker_stopped',
|
|
233
|
+
task_type: normalizedTaskType,
|
|
234
|
+
label: label || null,
|
|
235
|
+
});
|
|
236
|
+
},
|
|
237
|
+
};
|
|
238
|
+
};
|