mcp-scraper 0.69.0 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -2
- package/README.md +6 -2
- package/dist/{analytics-repository-62CAZAIP.js → analytics-repository-6YLRSWYO.js} +3 -2
- package/dist/bin/api-server.cjs +18592 -12680
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +10492 -9129
- package/dist/bin/mcp-stdio-server.js +9 -7
- package/dist/bin/paa-harvest.cjs +308 -41
- package/dist/bin/paa-harvest.js +4 -3
- package/dist/{chunk-M7D7WO75.js → chunk-2N4V4LVU.js} +3178 -2019
- package/dist/chunk-7RQULQF3.js +263 -0
- package/dist/{chunk-UL4ZKAWZ.js → chunk-A66DGFOU.js} +1 -1
- package/dist/{chunk-ES25GP6C.js → chunk-CXY5WV45.js} +24 -34
- package/dist/{chunk-Y46YNQMM.js → chunk-F5TK4KMT.js} +1 -1
- package/dist/{chunk-62LEXS5O.js → chunk-F6MUGMRN.js} +306 -44
- package/dist/{chunk-5RFQGIYC.js → chunk-FVL4GUTP.js} +2 -2
- package/dist/{chunk-OZIUFN6B.js → chunk-I45JV4EU.js} +2 -2
- package/dist/{chunk-2LXTOAKS.js → chunk-KIPIBPUB.js} +9 -1
- package/dist/chunk-MB72PA6S.js +2003 -0
- package/dist/chunk-OM7HVEJ3.js +26 -0
- package/dist/chunk-PJEEKOUM.js +404 -0
- package/dist/{chunk-RX2QAHML.js → chunk-RAFQEPJ4.js} +2 -2
- package/dist/{chunk-KO5CK5ZT.js → chunk-SR7GSLEA.js} +202 -441
- package/dist/chunk-T3MZISOF.js +240 -0
- package/dist/{chunk-ULYOCJUJ.js → chunk-TUTQHP4O.js} +1 -1
- package/dist/{chunk-65FTMB7G.js → chunk-WEFPBAAG.js} +180 -9
- package/dist/{chunk-LN6N3YLK.js → chunk-WFQ2E4WE.js} +1 -1
- package/dist/{db-ZGZAKYBW.js → db-566MOHHD.js} +7 -1
- package/dist/{extract-bundle-SXSJCPQI.js → extract-bundle-337ZDIIQ.js} +4 -4
- package/dist/gmail-service-E276SBJU.js +25 -0
- package/dist/index.cjs +322 -49
- package/dist/index.d.cts +15 -2
- package/dist/index.d.ts +15 -2
- package/dist/index.js +4 -3
- package/dist/{lead-list-enrichment-repository-CXLVIV3R.js → lead-list-enrichment-repository-WWEMJ7EN.js} +2 -2
- package/dist/{location-data-repository-G72M7DTN.js → location-data-repository-H4B2BWGY.js} +2 -2
- package/dist/{server-WCNYXILA.js → server-UGWC5YMD.js} +8376 -6616
- package/dist/{site-extract-repository-RLB3TFJ3.js → site-extract-repository-ACNAVNDN.js} +3 -3
- package/dist/{worker-KHSJG365.js → worker-JQORPCFQ.js} +9 -7
- package/package.json +1 -1
- package/dist/chunk-2L4C4DAZ.js +0 -103
|
@@ -1,46 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
} from "./chunk-
|
|
2
|
+
getSessionSecret,
|
|
3
|
+
serviceConnectionAdvertisesAction
|
|
4
|
+
} from "./chunk-PJEEKOUM.js";
|
|
5
5
|
|
|
6
6
|
// src/api/analytics-repository.ts
|
|
7
|
-
import { createHash, createHmac
|
|
7
|
+
import { createHash, createHmac, randomBytes, randomUUID } from "crypto";
|
|
8
8
|
import { Pool } from "pg";
|
|
9
9
|
|
|
10
|
-
// src/api/session.ts
|
|
11
|
-
import { createHmac, timingSafeEqual } from "crypto";
|
|
12
|
-
var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
13
|
-
function getSessionSecret() {
|
|
14
|
-
const configured = process.env.SESSION_SECRET?.trim();
|
|
15
|
-
if (configured) return configured;
|
|
16
|
-
if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
|
|
17
|
-
return "dev-secret-change-me";
|
|
18
|
-
}
|
|
19
|
-
var secret = () => getSessionSecret();
|
|
20
|
-
function safeEqualHex(a, b) {
|
|
21
|
-
if (a.length !== b.length) return false;
|
|
22
|
-
try {
|
|
23
|
-
return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
24
|
-
} catch {
|
|
25
|
-
return false;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
function signSession(userId) {
|
|
29
|
-
const payload = String(userId);
|
|
30
|
-
const sig = createHmac("sha256", secret()).update(payload).digest("hex");
|
|
31
|
-
return `${payload}.${sig}`;
|
|
32
|
-
}
|
|
33
|
-
function verifySession(token) {
|
|
34
|
-
const dot = token.lastIndexOf(".");
|
|
35
|
-
if (dot === -1) return null;
|
|
36
|
-
const payload = token.slice(0, dot);
|
|
37
|
-
const sig = token.slice(dot + 1);
|
|
38
|
-
const expected = createHmac("sha256", secret()).update(payload).digest("hex");
|
|
39
|
-
if (!safeEqualHex(sig, expected)) return null;
|
|
40
|
-
const id = parseInt(payload);
|
|
41
|
-
return isNaN(id) ? null : id;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
10
|
// src/api/analytics-attribution.ts
|
|
45
11
|
var aliases = [
|
|
46
12
|
[/perplexity/i, "llm", "Perplexity"],
|
|
@@ -116,353 +82,6 @@ function resolveAnalyticsAttribution(input) {
|
|
|
116
82
|
};
|
|
117
83
|
}
|
|
118
84
|
|
|
119
|
-
// src/api/service-connections.ts
|
|
120
|
-
import { randomUUID } from "crypto";
|
|
121
|
-
function serviceConnectionAdvertisesAction(connection, providerConfigKeys, actionName) {
|
|
122
|
-
return connection.lifecycleStatus === "connected" && connection.actionsEnabled && providerConfigKeys.includes(connection.providerConfigKey) && connection.actionTools.includes(actionName);
|
|
123
|
-
}
|
|
124
|
-
var schemaReady = null;
|
|
125
|
-
var schemaDb = null;
|
|
126
|
-
function ensureServiceConnectionsSchema() {
|
|
127
|
-
const currentDb = getDb();
|
|
128
|
-
if (schemaReady && schemaDb === currentDb) return schemaReady;
|
|
129
|
-
schemaDb = currentDb;
|
|
130
|
-
schemaReady = (async () => {
|
|
131
|
-
const db = currentDb;
|
|
132
|
-
await db.execute(`
|
|
133
|
-
CREATE TABLE IF NOT EXISTS service_connections (
|
|
134
|
-
id TEXT PRIMARY KEY,
|
|
135
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
136
|
-
provider_config_key TEXT NOT NULL,
|
|
137
|
-
provider TEXT,
|
|
138
|
-
transport TEXT NOT NULL,
|
|
139
|
-
upstream_connection_id TEXT,
|
|
140
|
-
label TEXT,
|
|
141
|
-
user_label TEXT,
|
|
142
|
-
provider_account_id TEXT,
|
|
143
|
-
provider_account_email TEXT,
|
|
144
|
-
provider_account_name TEXT,
|
|
145
|
-
provider_identity_checked_at TEXT,
|
|
146
|
-
provider_identity_source_updated_at TEXT,
|
|
147
|
-
lifecycle_status TEXT NOT NULL DEFAULT 'pending',
|
|
148
|
-
operational_status TEXT NOT NULL DEFAULT 'unknown',
|
|
149
|
-
reconnect_required INTEGER NOT NULL DEFAULT 0,
|
|
150
|
-
actions_enabled INTEGER NOT NULL DEFAULT 0,
|
|
151
|
-
read_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
152
|
-
action_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
153
|
-
tool_revision TEXT,
|
|
154
|
-
last_checked_at TEXT,
|
|
155
|
-
last_successful_call_at TEXT,
|
|
156
|
-
last_failure_at TEXT,
|
|
157
|
-
last_failure_code TEXT,
|
|
158
|
-
last_failure_retryable INTEGER,
|
|
159
|
-
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
160
|
-
vault_name TEXT,
|
|
161
|
-
table_name TEXT,
|
|
162
|
-
legacy_owner_identity TEXT,
|
|
163
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
164
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
165
|
-
UNIQUE(user_id, provider_config_key, upstream_connection_id)
|
|
166
|
-
)
|
|
167
|
-
`);
|
|
168
|
-
try {
|
|
169
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN user_label TEXT`);
|
|
170
|
-
} catch {
|
|
171
|
-
}
|
|
172
|
-
try {
|
|
173
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT`);
|
|
174
|
-
} catch {
|
|
175
|
-
}
|
|
176
|
-
try {
|
|
177
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT`);
|
|
178
|
-
} catch {
|
|
179
|
-
}
|
|
180
|
-
try {
|
|
181
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT`);
|
|
182
|
-
} catch {
|
|
183
|
-
}
|
|
184
|
-
try {
|
|
185
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT`);
|
|
186
|
-
} catch {
|
|
187
|
-
}
|
|
188
|
-
try {
|
|
189
|
-
await db.execute(`ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT`);
|
|
190
|
-
} catch {
|
|
191
|
-
}
|
|
192
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)`);
|
|
193
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)`);
|
|
194
|
-
await db.execute(`
|
|
195
|
-
CREATE TABLE IF NOT EXISTS service_connection_health_events (
|
|
196
|
-
id TEXT PRIMARY KEY,
|
|
197
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
198
|
-
operational_status TEXT NOT NULL,
|
|
199
|
-
failure_code TEXT,
|
|
200
|
-
retryable INTEGER,
|
|
201
|
-
evidence_source TEXT NOT NULL,
|
|
202
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
203
|
-
)
|
|
204
|
-
`);
|
|
205
|
-
await db.execute(`CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)`);
|
|
206
|
-
await db.execute(`
|
|
207
|
-
CREATE TABLE IF NOT EXISTS service_connection_action_audit (
|
|
208
|
-
id TEXT PRIMARY KEY,
|
|
209
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
210
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
211
|
-
tool TEXT NOT NULL,
|
|
212
|
-
request_id TEXT NOT NULL,
|
|
213
|
-
status TEXT NOT NULL,
|
|
214
|
-
error_code TEXT,
|
|
215
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
216
|
-
UNIQUE(user_id, request_id)
|
|
217
|
-
)
|
|
218
|
-
`);
|
|
219
|
-
try {
|
|
220
|
-
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT`);
|
|
221
|
-
} catch {
|
|
222
|
-
}
|
|
223
|
-
try {
|
|
224
|
-
await db.execute(`ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT`);
|
|
225
|
-
} catch {
|
|
226
|
-
}
|
|
227
|
-
})().catch((error) => {
|
|
228
|
-
schemaReady = null;
|
|
229
|
-
schemaDb = null;
|
|
230
|
-
throw error;
|
|
231
|
-
});
|
|
232
|
-
return schemaReady;
|
|
233
|
-
}
|
|
234
|
-
function parseTools(value) {
|
|
235
|
-
if (typeof value !== "string") return [];
|
|
236
|
-
try {
|
|
237
|
-
const parsed = JSON.parse(value);
|
|
238
|
-
return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
|
|
239
|
-
} catch {
|
|
240
|
-
return [];
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
function rowToRecord(row) {
|
|
244
|
-
return {
|
|
245
|
-
id: String(row.id),
|
|
246
|
-
userId: Number(row.user_id),
|
|
247
|
-
providerConfigKey: String(row.provider_config_key),
|
|
248
|
-
provider: typeof row.provider === "string" ? row.provider : null,
|
|
249
|
-
transport: row.transport === "remote_mcp" ? "remote_mcp" : "nango",
|
|
250
|
-
upstreamConnectionId: typeof row.upstream_connection_id === "string" ? row.upstream_connection_id : null,
|
|
251
|
-
label: typeof row.label === "string" ? row.label : null,
|
|
252
|
-
userLabel: typeof row.user_label === "string" ? row.user_label : null,
|
|
253
|
-
providerAccountId: typeof row.provider_account_id === "string" ? row.provider_account_id : null,
|
|
254
|
-
providerAccountEmail: typeof row.provider_account_email === "string" ? row.provider_account_email : null,
|
|
255
|
-
providerAccountName: typeof row.provider_account_name === "string" ? row.provider_account_name : null,
|
|
256
|
-
providerIdentityCheckedAt: typeof row.provider_identity_checked_at === "string" ? row.provider_identity_checked_at : null,
|
|
257
|
-
providerIdentitySourceUpdatedAt: typeof row.provider_identity_source_updated_at === "string" ? row.provider_identity_source_updated_at : null,
|
|
258
|
-
lifecycleStatus: String(row.lifecycle_status),
|
|
259
|
-
operationalStatus: String(row.operational_status),
|
|
260
|
-
reconnectRequired: Number(row.reconnect_required) === 1,
|
|
261
|
-
actionsEnabled: Number(row.actions_enabled) === 1,
|
|
262
|
-
readTools: parseTools(row.read_tools_json),
|
|
263
|
-
actionTools: parseTools(row.action_tools_json),
|
|
264
|
-
toolRevision: typeof row.tool_revision === "string" ? row.tool_revision : null,
|
|
265
|
-
lastCheckedAt: typeof row.last_checked_at === "string" ? row.last_checked_at : null,
|
|
266
|
-
lastSuccessfulCallAt: typeof row.last_successful_call_at === "string" ? row.last_successful_call_at : null,
|
|
267
|
-
lastFailureAt: typeof row.last_failure_at === "string" ? row.last_failure_at : null,
|
|
268
|
-
lastFailureCode: typeof row.last_failure_code === "string" ? row.last_failure_code : null,
|
|
269
|
-
lastFailureRetryable: row.last_failure_retryable === null || row.last_failure_retryable === void 0 ? null : Number(row.last_failure_retryable) === 1,
|
|
270
|
-
consecutiveFailures: Number(row.consecutive_failures ?? 0),
|
|
271
|
-
vaultName: typeof row.vault_name === "string" ? row.vault_name : null,
|
|
272
|
-
tableName: typeof row.table_name === "string" ? row.table_name : null,
|
|
273
|
-
createdAt: String(row.created_at),
|
|
274
|
-
updatedAt: String(row.updated_at)
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
async function userIdForIdentity(identity) {
|
|
278
|
-
const user = await getUserByEmail(identity.normalize("NFKC").trim().toLowerCase());
|
|
279
|
-
if (!user) throw new Error("service_connection_user_not_found");
|
|
280
|
-
return Number(user.id);
|
|
281
|
-
}
|
|
282
|
-
async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
283
|
-
await ensureServiceConnectionsSchema();
|
|
284
|
-
const db = getDb();
|
|
285
|
-
const userId = await userIdForIdentity(identity);
|
|
286
|
-
for (const connection of discovered) {
|
|
287
|
-
await db.execute({
|
|
288
|
-
sql: `
|
|
289
|
-
INSERT INTO service_connections (
|
|
290
|
-
id, user_id, provider_config_key, provider, transport, upstream_connection_id,
|
|
291
|
-
label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
|
|
292
|
-
) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
|
|
293
|
-
ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
|
|
294
|
-
provider = excluded.provider,
|
|
295
|
-
label = COALESCE(excluded.label, service_connections.label),
|
|
296
|
-
lifecycle_status = excluded.lifecycle_status,
|
|
297
|
-
reconnect_required = excluded.reconnect_required,
|
|
298
|
-
updated_at = excluded.updated_at
|
|
299
|
-
`,
|
|
300
|
-
args: [
|
|
301
|
-
randomUUID(),
|
|
302
|
-
userId,
|
|
303
|
-
connection.providerConfigKey,
|
|
304
|
-
connection.provider,
|
|
305
|
-
connection.upstreamConnectionId,
|
|
306
|
-
connection.label,
|
|
307
|
-
connection.lifecycleStatus,
|
|
308
|
-
connection.reconnectRequired ? 1 : 0,
|
|
309
|
-
identity,
|
|
310
|
-
connection.createdAt,
|
|
311
|
-
connection.updatedAt
|
|
312
|
-
]
|
|
313
|
-
});
|
|
314
|
-
}
|
|
315
|
-
return listServiceConnections(identity, "nango");
|
|
316
|
-
}
|
|
317
|
-
async function listServiceConnections(identity, transport) {
|
|
318
|
-
await ensureServiceConnectionsSchema();
|
|
319
|
-
const userId = await userIdForIdentity(identity);
|
|
320
|
-
const result = await getDb().execute({
|
|
321
|
-
sql: `SELECT * FROM service_connections WHERE user_id = ?${transport ? " AND transport = ?" : ""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,
|
|
322
|
-
args: transport ? [userId, transport] : [userId]
|
|
323
|
-
});
|
|
324
|
-
return result.rows.map((row) => rowToRecord(row));
|
|
325
|
-
}
|
|
326
|
-
async function getOwnedServiceConnection(identity, connectionId) {
|
|
327
|
-
await ensureServiceConnectionsSchema();
|
|
328
|
-
const userId = await userIdForIdentity(identity);
|
|
329
|
-
const result = await getDb().execute({
|
|
330
|
-
sql: "SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",
|
|
331
|
-
args: [connectionId, userId]
|
|
332
|
-
});
|
|
333
|
-
const row = result.rows[0];
|
|
334
|
-
return row ? rowToRecord(row) : null;
|
|
335
|
-
}
|
|
336
|
-
async function updateServiceConnectionTools(connectionId, readTools, actionTools, toolRevision) {
|
|
337
|
-
await ensureServiceConnectionsSchema();
|
|
338
|
-
await getDb().execute({
|
|
339
|
-
sql: `UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
340
|
-
args: [JSON.stringify(readTools), JSON.stringify(actionTools), toolRevision, connectionId]
|
|
341
|
-
});
|
|
342
|
-
}
|
|
343
|
-
async function setServiceConnectionProviderIdentity(args) {
|
|
344
|
-
await ensureServiceConnectionsSchema();
|
|
345
|
-
await getDb().execute({
|
|
346
|
-
sql: `
|
|
347
|
-
UPDATE service_connections SET
|
|
348
|
-
provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
|
|
349
|
-
provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
|
|
350
|
-
updated_at = datetime('now')
|
|
351
|
-
WHERE id = ?
|
|
352
|
-
`,
|
|
353
|
-
args: [
|
|
354
|
-
args.providerAccountId ?? null,
|
|
355
|
-
args.providerAccountEmail?.toLowerCase() ?? null,
|
|
356
|
-
args.providerAccountName ?? null,
|
|
357
|
-
args.sourceUpdatedAt,
|
|
358
|
-
args.connectionId
|
|
359
|
-
]
|
|
360
|
-
});
|
|
361
|
-
}
|
|
362
|
-
async function setServiceConnectionUserLabel(identity, connectionId, userLabel) {
|
|
363
|
-
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
364
|
-
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
365
|
-
await getDb().execute({
|
|
366
|
-
sql: `UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
367
|
-
args: [userLabel, connectionId, connection.userId]
|
|
368
|
-
});
|
|
369
|
-
return userLabel;
|
|
370
|
-
}
|
|
371
|
-
async function recordServiceConnectionHealth(args) {
|
|
372
|
-
await ensureServiceConnectionsSchema();
|
|
373
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
374
|
-
await getDb().execute({
|
|
375
|
-
sql: `
|
|
376
|
-
UPDATE service_connections SET
|
|
377
|
-
operational_status = ?, last_checked_at = ?,
|
|
378
|
-
last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
|
|
379
|
-
last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
|
|
380
|
-
last_failure_code = ?, last_failure_retryable = ?,
|
|
381
|
-
consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
|
|
382
|
-
updated_at = datetime('now')
|
|
383
|
-
WHERE id = ?
|
|
384
|
-
`,
|
|
385
|
-
args: [
|
|
386
|
-
args.operationalStatus,
|
|
387
|
-
now,
|
|
388
|
-
args.operationalStatus,
|
|
389
|
-
now,
|
|
390
|
-
args.operationalStatus,
|
|
391
|
-
now,
|
|
392
|
-
args.failureCode ?? null,
|
|
393
|
-
args.retryable == null ? null : args.retryable ? 1 : 0,
|
|
394
|
-
args.operationalStatus,
|
|
395
|
-
args.connectionId
|
|
396
|
-
]
|
|
397
|
-
});
|
|
398
|
-
await getDb().execute({
|
|
399
|
-
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
400
|
-
args: [randomUUID(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
401
|
-
});
|
|
402
|
-
}
|
|
403
|
-
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
404
|
-
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
405
|
-
if (!connection || connection.lifecycleStatus === "disconnected") throw new Error("service_connection_not_found");
|
|
406
|
-
await getDb().execute({
|
|
407
|
-
sql: `UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
408
|
-
args: [enabled ? 1 : 0, connectionId, connection.userId]
|
|
409
|
-
});
|
|
410
|
-
return enabled;
|
|
411
|
-
}
|
|
412
|
-
async function markServiceConnectionDisconnected(identity, connectionId) {
|
|
413
|
-
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
414
|
-
if (!connection) throw new Error("service_connection_not_found");
|
|
415
|
-
await getDb().execute({
|
|
416
|
-
sql: `UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?`,
|
|
417
|
-
args: [connectionId, connection.userId]
|
|
418
|
-
});
|
|
419
|
-
const remaining = await getDb().execute({
|
|
420
|
-
sql: `SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'`,
|
|
421
|
-
args: [connection.userId]
|
|
422
|
-
});
|
|
423
|
-
return Number(remaining.rows[0]?.count ?? 0);
|
|
424
|
-
}
|
|
425
|
-
async function recordServiceConnectionAction(args) {
|
|
426
|
-
await ensureServiceConnectionsSchema();
|
|
427
|
-
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
428
|
-
if (!connection) throw new Error("service_connection_not_found");
|
|
429
|
-
await getDb().execute({
|
|
430
|
-
sql: `UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?`,
|
|
431
|
-
args: [args.status, args.errorCode ?? null, args.result === void 0 ? null : JSON.stringify(args.result), connection.userId, args.requestId, connection.id]
|
|
432
|
-
});
|
|
433
|
-
}
|
|
434
|
-
async function claimServiceConnectionAction(args) {
|
|
435
|
-
await ensureServiceConnectionsSchema();
|
|
436
|
-
const connection = await getOwnedServiceConnection(args.identity, args.connectionId);
|
|
437
|
-
if (!connection) throw new Error("service_connection_not_found");
|
|
438
|
-
const inserted = await getDb().execute({
|
|
439
|
-
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
440
|
-
args: [randomUUID(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
441
|
-
});
|
|
442
|
-
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
443
|
-
const existing = await getDb().execute({
|
|
444
|
-
sql: `SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1`,
|
|
445
|
-
args: [connection.userId, args.requestId]
|
|
446
|
-
});
|
|
447
|
-
const row = existing.rows[0];
|
|
448
|
-
const conflict = String(row?.connection_id ?? "") !== connection.id || String(row?.tool ?? "") !== args.tool || String(row?.request_digest ?? "") !== args.requestDigest;
|
|
449
|
-
let result = void 0;
|
|
450
|
-
if (!conflict && typeof row?.result_json === "string") {
|
|
451
|
-
try {
|
|
452
|
-
result = JSON.parse(row.result_json);
|
|
453
|
-
} catch {
|
|
454
|
-
result = void 0;
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
return {
|
|
458
|
-
claimed: false,
|
|
459
|
-
status: String(row?.status ?? "unknown"),
|
|
460
|
-
errorCode: typeof row?.error_code === "string" ? row.error_code : null,
|
|
461
|
-
conflict,
|
|
462
|
-
...result !== void 0 ? { result } : {}
|
|
463
|
-
};
|
|
464
|
-
}
|
|
465
|
-
|
|
466
85
|
// src/api/analytics-provider-registry.ts
|
|
467
86
|
var META_ACTION_SOURCES = /* @__PURE__ */ new Set([
|
|
468
87
|
"email",
|
|
@@ -2237,6 +1856,144 @@ async function migrateAnalytics() {
|
|
|
2237
1856
|
occurred_at timestamptz NOT NULL DEFAULT now()
|
|
2238
1857
|
)
|
|
2239
1858
|
`);
|
|
1859
|
+
await client.query("ALTER TABLE analytics_crm_provisioning_plans ADD COLUMN IF NOT EXISTS public_plan_id text");
|
|
1860
|
+
await client.query("ALTER TABLE analytics_crm_provisioning_plans ADD COLUMN IF NOT EXISTS schema_fingerprint text");
|
|
1861
|
+
await client.query("ALTER TABLE analytics_crm_provisioning_plans ADD COLUMN IF NOT EXISTS request_fingerprint text");
|
|
1862
|
+
await client.query("ALTER TABLE analytics_crm_provisioning_plans ADD COLUMN IF NOT EXISTS approved_field_ids jsonb NOT NULL DEFAULT '[]'::jsonb");
|
|
1863
|
+
await client.query("ALTER TABLE analytics_crm_provisioning_receipts ADD COLUMN IF NOT EXISTS public_receipt_id text");
|
|
1864
|
+
await client.query("ALTER TABLE analytics_crm_provisioning_receipts ADD COLUMN IF NOT EXISTS receipt jsonb NOT NULL DEFAULT '{}'::jsonb");
|
|
1865
|
+
await client.query("CREATE UNIQUE INDEX IF NOT EXISTS analytics_crm_plan_public_scope ON analytics_crm_provisioning_plans(site_id, public_plan_id) WHERE public_plan_id IS NOT NULL");
|
|
1866
|
+
await client.query("CREATE UNIQUE INDEX IF NOT EXISTS analytics_crm_receipt_public_id ON analytics_crm_provisioning_receipts(public_receipt_id) WHERE public_receipt_id IS NOT NULL");
|
|
1867
|
+
await client.query(`
|
|
1868
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_connection_capabilities (
|
|
1869
|
+
site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
|
|
1870
|
+
connection_id uuid NOT NULL REFERENCES analytics_connections(id) ON DELETE CASCADE,
|
|
1871
|
+
provider text NOT NULL CHECK (provider IN ('hubspot','salesforce','highlevel','zoho','pipedrive','keap')),
|
|
1872
|
+
contract_version integer NOT NULL DEFAULT 1 CHECK (contract_version > 0),
|
|
1873
|
+
mode text NOT NULL CHECK (mode IN ('incremental_and_webhook','incremental_only','webhook_only','manual_backfill','unavailable')),
|
|
1874
|
+
schema_fingerprint text,
|
|
1875
|
+
allowed_operations jsonb NOT NULL DEFAULT '[]'::jsonb,
|
|
1876
|
+
evidence jsonb NOT NULL,
|
|
1877
|
+
verified_at timestamptz NOT NULL,
|
|
1878
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
1879
|
+
PRIMARY KEY(connection_id, contract_version),
|
|
1880
|
+
UNIQUE(site_id, connection_id, provider, contract_version)
|
|
1881
|
+
)
|
|
1882
|
+
`);
|
|
1883
|
+
await client.query(`
|
|
1884
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_sync_runs (
|
|
1885
|
+
sync_id text PRIMARY KEY,
|
|
1886
|
+
site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
|
|
1887
|
+
connection_id uuid NOT NULL REFERENCES analytics_connections(id) ON DELETE CASCADE,
|
|
1888
|
+
provider text NOT NULL CHECK (provider IN ('hubspot','salesforce','highlevel','zoho','pipedrive','keap')),
|
|
1889
|
+
mode text NOT NULL CHECK (mode IN ('initial','incremental','webhook')),
|
|
1890
|
+
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','running','completed','degraded','cancelled','webhook_only')),
|
|
1891
|
+
mapping jsonb NOT NULL,
|
|
1892
|
+
mapping_version integer NOT NULL DEFAULT 1 CHECK (mapping_version > 0),
|
|
1893
|
+
capability_evidence jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
1894
|
+
cursor jsonb,
|
|
1895
|
+
idempotency_key text NOT NULL,
|
|
1896
|
+
request_fingerprint text NOT NULL,
|
|
1897
|
+
last_started_at timestamptz,
|
|
1898
|
+
last_success_at timestamptz,
|
|
1899
|
+
next_run_at timestamptz,
|
|
1900
|
+
imported integer NOT NULL DEFAULT 0 CHECK (imported >= 0),
|
|
1901
|
+
deduplicated integer NOT NULL DEFAULT 0 CHECK (deduplicated >= 0),
|
|
1902
|
+
rejected integer NOT NULL DEFAULT 0 CHECK (rejected >= 0),
|
|
1903
|
+
last_error_code text,
|
|
1904
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1905
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
1906
|
+
UNIQUE(site_id, idempotency_key)
|
|
1907
|
+
)
|
|
1908
|
+
`);
|
|
1909
|
+
await client.query("CREATE INDEX IF NOT EXISTS analytics_crm_sync_runs_due ON analytics_crm_sync_runs(state, next_run_at) WHERE next_run_at IS NOT NULL");
|
|
1910
|
+
await client.query(`
|
|
1911
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_outbound_policies (
|
|
1912
|
+
policy_id text PRIMARY KEY,
|
|
1913
|
+
site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
|
|
1914
|
+
connection_id uuid NOT NULL REFERENCES analytics_connections(id) ON DELETE CASCADE,
|
|
1915
|
+
provider text NOT NULL CHECK (provider IN ('hubspot','salesforce','highlevel','zoho','pipedrive','keap')),
|
|
1916
|
+
kind text NOT NULL CHECK (kind IN ('person_summary','pipeline_event')),
|
|
1917
|
+
enabled boolean NOT NULL DEFAULT false,
|
|
1918
|
+
version integer NOT NULL CHECK (version > 0),
|
|
1919
|
+
provisioning_receipt_id text NOT NULL,
|
|
1920
|
+
schema_fingerprint text NOT NULL,
|
|
1921
|
+
mapping_version integer CHECK (mapping_version > 0),
|
|
1922
|
+
mapping jsonb,
|
|
1923
|
+
allowed_fields jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(allowed_fields) = 'array'),
|
|
1924
|
+
selected_events jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(selected_events) = 'array'),
|
|
1925
|
+
purpose text NOT NULL CHECK (purpose IN ('necessary_form_fulfillment','site_analytics','advertising_measurement')),
|
|
1926
|
+
created_by_user_id bigint NOT NULL,
|
|
1927
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1928
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
1929
|
+
UNIQUE(site_id, connection_id, kind, version),
|
|
1930
|
+
CHECK ((kind='person_summary' AND mapping_version IS NULL AND mapping IS NULL) OR
|
|
1931
|
+
(kind='pipeline_event' AND mapping_version IS NOT NULL AND mapping IS NOT NULL))
|
|
1932
|
+
)
|
|
1933
|
+
`);
|
|
1934
|
+
await client.query(`
|
|
1935
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_outbound_jobs (
|
|
1936
|
+
job_id text PRIMARY KEY,
|
|
1937
|
+
policy_id text NOT NULL REFERENCES analytics_crm_outbound_policies(policy_id) ON DELETE CASCADE,
|
|
1938
|
+
site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
|
|
1939
|
+
person_id uuid NOT NULL REFERENCES analytics_people(id) ON DELETE CASCADE,
|
|
1940
|
+
kind text NOT NULL CHECK (kind IN ('person_summary','pipeline_event')),
|
|
1941
|
+
state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','running','retrying','delivered','dead_letter','cancelled')),
|
|
1942
|
+
policy_version integer NOT NULL CHECK (policy_version > 0),
|
|
1943
|
+
mapping_version integer CHECK (mapping_version > 0),
|
|
1944
|
+
source_event_id text,
|
|
1945
|
+
payload_ciphertext text NOT NULL,
|
|
1946
|
+
idempotency_key text NOT NULL,
|
|
1947
|
+
request_fingerprint text NOT NULL,
|
|
1948
|
+
attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0),
|
|
1949
|
+
max_attempts integer NOT NULL DEFAULT 5 CHECK (max_attempts BETWEEN 1 AND 20),
|
|
1950
|
+
next_attempt_at timestamptz,
|
|
1951
|
+
lease_expires_at timestamptz,
|
|
1952
|
+
safe_error_code text,
|
|
1953
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1954
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
1955
|
+
delivered_at timestamptz,
|
|
1956
|
+
cancelled_at timestamptz,
|
|
1957
|
+
UNIQUE(site_id, idempotency_key)
|
|
1958
|
+
)
|
|
1959
|
+
`);
|
|
1960
|
+
await client.query(`
|
|
1961
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_outbound_attempts (
|
|
1962
|
+
id uuid PRIMARY KEY,
|
|
1963
|
+
job_id text NOT NULL REFERENCES analytics_crm_outbound_jobs(job_id) ON DELETE CASCADE,
|
|
1964
|
+
attempt integer NOT NULL CHECK (attempt > 0),
|
|
1965
|
+
status text NOT NULL CHECK (status IN ('started','accepted','retrying','failed','cancelled')),
|
|
1966
|
+
safe_error_code text,
|
|
1967
|
+
started_at timestamptz NOT NULL DEFAULT now(),
|
|
1968
|
+
completed_at timestamptz,
|
|
1969
|
+
UNIQUE(job_id, attempt)
|
|
1970
|
+
)
|
|
1971
|
+
`);
|
|
1972
|
+
await client.query(`
|
|
1973
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_outbound_receipts (
|
|
1974
|
+
receipt_id text PRIMARY KEY,
|
|
1975
|
+
job_id text NOT NULL REFERENCES analytics_crm_outbound_jobs(job_id) ON DELETE CASCADE,
|
|
1976
|
+
status text NOT NULL CHECK (status IN ('accepted','rejected','failed','retrying')),
|
|
1977
|
+
external_receipt_id text,
|
|
1978
|
+
safe_error_code text,
|
|
1979
|
+
occurred_at timestamptz NOT NULL DEFAULT now()
|
|
1980
|
+
)
|
|
1981
|
+
`);
|
|
1982
|
+
await client.query(`
|
|
1983
|
+
CREATE TABLE IF NOT EXISTS analytics_crm_outbound_worker_state (
|
|
1984
|
+
worker_key text PRIMARY KEY,
|
|
1985
|
+
last_started_at timestamptz,
|
|
1986
|
+
last_completed_at timestamptz,
|
|
1987
|
+
last_success_at timestamptz,
|
|
1988
|
+
last_error_code text,
|
|
1989
|
+
last_claimed integer NOT NULL DEFAULT 0,
|
|
1990
|
+
last_remaining_due integer NOT NULL DEFAULT 0,
|
|
1991
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
1992
|
+
)
|
|
1993
|
+
`);
|
|
1994
|
+
await client.query("CREATE INDEX IF NOT EXISTS analytics_crm_outbound_jobs_due ON analytics_crm_outbound_jobs(state,next_attempt_at) WHERE state IN ('pending','retrying')");
|
|
1995
|
+
await client.query("CREATE INDEX IF NOT EXISTS analytics_crm_outbound_jobs_person ON analytics_crm_outbound_jobs(site_id,person_id,created_at DESC)");
|
|
1996
|
+
await client.query("CREATE INDEX IF NOT EXISTS analytics_crm_outbound_receipts_job ON analytics_crm_outbound_receipts(job_id,occurred_at DESC)");
|
|
2240
1997
|
await client.query(`
|
|
2241
1998
|
CREATE TABLE IF NOT EXISTS analytics_score_definitions (
|
|
2242
1999
|
id uuid PRIMARY KEY,
|
|
@@ -2559,7 +2316,7 @@ async function requireAnalyticsEditor(client, siteId, userId) {
|
|
|
2559
2316
|
async function createAnalyticsSite(input) {
|
|
2560
2317
|
const db = getAnalyticsPool();
|
|
2561
2318
|
const client = await db.connect();
|
|
2562
|
-
const id =
|
|
2319
|
+
const id = randomUUID();
|
|
2563
2320
|
const baseSlug = normalizeSlug(input.slug || input.name);
|
|
2564
2321
|
try {
|
|
2565
2322
|
await client.query("BEGIN");
|
|
@@ -2790,7 +2547,7 @@ async function createAnalyticsPixel(input) {
|
|
|
2790
2547
|
VALUES ($1, $2, $3, $4, $5)
|
|
2791
2548
|
RETURNING id, site_id, public_id, name, environment, status, created_at::text, NULL::text AS last_event_at`,
|
|
2792
2549
|
[
|
|
2793
|
-
|
|
2550
|
+
randomUUID(),
|
|
2794
2551
|
input.siteId,
|
|
2795
2552
|
publicPixelId(),
|
|
2796
2553
|
input.name.trim(),
|
|
@@ -2867,7 +2624,7 @@ async function setAnalyticsPixelDomainState(input) {
|
|
|
2867
2624
|
SELECT $1, p.id, $4, $5 FROM analytics_pixels p WHERE p.id = $2 AND p.site_id = $3
|
|
2868
2625
|
ON CONFLICT(pixel_id, hostname) DO UPDATE SET state = EXCLUDED.state, updated_at = now()
|
|
2869
2626
|
RETURNING id`,
|
|
2870
|
-
[
|
|
2627
|
+
[randomUUID(), input.pixelId, input.siteId, hostname, input.state]
|
|
2871
2628
|
);
|
|
2872
2629
|
if (!result.rowCount)
|
|
2873
2630
|
throw new AnalyticsRepositoryError(
|
|
@@ -2909,7 +2666,7 @@ async function upsertAnalyticsHostGroup(input) {
|
|
|
2909
2666
|
ON CONFLICT(pixel_id, name) DO UPDATE SET
|
|
2910
2667
|
auto_decorate=EXCLUDED.auto_decorate, updated_at=now()
|
|
2911
2668
|
RETURNING id`,
|
|
2912
|
-
[
|
|
2669
|
+
[randomUUID(), input.siteId, input.pixelId, input.name.trim().slice(0, 120), Boolean(input.autoDecorate)]
|
|
2913
2670
|
);
|
|
2914
2671
|
await client.query("DELETE FROM analytics_host_group_members WHERE group_id=$1", [group.rows[0].id]);
|
|
2915
2672
|
for (const hostname of hostnames)
|
|
@@ -2992,7 +2749,7 @@ async function prepareAnalyticsLinkerIssue(input) {
|
|
|
2992
2749
|
VALUES ($1,$2,'visitor_id',$3)
|
|
2993
2750
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at=now()
|
|
2994
2751
|
RETURNING id`,
|
|
2995
|
-
[
|
|
2752
|
+
[randomUUID(), siteId, identityHmac("visitor_id", input.visitorId)]
|
|
2996
2753
|
);
|
|
2997
2754
|
const existingPerson = await client.query(
|
|
2998
2755
|
`SELECT person_id FROM analytics_identity_edges
|
|
@@ -3005,17 +2762,17 @@ async function prepareAnalyticsLinkerIssue(input) {
|
|
|
3005
2762
|
const person = await client.query(
|
|
3006
2763
|
`INSERT INTO analytics_people(id, site_id, crm_person_ref) VALUES ($1,$2,$3)
|
|
3007
2764
|
ON CONFLICT(site_id, crm_person_ref) DO UPDATE SET last_seen_at=now() RETURNING id`,
|
|
3008
|
-
[
|
|
2765
|
+
[randomUUID(), siteId, `XRay::anonymous::${identityHmac("visitor_id", input.visitorId).slice(0, 24)}`]
|
|
3009
2766
|
);
|
|
3010
2767
|
personId = person.rows[0].id;
|
|
3011
2768
|
await client.query(
|
|
3012
2769
|
`INSERT INTO analytics_identity_edges(id, site_id, person_id, identity_node_id, evidence_kind, confidence)
|
|
3013
2770
|
VALUES ($1,$2,$3,$4,'first_party_linker',0.980)
|
|
3014
2771
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at=now()`,
|
|
3015
|
-
[
|
|
2772
|
+
[randomUUID(), siteId, personId, node.rows[0].id]
|
|
3016
2773
|
);
|
|
3017
2774
|
}
|
|
3018
|
-
const tokenId =
|
|
2775
|
+
const tokenId = randomUUID();
|
|
3019
2776
|
await client.query(
|
|
3020
2777
|
`INSERT INTO analytics_linker_tokens(
|
|
3021
2778
|
id, site_id, pixel_id, host_group_id, identity_node_id,
|
|
@@ -3071,13 +2828,13 @@ async function redeemAnalyticsLinkerRecord(input) {
|
|
|
3071
2828
|
VALUES ($1,$2,'visitor_id',$3)
|
|
3072
2829
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at=now()
|
|
3073
2830
|
RETURNING id`,
|
|
3074
|
-
[
|
|
2831
|
+
[randomUUID(), input.siteId, identityHmac("visitor_id", input.visitorId)]
|
|
3075
2832
|
);
|
|
3076
2833
|
await client.query(
|
|
3077
2834
|
`INSERT INTO analytics_identity_edges(id, site_id, person_id, identity_node_id, evidence_kind, confidence)
|
|
3078
2835
|
VALUES ($1,$2,$3,$4,'cross_domain_linker',0.980)
|
|
3079
2836
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at=now()`,
|
|
3080
|
-
[
|
|
2837
|
+
[randomUUID(), input.siteId, personId, targetNode.rows[0].id]
|
|
3081
2838
|
);
|
|
3082
2839
|
await client.query(
|
|
3083
2840
|
"UPDATE analytics_events SET person_id=$1 WHERE site_id=$2 AND visitor_id=$3 AND person_id IS NULL",
|
|
@@ -3169,7 +2926,7 @@ async function materializeAnalyticsSessionAndTouch(input) {
|
|
|
3169
2926
|
updated_at = now()
|
|
3170
2927
|
RETURNING id`,
|
|
3171
2928
|
[
|
|
3172
|
-
|
|
2929
|
+
randomUUID(),
|
|
3173
2930
|
input.siteId,
|
|
3174
2931
|
input.pixelId,
|
|
3175
2932
|
visitorHmac,
|
|
@@ -3206,7 +2963,7 @@ async function materializeAnalyticsSessionAndTouch(input) {
|
|
|
3206
2963
|
person_id = COALESCE(analytics_attribution_touches.person_id, EXCLUDED.person_id)
|
|
3207
2964
|
RETURNING id`,
|
|
3208
2965
|
[
|
|
3209
|
-
|
|
2966
|
+
randomUUID(),
|
|
3210
2967
|
input.siteId,
|
|
3211
2968
|
session.rows[0].id,
|
|
3212
2969
|
input.personId ?? null,
|
|
@@ -3230,7 +2987,7 @@ async function materializeAnalyticsSessionAndTouch(input) {
|
|
|
3230
2987
|
}
|
|
3231
2988
|
async function ingestAnalyticsEvents(input) {
|
|
3232
2989
|
const db = getAnalyticsPool();
|
|
3233
|
-
const requestId = `air_${
|
|
2990
|
+
const requestId = `air_${randomUUID()}`;
|
|
3234
2991
|
const hostname = normalizeObservedHostname(input.origin);
|
|
3235
2992
|
const pixelResult = await db.query(
|
|
3236
2993
|
`SELECT p.id, p.site_id, p.status,
|
|
@@ -3251,7 +3008,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3251
3008
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
3252
3009
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
3253
3010
|
[
|
|
3254
|
-
|
|
3011
|
+
randomUUID(),
|
|
3255
3012
|
requestId,
|
|
3256
3013
|
pixel?.site_id ?? null,
|
|
3257
3014
|
pixel?.id ?? null,
|
|
@@ -3276,7 +3033,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3276
3033
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
3277
3034
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
3278
3035
|
[
|
|
3279
|
-
|
|
3036
|
+
randomUUID(),
|
|
3280
3037
|
requestId,
|
|
3281
3038
|
pixel.site_id,
|
|
3282
3039
|
pixel.id,
|
|
@@ -3301,7 +3058,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3301
3058
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, rejected_count, reason_codes)
|
|
3302
3059
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb)`,
|
|
3303
3060
|
[
|
|
3304
|
-
|
|
3061
|
+
randomUUID(),
|
|
3305
3062
|
requestId,
|
|
3306
3063
|
pixel.site_id,
|
|
3307
3064
|
pixel.id,
|
|
@@ -3325,7 +3082,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3325
3082
|
ON CONFLICT(pixel_id, hostname) DO UPDATE
|
|
3326
3083
|
SET last_seen_at = now(), updated_at = now()
|
|
3327
3084
|
RETURNING state`,
|
|
3328
|
-
[
|
|
3085
|
+
[randomUUID(), pixel.id, hostname]
|
|
3329
3086
|
);
|
|
3330
3087
|
if (domainResult.rows[0]?.state !== "approved") {
|
|
3331
3088
|
await db.query(
|
|
@@ -3339,7 +3096,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3339
3096
|
`INSERT INTO analytics_ingestion_receipts(id, request_id, site_id, pixel_id, hostname, rejected_count, reason_codes)
|
|
3340
3097
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
|
3341
3098
|
[
|
|
3342
|
-
|
|
3099
|
+
randomUUID(),
|
|
3343
3100
|
requestId,
|
|
3344
3101
|
pixel.site_id,
|
|
3345
3102
|
pixel.id,
|
|
@@ -3399,7 +3156,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3399
3156
|
$23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33
|
|
3400
3157
|
) ON CONFLICT(site_id, event_id) DO NOTHING`,
|
|
3401
3158
|
[
|
|
3402
|
-
|
|
3159
|
+
randomUUID(),
|
|
3403
3160
|
event.eventId,
|
|
3404
3161
|
pixel.site_id,
|
|
3405
3162
|
pixel.id,
|
|
@@ -3465,7 +3222,7 @@ async function ingestAnalyticsEvents(input) {
|
|
|
3465
3222
|
id, request_id, site_id, pixel_id, hostname, accepted_count, rejected_count, reason_codes
|
|
3466
3223
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`,
|
|
3467
3224
|
[
|
|
3468
|
-
|
|
3225
|
+
randomUUID(),
|
|
3469
3226
|
requestId,
|
|
3470
3227
|
pixel.site_id,
|
|
3471
3228
|
pixel.id,
|
|
@@ -3556,7 +3313,7 @@ async function createAnalyticsConversion(input) {
|
|
|
3556
3313
|
ORDER BY e.confidence DESC LIMIT 1`,
|
|
3557
3314
|
[input.siteId, identityHmac("session_id", input.sessionId)]
|
|
3558
3315
|
) : { rows: [] };
|
|
3559
|
-
const proposedId =
|
|
3316
|
+
const proposedId = randomUUID();
|
|
3560
3317
|
const inserted = await client.query(
|
|
3561
3318
|
`INSERT INTO analytics_conversions(
|
|
3562
3319
|
id, source_event_id, site_id, pixel_id, session_id, conversion_kind,
|
|
@@ -3596,7 +3353,7 @@ async function createAnalyticsConversion(input) {
|
|
|
3596
3353
|
) VALUES ($1,$2,$3,$4,$5)
|
|
3597
3354
|
ON CONFLICT(destination_id, conversion_id) DO NOTHING`,
|
|
3598
3355
|
[
|
|
3599
|
-
|
|
3356
|
+
randomUUID(),
|
|
3600
3357
|
destination.id,
|
|
3601
3358
|
conversionId,
|
|
3602
3359
|
resolvedPerson.rows[0]?.person_id ?? null,
|
|
@@ -4369,7 +4126,7 @@ async function createAnalyticsCampaignLink(input) {
|
|
|
4369
4126
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
|
|
4370
4127
|
RETURNING *, 0::int AS click_count`,
|
|
4371
4128
|
[
|
|
4372
|
-
|
|
4129
|
+
randomUUID(),
|
|
4373
4130
|
input.siteId,
|
|
4374
4131
|
input.pixelId ?? null,
|
|
4375
4132
|
input.name.trim(),
|
|
@@ -4440,7 +4197,7 @@ async function resolveAnalyticsCampaignLink(shortCode, referrer) {
|
|
|
4440
4197
|
if (!row) return null;
|
|
4441
4198
|
await db.query(
|
|
4442
4199
|
`INSERT INTO analytics_campaign_clicks(id, link_id, referrer) VALUES ($1, $2, $3)`,
|
|
4443
|
-
[
|
|
4200
|
+
[randomUUID(), row.id, normalizeAnalyticsUrl(referrer || void 0)]
|
|
4444
4201
|
);
|
|
4445
4202
|
return buildTaggedCampaignUrl(row);
|
|
4446
4203
|
}
|
|
@@ -4465,7 +4222,7 @@ async function createAnalyticsForm(input) {
|
|
|
4465
4222
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10,$11,$12,$13,$14)
|
|
4466
4223
|
RETURNING *, 0::int AS submission_count`,
|
|
4467
4224
|
[
|
|
4468
|
-
|
|
4225
|
+
randomUUID(),
|
|
4469
4226
|
publicId,
|
|
4470
4227
|
input.siteId,
|
|
4471
4228
|
input.pixelId,
|
|
@@ -4510,7 +4267,7 @@ async function getPublicAnalyticsForm(publicId) {
|
|
|
4510
4267
|
return result.rows[0] ?? null;
|
|
4511
4268
|
}
|
|
4512
4269
|
async function recordAnalyticsFormSubmission(input) {
|
|
4513
|
-
const id =
|
|
4270
|
+
const id = randomUUID();
|
|
4514
4271
|
await getAnalyticsPool().query(
|
|
4515
4272
|
`INSERT INTO analytics_form_submissions(id, form_id, site_id, pixel_id, visitor_id, session_id, crm_person_ref, source, medium, campaign, crm_delivery_status, click_ids)
|
|
4516
4273
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::jsonb)`,
|
|
@@ -4554,7 +4311,7 @@ function sanitizeClickIds(value) {
|
|
|
4554
4311
|
return output;
|
|
4555
4312
|
}
|
|
4556
4313
|
function identityHmac(kind, value) {
|
|
4557
|
-
return
|
|
4314
|
+
return createHmac("sha256", getSessionSecret()).update(`${kind}:${value.trim().toLowerCase()}`).digest("hex");
|
|
4558
4315
|
}
|
|
4559
4316
|
async function linkAnalyticsIdentityInTransaction(client, input) {
|
|
4560
4317
|
const stable = input.customerId || input.orderId || input.email || input.phone;
|
|
@@ -4563,7 +4320,7 @@ async function linkAnalyticsIdentityInTransaction(client, input) {
|
|
|
4563
4320
|
const person = await client.query(
|
|
4564
4321
|
`INSERT INTO analytics_people(id, site_id, crm_person_ref) VALUES ($1,$2,$3)
|
|
4565
4322
|
ON CONFLICT(site_id, crm_person_ref) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
4566
|
-
[
|
|
4323
|
+
[randomUUID(), input.siteId, crmPersonRef]
|
|
4567
4324
|
);
|
|
4568
4325
|
const personId = person.rows[0].id;
|
|
4569
4326
|
const signals = [];
|
|
@@ -4585,14 +4342,14 @@ async function linkAnalyticsIdentityInTransaction(client, input) {
|
|
|
4585
4342
|
const node = await client.query(
|
|
4586
4343
|
`INSERT INTO analytics_identity_nodes(id, site_id, kind, value_hmac) VALUES ($1,$2,$3,$4)
|
|
4587
4344
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
4588
|
-
[
|
|
4345
|
+
[randomUUID(), input.siteId, signal.kind, identityHmac(signal.kind, signal.value)]
|
|
4589
4346
|
);
|
|
4590
4347
|
await client.query(
|
|
4591
4348
|
`INSERT INTO analytics_identity_edges(id, site_id, person_id, identity_node_id, evidence_kind, confidence)
|
|
4592
4349
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
4593
4350
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE
|
|
4594
4351
|
SET last_seen_at = now(), confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence)`,
|
|
4595
|
-
[
|
|
4352
|
+
[randomUUID(), input.siteId, personId, node.rows[0].id, signal.evidence, signal.confidence]
|
|
4596
4353
|
);
|
|
4597
4354
|
}
|
|
4598
4355
|
return personId;
|
|
@@ -4604,7 +4361,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
4604
4361
|
const person = await client.query(
|
|
4605
4362
|
`INSERT INTO analytics_people(id, site_id, crm_person_ref) VALUES ($1,$2,$3)
|
|
4606
4363
|
ON CONFLICT(site_id, crm_person_ref) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
4607
|
-
[
|
|
4364
|
+
[randomUUID(), input.siteId, input.crmPersonRef]
|
|
4608
4365
|
);
|
|
4609
4366
|
const personId = person.rows[0].id;
|
|
4610
4367
|
const signals = [];
|
|
@@ -4651,7 +4408,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
4651
4408
|
`INSERT INTO analytics_identity_nodes(id, site_id, kind, value_hmac) VALUES ($1,$2,$3,$4)
|
|
4652
4409
|
ON CONFLICT(site_id, kind, value_hmac) DO UPDATE SET last_seen_at = now() RETURNING id`,
|
|
4653
4410
|
[
|
|
4654
|
-
|
|
4411
|
+
randomUUID(),
|
|
4655
4412
|
input.siteId,
|
|
4656
4413
|
signal.kind,
|
|
4657
4414
|
identityHmac(signal.kind, signal.value)
|
|
@@ -4662,7 +4419,7 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
4662
4419
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
4663
4420
|
ON CONFLICT(person_id, identity_node_id, evidence_kind) DO UPDATE SET last_seen_at = now(), confidence = greatest(analytics_identity_edges.confidence, EXCLUDED.confidence)`,
|
|
4664
4421
|
[
|
|
4665
|
-
|
|
4422
|
+
randomUUID(),
|
|
4666
4423
|
input.siteId,
|
|
4667
4424
|
personId,
|
|
4668
4425
|
node.rows[0].id,
|
|
@@ -4677,6 +4434,8 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
4677
4434
|
[personId, input.submissionId, input.siteId]
|
|
4678
4435
|
);
|
|
4679
4436
|
if (input.visitorId || input.sessionId) {
|
|
4437
|
+
const visitorHmac = input.visitorId ? identityHmac("visitor_id", input.visitorId) : null;
|
|
4438
|
+
const sessionHmac = input.sessionId ? identityHmac("session_id", input.sessionId) : null;
|
|
4680
4439
|
await client.query(
|
|
4681
4440
|
`UPDATE analytics_events SET person_id = $1 WHERE site_id = $2 AND person_id IS NULL
|
|
4682
4441
|
AND (($3::text IS NOT NULL AND visitor_id = $3) OR ($4::text IS NOT NULL AND session_id = $4))`,
|
|
@@ -4691,6 +4450,23 @@ async function linkAnalyticsFormIdentity(input) {
|
|
|
4691
4450
|
`UPDATE analytics_conversions SET person_id = $1 WHERE site_id = $2 AND person_id IS NULL AND $3::text IS NOT NULL AND session_id = $3`,
|
|
4692
4451
|
[personId, input.siteId, input.sessionId ?? null]
|
|
4693
4452
|
);
|
|
4453
|
+
await client.query(
|
|
4454
|
+
`UPDATE analytics_sessions SET person_id = $1
|
|
4455
|
+
WHERE site_id = $2 AND person_id IS NULL
|
|
4456
|
+
AND (($3::text IS NOT NULL AND visitor_hmac = $3)
|
|
4457
|
+
OR ($4::text IS NOT NULL AND session_hmac = $4))`,
|
|
4458
|
+
[personId, input.siteId, visitorHmac, sessionHmac]
|
|
4459
|
+
);
|
|
4460
|
+
await client.query(
|
|
4461
|
+
`UPDATE analytics_attribution_touches SET person_id = $1
|
|
4462
|
+
WHERE site_id = $2 AND person_id IS NULL
|
|
4463
|
+
AND (($3::text IS NOT NULL AND visitor_hmac = $3)
|
|
4464
|
+
OR ($4::text IS NOT NULL AND session_id IN (
|
|
4465
|
+
SELECT id FROM analytics_sessions
|
|
4466
|
+
WHERE site_id = $2 AND session_hmac = $4
|
|
4467
|
+
)))`,
|
|
4468
|
+
[personId, input.siteId, visitorHmac, sessionHmac]
|
|
4469
|
+
);
|
|
4694
4470
|
}
|
|
4695
4471
|
await client.query("COMMIT");
|
|
4696
4472
|
return { personId, linkedSignals: signals.length };
|
|
@@ -4760,7 +4536,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
4760
4536
|
const db = getAnalyticsPool();
|
|
4761
4537
|
await requireEditor(db, input.siteId, input.userId);
|
|
4762
4538
|
const client = await db.connect();
|
|
4763
|
-
const id =
|
|
4539
|
+
const id = randomUUID();
|
|
4764
4540
|
try {
|
|
4765
4541
|
await client.query("BEGIN");
|
|
4766
4542
|
await client.query(
|
|
@@ -4780,7 +4556,7 @@ async function createAnalyticsCrmImport(input) {
|
|
|
4780
4556
|
await client.query(
|
|
4781
4557
|
`INSERT INTO analytics_crm_import_rows(id, import_id, crm_person_ref, payload_ciphertext)
|
|
4782
4558
|
VALUES ($1,$2,$3,$4) ON CONFLICT(import_id, crm_person_ref) DO NOTHING`,
|
|
4783
|
-
[
|
|
4559
|
+
[randomUUID(), id, row.crmPersonRef, row.payloadCiphertext]
|
|
4784
4560
|
);
|
|
4785
4561
|
}
|
|
4786
4562
|
await client.query("COMMIT");
|
|
@@ -4903,7 +4679,7 @@ async function createAnalyticsActivationDestination(input) {
|
|
|
4903
4679
|
`INSERT INTO analytics_activation_destinations(id, site_id, platform, name, connection_ref, external_dataset_id, event_mapping, provider_config, created_by_user_id)
|
|
4904
4680
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9) RETURNING *`,
|
|
4905
4681
|
[
|
|
4906
|
-
|
|
4682
|
+
randomUUID(),
|
|
4907
4683
|
input.siteId,
|
|
4908
4684
|
input.platform,
|
|
4909
4685
|
input.name.trim(),
|
|
@@ -5050,7 +4826,7 @@ async function testAnalyticsActivationDestination(input) {
|
|
|
5050
4826
|
400
|
|
5051
4827
|
);
|
|
5052
4828
|
}
|
|
5053
|
-
const testId =
|
|
4829
|
+
const testId = randomUUID();
|
|
5054
4830
|
const syntheticHash = createHash("sha256").update(`xray-provider-test:${testId}`).digest("hex");
|
|
5055
4831
|
const providerConfig = destination.provider_config ?? {};
|
|
5056
4832
|
const payload = {
|
|
@@ -5288,7 +5064,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
5288
5064
|
`INSERT INTO analytics_activation_jobs(id, destination_id, conversion_id, person_id, payload_ciphertext)
|
|
5289
5065
|
VALUES ($1,$2,$3,$4,$5) ON CONFLICT(destination_id, conversion_id) DO NOTHING`,
|
|
5290
5066
|
[
|
|
5291
|
-
|
|
5067
|
+
randomUUID(),
|
|
5292
5068
|
destination.id,
|
|
5293
5069
|
input.conversionId,
|
|
5294
5070
|
input.personId ?? null,
|
|
@@ -5300,7 +5076,7 @@ async function queueAnalyticsActivation(input) {
|
|
|
5300
5076
|
return queued;
|
|
5301
5077
|
}
|
|
5302
5078
|
async function queueAnalyticsFormDelivery(input) {
|
|
5303
|
-
const id =
|
|
5079
|
+
const id = randomUUID();
|
|
5304
5080
|
await getAnalyticsPool().query(
|
|
5305
5081
|
`INSERT INTO analytics_form_delivery_jobs(id, submission_id, owner_user_id, payload_ciphertext, last_error_code)
|
|
5306
5082
|
VALUES ($1,$2,$3,$4,$5)`,
|
|
@@ -5457,7 +5233,7 @@ async function analyticsHealth(siteId, userId) {
|
|
|
5457
5233
|
}
|
|
5458
5234
|
async function refreshAnalyticsDailyRollups(input) {
|
|
5459
5235
|
const db = getAnalyticsPool();
|
|
5460
|
-
const runId =
|
|
5236
|
+
const runId = randomUUID();
|
|
5461
5237
|
await db.query(
|
|
5462
5238
|
`INSERT INTO analytics_rollup_runs(id, window_start, window_end, status) VALUES ($1, $2, $3, 'running')`,
|
|
5463
5239
|
[runId, input.start, input.end]
|
|
@@ -5654,7 +5430,7 @@ async function createAnalyticsExport(input) {
|
|
|
5654
5430
|
`Generated ${(/* @__PURE__ */ new Date()).toISOString()} from the governed ${input.report} report contract.`
|
|
5655
5431
|
].join("\n");
|
|
5656
5432
|
}
|
|
5657
|
-
const id =
|
|
5433
|
+
const id = randomUUID();
|
|
5658
5434
|
const inserted = await getAnalyticsPool().query(
|
|
5659
5435
|
`INSERT INTO analytics_exports(id, site_id, requested_by_user_id, idempotency_key, report, format, filters, content)
|
|
5660
5436
|
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)
|
|
@@ -5689,21 +5465,6 @@ async function createAnalyticsExport(input) {
|
|
|
5689
5465
|
}
|
|
5690
5466
|
|
|
5691
5467
|
export {
|
|
5692
|
-
getSessionSecret,
|
|
5693
|
-
signSession,
|
|
5694
|
-
verifySession,
|
|
5695
|
-
ensureServiceConnectionsSchema,
|
|
5696
|
-
reconcileDiscoveredNangoConnections,
|
|
5697
|
-
listServiceConnections,
|
|
5698
|
-
getOwnedServiceConnection,
|
|
5699
|
-
updateServiceConnectionTools,
|
|
5700
|
-
setServiceConnectionProviderIdentity,
|
|
5701
|
-
setServiceConnectionUserLabel,
|
|
5702
|
-
recordServiceConnectionHealth,
|
|
5703
|
-
setServiceConnectionActions,
|
|
5704
|
-
markServiceConnectionDisconnected,
|
|
5705
|
-
recordServiceConnectionAction,
|
|
5706
|
-
claimServiceConnectionAction,
|
|
5707
5468
|
AnalyticsProviderRegistryError,
|
|
5708
5469
|
getAnalyticsProviderDefinition,
|
|
5709
5470
|
assertConnectionSupportsProvider,
|