@shuffleio/shuffle-mcps 0.0.64 → 0.0.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{appsCache-A4OIDXIN.mjs → appsCache-QAL2GC5W.mjs} +1 -2
- package/dist/{chunk-2PFFYHRO.mjs → chunk-V3R4HDJT.mjs} +65 -16
- package/dist/index.d.mts +339 -334
- package/dist/index.d.ts +339 -334
- package/dist/index.js +12264 -69346
- package/dist/index.mjs +13557 -69914
- package/package.json +13 -3
- package/dist/analytics-JGBV375U.mjs +0 -20
- package/dist/auth-utils-HYVXQKTL.mjs +0 -12
- package/dist/chunk-2NCPSII6.mjs +0 -105
- package/dist/chunk-C7HCHOHU.mjs +0 -1208
- package/dist/chunk-PL74JBB4.mjs +0 -64
- package/dist/chunk-TUSSDUVQ.mjs +0 -155
- package/dist/datastore-TTQFTTDP.mjs +0 -27
package/dist/chunk-C7HCHOHU.mjs
DELETED
|
@@ -1,1208 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
getApiUrl,
|
|
3
|
-
getAuthHeader
|
|
4
|
-
} from "./chunk-2PFFYHRO.mjs";
|
|
5
|
-
|
|
6
|
-
// requestScheduler.ts
|
|
7
|
-
var MAX_CONCURRENT = 6;
|
|
8
|
-
var MAX_PER_SECOND = 16;
|
|
9
|
-
var priorityQueue = [];
|
|
10
|
-
var backgroundQueue = [];
|
|
11
|
-
var active = 0;
|
|
12
|
-
var recentStarts = [];
|
|
13
|
-
var pump = () => {
|
|
14
|
-
const now = Date.now();
|
|
15
|
-
recentStarts = recentStarts.filter((t) => now - t < 1e3);
|
|
16
|
-
while (active < MAX_CONCURRENT && recentStarts.length < MAX_PER_SECOND) {
|
|
17
|
-
const next = priorityQueue.shift() || backgroundQueue.shift();
|
|
18
|
-
if (!next) return;
|
|
19
|
-
active += 1;
|
|
20
|
-
recentStarts.push(Date.now());
|
|
21
|
-
next();
|
|
22
|
-
}
|
|
23
|
-
if ((priorityQueue.length || backgroundQueue.length) && recentStarts.length >= MAX_PER_SECOND) {
|
|
24
|
-
const wait2 = Math.max(20, 1e3 - (Date.now() - recentStarts[0]));
|
|
25
|
-
setTimeout(pump, wait2);
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
var acquireDatastoreSlot = (priority = false) => new Promise((resolve) => {
|
|
29
|
-
let released = false;
|
|
30
|
-
const release = () => {
|
|
31
|
-
if (released) return;
|
|
32
|
-
released = true;
|
|
33
|
-
active = Math.max(0, active - 1);
|
|
34
|
-
pump();
|
|
35
|
-
};
|
|
36
|
-
const start = () => resolve(release);
|
|
37
|
-
if (priority) priorityQueue.push(start);
|
|
38
|
-
else backgroundQueue.push(start);
|
|
39
|
-
pump();
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
// ../Shuffle-Core/fetchBreaker.ts
|
|
43
|
-
var rewriteShufflerApiToShuffleSecurity = (input) => {
|
|
44
|
-
if (typeof window === "undefined") return input;
|
|
45
|
-
const currentHost = window.location.hostname.toLowerCase();
|
|
46
|
-
const isOnShuffleSecurity = currentHost === "shuffle.security" || currentHost.endsWith(".shuffle.security") || currentHost === "localhost" || currentHost === "127.0.0.1" || currentHost.includes("lovable");
|
|
47
|
-
if (!isOnShuffleSecurity) return input;
|
|
48
|
-
let urlStr = null;
|
|
49
|
-
if (typeof input === "string") urlStr = input;
|
|
50
|
-
else if (input instanceof URL) urlStr = input.toString();
|
|
51
|
-
else if (input && typeof input.url === "string") urlStr = input.url;
|
|
52
|
-
if (!urlStr) return input;
|
|
53
|
-
if (!urlStr.includes("/api/") || urlStr.includes("/api/v1/auth/exchange")) {
|
|
54
|
-
return input;
|
|
55
|
-
}
|
|
56
|
-
try {
|
|
57
|
-
const u = new URL(urlStr, window.location.href);
|
|
58
|
-
const host = u.hostname.toLowerCase();
|
|
59
|
-
if (host === "shuffler.io" || host === "www.shuffler.io") {
|
|
60
|
-
u.protocol = "https:";
|
|
61
|
-
u.hostname = "uk.shuffle.security";
|
|
62
|
-
const rewritten = u.toString();
|
|
63
|
-
if (typeof input === "string") return rewritten;
|
|
64
|
-
if (input instanceof URL) return new URL(rewritten);
|
|
65
|
-
if (typeof Request !== "undefined" && input instanceof Request) return new Request(rewritten, input);
|
|
66
|
-
} else if (host.endsWith(".shuffler.io")) {
|
|
67
|
-
const subdomain = host.slice(0, -".shuffler.io".length);
|
|
68
|
-
u.protocol = "https:";
|
|
69
|
-
u.hostname = `${subdomain}.shuffle.security`;
|
|
70
|
-
const rewritten = u.toString();
|
|
71
|
-
if (typeof input === "string") return rewritten;
|
|
72
|
-
if (input instanceof URL) return new URL(rewritten);
|
|
73
|
-
if (typeof Request !== "undefined" && input instanceof Request) return new Request(rewritten, input);
|
|
74
|
-
}
|
|
75
|
-
} catch {
|
|
76
|
-
}
|
|
77
|
-
return input;
|
|
78
|
-
};
|
|
79
|
-
var FAIL_THRESHOLD = 12;
|
|
80
|
-
var ROLLING_WINDOW_MS = 5e3;
|
|
81
|
-
var COOLDOWN_MS = 1e4;
|
|
82
|
-
var PROBE_AFTER_MS = 1500;
|
|
83
|
-
var MAX_PROBE_BACKOFF_MS = 5e3;
|
|
84
|
-
var HARD_BURST_THRESHOLD = 30;
|
|
85
|
-
var HARD_BURST_WINDOW_MS = 1e3;
|
|
86
|
-
var state = /* @__PURE__ */ new Map();
|
|
87
|
-
var installed = false;
|
|
88
|
-
var allowedOrigins = /* @__PURE__ */ new Set();
|
|
89
|
-
var getEntry = (key) => {
|
|
90
|
-
let entry = state.get(key);
|
|
91
|
-
if (!entry) {
|
|
92
|
-
entry = { failures: [], attempts: [], blockedUntil: 0, probeAfter: 0, probing: false, warned: false };
|
|
93
|
-
state.set(key, entry);
|
|
94
|
-
}
|
|
95
|
-
return entry;
|
|
96
|
-
};
|
|
97
|
-
var trip = (entry, now, cooldownMs = COOLDOWN_MS) => {
|
|
98
|
-
entry.blockedUntil = now + cooldownMs;
|
|
99
|
-
entry.probeAfter = now + PROBE_AFTER_MS;
|
|
100
|
-
entry.probing = false;
|
|
101
|
-
};
|
|
102
|
-
var reset = (entry) => {
|
|
103
|
-
entry.failures.length = 0;
|
|
104
|
-
entry.attempts.length = 0;
|
|
105
|
-
entry.blockedUntil = 0;
|
|
106
|
-
entry.probeAfter = 0;
|
|
107
|
-
entry.probing = false;
|
|
108
|
-
entry.warned = false;
|
|
109
|
-
};
|
|
110
|
-
var prune = (arr, now, windowMs) => {
|
|
111
|
-
const cutoff = now - windowMs;
|
|
112
|
-
while (arr.length && arr[0] < cutoff) arr.shift();
|
|
113
|
-
};
|
|
114
|
-
var keyFor = (input, init) => {
|
|
115
|
-
let urlStr;
|
|
116
|
-
let method = (init?.method || (typeof input !== "string" && "method" in input ? input.method : "GET") || "GET").toUpperCase();
|
|
117
|
-
if (typeof input === "string") urlStr = input;
|
|
118
|
-
else if (input instanceof URL) urlStr = input.toString();
|
|
119
|
-
else if (input && typeof input.url === "string") urlStr = input.url;
|
|
120
|
-
else return null;
|
|
121
|
-
try {
|
|
122
|
-
const u = new URL(urlStr, typeof window !== "undefined" ? window.location.href : "http://localhost");
|
|
123
|
-
if (allowedOrigins.size && !allowedOrigins.has(u.origin)) return null;
|
|
124
|
-
if (!u.pathname.startsWith("/api/")) return null;
|
|
125
|
-
return `${method} ${u.origin}${u.pathname}`;
|
|
126
|
-
} catch {
|
|
127
|
-
return null;
|
|
128
|
-
}
|
|
129
|
-
};
|
|
130
|
-
var withStoredAuth = (input, init) => {
|
|
131
|
-
if (typeof window === "undefined") return init;
|
|
132
|
-
let urlStr;
|
|
133
|
-
if (typeof input === "string") urlStr = input;
|
|
134
|
-
else if (input instanceof URL) urlStr = input.toString();
|
|
135
|
-
else if (input && typeof input.url === "string") urlStr = input.url;
|
|
136
|
-
else return init;
|
|
137
|
-
try {
|
|
138
|
-
const url = new URL(urlStr, window.location.href);
|
|
139
|
-
const isKnownShuffleOrigin = allowedOrigins.has(url.origin) || url.origin === window.location.origin || url.hostname === "shuffler.io" || url.hostname.endsWith(".shuffler.io") || url.hostname === "tunnel.schemaless.org" || url.hostname === "shuffle.security" || url.hostname.endsWith(".shuffle.security");
|
|
140
|
-
if (!url.pathname.startsWith("/api/") || !isKnownShuffleOrigin) return init;
|
|
141
|
-
if (url.pathname === "/api/v1/login" || url.pathname === "/api/v1/users/register") return init;
|
|
142
|
-
const sessionToken = window.localStorage.getItem("session_token");
|
|
143
|
-
const token = sessionToken && sessionToken.trim().length > 0 ? sessionToken.trim() : null;
|
|
144
|
-
const authMode = window.localStorage.getItem("shuffle_auth_mode");
|
|
145
|
-
const isCapacitor = Boolean(
|
|
146
|
-
window.Capacitor?.isNativePlatform?.() || window._isCapacitorNative
|
|
147
|
-
);
|
|
148
|
-
const requestHeaders = typeof Request !== "undefined" && input instanceof Request ? input.headers : void 0;
|
|
149
|
-
const headers = new Headers(requestHeaders);
|
|
150
|
-
if (init?.headers) {
|
|
151
|
-
new Headers(init.headers).forEach((value, name) => headers.set(name, value));
|
|
152
|
-
}
|
|
153
|
-
const rootDomain = (host) => {
|
|
154
|
-
const parts = host.split(".");
|
|
155
|
-
return parts.length <= 2 ? host : parts.slice(-2).join(".");
|
|
156
|
-
};
|
|
157
|
-
const backendHost = url.hostname.toLowerCase();
|
|
158
|
-
const frontendHost = window.location.hostname.toLowerCase();
|
|
159
|
-
const isLocalPair = ["localhost", "127.0.0.1"].includes(backendHost) && ["localhost", "127.0.0.1"].includes(frontendHost);
|
|
160
|
-
const isCrossDomain = backendHost !== frontendHost && !isLocalPair && rootDomain(backendHost) !== rootDomain(frontendHost);
|
|
161
|
-
if (token && (authMode === "bearer" || isCapacitor || isCrossDomain) && !headers.has("Authorization")) {
|
|
162
|
-
headers.set("Authorization", `Bearer ${token}`);
|
|
163
|
-
}
|
|
164
|
-
if (!headers.has("Org-Id")) {
|
|
165
|
-
try {
|
|
166
|
-
const raw = window.localStorage.getItem("shuffle_user_info");
|
|
167
|
-
if (raw) {
|
|
168
|
-
const parsed = JSON.parse(raw);
|
|
169
|
-
const orgId = parsed?.active_org?.id;
|
|
170
|
-
if (orgId) headers.set("Org-Id", orgId);
|
|
171
|
-
}
|
|
172
|
-
} catch {
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
return {
|
|
176
|
-
...init,
|
|
177
|
-
credentials: init?.credentials || "include",
|
|
178
|
-
headers
|
|
179
|
-
};
|
|
180
|
-
} catch {
|
|
181
|
-
return init;
|
|
182
|
-
}
|
|
183
|
-
};
|
|
184
|
-
var synthetic503 = (key, retryInMs) => new Response(
|
|
185
|
-
JSON.stringify({
|
|
186
|
-
error: "circuit_breaker_open",
|
|
187
|
-
message: `Too many failures for ${key}. Cooling down for ${Math.ceil(retryInMs / 1e3)}s.`
|
|
188
|
-
}),
|
|
189
|
-
{
|
|
190
|
-
status: 503,
|
|
191
|
-
statusText: "Service Unavailable (circuit breaker)",
|
|
192
|
-
headers: {
|
|
193
|
-
"Content-Type": "application/json",
|
|
194
|
-
"Retry-After": String(Math.ceil(retryInMs / 1e3)),
|
|
195
|
-
"X-Circuit-Breaker": "open"
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
);
|
|
199
|
-
var isCircuitBreakerResponse = (res) => {
|
|
200
|
-
if (!res) return false;
|
|
201
|
-
if (res.headers?.get?.("X-Circuit-Breaker") === "open") return true;
|
|
202
|
-
return res.status === 503 && /circuit breaker/i.test(res.statusText || "");
|
|
203
|
-
};
|
|
204
|
-
var registerProtectedOrigin = (url) => {
|
|
205
|
-
if (!url) return;
|
|
206
|
-
try {
|
|
207
|
-
const origin = new URL(url, typeof window !== "undefined" ? window.location.href : "http://localhost").origin;
|
|
208
|
-
allowedOrigins.add(origin);
|
|
209
|
-
} catch {
|
|
210
|
-
}
|
|
211
|
-
};
|
|
212
|
-
var installFetchBreaker = () => {
|
|
213
|
-
if (installed) return;
|
|
214
|
-
if (typeof window === "undefined" || typeof window.fetch !== "function") return;
|
|
215
|
-
installed = true;
|
|
216
|
-
const original = window.fetch.bind(window);
|
|
217
|
-
window.fetch = (async (input, init) => {
|
|
218
|
-
const finalInput = rewriteShufflerApiToShuffleSecurity(input);
|
|
219
|
-
const authenticatedInit = withStoredAuth(finalInput, init);
|
|
220
|
-
const key = keyFor(finalInput, authenticatedInit);
|
|
221
|
-
if (!key) return original(finalInput, authenticatedInit);
|
|
222
|
-
const entry = getEntry(key);
|
|
223
|
-
const now = Date.now();
|
|
224
|
-
let isProbe = false;
|
|
225
|
-
if (entry.blockedUntil > now) {
|
|
226
|
-
if (!entry.probing && now >= entry.probeAfter) {
|
|
227
|
-
entry.probing = true;
|
|
228
|
-
isProbe = true;
|
|
229
|
-
} else {
|
|
230
|
-
return synthetic503(key, entry.blockedUntil - now);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
if (!isProbe) {
|
|
234
|
-
entry.attempts.push(now);
|
|
235
|
-
prune(entry.attempts, now, HARD_BURST_WINDOW_MS);
|
|
236
|
-
if (entry.attempts.length > HARD_BURST_THRESHOLD) {
|
|
237
|
-
trip(entry, now);
|
|
238
|
-
if (!entry.warned) {
|
|
239
|
-
entry.warned = true;
|
|
240
|
-
console.error(
|
|
241
|
-
`[fetchBreaker] Burst detected on ${key} (${entry.attempts.length} calls / ${HARD_BURST_WINDOW_MS}ms). Blocking for up to ${COOLDOWN_MS / 1e3}s (probing again in ${PROBE_AFTER_MS / 1e3}s). Fix the caller \u2014 this is almost always a missing useEffect dep or a render loop.`
|
|
242
|
-
);
|
|
243
|
-
}
|
|
244
|
-
return synthetic503(key, COOLDOWN_MS);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
const onProbeFailure = (ts) => {
|
|
248
|
-
if (!isProbe) return;
|
|
249
|
-
entry.probing = false;
|
|
250
|
-
entry.probeAfter = ts + Math.min(MAX_PROBE_BACKOFF_MS, PROBE_AFTER_MS * 2);
|
|
251
|
-
entry.blockedUntil = Math.max(entry.blockedUntil, ts + PROBE_AFTER_MS);
|
|
252
|
-
};
|
|
253
|
-
let response;
|
|
254
|
-
try {
|
|
255
|
-
response = await original(finalInput, authenticatedInit);
|
|
256
|
-
} catch (err) {
|
|
257
|
-
const ts = Date.now();
|
|
258
|
-
entry.failures.push(ts);
|
|
259
|
-
prune(entry.failures, ts, ROLLING_WINDOW_MS);
|
|
260
|
-
onProbeFailure(ts);
|
|
261
|
-
if (!isProbe && entry.failures.length >= FAIL_THRESHOLD) {
|
|
262
|
-
trip(entry, ts);
|
|
263
|
-
if (!entry.warned) {
|
|
264
|
-
entry.warned = true;
|
|
265
|
-
console.error(
|
|
266
|
-
`[fetchBreaker] ${entry.failures.length} network errors on ${key} in ${ROLLING_WINDOW_MS / 1e3}s. Blocking for up to ${COOLDOWN_MS / 1e3}s.`
|
|
267
|
-
);
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
throw err;
|
|
271
|
-
}
|
|
272
|
-
if (response.status >= 500 || response.status === 0) {
|
|
273
|
-
const ts = Date.now();
|
|
274
|
-
entry.failures.push(ts);
|
|
275
|
-
prune(entry.failures, ts, ROLLING_WINDOW_MS);
|
|
276
|
-
onProbeFailure(ts);
|
|
277
|
-
if (!isProbe && entry.failures.length >= FAIL_THRESHOLD) {
|
|
278
|
-
trip(entry, ts);
|
|
279
|
-
if (!entry.warned) {
|
|
280
|
-
entry.warned = true;
|
|
281
|
-
console.error(
|
|
282
|
-
`[fetchBreaker] ${entry.failures.length} ${response.status}s on ${key} in ${ROLLING_WINDOW_MS / 1e3}s. Blocking for up to ${COOLDOWN_MS / 1e3}s.`
|
|
283
|
-
);
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
} else if (response.status < 400) {
|
|
287
|
-
reset(entry);
|
|
288
|
-
} else {
|
|
289
|
-
if (isProbe) reset(entry);
|
|
290
|
-
}
|
|
291
|
-
return response;
|
|
292
|
-
});
|
|
293
|
-
};
|
|
294
|
-
|
|
295
|
-
// datastore.ts
|
|
296
|
-
var _runtimeOrgId = null;
|
|
297
|
-
var setRuntimeOrgId = (orgId) => {
|
|
298
|
-
_runtimeOrgId = orgId || null;
|
|
299
|
-
};
|
|
300
|
-
var getOrgId = () => {
|
|
301
|
-
if (_runtimeOrgId) return _runtimeOrgId;
|
|
302
|
-
try {
|
|
303
|
-
const userInfo = localStorage.getItem("shuffle_user_info") || localStorage.getItem("userinfo") || localStorage.getItem("user_info");
|
|
304
|
-
if (userInfo) {
|
|
305
|
-
const parsed = JSON.parse(userInfo);
|
|
306
|
-
return parsed.active_org?.id || parsed.org_id || parsed.active_org_id || parsed.orgs && parsed.orgs[0]?.id || null;
|
|
307
|
-
}
|
|
308
|
-
} catch {
|
|
309
|
-
}
|
|
310
|
-
return null;
|
|
311
|
-
};
|
|
312
|
-
var ORG_ID_WAIT_MS = 8e3;
|
|
313
|
-
var ORG_ID_POLL_MS = 100;
|
|
314
|
-
var waitForOrgId = async () => {
|
|
315
|
-
const immediate = getOrgId();
|
|
316
|
-
if (immediate) return immediate;
|
|
317
|
-
if (typeof window === "undefined") return null;
|
|
318
|
-
const deadline = Date.now() + ORG_ID_WAIT_MS;
|
|
319
|
-
while (Date.now() < deadline) {
|
|
320
|
-
await new Promise((resolve) => setTimeout(resolve, ORG_ID_POLL_MS));
|
|
321
|
-
const orgId = getOrgId();
|
|
322
|
-
if (orgId) return orgId;
|
|
323
|
-
}
|
|
324
|
-
return null;
|
|
325
|
-
};
|
|
326
|
-
var decodeDatastoreKey = (key) => {
|
|
327
|
-
if (!key) return "";
|
|
328
|
-
if (!key.includes("%")) return key;
|
|
329
|
-
try {
|
|
330
|
-
return decodeURIComponent(key);
|
|
331
|
-
} catch {
|
|
332
|
-
return key.replace(/%3a/gi, ":");
|
|
333
|
-
}
|
|
334
|
-
};
|
|
335
|
-
var normalizeDatastoreKey = (key) => {
|
|
336
|
-
if (!key) return key;
|
|
337
|
-
const decoded = decodeDatastoreKey(key);
|
|
338
|
-
if (!decoded.includes("::")) return decoded;
|
|
339
|
-
const parts = decoded.split("::").filter(Boolean);
|
|
340
|
-
return parts.length > 0 ? parts[parts.length - 1] : decoded;
|
|
341
|
-
};
|
|
342
|
-
var extractOrgIdFromDatastoreKey = (key) => {
|
|
343
|
-
if (!key) return null;
|
|
344
|
-
const decoded = decodeDatastoreKey(key);
|
|
345
|
-
if (!decoded.includes("::")) return null;
|
|
346
|
-
const parts = decoded.split("::").filter(Boolean);
|
|
347
|
-
return parts.length > 1 ? parts[0] : null;
|
|
348
|
-
};
|
|
349
|
-
var truncateResponsePreview = (value, maxLength = 280) => {
|
|
350
|
-
if (!value) return void 0;
|
|
351
|
-
const normalized = value.replace(/\s+/g, " ").trim();
|
|
352
|
-
if (!normalized) return void 0;
|
|
353
|
-
return normalized.length > maxLength ? `${normalized.slice(0, maxLength)}\u2026` : normalized;
|
|
354
|
-
};
|
|
355
|
-
var wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
356
|
-
var isTransientDatastoreStatus = (status) => {
|
|
357
|
-
return status === 408 || status === 425 || status === 429 || status >= 500;
|
|
358
|
-
};
|
|
359
|
-
var isSameDatastoreKey = (a, b) => {
|
|
360
|
-
if (!a) return false;
|
|
361
|
-
return normalizeDatastoreKey(a) === normalizeDatastoreKey(b);
|
|
362
|
-
};
|
|
363
|
-
var serializeDatastoreValue = (value) => {
|
|
364
|
-
if (typeof value === "string") {
|
|
365
|
-
const trimmed = value.trim();
|
|
366
|
-
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
367
|
-
try {
|
|
368
|
-
return JSON.stringify(JSON.parse(trimmed));
|
|
369
|
-
} catch {
|
|
370
|
-
return value;
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
return value;
|
|
374
|
-
}
|
|
375
|
-
try {
|
|
376
|
-
return JSON.stringify(value);
|
|
377
|
-
} catch {
|
|
378
|
-
return String(value);
|
|
379
|
-
}
|
|
380
|
-
};
|
|
381
|
-
var DATASTORE_VALUE_SOFT_LIMIT_BYTES = 9e5;
|
|
382
|
-
var DATASTORE_VALUE_HARD_LIMIT_BYTES = 14e5;
|
|
383
|
-
var byteLength = (s) => {
|
|
384
|
-
try {
|
|
385
|
-
return new Blob([s]).size;
|
|
386
|
-
} catch {
|
|
387
|
-
return s.length;
|
|
388
|
-
}
|
|
389
|
-
};
|
|
390
|
-
var looksLikeValueTooBig = (status, body) => {
|
|
391
|
-
if (status < 500) return false;
|
|
392
|
-
const b = (body || "").toLowerCase();
|
|
393
|
-
return b.includes("value") && (b.includes("too big") || b.includes("too large") || b.includes("size") || b.includes("limit"));
|
|
394
|
-
};
|
|
395
|
-
var slimIncidentValueForWrite = (raw) => {
|
|
396
|
-
let obj;
|
|
397
|
-
try {
|
|
398
|
-
obj = JSON.parse(raw);
|
|
399
|
-
} catch {
|
|
400
|
-
return null;
|
|
401
|
-
}
|
|
402
|
-
if (!obj || typeof obj !== "object") return null;
|
|
403
|
-
const dropped = [];
|
|
404
|
-
const stripBase64Attachments = (node) => {
|
|
405
|
-
if (!node || typeof node !== "object") return;
|
|
406
|
-
if (Array.isArray(node.attachments)) {
|
|
407
|
-
for (const a of node.attachments) {
|
|
408
|
-
if (a && typeof a === "object" && typeof a.data === "string" && a.data.length > 4e3) {
|
|
409
|
-
a.data = "";
|
|
410
|
-
a._truncated = true;
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
for (const k of Object.keys(node)) {
|
|
415
|
-
const v = node[k];
|
|
416
|
-
if (v && typeof v === "object") stripBase64Attachments(v);
|
|
417
|
-
}
|
|
418
|
-
};
|
|
419
|
-
stripBase64Attachments(obj);
|
|
420
|
-
dropped.push("attachments.data");
|
|
421
|
-
const capString = (s, cap = 2e5) => typeof s === "string" && s.length > cap ? s.slice(0, cap) + "\u2026[truncated]" : s;
|
|
422
|
-
if (obj.email && typeof obj.email === "object") {
|
|
423
|
-
if (typeof obj.email.body_html === "string") obj.email.body_html = capString(obj.email.body_html);
|
|
424
|
-
if (typeof obj.email.body === "string") obj.email.body = capString(obj.email.body);
|
|
425
|
-
if (Array.isArray(obj.email.messages)) {
|
|
426
|
-
for (const m of obj.email.messages) {
|
|
427
|
-
if (m && typeof m === "object") {
|
|
428
|
-
if (typeof m.body_html === "string") m.body_html = capString(m.body_html);
|
|
429
|
-
if (typeof m.body === "string") m.body = capString(m.body);
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
dropped.push("email.body_html");
|
|
434
|
-
}
|
|
435
|
-
if (obj.rawOCSF?.unmapped_original) {
|
|
436
|
-
obj.rawOCSF = { ...obj.rawOCSF };
|
|
437
|
-
delete obj.rawOCSF.unmapped_original;
|
|
438
|
-
dropped.push("rawOCSF.unmapped_original");
|
|
439
|
-
}
|
|
440
|
-
if (Array.isArray(obj.activity) && obj.activity.length > 200) {
|
|
441
|
-
obj.activity = obj.activity.slice(-200);
|
|
442
|
-
dropped.push("activity[older]");
|
|
443
|
-
}
|
|
444
|
-
if (Array.isArray(obj.timeline) && obj.timeline.length > 200) {
|
|
445
|
-
obj.timeline = obj.timeline.slice(-200);
|
|
446
|
-
dropped.push("timeline[older]");
|
|
447
|
-
}
|
|
448
|
-
return { value: JSON.stringify(obj), dropped };
|
|
449
|
-
};
|
|
450
|
-
var parseWriteResponseBody = (bodyText) => {
|
|
451
|
-
if (!bodyText) return {};
|
|
452
|
-
try {
|
|
453
|
-
const parsed = JSON.parse(bodyText);
|
|
454
|
-
if (!parsed || typeof parsed !== "object") return {};
|
|
455
|
-
const rawKeys = Array.isArray(parsed.keys_existed) ? parsed.keys_existed : void 0;
|
|
456
|
-
return {
|
|
457
|
-
success: typeof parsed.success === "boolean" ? parsed.success : void 0,
|
|
458
|
-
keysExisted: rawKeys?.map((k) => ({
|
|
459
|
-
key: String(k?.key ?? ""),
|
|
460
|
-
existed: !!k?.existed,
|
|
461
|
-
changed: !!k?.changed
|
|
462
|
-
}))
|
|
463
|
-
};
|
|
464
|
-
} catch {
|
|
465
|
-
return {};
|
|
466
|
-
}
|
|
467
|
-
};
|
|
468
|
-
var isUnchangedWrite = (keysExisted) => !!keysExisted && keysExisted.length > 0 && keysExisted.every((k) => k.existed && !k.changed);
|
|
469
|
-
var datastoreUrl = (path, regionUrl) => regionUrl ? `${regionUrl.replace(/\/+$/, "")}${path}` : getApiUrl(path);
|
|
470
|
-
var setDatastoreItem = async (key, value, category, overrideOrgId, options) => {
|
|
471
|
-
const extractedOrgId = !overrideOrgId ? extractOrgIdFromDatastoreKey(key) : null;
|
|
472
|
-
const orgId = overrideOrgId || extractedOrgId || await waitForOrgId();
|
|
473
|
-
if (!orgId) {
|
|
474
|
-
return { success: false, error: "No organization ID found" };
|
|
475
|
-
}
|
|
476
|
-
const rawKey = normalizeDatastoreKey(key);
|
|
477
|
-
let serialized = serializeDatastoreValue(value);
|
|
478
|
-
if (category === "shuffle-security_incidents" && byteLength(serialized) > DATASTORE_VALUE_SOFT_LIMIT_BYTES) {
|
|
479
|
-
const slim = slimIncidentValueForWrite(serialized);
|
|
480
|
-
if (slim) {
|
|
481
|
-
console.warn(
|
|
482
|
-
`[datastore.set] pre-slim incident ${rawKey} bytes=${byteLength(serialized)} -> ${byteLength(slim.value)} dropped=${slim.dropped.join(",")}`
|
|
483
|
-
);
|
|
484
|
-
serialized = slim.value;
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
if (byteLength(serialized) > DATASTORE_VALUE_HARD_LIMIT_BYTES) {
|
|
488
|
-
return {
|
|
489
|
-
success: false,
|
|
490
|
-
error: `Datastore value too large (${byteLength(serialized)} bytes) after slimming; write aborted for ${rawKey}`
|
|
491
|
-
};
|
|
492
|
-
}
|
|
493
|
-
const buildPayload = (v) => [{
|
|
494
|
-
key: rawKey,
|
|
495
|
-
value: v,
|
|
496
|
-
category,
|
|
497
|
-
org_id: orgId,
|
|
498
|
-
...category === "shuffle-security_incidents" ? { ignore_security_rules: true } : {}
|
|
499
|
-
}];
|
|
500
|
-
const headers = {
|
|
501
|
-
"Content-Type": "application/json",
|
|
502
|
-
...getAuthHeader(orgId)
|
|
503
|
-
};
|
|
504
|
-
console.log(`[datastore.set] key=${rawKey} category=${category} orgId=${orgId} bytes=${byteLength(serialized)}${overrideOrgId ? " (override)" : ""}`);
|
|
505
|
-
const send = (v) => fetch(datastoreUrl("/api/v2/datastore", options?.regionUrl), {
|
|
506
|
-
method: "POST",
|
|
507
|
-
credentials: "include",
|
|
508
|
-
headers,
|
|
509
|
-
body: JSON.stringify(buildPayload(v))
|
|
510
|
-
});
|
|
511
|
-
let response = await send(serialized);
|
|
512
|
-
if (!response.ok) {
|
|
513
|
-
const bodyText = await response.text().catch(() => "");
|
|
514
|
-
const parsed = parseWriteResponseBody(bodyText);
|
|
515
|
-
if (isUnchangedWrite(parsed.keysExisted)) {
|
|
516
|
-
console.log(`[datastore.set] key=${rawKey} unchanged (value identical) \u2014 treated as no-op`);
|
|
517
|
-
return { success: true, changed: false, keysExisted: parsed.keysExisted };
|
|
518
|
-
}
|
|
519
|
-
if (category === "shuffle-security_incidents" && looksLikeValueTooBig(response.status, bodyText)) {
|
|
520
|
-
const slim = slimIncidentValueForWrite(serialized);
|
|
521
|
-
if (slim && slim.value.length < serialized.length) {
|
|
522
|
-
console.warn(
|
|
523
|
-
`[datastore.set] backend rejected large value for ${rawKey}; retrying slimmed bytes=${byteLength(slim.value)} dropped=${slim.dropped.join(",")}`
|
|
524
|
-
);
|
|
525
|
-
response = await send(slim.value);
|
|
526
|
-
if (response.ok) {
|
|
527
|
-
const retryOk = parseWriteResponseBody(await response.text().catch(() => ""));
|
|
528
|
-
return {
|
|
529
|
-
success: true,
|
|
530
|
-
keysExisted: retryOk.keysExisted,
|
|
531
|
-
changed: retryOk.keysExisted ? retryOk.keysExisted.some((k) => k.changed) : void 0
|
|
532
|
-
};
|
|
533
|
-
}
|
|
534
|
-
const retryBody = await response.text().catch(() => "");
|
|
535
|
-
const retryParsed = parseWriteResponseBody(retryBody);
|
|
536
|
-
if (isUnchangedWrite(retryParsed.keysExisted)) {
|
|
537
|
-
return { success: true, changed: false, keysExisted: retryParsed.keysExisted };
|
|
538
|
-
}
|
|
539
|
-
return {
|
|
540
|
-
success: false,
|
|
541
|
-
error: `Datastore value too large for ${rawKey} even after slimming: ${response.status} ${truncateResponsePreview(retryBody) || response.statusText}`,
|
|
542
|
-
keysExisted: retryParsed.keysExisted
|
|
543
|
-
};
|
|
544
|
-
}
|
|
545
|
-
return {
|
|
546
|
-
success: false,
|
|
547
|
-
error: `Datastore value too large for ${rawKey}: ${response.status} ${truncateResponsePreview(bodyText) || response.statusText}`,
|
|
548
|
-
keysExisted: parsed.keysExisted
|
|
549
|
-
};
|
|
550
|
-
}
|
|
551
|
-
return {
|
|
552
|
-
success: false,
|
|
553
|
-
error: `Failed to set datastore item: ${response.status} ${truncateResponsePreview(bodyText) || response.statusText}`,
|
|
554
|
-
keysExisted: parsed.keysExisted
|
|
555
|
-
};
|
|
556
|
-
}
|
|
557
|
-
const okParsed = parseWriteResponseBody(await response.text().catch(() => ""));
|
|
558
|
-
if (okParsed.success === false && !isUnchangedWrite(okParsed.keysExisted)) {
|
|
559
|
-
return {
|
|
560
|
-
success: false,
|
|
561
|
-
error: `Failed to set datastore item ${rawKey}: backend reported success=false`,
|
|
562
|
-
keysExisted: okParsed.keysExisted
|
|
563
|
-
};
|
|
564
|
-
}
|
|
565
|
-
return {
|
|
566
|
-
success: true,
|
|
567
|
-
keysExisted: okParsed.keysExisted,
|
|
568
|
-
changed: okParsed.keysExisted ? okParsed.keysExisted.some((k) => k.changed) : void 0
|
|
569
|
-
};
|
|
570
|
-
};
|
|
571
|
-
var setDatastoreItems = async (items, category) => {
|
|
572
|
-
const orgId = await waitForOrgId();
|
|
573
|
-
if (!orgId) {
|
|
574
|
-
return { success: false, error: "No organization ID found" };
|
|
575
|
-
}
|
|
576
|
-
const payload = items.map((item) => ({
|
|
577
|
-
key: item.key,
|
|
578
|
-
value: serializeDatastoreValue(item.value),
|
|
579
|
-
category,
|
|
580
|
-
...category === "shuffle-security_incidents" ? { ignore_security_rules: true } : {}
|
|
581
|
-
}));
|
|
582
|
-
const response = await fetch(getApiUrl("/api/v2/datastore"), {
|
|
583
|
-
method: "POST",
|
|
584
|
-
credentials: "include",
|
|
585
|
-
headers: {
|
|
586
|
-
"Content-Type": "application/json",
|
|
587
|
-
...getAuthHeader(orgId)
|
|
588
|
-
},
|
|
589
|
-
body: JSON.stringify(payload)
|
|
590
|
-
});
|
|
591
|
-
if (!response.ok) {
|
|
592
|
-
const bodyText = await response.text().catch(() => "");
|
|
593
|
-
const parsed = parseWriteResponseBody(bodyText);
|
|
594
|
-
if (isUnchangedWrite(parsed.keysExisted)) {
|
|
595
|
-
return { success: true, changed: false, keysExisted: parsed.keysExisted };
|
|
596
|
-
}
|
|
597
|
-
const fallbackResults = await Promise.all(items.map(
|
|
598
|
-
(item) => setDatastoreItem(item.key, item.value, category)
|
|
599
|
-
));
|
|
600
|
-
const failedWrites = fallbackResults.filter((result) => !result.success);
|
|
601
|
-
if (failedWrites.length > 0) {
|
|
602
|
-
return {
|
|
603
|
-
success: false,
|
|
604
|
-
error: failedWrites[0]?.error || `Failed to set datastore items: ${response.statusText}`,
|
|
605
|
-
keysExisted: fallbackResults.flatMap((r) => r.keysExisted || [])
|
|
606
|
-
};
|
|
607
|
-
}
|
|
608
|
-
const fallbackKeys = fallbackResults.flatMap((r) => r.keysExisted || []);
|
|
609
|
-
return {
|
|
610
|
-
success: true,
|
|
611
|
-
keysExisted: fallbackKeys.length ? fallbackKeys : void 0,
|
|
612
|
-
changed: fallbackKeys.length ? fallbackKeys.some((k) => k.changed) : void 0
|
|
613
|
-
};
|
|
614
|
-
}
|
|
615
|
-
const okParsed = parseWriteResponseBody(await response.text().catch(() => ""));
|
|
616
|
-
if (okParsed.keysExisted?.some((k) => !k.changed)) {
|
|
617
|
-
console.warn(
|
|
618
|
-
`[datastore.setMany] category=${category} unchanged keys: ${okParsed.keysExisted.filter((k) => !k.changed).map((k) => k.key).join(", ")}`
|
|
619
|
-
);
|
|
620
|
-
}
|
|
621
|
-
return {
|
|
622
|
-
success: okParsed.success !== false,
|
|
623
|
-
keysExisted: okParsed.keysExisted,
|
|
624
|
-
changed: okParsed.keysExisted ? okParsed.keysExisted.some((k) => k.changed) : void 0,
|
|
625
|
-
...okParsed.success === false ? { error: "Backend reported success=false for bulk datastore write" } : {}
|
|
626
|
-
};
|
|
627
|
-
};
|
|
628
|
-
var getDatastoreItem = async (key, category, overrideOrgId, options) => {
|
|
629
|
-
const extractedOrgId = !overrideOrgId ? extractOrgIdFromDatastoreKey(key) : null;
|
|
630
|
-
const orgId = overrideOrgId || extractedOrgId || await waitForOrgId();
|
|
631
|
-
if (!orgId) {
|
|
632
|
-
return { success: false, error: "No organization ID found" };
|
|
633
|
-
}
|
|
634
|
-
const rawKey = normalizeDatastoreKey(key);
|
|
635
|
-
const payload = {
|
|
636
|
-
key: rawKey,
|
|
637
|
-
org_id: orgId
|
|
638
|
-
};
|
|
639
|
-
if (category) {
|
|
640
|
-
payload.category = category;
|
|
641
|
-
}
|
|
642
|
-
const isVulnsCategory = category === "shuffle-security_vulns" || category === "shuffle-security_vulnerabilities" || category === "vulns";
|
|
643
|
-
const requestUrl = isVulnsCategory ? datastoreUrl(`/api/v2/vulns/${encodeURIComponent(rawKey)}`, options?.regionUrl) : datastoreUrl(`/api/v1/orgs/${orgId}/get_cache`, options?.regionUrl);
|
|
644
|
-
const baseDiagnostics = {
|
|
645
|
-
operation: "get",
|
|
646
|
-
category,
|
|
647
|
-
orgId,
|
|
648
|
-
url: requestUrl,
|
|
649
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
650
|
-
};
|
|
651
|
-
const headers = {
|
|
652
|
-
"Content-Type": "application/json",
|
|
653
|
-
...getAuthHeader(orgId)
|
|
654
|
-
};
|
|
655
|
-
console.log(`[datastore.get] key=${rawKey} category=${category} orgId=${orgId}${overrideOrgId ? " (override)" : ""}`);
|
|
656
|
-
let response = null;
|
|
657
|
-
let rawBody = "";
|
|
658
|
-
const maxAttempts = category === "shuffle-security_incidents" ? 3 : 2;
|
|
659
|
-
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
660
|
-
const release = await acquireDatastoreSlot(options?.priority === true);
|
|
661
|
-
try {
|
|
662
|
-
response = await fetch(requestUrl, {
|
|
663
|
-
method: isVulnsCategory ? "GET" : "POST",
|
|
664
|
-
credentials: "include",
|
|
665
|
-
headers,
|
|
666
|
-
body: isVulnsCategory ? void 0 : JSON.stringify(payload)
|
|
667
|
-
});
|
|
668
|
-
rawBody = await response.text();
|
|
669
|
-
if (response.ok || response.status === 404 || !isTransientDatastoreStatus(response.status) || attempt === maxAttempts - 1) {
|
|
670
|
-
break;
|
|
671
|
-
}
|
|
672
|
-
} catch (error) {
|
|
673
|
-
if (attempt === maxAttempts - 1) {
|
|
674
|
-
release();
|
|
675
|
-
return {
|
|
676
|
-
success: false,
|
|
677
|
-
error: error instanceof Error ? error.message : "Failed to get datastore item",
|
|
678
|
-
diagnostics: {
|
|
679
|
-
...baseDiagnostics,
|
|
680
|
-
errorStage: "request",
|
|
681
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
682
|
-
}
|
|
683
|
-
};
|
|
684
|
-
}
|
|
685
|
-
} finally {
|
|
686
|
-
release();
|
|
687
|
-
}
|
|
688
|
-
await wait(400 * (attempt + 1));
|
|
689
|
-
}
|
|
690
|
-
if (!response) {
|
|
691
|
-
return {
|
|
692
|
-
success: false,
|
|
693
|
-
error: "Failed to get datastore item",
|
|
694
|
-
diagnostics: {
|
|
695
|
-
...baseDiagnostics,
|
|
696
|
-
errorStage: "request",
|
|
697
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
698
|
-
}
|
|
699
|
-
};
|
|
700
|
-
}
|
|
701
|
-
if (!response.ok) {
|
|
702
|
-
if (response.status === 404) {
|
|
703
|
-
return {
|
|
704
|
-
success: true,
|
|
705
|
-
item: void 0,
|
|
706
|
-
diagnostics: {
|
|
707
|
-
...baseDiagnostics,
|
|
708
|
-
status: response.status,
|
|
709
|
-
statusText: response.statusText,
|
|
710
|
-
contentType: response.headers.get("content-type"),
|
|
711
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
712
|
-
errorStage: "response",
|
|
713
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
714
|
-
}
|
|
715
|
-
};
|
|
716
|
-
}
|
|
717
|
-
if (category === "shuffle-security_incidents" && isTransientDatastoreStatus(response.status) && !isCircuitBreakerResponse(response)) {
|
|
718
|
-
const fallbackItem = await findDatastoreItemInCategoryPages(rawKey, category, orgId);
|
|
719
|
-
if (fallbackItem) {
|
|
720
|
-
return {
|
|
721
|
-
success: true,
|
|
722
|
-
item: fallbackItem,
|
|
723
|
-
diagnostics: {
|
|
724
|
-
...baseDiagnostics,
|
|
725
|
-
status: response.status,
|
|
726
|
-
statusText: `${response.statusText || "transient"}; recovered via list_cache`,
|
|
727
|
-
contentType: response.headers.get("content-type"),
|
|
728
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
729
|
-
errorStage: "response",
|
|
730
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
731
|
-
}
|
|
732
|
-
};
|
|
733
|
-
}
|
|
734
|
-
}
|
|
735
|
-
if (isCircuitBreakerResponse(response)) {
|
|
736
|
-
return {
|
|
737
|
-
success: false,
|
|
738
|
-
error: "circuit_breaker_open",
|
|
739
|
-
diagnostics: {
|
|
740
|
-
...baseDiagnostics,
|
|
741
|
-
status: response.status,
|
|
742
|
-
statusText: response.statusText,
|
|
743
|
-
contentType: response.headers.get("content-type"),
|
|
744
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
745
|
-
errorStage: "response",
|
|
746
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
747
|
-
}
|
|
748
|
-
};
|
|
749
|
-
}
|
|
750
|
-
return {
|
|
751
|
-
success: false,
|
|
752
|
-
error: `Failed to get datastore item: ${response.status} ${response.statusText}`.trim(),
|
|
753
|
-
diagnostics: {
|
|
754
|
-
...baseDiagnostics,
|
|
755
|
-
status: response.status,
|
|
756
|
-
statusText: response.statusText,
|
|
757
|
-
contentType: response.headers.get("content-type"),
|
|
758
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
759
|
-
errorStage: "response",
|
|
760
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
761
|
-
}
|
|
762
|
-
};
|
|
763
|
-
}
|
|
764
|
-
let data;
|
|
765
|
-
try {
|
|
766
|
-
data = rawBody ? JSON.parse(rawBody) : {};
|
|
767
|
-
} catch (error) {
|
|
768
|
-
return {
|
|
769
|
-
success: false,
|
|
770
|
-
error: error instanceof Error ? `Failed to parse datastore item response: ${error.message}` : "Failed to parse datastore item response",
|
|
771
|
-
diagnostics: {
|
|
772
|
-
...baseDiagnostics,
|
|
773
|
-
status: response.status,
|
|
774
|
-
statusText: response.statusText,
|
|
775
|
-
contentType: response.headers.get("content-type"),
|
|
776
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
777
|
-
errorStage: "parse",
|
|
778
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
779
|
-
}
|
|
780
|
-
};
|
|
781
|
-
}
|
|
782
|
-
if (data.success === false && !data.value) {
|
|
783
|
-
return {
|
|
784
|
-
success: true,
|
|
785
|
-
item: void 0,
|
|
786
|
-
diagnostics: {
|
|
787
|
-
...baseDiagnostics,
|
|
788
|
-
status: response.status,
|
|
789
|
-
statusText: response.statusText,
|
|
790
|
-
contentType: response.headers.get("content-type"),
|
|
791
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
792
|
-
responseShape: "unknown",
|
|
793
|
-
itemCount: 0,
|
|
794
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
795
|
-
}
|
|
796
|
-
};
|
|
797
|
-
}
|
|
798
|
-
if (typeof data.value !== "string") {
|
|
799
|
-
if (data && typeof data === "object" && (data.id || data.title || data.cve_id || data.affected)) {
|
|
800
|
-
return {
|
|
801
|
-
success: true,
|
|
802
|
-
item: {
|
|
803
|
-
key: data.id || rawKey,
|
|
804
|
-
value: JSON.stringify(data),
|
|
805
|
-
category: category || "shuffle-security_vulns",
|
|
806
|
-
org_id: orgId,
|
|
807
|
-
...data
|
|
808
|
-
},
|
|
809
|
-
diagnostics: {
|
|
810
|
-
...baseDiagnostics,
|
|
811
|
-
status: response.status,
|
|
812
|
-
statusText: response.statusText,
|
|
813
|
-
contentType: response.headers.get("content-type"),
|
|
814
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
815
|
-
responseShape: "object",
|
|
816
|
-
itemCount: 1,
|
|
817
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
818
|
-
}
|
|
819
|
-
};
|
|
820
|
-
}
|
|
821
|
-
return {
|
|
822
|
-
success: true,
|
|
823
|
-
item: void 0,
|
|
824
|
-
diagnostics: {
|
|
825
|
-
...baseDiagnostics,
|
|
826
|
-
status: response.status,
|
|
827
|
-
statusText: response.statusText,
|
|
828
|
-
contentType: response.headers.get("content-type"),
|
|
829
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
830
|
-
responseShape: "unknown",
|
|
831
|
-
itemCount: 0,
|
|
832
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
833
|
-
}
|
|
834
|
-
};
|
|
835
|
-
}
|
|
836
|
-
return {
|
|
837
|
-
success: true,
|
|
838
|
-
item: data,
|
|
839
|
-
diagnostics: {
|
|
840
|
-
...baseDiagnostics,
|
|
841
|
-
status: response.status,
|
|
842
|
-
statusText: response.statusText,
|
|
843
|
-
contentType: response.headers.get("content-type"),
|
|
844
|
-
responseShape: "unknown",
|
|
845
|
-
itemCount: data?.key ? 1 : 0,
|
|
846
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
847
|
-
}
|
|
848
|
-
};
|
|
849
|
-
};
|
|
850
|
-
async function findDatastoreItemInCategoryPages(rawKey, category, orgId) {
|
|
851
|
-
let cursor;
|
|
852
|
-
for (let page = 0; page < 12; page += 1) {
|
|
853
|
-
const response = await getDatastoreByCategory(category, cursor, 100, orgId);
|
|
854
|
-
if (!response.success) return null;
|
|
855
|
-
const found = (response.data || []).find((item) => isSameDatastoreKey(item.key, rawKey));
|
|
856
|
-
if (found) return found;
|
|
857
|
-
if (!response.cursor) return null;
|
|
858
|
-
cursor = response.cursor;
|
|
859
|
-
}
|
|
860
|
-
return null;
|
|
861
|
-
}
|
|
862
|
-
var getDatastoreItemPublic = async (key, orgId, authorization) => {
|
|
863
|
-
const response = await fetch(
|
|
864
|
-
getApiUrl(`/api/v1/orgs/${orgId}/cache/${key}?authorization=${authorization}`),
|
|
865
|
-
{ method: "GET" }
|
|
866
|
-
);
|
|
867
|
-
if (!response.ok) {
|
|
868
|
-
if (response.status === 404) {
|
|
869
|
-
return { success: true, item: void 0 };
|
|
870
|
-
}
|
|
871
|
-
return { success: false, error: `Failed to get public datastore item: ${response.statusText}` };
|
|
872
|
-
}
|
|
873
|
-
const data = await response.json();
|
|
874
|
-
if (data.success === false && !data.value) {
|
|
875
|
-
return { success: true, item: void 0 };
|
|
876
|
-
}
|
|
877
|
-
return { success: true, item: data };
|
|
878
|
-
};
|
|
879
|
-
var tryExtractItemsFromBody = (rawBody) => {
|
|
880
|
-
try {
|
|
881
|
-
const data = rawBody ? JSON.parse(rawBody) : null;
|
|
882
|
-
if (!data || typeof data !== "object") return null;
|
|
883
|
-
const items = Array.isArray(data) ? data : data.keys || data.data || [];
|
|
884
|
-
if (!Array.isArray(items) || items.length === 0) return null;
|
|
885
|
-
if (!items.some((i) => i && typeof i === "object" && typeof i.key === "string")) return null;
|
|
886
|
-
const shape = Array.isArray(data) ? "array" : Array.isArray(data?.keys) ? "keys" : Array.isArray(data?.data) ? "data" : "unknown";
|
|
887
|
-
return {
|
|
888
|
-
items,
|
|
889
|
-
categoryConfig: data.category_config,
|
|
890
|
-
cursor: data.cursor,
|
|
891
|
-
totalAmount: data.total_amount,
|
|
892
|
-
shape
|
|
893
|
-
};
|
|
894
|
-
} catch {
|
|
895
|
-
return null;
|
|
896
|
-
}
|
|
897
|
-
};
|
|
898
|
-
var scopeItemsToOrg = (items, orgId) => {
|
|
899
|
-
if (!Array.isArray(items)) return items;
|
|
900
|
-
const mismatched = items.filter((item) => {
|
|
901
|
-
const owner = item?.org_id || item?.orgId;
|
|
902
|
-
return typeof owner === "string" && owner && owner !== orgId;
|
|
903
|
-
}).length;
|
|
904
|
-
if (mismatched > 0) {
|
|
905
|
-
console.debug(`[Datastore] ${mismatched}/${items.length} items have a different org_id than ${orgId} \u2014 showing them anyway`);
|
|
906
|
-
}
|
|
907
|
-
return items;
|
|
908
|
-
};
|
|
909
|
-
var filterItemsByCategory = (items, category) => {
|
|
910
|
-
if (!Array.isArray(items) || !category || category === "all") return items;
|
|
911
|
-
const targetCategory = category.trim().toLowerCase();
|
|
912
|
-
const isVulns = targetCategory === "shuffle-security_vulns" || targetCategory === "shuffle-security_vulnerabilities" || targetCategory === "vulns";
|
|
913
|
-
const vulnAliases = /* @__PURE__ */ new Set(["shuffle-security_vulns", "shuffle-security_vulnerabilities", "vulns"]);
|
|
914
|
-
const filtered = items.filter((item) => {
|
|
915
|
-
if (!item || typeof item !== "object") return false;
|
|
916
|
-
const itemCat = typeof item.category === "string" ? item.category.trim().toLowerCase() : "";
|
|
917
|
-
if (!itemCat) return true;
|
|
918
|
-
if (isVulns) return vulnAliases.has(itemCat);
|
|
919
|
-
return itemCat === targetCategory;
|
|
920
|
-
});
|
|
921
|
-
const leakedCount = items.length - filtered.length;
|
|
922
|
-
if (leakedCount > 0) {
|
|
923
|
-
console.warn(`[Datastore] Filtered out ${leakedCount}/${items.length} items with mismatched category (requested: "${category}") due to backend analyzed-match leakage`);
|
|
924
|
-
}
|
|
925
|
-
return filtered;
|
|
926
|
-
};
|
|
927
|
-
var getDatastorePageSize = (category, limit) => limit ?? (category === DATASTORE_CATEGORIES.INCIDENTS ? 50 : 100);
|
|
928
|
-
var getDatastoreByCategory = async (category, cursor, limit, overrideOrgId) => {
|
|
929
|
-
const orgId = overrideOrgId || await waitForOrgId();
|
|
930
|
-
if (!orgId) {
|
|
931
|
-
return {
|
|
932
|
-
success: false,
|
|
933
|
-
error: "No organization ID found",
|
|
934
|
-
diagnostics: {
|
|
935
|
-
operation: "list",
|
|
936
|
-
category,
|
|
937
|
-
orgId: null,
|
|
938
|
-
cursor,
|
|
939
|
-
errorStage: "request",
|
|
940
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
941
|
-
}
|
|
942
|
-
};
|
|
943
|
-
}
|
|
944
|
-
const effectiveLimit = getDatastorePageSize(category, limit);
|
|
945
|
-
const isVulnsCategory = category === "shuffle-security_vulns" || category === "shuffle-security_vulnerabilities" || category === "vulns";
|
|
946
|
-
let url = isVulnsCategory ? `/api/v2/vulns?skip_fields=false&top=${effectiveLimit}` : `/api/v1/orgs/${orgId}/list_cache?category=${encodeURIComponent(category)}&top=${effectiveLimit}`;
|
|
947
|
-
if (cursor) {
|
|
948
|
-
url += `&cursor=${encodeURIComponent(cursor)}`;
|
|
949
|
-
}
|
|
950
|
-
const requestUrl = getApiUrl(url);
|
|
951
|
-
const baseDiagnostics = {
|
|
952
|
-
operation: "list",
|
|
953
|
-
category,
|
|
954
|
-
orgId,
|
|
955
|
-
url: requestUrl,
|
|
956
|
-
cursor,
|
|
957
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
958
|
-
};
|
|
959
|
-
let response;
|
|
960
|
-
try {
|
|
961
|
-
response = await fetch(
|
|
962
|
-
requestUrl,
|
|
963
|
-
{
|
|
964
|
-
method: "GET",
|
|
965
|
-
credentials: "include",
|
|
966
|
-
headers: {
|
|
967
|
-
"Content-Type": "application/json",
|
|
968
|
-
...getAuthHeader(orgId)
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
);
|
|
972
|
-
} catch (error) {
|
|
973
|
-
try {
|
|
974
|
-
response = await fetch(
|
|
975
|
-
requestUrl,
|
|
976
|
-
{
|
|
977
|
-
method: "GET",
|
|
978
|
-
credentials: "include",
|
|
979
|
-
headers: {
|
|
980
|
-
"Content-Type": "application/json",
|
|
981
|
-
...getAuthHeader(orgId)
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
);
|
|
985
|
-
} catch (retryError) {
|
|
986
|
-
return {
|
|
987
|
-
success: false,
|
|
988
|
-
error: retryError instanceof Error ? retryError.message : "Failed to request datastore items",
|
|
989
|
-
diagnostics: {
|
|
990
|
-
...baseDiagnostics,
|
|
991
|
-
errorStage: "request"
|
|
992
|
-
}
|
|
993
|
-
};
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
const contentType = response.headers.get("content-type");
|
|
997
|
-
const rawBody = await response.text();
|
|
998
|
-
if (!response.ok) {
|
|
999
|
-
const extracted = tryExtractItemsFromBody(rawBody);
|
|
1000
|
-
if (extracted && extracted.items.length > 0) {
|
|
1001
|
-
console.warn(`[Datastore] ${response.status} response for category=${category} but body contained ${extracted.items.length} valid items \u2014 treating as success`);
|
|
1002
|
-
const rawCount = extracted.items.length;
|
|
1003
|
-
const scoped = scopeItemsToOrg(extracted.items, orgId);
|
|
1004
|
-
const filtered = filterItemsByCategory(scoped, category);
|
|
1005
|
-
return {
|
|
1006
|
-
success: true,
|
|
1007
|
-
data: filtered,
|
|
1008
|
-
rawItemCount: rawCount,
|
|
1009
|
-
categoryConfig: extracted.categoryConfig,
|
|
1010
|
-
cursor: extracted.cursor,
|
|
1011
|
-
totalAmount: extracted.totalAmount,
|
|
1012
|
-
diagnostics: {
|
|
1013
|
-
...baseDiagnostics,
|
|
1014
|
-
status: response.status,
|
|
1015
|
-
statusText: response.statusText,
|
|
1016
|
-
contentType,
|
|
1017
|
-
responseShape: extracted.shape,
|
|
1018
|
-
itemCount: filtered.length,
|
|
1019
|
-
rawItemCount: rawCount,
|
|
1020
|
-
totalAmount: extracted.totalAmount ?? null
|
|
1021
|
-
}
|
|
1022
|
-
};
|
|
1023
|
-
}
|
|
1024
|
-
if (response.status >= 400 && response.status < 500) {
|
|
1025
|
-
try {
|
|
1026
|
-
const retryResponse = await fetch(
|
|
1027
|
-
requestUrl,
|
|
1028
|
-
{
|
|
1029
|
-
method: "GET",
|
|
1030
|
-
credentials: "include",
|
|
1031
|
-
headers: {
|
|
1032
|
-
"Content-Type": "application/json",
|
|
1033
|
-
...getAuthHeader(orgId)
|
|
1034
|
-
}
|
|
1035
|
-
}
|
|
1036
|
-
);
|
|
1037
|
-
const retryBody = await retryResponse.text();
|
|
1038
|
-
const retryExtracted = tryExtractItemsFromBody(retryBody);
|
|
1039
|
-
if (retryResponse.ok || retryExtracted && retryExtracted.items.length > 0) {
|
|
1040
|
-
const rawCount = (retryExtracted?.items || []).length;
|
|
1041
|
-
const scoped = scopeItemsToOrg(retryExtracted?.items || [], orgId);
|
|
1042
|
-
const filtered = filterItemsByCategory(scoped, category);
|
|
1043
|
-
console.warn(`[Datastore] Retry succeeded for category=${category} (status=${retryResponse.status}, items=${filtered.length})`);
|
|
1044
|
-
return {
|
|
1045
|
-
success: true,
|
|
1046
|
-
data: filtered,
|
|
1047
|
-
rawItemCount: rawCount,
|
|
1048
|
-
categoryConfig: retryExtracted?.categoryConfig,
|
|
1049
|
-
cursor: retryExtracted?.cursor,
|
|
1050
|
-
totalAmount: retryExtracted?.totalAmount,
|
|
1051
|
-
diagnostics: {
|
|
1052
|
-
...baseDiagnostics,
|
|
1053
|
-
status: retryResponse.status,
|
|
1054
|
-
statusText: retryResponse.statusText,
|
|
1055
|
-
contentType: retryResponse.headers.get("content-type"),
|
|
1056
|
-
responseShape: retryExtracted?.shape || "unknown",
|
|
1057
|
-
itemCount: filtered.length,
|
|
1058
|
-
rawItemCount: rawCount,
|
|
1059
|
-
totalAmount: retryExtracted?.totalAmount ?? null
|
|
1060
|
-
}
|
|
1061
|
-
};
|
|
1062
|
-
}
|
|
1063
|
-
} catch {
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
return {
|
|
1067
|
-
success: false,
|
|
1068
|
-
error: isCircuitBreakerResponse(response) ? "circuit_breaker_open" : `Failed to get datastore items: ${response.status} ${response.statusText}`.trim(),
|
|
1069
|
-
diagnostics: {
|
|
1070
|
-
...baseDiagnostics,
|
|
1071
|
-
status: response.status,
|
|
1072
|
-
statusText: response.statusText,
|
|
1073
|
-
contentType,
|
|
1074
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
1075
|
-
errorStage: "response"
|
|
1076
|
-
}
|
|
1077
|
-
};
|
|
1078
|
-
}
|
|
1079
|
-
try {
|
|
1080
|
-
const data = rawBody ? JSON.parse(rawBody) : {};
|
|
1081
|
-
const responseShape = Array.isArray(data) ? "array" : Array.isArray(data?.keys) ? "keys" : Array.isArray(data?.data) ? "data" : "unknown";
|
|
1082
|
-
const items = Array.isArray(data) ? data : data.keys || data.data || [];
|
|
1083
|
-
const rawCount = Array.isArray(items) ? items.length : 0;
|
|
1084
|
-
const scoped = scopeItemsToOrg(items, orgId);
|
|
1085
|
-
const filtered = filterItemsByCategory(scoped, category);
|
|
1086
|
-
const totalAmount = data.total_amount ?? data.total ?? data.amount;
|
|
1087
|
-
return {
|
|
1088
|
-
success: true,
|
|
1089
|
-
data: filtered,
|
|
1090
|
-
rawItemCount: rawCount,
|
|
1091
|
-
categoryConfig: data.category_config,
|
|
1092
|
-
cursor: data.cursor,
|
|
1093
|
-
totalAmount,
|
|
1094
|
-
diagnostics: {
|
|
1095
|
-
...baseDiagnostics,
|
|
1096
|
-
status: response.status,
|
|
1097
|
-
statusText: response.statusText,
|
|
1098
|
-
contentType,
|
|
1099
|
-
responseShape,
|
|
1100
|
-
itemCount: filtered.length,
|
|
1101
|
-
rawItemCount: rawCount,
|
|
1102
|
-
totalAmount: totalAmount ?? null
|
|
1103
|
-
}
|
|
1104
|
-
};
|
|
1105
|
-
} catch (error) {
|
|
1106
|
-
return {
|
|
1107
|
-
success: false,
|
|
1108
|
-
error: error instanceof Error ? `Failed to parse datastore response: ${error.message}` : "Failed to parse datastore response",
|
|
1109
|
-
diagnostics: {
|
|
1110
|
-
...baseDiagnostics,
|
|
1111
|
-
status: response.status,
|
|
1112
|
-
statusText: response.statusText,
|
|
1113
|
-
contentType,
|
|
1114
|
-
bodyPreview: truncateResponsePreview(rawBody),
|
|
1115
|
-
errorStage: "parse"
|
|
1116
|
-
}
|
|
1117
|
-
};
|
|
1118
|
-
}
|
|
1119
|
-
};
|
|
1120
|
-
var deleteDatastoreItem = async (key, category, overrideOrgId, options) => {
|
|
1121
|
-
const extractedOrgId = !overrideOrgId ? extractOrgIdFromDatastoreKey(key) : null;
|
|
1122
|
-
const orgId = overrideOrgId || extractedOrgId || await waitForOrgId();
|
|
1123
|
-
if (!orgId) {
|
|
1124
|
-
return { success: false, error: "No organization ID found" };
|
|
1125
|
-
}
|
|
1126
|
-
const rawKey = normalizeDatastoreKey(key);
|
|
1127
|
-
const payload = {
|
|
1128
|
-
key: rawKey,
|
|
1129
|
-
org_id: orgId
|
|
1130
|
-
};
|
|
1131
|
-
if (category) {
|
|
1132
|
-
payload.category = category;
|
|
1133
|
-
}
|
|
1134
|
-
const headers = {
|
|
1135
|
-
"Content-Type": "application/json",
|
|
1136
|
-
...getAuthHeader(orgId)
|
|
1137
|
-
};
|
|
1138
|
-
console.log(`[datastore.delete] key=${rawKey} category=${category} orgId=${orgId}${overrideOrgId ? " (override)" : ""}`);
|
|
1139
|
-
const response = await fetch(datastoreUrl(`/api/v1/orgs/${orgId}/delete_cache`, options?.regionUrl), {
|
|
1140
|
-
method: "POST",
|
|
1141
|
-
credentials: "include",
|
|
1142
|
-
headers,
|
|
1143
|
-
body: JSON.stringify(payload)
|
|
1144
|
-
});
|
|
1145
|
-
if (!response.ok) {
|
|
1146
|
-
return { success: false, error: `Failed to delete datastore item: ${response.statusText}` };
|
|
1147
|
-
}
|
|
1148
|
-
return { success: true };
|
|
1149
|
-
};
|
|
1150
|
-
var deleteDatastoreItems = async (keys, category) => {
|
|
1151
|
-
const orgId = await waitForOrgId();
|
|
1152
|
-
if (!orgId) {
|
|
1153
|
-
return { success: false, deleted: 0, failed: keys, error: "No organization ID found" };
|
|
1154
|
-
}
|
|
1155
|
-
const results = await Promise.allSettled(
|
|
1156
|
-
keys.map((key) => deleteDatastoreItem(key, category))
|
|
1157
|
-
);
|
|
1158
|
-
const failed = [];
|
|
1159
|
-
let deleted = 0;
|
|
1160
|
-
results.forEach((result, index) => {
|
|
1161
|
-
if (result.status === "fulfilled" && result.value.success) {
|
|
1162
|
-
deleted++;
|
|
1163
|
-
} else {
|
|
1164
|
-
failed.push(keys[index]);
|
|
1165
|
-
}
|
|
1166
|
-
});
|
|
1167
|
-
return {
|
|
1168
|
-
success: failed.length === 0,
|
|
1169
|
-
deleted,
|
|
1170
|
-
failed
|
|
1171
|
-
};
|
|
1172
|
-
};
|
|
1173
|
-
var DATASTORE_CATEGORIES = {
|
|
1174
|
-
INCIDENTS: "shuffle-security_incidents",
|
|
1175
|
-
EVENTS: "shuffle-security_events",
|
|
1176
|
-
VULNERABILITIES: "shuffle-security_vulns",
|
|
1177
|
-
VULNS: "shuffle-security_vulns",
|
|
1178
|
-
ASSETS: "shuffle-security_assets",
|
|
1179
|
-
PACKAGES: "shuffle-security_packages",
|
|
1180
|
-
SOFTWARE: "shuffle-security_software",
|
|
1181
|
-
TEMPLATES: "shuffle-security_templates",
|
|
1182
|
-
CONFIGURATION: "shuffle-security_configuration",
|
|
1183
|
-
IOCS: "shuffle-security_ioc-config",
|
|
1184
|
-
CUSTOM_FIELDS: "shuffle-security_custom-fields",
|
|
1185
|
-
THREAT_FEEDS: "shuffle-security_threat-feeds",
|
|
1186
|
-
INFRASTRUCTURE: "shuffle-security_infrastructure",
|
|
1187
|
-
USERS: "shuffle-security_users",
|
|
1188
|
-
REPORTS: "shuffle-security_reports",
|
|
1189
|
-
// Legacy - for migration purposes
|
|
1190
|
-
LEGACY_ALERTS: "shuffle-security_alerts",
|
|
1191
|
-
LEGACY_CASES: "shuffle-security_cases"
|
|
1192
|
-
};
|
|
1193
|
-
|
|
1194
|
-
export {
|
|
1195
|
-
registerProtectedOrigin,
|
|
1196
|
-
installFetchBreaker,
|
|
1197
|
-
setRuntimeOrgId,
|
|
1198
|
-
setDatastoreItem,
|
|
1199
|
-
setDatastoreItems,
|
|
1200
|
-
getDatastoreItem,
|
|
1201
|
-
getDatastoreItemPublic,
|
|
1202
|
-
filterItemsByCategory,
|
|
1203
|
-
getDatastorePageSize,
|
|
1204
|
-
getDatastoreByCategory,
|
|
1205
|
-
deleteDatastoreItem,
|
|
1206
|
-
deleteDatastoreItems,
|
|
1207
|
-
DATASTORE_CATEGORIES
|
|
1208
|
-
};
|