@agent-native/core 0.122.1 → 0.122.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -13
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +6 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/production-agent.ts +65 -1
- package/corpus/core/src/agent/run-store.ts +61 -1
- package/corpus/core/src/server/builder-design-systems.ts +1 -1
- package/corpus/templates/analytics/.agents/skills/dashboard-management/SKILL.md +15 -0
- package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/report-panel-window.ts +3 -3
- package/corpus/templates/analytics/changelog/2026-07-25-analytics-dashboards-and-scheduled-reports-are-faster-and-mo.md +6 -0
- package/corpus/templates/analytics/server/db/schema.ts +12 -0
- package/corpus/templates/analytics/server/lib/dashboard-panel-query.ts +8 -4
- package/corpus/templates/analytics/server/lib/dashboard-report.ts +136 -5
- package/corpus/templates/analytics/server/lib/dashboard-time-scope.ts +250 -1
- package/corpus/templates/analytics/server/lib/first-party-analytics-cache.ts +167 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +34 -9
- package/corpus/templates/analytics/server/lib/first-party-dashboard-repair.ts +105 -23
- package/corpus/templates/analytics/server/lib/first-party-unbounded-panel-repair.ts +169 -0
- package/corpus/templates/analytics/server/plugins/db.ts +41 -1
- package/corpus/templates/analytics/shared/dashboard-report-timeouts.ts +6 -3
- package/corpus/templates/design/actions/edit-design.ts +66 -27
- package/corpus/templates/design/changelog/2026-07-25-design-edits-retry-concurrent-changes.md +6 -0
- package/corpus/templates/design/server/source-workspace.ts +5 -1
- package/corpus/templates/forms/.agents/skills/form-responses/SKILL.md +6 -4
- package/corpus/templates/forms/actions/create-form.ts +5 -1
- package/corpus/templates/forms/actions/export-responses.ts +22 -11
- package/corpus/templates/forms/actions/update-form.ts +5 -1
- package/corpus/templates/forms/changelog/2026-07-25-fields-can-be-created-without-ids.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-25-response-exports-use-file-storage.md +6 -0
- package/corpus/templates/forms/server/lib/validate-fields.ts +43 -0
- package/corpus/templates/slides/actions/extract-pdf.ts +4 -4
- package/corpus/templates/slides/actions/import-file.ts +3 -4
- package/corpus/templates/slides/changelog/2026-07-25-gemini-image-models-updated.md +6 -0
- package/corpus/templates/slides/changelog/2026-07-25-pdf-imports-tolerate-missing-canvas.md +6 -0
- package/corpus/templates/slides/server/handlers/image-providers/gemini.ts +5 -2
- package/corpus/templates/slides/server/lib/pdf-parse-setup.ts +150 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +63 -1
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +49 -0
- package/dist/agent/run-store.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/builder-design-systems.js +1 -1
- package/dist/server/builder-design-systems.js.map +1 -1
- package/package.json +3 -3
- package/src/agent/production-agent.ts +65 -1
- package/src/agent/run-store.ts +61 -1
- package/src/server/builder-design-systems.ts +1 -1
|
@@ -14,7 +14,7 @@ type DashboardFilterLike = {
|
|
|
14
14
|
default?: unknown;
|
|
15
15
|
};
|
|
16
16
|
|
|
17
|
-
type DashboardPanelLike = {
|
|
17
|
+
export type DashboardPanelLike = {
|
|
18
18
|
id?: unknown;
|
|
19
19
|
title?: unknown;
|
|
20
20
|
chartType?: unknown;
|
|
@@ -64,6 +64,241 @@ function hasExplicitLowerBound(sql: string): boolean {
|
|
|
64
64
|
);
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
const ANALYTICS_SCAN_RE = /\b(?:FROM|JOIN)\s+analytics_events\b/gi;
|
|
68
|
+
const TOP_LEVEL_CTE_HEAD_RE = /^\s*WITH\b/i;
|
|
69
|
+
const CTE_NAME_RE =
|
|
70
|
+
/^\s*((?:[A-Za-z_]\w*|"(?:[^"]|"")*"|`(?:[^`]|``)*`|\[(?:[^\]]|\]\])*\]))\s*(?:\([^)]*\)\s*)?AS\s*(?:(?:NOT\s+)?MATERIALIZED\s*)?\(/i;
|
|
71
|
+
|
|
72
|
+
type TopLevelCte = { name: string; bodyStart: number; bodyEnd: number };
|
|
73
|
+
|
|
74
|
+
type TopLevelCteParse = {
|
|
75
|
+
ctes: TopLevelCte[];
|
|
76
|
+
outerQueryStart: number;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
function findClosingParenthesis(sql: string, bodyStart: number): number {
|
|
80
|
+
let depth = 1;
|
|
81
|
+
let quote: "'" | '"' | "`" | null = null;
|
|
82
|
+
let lineComment = false;
|
|
83
|
+
let blockComment = false;
|
|
84
|
+
|
|
85
|
+
for (let i = bodyStart; i < sql.length; i += 1) {
|
|
86
|
+
const char = sql[i];
|
|
87
|
+
const next = sql[i + 1];
|
|
88
|
+
|
|
89
|
+
if (lineComment) {
|
|
90
|
+
if (char === "\n") lineComment = false;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (blockComment) {
|
|
94
|
+
if (char === "*" && next === "/") {
|
|
95
|
+
blockComment = false;
|
|
96
|
+
i += 1;
|
|
97
|
+
}
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (quote) {
|
|
101
|
+
if (char === quote) {
|
|
102
|
+
if (next === quote) {
|
|
103
|
+
i += 1;
|
|
104
|
+
} else {
|
|
105
|
+
quote = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (char === "-" && next === "-") {
|
|
111
|
+
lineComment = true;
|
|
112
|
+
i += 1;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (char === "/" && next === "*") {
|
|
116
|
+
blockComment = true;
|
|
117
|
+
i += 1;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (char === "'" || char === '"' || char === "`") {
|
|
121
|
+
quote = char;
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (char === "(") depth += 1;
|
|
125
|
+
else if (char === ")") {
|
|
126
|
+
depth -= 1;
|
|
127
|
+
if (depth === 0) return i;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return -1;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function skipSqlTrivia(sql: string, start: number): number {
|
|
135
|
+
let i = start;
|
|
136
|
+
while (i < sql.length) {
|
|
137
|
+
if (/\s/.test(sql[i])) {
|
|
138
|
+
i += 1;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (sql[i] === "-" && sql[i + 1] === "-") {
|
|
142
|
+
i += 2;
|
|
143
|
+
while (i < sql.length && sql[i] !== "\n") i += 1;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (sql[i] === "/" && sql[i + 1] === "*") {
|
|
147
|
+
const end = sql.indexOf("*/", i + 2);
|
|
148
|
+
if (end === -1) return sql.length;
|
|
149
|
+
i = end + 2;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
return i;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function splitTopLevelUnionBranches(sql: string): string[] {
|
|
158
|
+
const branches: string[] = [];
|
|
159
|
+
let start = 0;
|
|
160
|
+
let depth = 0;
|
|
161
|
+
let quote: "'" | '"' | "`" | null = null;
|
|
162
|
+
let lineComment = false;
|
|
163
|
+
let blockComment = false;
|
|
164
|
+
|
|
165
|
+
for (let i = 0; i < sql.length; i += 1) {
|
|
166
|
+
const char = sql[i];
|
|
167
|
+
const next = sql[i + 1];
|
|
168
|
+
|
|
169
|
+
if (lineComment) {
|
|
170
|
+
if (char === "\n") lineComment = false;
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
if (blockComment) {
|
|
174
|
+
if (char === "*" && next === "/") {
|
|
175
|
+
blockComment = false;
|
|
176
|
+
i += 1;
|
|
177
|
+
}
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (quote) {
|
|
181
|
+
if (char === quote) {
|
|
182
|
+
if (next === quote) i += 1;
|
|
183
|
+
else quote = null;
|
|
184
|
+
}
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
if (char === "-" && next === "-") {
|
|
188
|
+
lineComment = true;
|
|
189
|
+
i += 1;
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
if (char === "/" && next === "*") {
|
|
193
|
+
blockComment = true;
|
|
194
|
+
i += 1;
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
if (char === "'" || char === '"' || char === "`") {
|
|
198
|
+
quote = char;
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (char === "(") {
|
|
202
|
+
depth += 1;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
if (char === ")") {
|
|
206
|
+
depth = Math.max(0, depth - 1);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (depth !== 0) continue;
|
|
210
|
+
|
|
211
|
+
const union = /^UNION\b/i.exec(sql.slice(i));
|
|
212
|
+
if (!union) continue;
|
|
213
|
+
branches.push(sql.slice(start, i));
|
|
214
|
+
start = skipSqlTrivia(sql, i + union[0].length);
|
|
215
|
+
const modifier = /^(?:ALL|DISTINCT)\b/i.exec(sql.slice(start));
|
|
216
|
+
if (modifier) start = skipSqlTrivia(sql, start + modifier[0].length);
|
|
217
|
+
i = start - 1;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
branches.push(sql.slice(start));
|
|
221
|
+
return branches;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** One top-level `name AS (...)` common-table-expression's body span. */
|
|
225
|
+
function topLevelCtes(sql: string): TopLevelCteParse | null {
|
|
226
|
+
const sqlStart = skipSqlTrivia(sql, 0);
|
|
227
|
+
const head = TOP_LEVEL_CTE_HEAD_RE.exec(sql.slice(sqlStart));
|
|
228
|
+
if (!head) return null;
|
|
229
|
+
const ctes: TopLevelCte[] = [];
|
|
230
|
+
let i = skipSqlTrivia(sql, sqlStart + head[0].length);
|
|
231
|
+
const recursive = /^RECURSIVE\b/i.exec(sql.slice(i));
|
|
232
|
+
if (recursive) i = skipSqlTrivia(sql, i + recursive[0].length);
|
|
233
|
+
while (i < sql.length) {
|
|
234
|
+
const nameMatch = CTE_NAME_RE.exec(sql.slice(i));
|
|
235
|
+
if (!nameMatch) return null;
|
|
236
|
+
const name = nameMatch[1];
|
|
237
|
+
const bodyStart = i + nameMatch[0].length;
|
|
238
|
+
const bodyEnd = findClosingParenthesis(sql, bodyStart);
|
|
239
|
+
if (bodyEnd === -1) return null;
|
|
240
|
+
ctes.push({ name, bodyStart, bodyEnd });
|
|
241
|
+
const k = skipSqlTrivia(sql, bodyEnd + 1);
|
|
242
|
+
if (sql[k] === ",") {
|
|
243
|
+
i = k + 1;
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
return { ctes, outerQueryStart: k };
|
|
247
|
+
}
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function hasAnyTimeBound(text: string): boolean {
|
|
252
|
+
// .search() ignores the shared global-flagged regex's lastIndex state,
|
|
253
|
+
// unlike .test(), which would otherwise give wrong results across calls.
|
|
254
|
+
return (
|
|
255
|
+
text.search(TEMPORAL_VARIABLE_RE) !== -1 || hasExplicitLowerBound(text)
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* True if every `analytics_events` scan in `sql` has its own time bound.
|
|
261
|
+
*
|
|
262
|
+
* `hasExplicitLowerBound`/time-variable checks used to run against the whole
|
|
263
|
+
* SQL string: a multi-CTE panel with a bound ANYWHERE (e.g. only on the
|
|
264
|
+
* final SELECT, or only on one of several sibling CTEs) passed validation
|
|
265
|
+
* even though an earlier, unbounded sibling CTE still did a full-table scan
|
|
266
|
+
* — the exact shape of several production incidents (2026-07-25 org-wide
|
|
267
|
+
* audit). This only tightens the check for that specific shape: sibling
|
|
268
|
+
* top-level CTEs each need their own bound. It deliberately does NOT require
|
|
269
|
+
* every nested subquery to be independently bounded — a correlated lookup
|
|
270
|
+
* nested inside an already-bounded outer query (e.g. "has this id EVER
|
|
271
|
+
* appeared as a referrer") is a legitimate, intentionally all-time pattern,
|
|
272
|
+
* and over-flagging it would make this check untrustworthy.
|
|
273
|
+
*/
|
|
274
|
+
function everyScanIsBounded(sql: string): boolean {
|
|
275
|
+
const parsed = topLevelCtes(sql);
|
|
276
|
+
if (!parsed) {
|
|
277
|
+
// A WITH query that we cannot parse must fail closed. Falling back to a
|
|
278
|
+
// whole-query bound lets a bounded sibling hide an unbounded CTE.
|
|
279
|
+
if (TOP_LEVEL_CTE_HEAD_RE.test(sql.slice(skipSqlTrivia(sql, 0)))) {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
return hasAnyTimeBound(sql);
|
|
283
|
+
}
|
|
284
|
+
const units = [
|
|
285
|
+
...parsed.ctes.map(({ bodyStart, bodyEnd }) =>
|
|
286
|
+
sql.slice(bodyStart, bodyEnd),
|
|
287
|
+
),
|
|
288
|
+
sql.slice(parsed.outerQueryStart),
|
|
289
|
+
];
|
|
290
|
+
return units.every((unit) =>
|
|
291
|
+
splitTopLevelUnionBranches(unit).every((branch) => {
|
|
292
|
+
const scans = branch.match(ANALYTICS_SCAN_RE) ?? [];
|
|
293
|
+
if (scans.length === 0) return true;
|
|
294
|
+
// A single lower bound cannot prove that every scan in a join or nested
|
|
295
|
+
// branch is bounded. Fail closed until the SQL has one scan per branch.
|
|
296
|
+
if (scans.length > 1) return false;
|
|
297
|
+
return hasAnyTimeBound(branch);
|
|
298
|
+
}),
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
|
|
67
302
|
function hasIntentionalHistoryDescription(
|
|
68
303
|
panel: DashboardPanelLike,
|
|
69
304
|
config: Record<string, unknown>,
|
|
@@ -166,5 +401,19 @@ export function validateFirstPartyDashboardTimeScope(
|
|
|
166
401
|
return `panel[${index}] ${label} reads first-party analytics without a time bound; use {{timeRange}} with a non-empty default filter, or explicitly set config.timeScope to "cohort-history" or "all-time" for intentional history scans`;
|
|
167
402
|
}
|
|
168
403
|
|
|
404
|
+
// A bound anywhere in the SQL text (checked above) is not the same as
|
|
405
|
+
// every CTE/subquery that reads analytics_events having its own bound — a
|
|
406
|
+
// multi-CTE panel can look bound overall while an earlier CTE still does a
|
|
407
|
+
// full-table scan. Skip this for "all-time" and "cohort-history": both are
|
|
408
|
+
// explicit escape hatches, and a cohort-defining CTE (e.g. "first ever
|
|
409
|
+
// active date per user") is legitimately unbounded by design.
|
|
410
|
+
if (
|
|
411
|
+
scope !== "all-time" &&
|
|
412
|
+
scope !== "cohort-history" &&
|
|
413
|
+
!everyScanIsBounded(sql)
|
|
414
|
+
) {
|
|
415
|
+
return `panel[${index}] ${label} has at least one analytics_events read (in a CTE or subquery) without its own time bound — a {{timeRange}} reference or literal bound elsewhere in the SQL does not cover it; add a bound to every analytics_events scan, or set config.timeScope to "all-time" for an intentional full-history scan`;
|
|
416
|
+
}
|
|
417
|
+
|
|
169
418
|
return null;
|
|
170
419
|
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { createHash } from "crypto";
|
|
2
|
+
|
|
3
|
+
import { getDbExec } from "@agent-native/core/db";
|
|
4
|
+
|
|
5
|
+
import type { AnalyticsQueryResult } from "./first-party-analytics.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* First-party dashboard panel query cache.
|
|
9
|
+
*
|
|
10
|
+
* `analytics_events` grows unbounded and several panels (rolling-window and
|
|
11
|
+
* cohort self-joins) are inherently expensive, so a page with many panels or
|
|
12
|
+
* a daily report screenshot capturing many panels at once repeatedly
|
|
13
|
+
* recomputes the same rows under concurrent load — that contention, not any
|
|
14
|
+
* single query alone, is what was pushing panels past their timeout budget.
|
|
15
|
+
* This mirrors bigquery.ts's L1 (in-process) + L2 (SQL-backed, shared across
|
|
16
|
+
* serverless invocations) cache, but keyed per scoped+interpolated SQL text
|
|
17
|
+
* (which already embeds org_id/owner_email via query args) and with a much
|
|
18
|
+
* shorter TTL since this is the app's own live data, not an immutable
|
|
19
|
+
* warehouse result.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
interface L1Entry {
|
|
23
|
+
result: AnalyticsQueryResult;
|
|
24
|
+
createdAt: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const CACHE_TTL_MS = 5 * 60 * 1000;
|
|
28
|
+
const MAX_L1_ENTRIES = 500;
|
|
29
|
+
|
|
30
|
+
const l1Cache = new Map<string, L1Entry>();
|
|
31
|
+
// De-dupes identical concurrent cache misses (e.g. the same panel query fired
|
|
32
|
+
// by several viewers, or by the report capture's 4-panel concurrent window)
|
|
33
|
+
// so only one of them actually hits the database.
|
|
34
|
+
const inFlight = new Map<string, Promise<AnalyticsQueryResult>>();
|
|
35
|
+
|
|
36
|
+
function inFlightKey(key: string, timeoutMs?: number): string {
|
|
37
|
+
return `${key}:${timeoutMs ?? "unbounded"}`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface FirstPartyCacheOptions {
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function firstPartyCacheKey(
|
|
45
|
+
scopedSql: string,
|
|
46
|
+
args: Array<string | null>,
|
|
47
|
+
): string {
|
|
48
|
+
return createHash("sha256")
|
|
49
|
+
.update(`${scopedSql}\n${JSON.stringify(args)}`)
|
|
50
|
+
.digest("hex");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function getL1(key: string): AnalyticsQueryResult | null {
|
|
54
|
+
const entry = l1Cache.get(key);
|
|
55
|
+
if (!entry) return null;
|
|
56
|
+
if (Date.now() - entry.createdAt > CACHE_TTL_MS) {
|
|
57
|
+
l1Cache.delete(key);
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
return entry.result;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function setL1(key: string, result: AnalyticsQueryResult): void {
|
|
64
|
+
if (l1Cache.size >= MAX_L1_ENTRIES) {
|
|
65
|
+
const oldest = l1Cache.keys().next().value;
|
|
66
|
+
if (oldest) l1Cache.delete(oldest);
|
|
67
|
+
}
|
|
68
|
+
l1Cache.set(key, { result, createdAt: Date.now() });
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function getL2(
|
|
72
|
+
key: string,
|
|
73
|
+
timeoutMs?: number,
|
|
74
|
+
): Promise<AnalyticsQueryResult | null> {
|
|
75
|
+
try {
|
|
76
|
+
const db = getDbExec();
|
|
77
|
+
const nowIso = new Date().toISOString();
|
|
78
|
+
const { rows } = await db.execute({
|
|
79
|
+
sql: "SELECT result FROM first_party_analytics_cache WHERE key = ? AND expires_at > ?",
|
|
80
|
+
args: [key, nowIso],
|
|
81
|
+
timeoutMs,
|
|
82
|
+
});
|
|
83
|
+
if (!rows.length) return null;
|
|
84
|
+
const raw = (rows[0] as { result: string }).result;
|
|
85
|
+
return JSON.parse(raw) as AnalyticsQueryResult;
|
|
86
|
+
} catch (err) {
|
|
87
|
+
console.warn("[first-party-analytics] L2 cache read failed:", err);
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function setL2(
|
|
93
|
+
key: string,
|
|
94
|
+
sql: string,
|
|
95
|
+
result: AnalyticsQueryResult,
|
|
96
|
+
timeoutMs?: number,
|
|
97
|
+
): Promise<void> {
|
|
98
|
+
try {
|
|
99
|
+
const db = getDbExec();
|
|
100
|
+
const now = new Date();
|
|
101
|
+
const expiresAt = new Date(now.getTime() + CACHE_TTL_MS);
|
|
102
|
+
const serialized = JSON.stringify(result);
|
|
103
|
+
// Upsert via delete+insert to stay dialect-agnostic (SQLite/Postgres).
|
|
104
|
+
await db.execute({
|
|
105
|
+
sql: "DELETE FROM first_party_analytics_cache WHERE key = ?",
|
|
106
|
+
args: [key],
|
|
107
|
+
timeoutMs,
|
|
108
|
+
});
|
|
109
|
+
await db.execute({
|
|
110
|
+
sql: "INSERT INTO first_party_analytics_cache (key, sql, result, created_at, expires_at) VALUES (?, ?, ?, ?, ?)",
|
|
111
|
+
args: [key, sql, serialized, now.toISOString(), expiresAt.toISOString()],
|
|
112
|
+
timeoutMs,
|
|
113
|
+
});
|
|
114
|
+
// Opportunistically prune expired rows so the table doesn't grow
|
|
115
|
+
// unbounded — the keyspace is effectively every distinct panel/filter
|
|
116
|
+
// combination. Run ~1% of the time to avoid thrashing on every write.
|
|
117
|
+
if (Math.random() < 0.01) {
|
|
118
|
+
await db.execute({
|
|
119
|
+
sql: "DELETE FROM first_party_analytics_cache WHERE expires_at <= ?",
|
|
120
|
+
args: [now.toISOString()],
|
|
121
|
+
timeoutMs,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
} catch (err) {
|
|
125
|
+
console.warn("[first-party-analytics] L2 cache write failed:", err);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Returns a cached result for (key, sql) if fresh, otherwise runs `compute`
|
|
131
|
+
* exactly once — even under concurrent callers with the same key — and
|
|
132
|
+
* caches the result.
|
|
133
|
+
*/
|
|
134
|
+
export async function withFirstPartyCache(
|
|
135
|
+
key: string,
|
|
136
|
+
sql: string,
|
|
137
|
+
compute: () => Promise<AnalyticsQueryResult>,
|
|
138
|
+
options: FirstPartyCacheOptions = {},
|
|
139
|
+
): Promise<AnalyticsQueryResult> {
|
|
140
|
+
const l1Hit = getL1(key);
|
|
141
|
+
if (l1Hit) return l1Hit;
|
|
142
|
+
|
|
143
|
+
// A report prewarm may have a shorter deadline than a normal panel request;
|
|
144
|
+
// never let those callers inherit one another's database timeout.
|
|
145
|
+
const requestKey = inFlightKey(key, options.timeoutMs);
|
|
146
|
+
const existing = inFlight.get(requestKey);
|
|
147
|
+
if (existing) return existing;
|
|
148
|
+
|
|
149
|
+
// Register the in-flight promise synchronously (before any `await`) so two
|
|
150
|
+
// callers racing on the same key can't both slip past the check above and
|
|
151
|
+
// both hit L2/compute.
|
|
152
|
+
const promise = (async () => {
|
|
153
|
+
const l2Hit = await getL2(key, options.timeoutMs);
|
|
154
|
+
if (l2Hit) {
|
|
155
|
+
setL1(key, l2Hit);
|
|
156
|
+
return l2Hit;
|
|
157
|
+
}
|
|
158
|
+
const result = await compute();
|
|
159
|
+
setL1(key, result);
|
|
160
|
+
await setL2(key, sql, result, options.timeoutMs);
|
|
161
|
+
return result;
|
|
162
|
+
})().finally(() => {
|
|
163
|
+
inFlight.delete(requestKey);
|
|
164
|
+
});
|
|
165
|
+
inFlight.set(requestKey, promise);
|
|
166
|
+
return promise;
|
|
167
|
+
}
|
|
@@ -8,6 +8,10 @@ import {
|
|
|
8
8
|
ingestAnalyticsExceptionEvents,
|
|
9
9
|
type DerivedExceptionFields,
|
|
10
10
|
} from "./error-capture.js";
|
|
11
|
+
import {
|
|
12
|
+
firstPartyCacheKey,
|
|
13
|
+
withFirstPartyCache,
|
|
14
|
+
} from "./first-party-analytics-cache.js";
|
|
11
15
|
|
|
12
16
|
export interface AnalyticsScope {
|
|
13
17
|
userEmail: string;
|
|
@@ -29,6 +33,13 @@ export interface AnalyticsQueryResult {
|
|
|
29
33
|
schema: { name: string; type: string }[];
|
|
30
34
|
}
|
|
31
35
|
|
|
36
|
+
export interface AnalyticsQueryOptions {
|
|
37
|
+
/** Cache only callers with a stable dashboard-panel lifecycle. */
|
|
38
|
+
cache?: boolean;
|
|
39
|
+
/** Bound the database work for callers with a smaller delivery deadline. */
|
|
40
|
+
timeoutMs?: number;
|
|
41
|
+
}
|
|
42
|
+
|
|
32
43
|
const MAX_EVENTS_PER_REQUEST = 100;
|
|
33
44
|
const MAX_QUERY_ROWS = 5_000;
|
|
34
45
|
const FIRST_PARTY_QUERY_TABLES = new Set([
|
|
@@ -636,16 +647,30 @@ function inferSchema(rows: Record<string, unknown>[]): {
|
|
|
636
647
|
export async function queryFirstPartyAnalytics(
|
|
637
648
|
sql: string,
|
|
638
649
|
scope: AnalyticsScope,
|
|
650
|
+
options: AnalyticsQueryOptions = {},
|
|
639
651
|
): Promise<AnalyticsQueryResult> {
|
|
640
652
|
validateFirstPartyAnalyticsSql(sql);
|
|
641
653
|
const scoped = scopedAnalyticsSql(sql, scope);
|
|
642
|
-
const
|
|
643
|
-
const
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
654
|
+
const wrappedSql = `SELECT * FROM (${scoped.sql}) AS first_party_analytics_query LIMIT ${MAX_QUERY_ROWS}`;
|
|
655
|
+
const timeoutMs = Math.max(
|
|
656
|
+
1,
|
|
657
|
+
options.timeoutMs ?? FIRST_PARTY_ANALYTICS_QUERY_TIMEOUT_MS,
|
|
658
|
+
);
|
|
659
|
+
// The cache key is the fully scoped SQL + args, which already embeds
|
|
660
|
+
// org_id/owner_email (see scopeClause) — a cache hit can only ever return
|
|
661
|
+
// rows the same tenant was already entitled to query.
|
|
662
|
+
const cacheKey = firstPartyCacheKey(wrappedSql, scoped.args);
|
|
663
|
+
const compute = async (): Promise<AnalyticsQueryResult> => {
|
|
664
|
+
const exec = getDbExec();
|
|
665
|
+
const result = await exec.execute({
|
|
666
|
+
sql: wrappedSql,
|
|
667
|
+
args: scoped.args,
|
|
668
|
+
timeoutMs,
|
|
669
|
+
maxAttempts: 1,
|
|
670
|
+
});
|
|
671
|
+
const rows = result.rows as Record<string, unknown>[];
|
|
672
|
+
return { rows, schema: inferSchema(rows) };
|
|
673
|
+
};
|
|
674
|
+
if (!options.cache) return compute();
|
|
675
|
+
return withFirstPartyCache(cacheKey, wrappedSql, compute, { timeoutMs });
|
|
651
676
|
}
|
|
@@ -1,40 +1,47 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
|
|
3
|
+
import { getDialect } from "@agent-native/core/db";
|
|
3
4
|
import { recordChange } from "@agent-native/core/server";
|
|
4
5
|
import { and, desc, eq } from "drizzle-orm";
|
|
5
6
|
|
|
6
7
|
import { getDb, schema } from "../db/index.js";
|
|
7
8
|
import { repairCanonicalFirstPartyDashboardQueries } from "./canonical-first-party-dashboard-repair";
|
|
8
9
|
import { FIRST_PARTY_DASHBOARD_ID } from "./first-party-metric-catalog";
|
|
10
|
+
import { repairUnboundedFirstPartyPanels } from "./first-party-unbounded-panel-repair.js";
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
updatedAt: schema.dashboards.updatedAt,
|
|
21
|
-
ownerEmail: schema.dashboards.ownerEmail,
|
|
22
|
-
orgId: schema.dashboards.orgId,
|
|
23
|
-
visibility: schema.dashboards.visibility,
|
|
24
|
-
})
|
|
25
|
-
.from(schema.dashboards)
|
|
26
|
-
.where(eq(schema.dashboards.id, FIRST_PARTY_DASHBOARD_ID));
|
|
27
|
-
if (!row || row.kind !== "sql" || typeof row.config !== "string") {
|
|
28
|
-
return false;
|
|
29
|
-
}
|
|
12
|
+
type DashboardRepairRow = {
|
|
13
|
+
id: string;
|
|
14
|
+
config: string;
|
|
15
|
+
kind: string;
|
|
16
|
+
title: string;
|
|
17
|
+
updatedAt: string;
|
|
18
|
+
ownerEmail: string;
|
|
19
|
+
orgId: string | null;
|
|
20
|
+
visibility: string;
|
|
21
|
+
};
|
|
30
22
|
|
|
23
|
+
/**
|
|
24
|
+
* Apply `repairFn` to one already-fetched dashboard row inside a transaction,
|
|
25
|
+
* guarded by an optimistic-concurrency fence on (config, updatedAt) so a
|
|
26
|
+
* concurrent human/agent edit always wins over the repair. Snapshots the
|
|
27
|
+
* pre-repair config into dashboard_revisions (bounded to the 50 most recent)
|
|
28
|
+
* before committing, so any repair is one click away from undo.
|
|
29
|
+
*/
|
|
30
|
+
async function applyRepairToDashboardRow(
|
|
31
|
+
row: DashboardRepairRow,
|
|
32
|
+
repairFn: (config: Record<string, unknown>) => {
|
|
33
|
+
config: Record<string, unknown>;
|
|
34
|
+
changed: boolean;
|
|
35
|
+
},
|
|
36
|
+
): Promise<boolean> {
|
|
37
|
+
const db = getDb() as any;
|
|
31
38
|
let config: Record<string, unknown>;
|
|
32
39
|
try {
|
|
33
40
|
config = JSON.parse(row.config) as Record<string, unknown>;
|
|
34
41
|
} catch {
|
|
35
42
|
return false;
|
|
36
43
|
}
|
|
37
|
-
const repaired =
|
|
44
|
+
const repaired = repairFn(config);
|
|
38
45
|
if (!repaired.changed) return false;
|
|
39
46
|
|
|
40
47
|
const repairedAt = new Date().toISOString();
|
|
@@ -49,7 +56,7 @@ export async function repairPersistedFirstPartyDashboardQueries(): Promise<boole
|
|
|
49
56
|
})
|
|
50
57
|
.where(
|
|
51
58
|
and(
|
|
52
|
-
eq(schema.dashboards.id,
|
|
59
|
+
eq(schema.dashboards.id, row.id),
|
|
53
60
|
eq(schema.dashboards.config, row.config),
|
|
54
61
|
eq(schema.dashboards.updatedAt, row.updatedAt),
|
|
55
62
|
),
|
|
@@ -104,9 +111,84 @@ export async function repairPersistedFirstPartyDashboardQueries(): Promise<boole
|
|
|
104
111
|
});
|
|
105
112
|
} catch (err) {
|
|
106
113
|
console.warn(
|
|
107
|
-
"[db]
|
|
114
|
+
"[db] Dashboard repair committed without a live change event:",
|
|
108
115
|
err instanceof Error ? err.message : err,
|
|
109
116
|
);
|
|
110
117
|
}
|
|
111
118
|
return true;
|
|
112
119
|
}
|
|
120
|
+
|
|
121
|
+
export async function repairPersistedFirstPartyDashboardQueries(): Promise<boolean> {
|
|
122
|
+
// guard:allow-unscoped — startup repair targets one fixed canonical dashboard
|
|
123
|
+
// and only replaces the exact shipped legacy SQL under an optimistic fence.
|
|
124
|
+
const db = getDb() as any;
|
|
125
|
+
const [row] = await db
|
|
126
|
+
.select({
|
|
127
|
+
id: schema.dashboards.id,
|
|
128
|
+
config: schema.dashboards.config,
|
|
129
|
+
kind: schema.dashboards.kind,
|
|
130
|
+
title: schema.dashboards.title,
|
|
131
|
+
updatedAt: schema.dashboards.updatedAt,
|
|
132
|
+
ownerEmail: schema.dashboards.ownerEmail,
|
|
133
|
+
orgId: schema.dashboards.orgId,
|
|
134
|
+
visibility: schema.dashboards.visibility,
|
|
135
|
+
})
|
|
136
|
+
.from(schema.dashboards)
|
|
137
|
+
.where(eq(schema.dashboards.id, FIRST_PARTY_DASHBOARD_ID));
|
|
138
|
+
if (!row || row.kind !== "sql" || typeof row.config !== "string") {
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
return applyRepairToDashboardRow(
|
|
142
|
+
row,
|
|
143
|
+
repairCanonicalFirstPartyDashboardQueries,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Scan every SQL dashboard (not just the one canonical dashboard above) for
|
|
149
|
+
* first-party panels whose SQL exactly matches a known-unbounded pattern —
|
|
150
|
+
* see first-party-unbounded-panel-repair.ts for how these were found (a
|
|
151
|
+
* full-org audit, 2026-07-25) and why exact-string matching, not a general
|
|
152
|
+
* SQL rewrite, is the safe way to fix panels this repair didn't author.
|
|
153
|
+
* Returns the number of dashboards actually changed.
|
|
154
|
+
*/
|
|
155
|
+
export async function repairUnboundedFirstPartyPanelsAcrossDashboards(): Promise<number> {
|
|
156
|
+
// guard:allow-unscoped — this is an org-wide startup repair: it may touch
|
|
157
|
+
// any dashboard's persisted panel SQL, but only ever replaces an exact
|
|
158
|
+
// known-bad SQL string under the same optimistic (config, updatedAt) fence
|
|
159
|
+
// used by the canonical repair above, so a concurrent edit always wins.
|
|
160
|
+
const db = getDb() as any;
|
|
161
|
+
const dialect = getDialect();
|
|
162
|
+
const rows = await db
|
|
163
|
+
.select({
|
|
164
|
+
id: schema.dashboards.id,
|
|
165
|
+
config: schema.dashboards.config,
|
|
166
|
+
kind: schema.dashboards.kind,
|
|
167
|
+
title: schema.dashboards.title,
|
|
168
|
+
updatedAt: schema.dashboards.updatedAt,
|
|
169
|
+
ownerEmail: schema.dashboards.ownerEmail,
|
|
170
|
+
orgId: schema.dashboards.orgId,
|
|
171
|
+
visibility: schema.dashboards.visibility,
|
|
172
|
+
})
|
|
173
|
+
.from(schema.dashboards)
|
|
174
|
+
.where(eq(schema.dashboards.kind, "sql"));
|
|
175
|
+
|
|
176
|
+
let repairedCount = 0;
|
|
177
|
+
for (const row of rows as DashboardRepairRow[]) {
|
|
178
|
+
if (typeof row.config !== "string") continue;
|
|
179
|
+
try {
|
|
180
|
+
const wasRepaired = await applyRepairToDashboardRow(row, (config) =>
|
|
181
|
+
repairUnboundedFirstPartyPanels(config, dialect),
|
|
182
|
+
);
|
|
183
|
+
if (wasRepaired) repairedCount += 1;
|
|
184
|
+
} catch (err) {
|
|
185
|
+
// One dashboard's malformed config or a lost optimistic-concurrency
|
|
186
|
+
// race must not block repairing every other dashboard.
|
|
187
|
+
console.warn(
|
|
188
|
+
`[db] Unbounded first-party panel repair failed for dashboard "${row.id}" (non-fatal):`,
|
|
189
|
+
err instanceof Error ? err.message : err,
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return repairedCount;
|
|
194
|
+
}
|