deepline 0.3.247 → 0.3.249

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +177 -13
  2. package/dist/bundling-sources/sdk/src/index.ts +8 -0
  3. package/dist/bundling-sources/sdk/src/plays/input-example.ts +40 -0
  4. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  5. package/dist/bundling-sources/sdk/src/runs/overview.ts +457 -0
  6. package/dist/bundling-sources/sdk/src/runs/receipts.ts +23 -0
  7. package/dist/bundling-sources/sdk/src/tool-calls.ts +21 -1
  8. package/dist/bundling-sources/sdk/src/types.ts +16 -1
  9. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +19 -1
  10. package/dist/bundling-sources/shared_libs/play-data-plane/column-names.ts +17 -3
  11. package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts +40 -8
  12. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +26 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +144 -14
  14. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +2 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +6 -3
  16. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +26 -21
  17. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +23 -21
  18. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +17 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +105 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +8 -1
  21. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +9 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +40 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +50 -5
  26. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +3 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +28 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +14 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +56 -16
  30. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-control-transport.ts +184 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +327 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +87 -30
  33. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +24 -13
  34. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts +23 -6
  35. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +76 -33
  36. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +90 -35
  37. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +16 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +112 -50
  39. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +447 -77
  40. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +22 -4
  41. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +105 -121
  42. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +1648 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +16 -19
  44. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +14 -6
  45. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +16 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +40 -0
  47. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +5 -2
  48. package/dist/bundling-sources/shared_libs/plays/dataset-summary.ts +11 -7
  49. package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts +5 -2
  50. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +11 -2
  51. package/dist/cli/index.js +3504 -2368
  52. package/dist/cli/index.mjs +3188 -2052
  53. package/dist/{compiler-manifest-n8ZtPv2h.d.mts → compiler-manifest-DavmLa8y.d.mts} +1 -0
  54. package/dist/{compiler-manifest-n8ZtPv2h.d.ts → compiler-manifest-DavmLa8y.d.ts} +1 -0
  55. package/dist/index.d.mts +97 -5
  56. package/dist/index.d.ts +97 -5
  57. package/dist/index.js +135 -15
  58. package/dist/index.mjs +135 -15
  59. package/dist/install-integrity.json +9 -2
  60. package/dist/plays/bundle-play-file.d.mts +2 -2
  61. package/dist/plays/bundle-play-file.d.ts +2 -2
  62. package/dist/plays/bundle-play-file.mjs +1 -1
  63. package/dist/release.d.mts +1 -1
  64. package/dist/release.d.ts +1 -1
  65. package/dist/release.js +1 -1
  66. package/dist/release.mjs +1 -1
  67. package/package.json +1 -1
@@ -0,0 +1,1648 @@
1
+ /**
2
+ * The customer-visible, append-only search projection of durable Play facts.
3
+ *
4
+ * Outbox and receipt triggers write small rows in their authoritative SQL
5
+ * transaction. They never deliver work, replay a Play, or copy receipt output.
6
+ * The API may read only through fixed, tenant-scoped queries in this module.
7
+ */
8
+ import { normalizeRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
9
+ import type { PostgresSchedulerQueryClient } from './postgres';
10
+
11
+ const EVENTS_TABLE = 'runtime_observability.events';
12
+ // Migration readiness tracks this projection body independently of scheduler
13
+ // wire and schema versions. Bump when a changed trigger must be reinstalled.
14
+ export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
15
+ 'runtime-event-search-log-context-v8-terminal-progress-counts-compact-first-progress-channel-aliases';
16
+ export const RUNTIME_EVENTS_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
17
+ const MAX_SEARCH_WINDOW_MS = RUNTIME_EVENTS_RETENTION_MS;
18
+ const MAX_SEARCH_RESULTS = 200;
19
+ const MAX_JSONPATH_BYTES = 512;
20
+
21
+ /** Explicit input validation; arbitrary TypeErrors are never public errors. */
22
+ export class RuntimeEventInputError extends TypeError {
23
+ constructor(message: string) {
24
+ super(message);
25
+ this.name = 'RuntimeEventInputError';
26
+ }
27
+ }
28
+
29
+ export type RuntimeSearchEvent = {
30
+ eventId: string;
31
+ occurredAt: string;
32
+ ingestedAt: string;
33
+ runId: string;
34
+ playName: string;
35
+ kind: string;
36
+ level: string | null;
37
+ message: string | null;
38
+ receiptRef: string | null;
39
+ searchDoc: Record<string, unknown>;
40
+ };
41
+
42
+ export type RuntimeSearchCursor = { occurredAt: string; eventId: string };
43
+
44
+ export type RuntimeEventSearchInput = {
45
+ runtimeScope: string;
46
+ orgId: string;
47
+ since: Date;
48
+ until: Date;
49
+ limit: number;
50
+ runId?: string;
51
+ playName?: string;
52
+ kind?: string;
53
+ jsonpath?: string;
54
+ cursor?: RuntimeSearchCursor;
55
+ };
56
+
57
+ export type RuntimeEventReceipt = {
58
+ /** Server-only; never serialize this physical key into a customer response. */
59
+ receiptKey: string;
60
+ status: 'completed' | 'failed' | 'skipped';
61
+ output: unknown | null;
62
+ outputRef: unknown | null;
63
+ error: string | null;
64
+ errorPayload: unknown | null;
65
+ };
66
+
67
+ export type RuntimeRunLogPage = {
68
+ totalLogCount: number;
69
+ logsTruncated: boolean;
70
+ lastStoredSeq: number;
71
+ firstStoredSeq: number | null;
72
+ entries: Array<{ seq: number; line: string }>;
73
+ };
74
+
75
+ function requiredScope(value: string): string {
76
+ const scope = normalizeRuntimeSchedulerSchema(value);
77
+ if (!scope)
78
+ throw new RuntimeEventInputError(
79
+ 'Runtime event search requires a valid runtime scope.',
80
+ );
81
+ return scope;
82
+ }
83
+
84
+ function requiredOrg(value: string): string {
85
+ const org = value?.trim();
86
+ if (!org || org.length > 256) {
87
+ throw new RuntimeEventInputError(
88
+ 'Runtime event search requires an organization.',
89
+ );
90
+ }
91
+ return org;
92
+ }
93
+
94
+ function quoteIdent(value: string): string {
95
+ return `"${value.replaceAll('"', '""')}"`;
96
+ }
97
+
98
+ function quoteLiteral(value: string): string {
99
+ return `'${value.replaceAll("'", "''")}'`;
100
+ }
101
+
102
+ /**
103
+ * Native PostgreSQL JSONPath equality, restricted to flat indexed fields or
104
+ * one scalar key under the bounded `context` object.
105
+ * No recursive paths, filters, methods, regex, comparisons or wildcards can
106
+ * turn an org-wide bounded search into an arbitrary JSON scan.
107
+ */
108
+ export function validateRuntimeEventJsonpath(value: string): string {
109
+ parseRuntimeEventJsonpath(value);
110
+ return value;
111
+ }
112
+
113
+ type RuntimeEventPredicate = {
114
+ path: string;
115
+ literal: string;
116
+ value: string | number | boolean;
117
+ };
118
+
119
+ function parseRuntimeEventJsonpath(value: string): RuntimeEventPredicate[] {
120
+ if (!value || Buffer.byteLength(value, 'utf8') > MAX_JSONPATH_BYTES) {
121
+ throw new RuntimeEventInputError(
122
+ 'Runtime event JSONPath is empty or exceeds 512 bytes.',
123
+ );
124
+ }
125
+ const clause =
126
+ /\$\.(orgId|runId|playName|runtimeScope|producerAttempt|kind|status|source|stepId|stepKind|datasetId|phase|provider|operation|context\.[A-Za-z_][A-Za-z0-9_]{0,63})\s*==\s*("(?:\\.|[^"\\])*"|true|false|-?\d+(?:\.\d+)?)/y;
127
+ let position = 0;
128
+ let clauses = 0;
129
+ const predicates: RuntimeEventPredicate[] = [];
130
+ while (position < value.length) {
131
+ clause.lastIndex = position;
132
+ const match = clause.exec(value);
133
+ if (!match)
134
+ throw new RuntimeEventInputError(
135
+ 'Runtime event JSONPath supports flat equality predicates only.',
136
+ );
137
+ let parsed: unknown;
138
+ try {
139
+ parsed = JSON.parse(match[2]);
140
+ } catch {
141
+ throw new RuntimeEventInputError(
142
+ 'Runtime event JSONPath requires a valid JSON scalar.',
143
+ );
144
+ }
145
+ if (match[2].startsWith('"')) {
146
+ if (typeof parsed !== 'string' || parsed.length > 256) {
147
+ throw new RuntimeEventInputError(
148
+ 'Runtime event JSONPath string value is too long.',
149
+ );
150
+ }
151
+ }
152
+ if (
153
+ (typeof parsed !== 'string' &&
154
+ typeof parsed !== 'number' &&
155
+ typeof parsed !== 'boolean') ||
156
+ (typeof parsed === 'number' && !Number.isFinite(parsed))
157
+ ) {
158
+ throw new RuntimeEventInputError(
159
+ 'Runtime event JSONPath requires a finite scalar.',
160
+ );
161
+ }
162
+ if (
163
+ ['orgId', 'runId', 'playName', 'runtimeScope'].includes(match[1]) &&
164
+ typeof parsed !== 'string'
165
+ ) {
166
+ throw new RuntimeEventInputError(
167
+ 'Runtime event identity predicates require strings.',
168
+ );
169
+ }
170
+ if (
171
+ match[1] === 'producerAttempt' &&
172
+ (typeof parsed !== 'number' ||
173
+ !Number.isSafeInteger(parsed) ||
174
+ parsed < 1)
175
+ ) {
176
+ throw new RuntimeEventInputError(
177
+ 'Runtime event producerAttempt requires a positive integer.',
178
+ );
179
+ }
180
+ predicates.push({ path: match[1], literal: match[2], value: parsed });
181
+ if (
182
+ match[1].startsWith('context.') &&
183
+ ['__proto__', 'constructor', 'prototype'].includes(
184
+ match[1].slice('context.'.length),
185
+ )
186
+ ) {
187
+ throw new RuntimeEventInputError(
188
+ 'Runtime event JSONPath has an unsupported context key.',
189
+ );
190
+ }
191
+ clauses += 1;
192
+ if (clauses > 8)
193
+ throw new RuntimeEventInputError(
194
+ 'Runtime event JSONPath has too many predicates.',
195
+ );
196
+ position = clause.lastIndex;
197
+ if (position === value.length) break;
198
+ const separator = /\s*&&\s*/y;
199
+ separator.lastIndex = position;
200
+ if (!separator.exec(value)) {
201
+ throw new RuntimeEventInputError(
202
+ 'Runtime event JSONPath supports conjunctions only.',
203
+ );
204
+ }
205
+ position = separator.lastIndex;
206
+ if (position === value.length) {
207
+ throw new RuntimeEventInputError(
208
+ 'Runtime event JSONPath ends with a conjunction.',
209
+ );
210
+ }
211
+ }
212
+ return predicates;
213
+ }
214
+
215
+ type EventRow = {
216
+ event_id: string;
217
+ occurred_at: Date | string;
218
+ ingested_at: Date | string;
219
+ run_id: string;
220
+ play_name: string;
221
+ kind: string;
222
+ level: string | null;
223
+ message: string | null;
224
+ receipt_ref: string | null;
225
+ search_doc: Record<string, unknown>;
226
+ };
227
+
228
+ function iso(value: Date | string): string {
229
+ const date = value instanceof Date ? value : new Date(value);
230
+ if (!Number.isFinite(date.getTime()))
231
+ throw new Error('Runtime event has invalid timestamp.');
232
+ return date.toISOString();
233
+ }
234
+
235
+ /** Client must be a dedicated reader in a transaction with SET LOCAL scope. */
236
+ export async function searchRuntimeEvents(
237
+ client: PostgresSchedulerQueryClient,
238
+ input: RuntimeEventSearchInput,
239
+ ): Promise<{
240
+ events: RuntimeSearchEvent[];
241
+ nextCursor: RuntimeSearchCursor | null;
242
+ }> {
243
+ const scope = requiredScope(input.runtimeScope);
244
+ const org = requiredOrg(input.orgId);
245
+ const since = input.since.getTime();
246
+ const until = input.until.getTime();
247
+ if (
248
+ !Number.isFinite(since) ||
249
+ !Number.isFinite(until) ||
250
+ since >= until ||
251
+ until - since > MAX_SEARCH_WINDOW_MS
252
+ ) {
253
+ throw new RuntimeEventInputError(
254
+ 'Runtime event search requires a window of at most seven days.',
255
+ );
256
+ }
257
+ if (
258
+ !Number.isSafeInteger(input.limit) ||
259
+ input.limit < 1 ||
260
+ input.limit > MAX_SEARCH_RESULTS
261
+ ) {
262
+ throw new RuntimeEventInputError(
263
+ 'Runtime event search limit must be between 1 and 200.',
264
+ );
265
+ }
266
+ const params: unknown[] = [
267
+ scope,
268
+ org,
269
+ input.since.toISOString(),
270
+ input.until.toISOString(),
271
+ ];
272
+ const predicates = [
273
+ 'runtime_scope = $1',
274
+ 'org_id = $2',
275
+ 'occurred_at >= $3::timestamptz',
276
+ 'occurred_at < $4::timestamptz',
277
+ "ingested_at >= clock_timestamp() - interval '7 days'",
278
+ ];
279
+ const addExact = (
280
+ column: 'run_id' | 'play_name' | 'kind',
281
+ value: string | undefined,
282
+ ) => {
283
+ if (value === undefined) return;
284
+ if (!value.trim() || value.length > 256)
285
+ throw new RuntimeEventInputError(
286
+ `Invalid runtime event ${column} filter.`,
287
+ );
288
+ params.push(value);
289
+ predicates.push(`${column} = $${params.length}`);
290
+ };
291
+ addExact('run_id', input.runId);
292
+ addExact('play_name', input.playName);
293
+ addExact('kind', input.kind);
294
+ if (input.jsonpath !== undefined) {
295
+ const identityColumns = {
296
+ orgId: 'org_id',
297
+ runId: 'run_id',
298
+ playName: 'play_name',
299
+ runtimeScope: 'runtime_scope',
300
+ } as const;
301
+ const documentPredicates: string[] = [];
302
+ for (const predicate of parseRuntimeEventJsonpath(input.jsonpath)) {
303
+ const column =
304
+ identityColumns[predicate.path as keyof typeof identityColumns];
305
+ if (column) {
306
+ params.push(predicate.value);
307
+ predicates.push(`${column} = $${params.length}`);
308
+ } else {
309
+ documentPredicates.push(`$.${predicate.path} == ${predicate.literal}`);
310
+ }
311
+ }
312
+ if (documentPredicates.length > 0) {
313
+ params.push(documentPredicates.join(' && '));
314
+ predicates.push(`search_doc @@ $${params.length}::jsonpath`);
315
+ }
316
+ }
317
+ if (input.cursor) {
318
+ const cursorAt = new Date(input.cursor.occurredAt);
319
+ if (
320
+ !Number.isFinite(cursorAt.getTime()) ||
321
+ !input.cursor.eventId ||
322
+ input.cursor.eventId.length > 512
323
+ ) {
324
+ throw new RuntimeEventInputError('Invalid runtime event cursor.');
325
+ }
326
+ params.push(cursorAt.toISOString(), input.cursor.eventId);
327
+ predicates.push(
328
+ `(occurred_at, event_id) < ($${params.length - 1}::timestamptz, $${params.length})`,
329
+ );
330
+ }
331
+ params.push(input.limit + 1);
332
+ const result = await client.query<EventRow>(
333
+ `SELECT event_id, occurred_at, ingested_at, run_id, play_name, kind, level,
334
+ coalesce(message, CASE WHEN kind = 'log.appended' THEN
335
+ left(CASE WHEN search_doc->>'source' = 'terminal' THEN
336
+ regexp_replace(log_body,
337
+ '^' || chr(1) || 'prov:[^' || chr(1) || ']*' || chr(1), '')
338
+ ELSE log_body END, 2048)
339
+ END) AS message,
340
+ receipt_ref, search_doc
341
+ FROM ${EVENTS_TABLE}
342
+ WHERE ${predicates.join(' AND ')}
343
+ ORDER BY occurred_at DESC, event_id DESC
344
+ LIMIT $${params.length}::integer`,
345
+ params,
346
+ );
347
+ const hasMore = result.rows.length > input.limit;
348
+ const events = result.rows.slice(0, input.limit).map((row) => ({
349
+ eventId: row.event_id,
350
+ occurredAt: iso(row.occurred_at),
351
+ ingestedAt: iso(row.ingested_at),
352
+ runId: row.run_id,
353
+ playName: row.play_name,
354
+ kind: row.kind,
355
+ level: row.level,
356
+ message: row.message,
357
+ receiptRef: row.receipt_ref,
358
+ searchDoc: {
359
+ ...row.search_doc,
360
+ orgId: org,
361
+ runId: row.run_id,
362
+ playName: row.play_name,
363
+ runtimeScope: scope,
364
+ },
365
+ }));
366
+ const last = events.at(-1);
367
+ return {
368
+ events,
369
+ nextCursor:
370
+ hasMore && last
371
+ ? { occurredAt: last.occurredAt, eventId: last.eventId }
372
+ : null,
373
+ };
374
+ }
375
+
376
+ /**
377
+ * Reads a receipt through a narrow SECURITY DEFINER function. The reader has
378
+ * no direct grant on any scheduler receipt table, and the function checks both
379
+ * the RLS transaction context and the event reference before dynamic SQL.
380
+ */
381
+ export async function getRuntimeEventReceipt(
382
+ client: PostgresSchedulerQueryClient,
383
+ input: {
384
+ runtimeScope: string;
385
+ orgId: string;
386
+ runId: string;
387
+ receiptRef: string;
388
+ },
389
+ ): Promise<RuntimeEventReceipt | null> {
390
+ const scope = requiredScope(input.runtimeScope);
391
+ const org = requiredOrg(input.orgId);
392
+ if (
393
+ !input.runId ||
394
+ input.runId.length > 256 ||
395
+ !/^[0-9a-f]{8}-[0-9a-f-]{27,}$/i.test(input.receiptRef)
396
+ ) {
397
+ throw new RuntimeEventInputError(
398
+ 'Invalid runtime event receipt reference.',
399
+ );
400
+ }
401
+ const result = await client.query<{ receipt: RuntimeEventReceipt | null }>(
402
+ `SELECT runtime_observability.fetch_receipt($1, $2, $3, $4::uuid) AS receipt`,
403
+ [scope, org, input.runId, input.receiptRef],
404
+ );
405
+ return result.rows[0]?.receipt ?? null;
406
+ }
407
+
408
+ /** Read the existing per-run log contract from the tenant-scoped event index. */
409
+ export async function getRuntimeRunLogPage(
410
+ client: PostgresSchedulerQueryClient,
411
+ input: {
412
+ runtimeScope: string;
413
+ orgId: string;
414
+ runId: string;
415
+ afterSeq: number;
416
+ limit: number;
417
+ /** Return the newest bounded window for a stream's initial connection. */
418
+ tail?: boolean;
419
+ },
420
+ ): Promise<RuntimeRunLogPage> {
421
+ const scope = requiredScope(input.runtimeScope);
422
+ const org = requiredOrg(input.orgId);
423
+ if (!input.runId || input.runId.length > 256) {
424
+ throw new RuntimeEventInputError(
425
+ 'Runtime run log read requires a valid run id.',
426
+ );
427
+ }
428
+ if (!Number.isSafeInteger(input.afterSeq) || input.afterSeq < 0) {
429
+ throw new RuntimeEventInputError(
430
+ 'Runtime run log cursor must be a non-negative integer.',
431
+ );
432
+ }
433
+ if (
434
+ !Number.isSafeInteger(input.limit) ||
435
+ input.limit < 1 ||
436
+ input.limit > 1000
437
+ ) {
438
+ throw new RuntimeEventInputError(
439
+ 'Runtime run log page limit must be between 1 and 1000.',
440
+ );
441
+ }
442
+ const tail = input.tail === true;
443
+ const pageWhere = tail ? '' : 'AND log_seq > $4';
444
+ const pageOrder = tail ? 'DESC' : 'ASC';
445
+ const pageLimit = tail ? '$4::integer' : '$5::integer';
446
+ const pageParams = tail
447
+ ? [scope, org, input.runId, input.limit]
448
+ : [scope, org, input.runId, input.afterSeq, input.limit];
449
+ const result = await client.query<{
450
+ total_log_count: string | number | null;
451
+ first_stored_seq: string | number | null;
452
+ last_stored_seq: string | number | null;
453
+ entries: Array<{ seq: string | number; line: string }>;
454
+ }>(
455
+ // Match the event-then-cursor relation lock order used by migrations and
456
+ // log projection. Cursor-first reads can deadlock a schema expansion.
457
+ `WITH retained AS (
458
+ SELECT min(log_seq) AS first_seq, max(log_seq) AS last_seq
459
+ FROM runtime_observability.events
460
+ WHERE runtime_scope = $1 AND org_id = $2 AND run_id = $3
461
+ AND kind = 'log.appended' AND log_seq IS NOT NULL
462
+ AND (log_body IS NOT NULL OR message IS NOT NULL)
463
+ AND ingested_at >= clock_timestamp() - interval '7 days'
464
+ ), run_cursor AS (
465
+ SELECT last_seq
466
+ FROM runtime_observability.run_log_cursors
467
+ WHERE runtime_scope = $1 AND org_id = $2 AND run_id = $3
468
+ ), page AS (
469
+ SELECT log_seq AS seq, coalesce(log_body, message) AS line
470
+ FROM runtime_observability.events
471
+ WHERE runtime_scope = $1 AND org_id = $2 AND run_id = $3
472
+ AND kind = 'log.appended' AND log_seq IS NOT NULL
473
+ AND (log_body IS NOT NULL OR message IS NOT NULL)
474
+ AND ingested_at >= clock_timestamp() - interval '7 days'
475
+ ${pageWhere}
476
+ ORDER BY log_seq ${pageOrder}
477
+ LIMIT ${pageLimit}
478
+ )
479
+ SELECT (SELECT last_seq FROM run_cursor) AS total_log_count,
480
+ retained.first_seq AS first_stored_seq,
481
+ retained.last_seq AS last_stored_seq,
482
+ coalesce((SELECT jsonb_agg(jsonb_build_object('seq', seq, 'line', line)
483
+ ORDER BY seq) FROM page), '[]'::jsonb) AS entries
484
+ FROM retained`,
485
+ pageParams,
486
+ );
487
+ const row = result.rows[0];
488
+ const totalLogCount = Number(row?.total_log_count ?? 0);
489
+ const firstStoredSeq =
490
+ row?.first_stored_seq == null ? null : Number(row.first_stored_seq);
491
+ const lastStoredSeq =
492
+ row?.last_stored_seq == null ? 0 : Number(row.last_stored_seq);
493
+ if (
494
+ !Number.isSafeInteger(totalLogCount) ||
495
+ !Number.isSafeInteger(lastStoredSeq) ||
496
+ (firstStoredSeq !== null && !Number.isSafeInteger(firstStoredSeq))
497
+ ) {
498
+ throw new Error(
499
+ 'Runtime run log cursor is outside the safe integer range.',
500
+ );
501
+ }
502
+ const rawEntries = Array.isArray(row?.entries) ? row.entries : [];
503
+ const entries = rawEntries.map((entry) => ({
504
+ seq: Number(entry.seq),
505
+ line: entry.line,
506
+ }));
507
+ if (
508
+ entries.some(
509
+ (entry) =>
510
+ !Number.isSafeInteger(entry.seq) || typeof entry.line !== 'string',
511
+ )
512
+ ) {
513
+ throw new Error('Runtime run log page contains an invalid row.');
514
+ }
515
+ return {
516
+ totalLogCount,
517
+ logsTruncated:
518
+ totalLogCount > 0 &&
519
+ (firstStoredSeq !== 1 || lastStoredSeq < totalLogCount),
520
+ lastStoredSeq,
521
+ firstStoredSeq,
522
+ entries,
523
+ };
524
+ }
525
+
526
+ /** Attach only on the explicit migration/bootstrap path, never on a hot read. */
527
+ export async function ensureRuntimeEventSearchSchema(
528
+ client: PostgresSchedulerQueryClient,
529
+ ): Promise<void> {
530
+ await client.query('CREATE SCHEMA IF NOT EXISTS runtime_observability');
531
+ await client.query(`
532
+ CREATE TABLE IF NOT EXISTS runtime_observability.events (
533
+ runtime_scope text NOT NULL,
534
+ org_id text NOT NULL,
535
+ event_id text NOT NULL,
536
+ log_seq bigint,
537
+ occurred_at timestamptz NOT NULL,
538
+ ingested_at timestamptz NOT NULL DEFAULT clock_timestamp(),
539
+ run_id text NOT NULL,
540
+ play_name text NOT NULL,
541
+ kind text NOT NULL,
542
+ level text,
543
+ message text,
544
+ log_body text,
545
+ receipt_ref uuid,
546
+ receipt_key text,
547
+ source_digest text,
548
+ search_doc jsonb NOT NULL DEFAULT '{}'::jsonb,
549
+ PRIMARY KEY (runtime_scope, org_id, event_id),
550
+ CONSTRAINT runtime_events_doc_size CHECK (octet_length(search_doc::text) <= 4096),
551
+ CONSTRAINT runtime_events_message_size CHECK (message IS NULL OR octet_length(message) <= 8192)
552
+ )
553
+ `);
554
+ await client.query(
555
+ `ALTER TABLE runtime_observability.events ADD COLUMN IF NOT EXISTS log_seq bigint`,
556
+ );
557
+ await client.query(
558
+ `ALTER TABLE runtime_observability.events ADD COLUMN IF NOT EXISTS log_body text`,
559
+ );
560
+ // Compact replay evidence survives body expiry. No writer uses it until the
561
+ // reader-first storage expansion has completed its deployment.
562
+ await client.query(
563
+ `ALTER TABLE runtime_observability.events ADD COLUMN IF NOT EXISTS log_text_digest text`,
564
+ );
565
+ await client.query(`CREATE TABLE IF NOT EXISTS runtime_observability.run_log_cursors (
566
+ runtime_scope text NOT NULL,
567
+ org_id text NOT NULL,
568
+ run_id text NOT NULL,
569
+ last_seq bigint NOT NULL CHECK (last_seq >= 1),
570
+ updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
571
+ PRIMARY KEY (runtime_scope, org_id, run_id)
572
+ )`);
573
+ await client.query(
574
+ `ALTER TABLE runtime_observability.run_log_cursors ENABLE ROW LEVEL SECURITY`,
575
+ );
576
+ await client.query(
577
+ `DROP POLICY IF EXISTS runtime_run_log_read_scope ON runtime_observability.run_log_cursors`,
578
+ );
579
+ await client.query(`CREATE POLICY runtime_run_log_read_scope ON runtime_observability.run_log_cursors
580
+ FOR SELECT TO PUBLIC USING (
581
+ org_id = current_setting('app.org_id', true)
582
+ AND runtime_scope = current_setting('app.runtime_scope', true)
583
+ )`);
584
+ await client.query(
585
+ `REVOKE ALL ON runtime_observability.run_log_cursors FROM PUBLIC`,
586
+ );
587
+ await client.query(`CREATE INDEX IF NOT EXISTS runtime_events_run_log_seq_idx
588
+ ON runtime_observability.events (runtime_scope, org_id, run_id, log_seq)
589
+ WHERE kind = 'log.appended' AND log_seq IS NOT NULL`);
590
+ await client.query(
591
+ `DROP INDEX IF EXISTS runtime_observability.runtime_run_log_cursors_retention_idx`,
592
+ );
593
+ await client.query(`CREATE INDEX IF NOT EXISTS runtime_events_org_time_idx
594
+ ON runtime_observability.events (runtime_scope, org_id, occurred_at DESC, event_id DESC)`);
595
+ await client.query(`CREATE INDEX IF NOT EXISTS runtime_events_run_time_idx
596
+ ON runtime_observability.events (runtime_scope, org_id, run_id, occurred_at DESC, event_id DESC)`);
597
+ await client.query(`CREATE INDEX IF NOT EXISTS runtime_events_doc_idx
598
+ ON runtime_observability.events USING gin (search_doc jsonb_path_ops)`);
599
+ await client.query(`CREATE INDEX IF NOT EXISTS runtime_events_retention_idx
600
+ ON runtime_observability.events (ingested_at, event_id)`);
601
+ await client.query(`CREATE UNIQUE INDEX IF NOT EXISTS runtime_events_receipt_ref_idx
602
+ ON runtime_observability.events (runtime_scope, org_id, receipt_ref)
603
+ WHERE receipt_ref IS NOT NULL`);
604
+ await client.query(
605
+ `ALTER TABLE runtime_observability.events ENABLE ROW LEVEL SECURITY`,
606
+ );
607
+ await client.query(
608
+ `DROP POLICY IF EXISTS runtime_events_read_scope ON runtime_observability.events`,
609
+ );
610
+ await client.query(`CREATE POLICY runtime_events_read_scope ON runtime_observability.events
611
+ FOR SELECT TO PUBLIC USING (
612
+ org_id = current_setting('app.org_id', true)
613
+ AND runtime_scope = current_setting('app.runtime_scope', true)
614
+ )`);
615
+ await client.query(`REVOKE ALL ON SCHEMA runtime_observability FROM PUBLIC`);
616
+ await client.query(`REVOKE ALL ON runtime_observability.events FROM PUBLIC`);
617
+ await client.query(`
618
+ CREATE OR REPLACE FUNCTION runtime_observability.fetch_receipt(
619
+ p_scope text, p_org text, p_run text, p_ref uuid
620
+ ) RETURNS jsonb
621
+ LANGUAGE plpgsql SECURITY DEFINER
622
+ SET search_path = pg_catalog
623
+ AS $function$
624
+ DECLARE answer jsonb; receipt_key_value text;
625
+ BEGIN
626
+ IF p_scope !~ '^[a-z_][a-z0-9_]{0,62}$'
627
+ OR p_org IS NULL OR p_run IS NULL OR p_ref IS NULL
628
+ OR p_org IS DISTINCT FROM current_setting('app.org_id', true)
629
+ OR p_scope IS DISTINCT FROM current_setting('app.runtime_scope', true)
630
+ THEN RETURN NULL; END IF;
631
+ SELECT event.receipt_key INTO receipt_key_value
632
+ FROM runtime_observability.events AS event
633
+ WHERE event.runtime_scope = p_scope AND event.org_id = p_org
634
+ AND event.run_id = p_run AND event.receipt_ref = p_ref
635
+ AND event.ingested_at >= clock_timestamp() - interval '7 days';
636
+ IF receipt_key_value IS NULL THEN RETURN NULL; END IF;
637
+ EXECUTE format(
638
+ 'SELECT jsonb_build_object(''receiptKey'', k, ''status'', status, ''output'', output_json, ''outputRef'', output_ref, ''error'', error, ''errorPayload'', error_payload) FROM %I.receipts WHERE org_id = $1 AND run_id = $2 AND k = $3 AND status IN (''completed'', ''failed'', ''skipped'')',
639
+ p_scope
640
+ ) INTO answer USING p_org, p_run, receipt_key_value;
641
+ RETURN answer;
642
+ END
643
+ $function$
644
+ `);
645
+ await client.query(
646
+ `REVOKE ALL ON FUNCTION runtime_observability.fetch_receipt(text,text,text,uuid) FROM PUBLIC`,
647
+ );
648
+ }
649
+
650
+ /** The exact provisioned PlanetScale reader role; never grant to a guessed role. */
651
+ export async function grantRuntimeEventReader(
652
+ client: PostgresSchedulerQueryClient,
653
+ readerRole: string,
654
+ ): Promise<void> {
655
+ if (!readerRole || !/^[A-Za-z_][A-Za-z0-9_.-]{0,127}$/.test(readerRole)) {
656
+ throw new RuntimeEventInputError(
657
+ 'Invalid provisioned runtime event reader role.',
658
+ );
659
+ }
660
+ const role = quoteIdent(readerRole);
661
+ await client.query(`GRANT USAGE ON SCHEMA runtime_observability TO ${role}`);
662
+ await client.query(`GRANT SELECT (
663
+ runtime_scope, org_id, event_id, occurred_at, ingested_at, run_id,
664
+ log_seq, log_body, play_name, kind, level, message, receipt_ref, search_doc
665
+ ) ON runtime_observability.events TO ${role}`);
666
+ await client.query(`GRANT SELECT (runtime_scope, org_id, run_id, last_seq)
667
+ ON runtime_observability.run_log_cursors TO ${role}`);
668
+ await client.query(
669
+ `GRANT EXECUTE ON FUNCTION runtime_observability.fetch_receipt(text,text,text,uuid) TO ${role}`,
670
+ );
671
+ }
672
+
673
+ export async function hasRuntimeEventReaderGrant(
674
+ client: PostgresSchedulerQueryClient,
675
+ readerRole: string,
676
+ runtimeScope: string,
677
+ ): Promise<boolean> {
678
+ if (!readerRole || !/^[A-Za-z_][A-Za-z0-9_.-]{0,127}$/.test(readerRole)) {
679
+ throw new RuntimeEventInputError(
680
+ 'Invalid provisioned runtime event reader role.',
681
+ );
682
+ }
683
+ const scope = requiredScope(runtimeScope);
684
+ const result = await client.query<{ ready: boolean }>(
685
+ `SELECT coalesce(roles.rolcanlogin AND NOT roles.rolsuper AND NOT roles.rolbypassrls
686
+ AND has_schema_privilege(roles.oid, 'runtime_observability', 'USAGE')
687
+ AND has_column_privilege(roles.oid, 'runtime_observability.events', 'event_id', 'SELECT')
688
+ AND has_column_privilege(roles.oid, 'runtime_observability.events', 'log_seq', 'SELECT')
689
+ AND has_column_privilege(roles.oid, 'runtime_observability.events', 'log_body', 'SELECT')
690
+ AND NOT has_column_privilege(roles.oid, 'runtime_observability.events', 'receipt_key', 'SELECT')
691
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.events', 'INSERT')
692
+ AND NOT has_any_column_privilege(roles.oid, 'runtime_observability.events', 'INSERT')
693
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.events', 'UPDATE')
694
+ AND NOT has_any_column_privilege(roles.oid, 'runtime_observability.events', 'UPDATE')
695
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.events', 'DELETE')
696
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.events', 'TRUNCATE')
697
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.events', 'REFERENCES')
698
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.events', 'TRIGGER')
699
+ AND has_column_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'runtime_scope', 'SELECT')
700
+ AND has_column_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'org_id', 'SELECT')
701
+ AND has_column_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'run_id', 'SELECT')
702
+ AND has_column_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'last_seq', 'SELECT')
703
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'INSERT')
704
+ AND NOT has_any_column_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'INSERT')
705
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'UPDATE')
706
+ AND NOT has_any_column_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'UPDATE')
707
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'DELETE')
708
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'TRUNCATE')
709
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'REFERENCES')
710
+ AND NOT has_table_privilege(roles.oid, 'runtime_observability.run_log_cursors', 'TRIGGER')
711
+ AND NOT has_schema_privilege(roles.oid, $2, 'USAGE')
712
+ AND NOT has_schema_privilege(roles.oid, $2, 'CREATE')
713
+ AND NOT EXISTS (SELECT 1 FROM pg_auth_members AS membership
714
+ WHERE pg_has_role(roles.oid, membership.roleid, 'MEMBER'))
715
+ AND NOT EXISTS (SELECT 1 FROM pg_class AS events
716
+ JOIN pg_namespace AS event_schema ON event_schema.oid = events.relnamespace
717
+ WHERE event_schema.nspname = 'runtime_observability'
718
+ AND events.relname = 'events'
719
+ AND pg_has_role(roles.oid, events.relowner, 'USAGE'))
720
+ AND NOT EXISTS (SELECT 1 FROM pg_class AS log_cursors
721
+ JOIN pg_namespace AS event_schema ON event_schema.oid = log_cursors.relnamespace
722
+ WHERE event_schema.nspname = 'runtime_observability'
723
+ AND log_cursors.relname = 'run_log_cursors'
724
+ AND pg_has_role(roles.oid, log_cursors.relowner, 'USAGE'))
725
+ AND (SELECT relrowsecurity FROM pg_class
726
+ WHERE oid = 'runtime_observability.events'::regclass)
727
+ AND (SELECT relrowsecurity FROM pg_class
728
+ WHERE oid = 'runtime_observability.run_log_cursors'::regclass)
729
+ AND NOT EXISTS (SELECT 1 FROM pg_class AS scoped_relation
730
+ JOIN pg_namespace AS scoped_schema ON scoped_schema.oid = scoped_relation.relnamespace
731
+ WHERE scoped_schema.nspname = $2
732
+ AND scoped_relation.relkind IN ('r', 'p', 'v', 'm', 'f')
733
+ AND (has_table_privilege(roles.oid, scoped_relation.oid, 'SELECT')
734
+ OR has_any_column_privilege(roles.oid, scoped_relation.oid, 'SELECT')))
735
+ AND NOT pg_has_role(roles.oid,
736
+ (SELECT receipt_function.proowner FROM pg_proc AS receipt_function
737
+ WHERE receipt_function.oid = 'runtime_observability.fetch_receipt(text,text,text,uuid)'::regprocedure),
738
+ 'USAGE')
739
+ AND has_function_privilege(roles.oid, 'runtime_observability.fetch_receipt(text,text,text,uuid)', 'EXECUTE'), false) AS ready
740
+ FROM pg_roles AS roles WHERE roles.rolname = $1`,
741
+ [readerRole, scope],
742
+ );
743
+ return result.rows[0]?.ready === true;
744
+ }
745
+
746
+ /**
747
+ * Triggers are attached to each scheduler namespace by the explicit migration.
748
+ * Search writes are atomic with the already-durable outbox/receipt write. A
749
+ * trigger failure is loud so no authoritative write can silently lose its
750
+ * corresponding search history after enablement.
751
+ */
752
+ export async function attachRuntimeEventSearchTriggers(
753
+ client: PostgresSchedulerQueryClient,
754
+ runtimeScope: string,
755
+ ): Promise<void> {
756
+ const scope = requiredScope(runtimeScope);
757
+ const schema = quoteIdent(scope);
758
+ await client.query(`
759
+ CREATE OR REPLACE FUNCTION ${schema}.search_receipt_event() RETURNS trigger
760
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
761
+ DECLARE play text; ref uuid;
762
+ BEGIN
763
+ IF NEW.status NOT IN ('completed', 'failed', 'skipped') THEN RETURN NEW; END IF;
764
+ IF TG_OP = 'UPDATE' THEN
765
+ IF NEW.status IS NOT DISTINCT FROM OLD.status
766
+ AND NEW.output_json IS NOT DISTINCT FROM OLD.output_json
767
+ AND NEW.output_ref IS NOT DISTINCT FROM OLD.output_ref
768
+ AND NEW.error IS NOT DISTINCT FROM OLD.error
769
+ THEN RETURN NEW; END IF;
770
+ END IF;
771
+ IF NEW.run_id IS NULL OR NEW.org_id IS NULL THEN
772
+ RAISE EXCEPTION 'Terminal receipt search projection requires run and organization';
773
+ END IF;
774
+ SELECT play_name INTO play FROM ${schema}.runs
775
+ WHERE run_id = NEW.run_id AND org_id = NEW.org_id;
776
+ IF play IS NULL THEN
777
+ RAISE EXCEPTION 'Terminal receipt search projection has no matching run';
778
+ END IF;
779
+ ref := gen_random_uuid();
780
+ INSERT INTO runtime_observability.events (
781
+ runtime_scope, org_id, event_id, occurred_at, run_id, play_name,
782
+ kind, receipt_ref, receipt_key, search_doc
783
+ ) VALUES (
784
+ ${quoteLiteral(scope)}, NEW.org_id, 'receipt:' || ref::text,
785
+ NEW.updated_at, NEW.run_id, play, 'receipt.' || NEW.status,
786
+ ref, NEW.k, jsonb_strip_nulls(jsonb_build_object(
787
+ 'kind', 'receipt.' || NEW.status, 'status', NEW.status,
788
+ 'source', 'receipt', 'producerAttempt', NEW.lease_owner_attempt
789
+ ))
790
+ );
791
+ RETURN NEW;
792
+ END
793
+ $function$
794
+ `);
795
+ await client.query(`DO $create$ BEGIN
796
+ IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.receipts'::regclass
797
+ AND tgname = 'runtime_receipt_event_v1') THEN
798
+ CREATE TRIGGER runtime_receipt_event_v1 AFTER INSERT OR UPDATE ON ${schema}.receipts
799
+ FOR EACH ROW EXECUTE FUNCTION ${schema}.search_receipt_event();
800
+ END IF;
801
+ END $create$`);
802
+ await client.query(`
803
+ CREATE OR REPLACE FUNCTION ${schema}.project_outbox_event(observation ${schema}.outbox) RETURNS void
804
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
805
+ DECLARE run_org text; play text; compact_logs boolean; run_status text; item jsonb; line jsonb; event_batch jsonb;
806
+ ordinal bigint; line_ordinal bigint; kind text; at_time timestamptz;
807
+ doc jsonb; line_doc jsonb; line_context jsonb;
808
+ eid text; existing_digest text; digest text; assigned_seq bigint; terminal_progress_identity boolean;
809
+ log_channel text; log_position bigint; matched_event_id text; matched_doc jsonb;
810
+ legacy_position_prefix text; log_alias jsonb; envelope_attempt jsonb;
811
+ line_count bigint; fresh_event_ids text[] := ARRAY[]::text[];
812
+ recent_log_keys jsonb := '{}'::jsonb; dedupe_key text;
813
+ terminal_overlap_counts jsonb := '{}'::jsonb;
814
+ recent_log_keys_loaded boolean := false; recent_log_attempt text;
815
+ BEGIN
816
+ -- Log ordering owns a transaction-scoped mutex, independent of the
817
+ -- scheduler's Run/attempt row-lock order. In particular, old completion
818
+ -- writers lock attempt before Run; this recorder must not invert that.
819
+ PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(observation.run_id));
820
+ SELECT org_id, play_name, run_log_store = 'planet_scale', status
821
+ INTO run_org, play, compact_logs, run_status
822
+ FROM ${schema}.runs WHERE run_id = observation.run_id;
823
+ IF run_org IS NULL OR play IS NULL THEN
824
+ RAISE EXCEPTION 'Outbox search projection has no matching run';
825
+ END IF;
826
+ IF observation.event_type = 'run.events' THEN
827
+ event_batch := observation.payload_json->'events';
828
+ IF jsonb_typeof(event_batch) IS DISTINCT FROM 'array' THEN
829
+ RAISE EXCEPTION 'Search projection requires run.events array';
830
+ END IF;
831
+ IF jsonb_array_length(event_batch) = 0
832
+ AND observation.payload_json->>'logBodyStore' IS DISTINCT FROM 'planet_scale' THEN
833
+ RAISE EXCEPTION 'Search projection requires nonempty run.events array';
834
+ END IF;
835
+ ELSIF observation.event_type IN ('run.completed', 'run.failed', 'runner.logs')
836
+ AND observation.payload_json ? 'terminalProgressEvents' THEN
837
+ event_batch := observation.payload_json->'terminalProgressEvents';
838
+ IF jsonb_typeof(event_batch) IS DISTINCT FROM 'array' THEN
839
+ RAISE EXCEPTION 'Search projection requires terminalProgressEvents array';
840
+ END IF;
841
+ END IF;
842
+ -- Ordinary terminal SQL already binds the attempt in its counter
843
+ -- segment; compact carriers also name it at the top level.
844
+ envelope_attempt := coalesce(NULLIF(observation.payload_json->'producerAttempt', 'null'::jsonb),
845
+ observation.payload_json->'terminalWorkCounterSegment'->'attempt');
846
+ -- The retained envelope has its own position space: it can contain
847
+ -- diagnostics absent from both live delivery and ctx-only replay.
848
+ IF observation.event_type IN ('run.completed', 'run.failed', 'runner.logs')
849
+ AND observation.payload_json ? 'terminalLogLines'
850
+ AND (envelope_attempt #>> '{}') ~ '^[1-9][0-9]{0,5}$' THEN
851
+ event_batch := coalesce(event_batch, '[]'::jsonb) || jsonb_build_array(
852
+ jsonb_strip_nulls(jsonb_build_object(
853
+ 'type', 'log.appended', 'runId', observation.run_id, 'source', 'terminal',
854
+ 'producerAttempt', envelope_attempt, 'channelOffset', 0,
855
+ 'lines', observation.payload_json->'terminalLogLines',
856
+ 'lineContexts', observation.payload_json->'terminalLogContexts'
857
+ ))
858
+ );
859
+ END IF;
860
+ IF event_batch IS NOT NULL THEN
861
+ -- Normalize old positional writers once per batch under the run mutex.
862
+ -- Per-line lookups must remain digest-selective: an OR fallback for
863
+ -- unmarked rows otherwise scans the whole attempt for every new line.
864
+ IF EXISTS (SELECT 1 FROM jsonb_array_elements(event_batch) AS entry
865
+ WHERE entry->>'type' = 'log.appended') THEN
866
+ UPDATE runtime_observability.events AS event SET search_doc = event.search_doc ||
867
+ jsonb_build_object('logContentDigest', event.source_digest) ||
868
+ CASE WHEN NOT (event.search_doc ? 'logPositions')
869
+ AND left(event.event_id, length('log:' || observation.run_id || ':' || (event.search_doc->>'producerAttempt') || ':')) =
870
+ 'log:' || observation.run_id || ':' || (event.search_doc->>'producerAttempt') || ':'
871
+ AND substring(event.event_id FROM length('log:' || observation.run_id || ':' || (event.search_doc->>'producerAttempt') || ':') + 1) ~ '^[0-9]{1,12}$'
872
+ THEN jsonb_build_object('logPositions', jsonb_build_object(
873
+ CASE WHEN event.search_doc->>'source' = 'coordinator' THEN 'replay' ELSE 'worker' END,
874
+ substring(event.event_id FROM length('log:' || observation.run_id || ':' || (event.search_doc->>'producerAttempt') || ':') + 1)::bigint))
875
+ ELSE '{}'::jsonb END
876
+ WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
877
+ AND event.run_id = observation.run_id AND event.kind = 'log.appended'
878
+ AND event.source_digest IS NOT NULL AND NOT (event.search_doc ? 'logContentDigest')
879
+ AND (event.search_doc->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$';
880
+ END IF;
881
+ FOR item, ordinal IN SELECT value, ordinality FROM jsonb_array_elements(event_batch) WITH ORDINALITY LOOP
882
+ IF jsonb_typeof(item) IS DISTINCT FROM 'object'
883
+ OR item->>'runId' IS DISTINCT FROM observation.run_id THEN
884
+ RAISE EXCEPTION 'Search projection event has invalid run identity';
885
+ END IF;
886
+ kind := coalesce(item->>'type', item->>'key');
887
+ IF kind NOT IN (
888
+ 'run.created', 'run.started', 'run.waiting', 'run.resumed',
889
+ 'run.completed', 'run.failed', 'run.cancelled',
890
+ 'step.started', 'step.progress', 'step.completed',
891
+ 'step.failed', 'step.skipped', 'log.appended',
892
+ 'dataset.lifecycle', 'activity.observed'
893
+ ) THEN
894
+ kind := 'runtime.event';
895
+ END IF;
896
+ at_time := CASE WHEN (item->>'occurredAt') ~ '^[0-9]{12,14}$'
897
+ THEN to_timestamp((item->>'occurredAt')::double precision / 1000)
898
+ ELSE observation.created_at END;
899
+ doc := jsonb_strip_nulls(jsonb_build_object(
900
+ 'kind', kind, 'source', left(item->>'source', 64),
901
+ 'producerAttempt', CASE WHEN (item->>'producerAttempt') ~ '^[0-9]{1,6}$'
902
+ THEN CASE WHEN (item->>'producerAttempt')::integer > 0
903
+ THEN (item->>'producerAttempt')::integer END END,
904
+ 'stepId', left(coalesce(item->>'stepId', item->'observation'->>'stepId'), 256),
905
+ 'stepKind', left(item->>'kind', 64),
906
+ 'status', left(coalesce(item->>'status', item->'observation'->'state'->>'kind'), 64),
907
+ 'datasetId', left(item->>'datasetId', 256),
908
+ 'phase', left(item->>'phase', 64),
909
+ 'provider', CASE WHEN item->'observation'->'target'->>'kind' = 'provider'
910
+ THEN left(item->'observation'->'target'->>'provider', 64) END,
911
+ 'operation', CASE WHEN item->'observation'->'target'->>'kind' = 'provider'
912
+ THEN left(item->'observation'->'target'->>'operation', 256) END
913
+ ));
914
+ IF kind = 'log.appended' AND jsonb_typeof(item->'lines') IS DISTINCT FROM 'array' THEN
915
+ RAISE EXCEPTION 'Search projection requires log.appended lines';
916
+ END IF;
917
+ IF kind = 'log.appended' AND item ? 'lineContexts'
918
+ AND (jsonb_typeof(item->'lineContexts') IS DISTINCT FROM 'array'
919
+ OR jsonb_array_length(item->'lineContexts') <> jsonb_array_length(item->'lines')) THEN
920
+ RAISE EXCEPTION 'Search projection requires lineContexts aligned to log lines';
921
+ END IF;
922
+ IF kind = 'log.appended' THEN
923
+ FOR line, line_ordinal IN SELECT value, ordinality FROM jsonb_array_elements(item->'lines') WITH ORDINALITY LOOP
924
+ IF jsonb_typeof(line) IS DISTINCT FROM 'string' THEN
925
+ RAISE EXCEPTION 'Search projection requires log line strings';
926
+ END IF;
927
+ IF (item->>'source' IS DISTINCT FROM 'coordinator')
928
+ AND NOT coalesce((item->>'channelOffset') ~ '^[0-9]{1,12}$', false) THEN
929
+ IF NOT recent_log_keys_loaded OR recent_log_attempt IS DISTINCT FROM item->>'producerAttempt' THEN
930
+ SELECT coalesce(jsonb_object_agg(recent.dedupe_key, true), '{}'::jsonb)
931
+ INTO recent_log_keys
932
+ FROM (
933
+ SELECT coalesce(event.log_text_digest, encode(sha256(convert_to(btrim(event.log_body), 'UTF8')), 'hex')) AS dedupe_key
934
+ FROM runtime_observability.events AS event
935
+ WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
936
+ AND event.run_id = observation.run_id AND event.kind = 'log.appended'
937
+ AND event.log_seq IS NOT NULL AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
938
+ AND (item->>'producerAttempt' IS NULL
939
+ OR event.search_doc->>'producerAttempt' = item->>'producerAttempt')
940
+ ORDER BY event.log_seq DESC
941
+ LIMIT 256
942
+ ) AS recent;
943
+ recent_log_keys_loaded := true;
944
+ recent_log_attempt := item->>'producerAttempt';
945
+ END IF;
946
+ dedupe_key := encode(
947
+ sha256(convert_to(btrim(line #>> '{}'), 'UTF8')), 'hex'
948
+ );
949
+ IF recent_log_keys ? dedupe_key THEN
950
+ CONTINUE;
951
+ END IF;
952
+ recent_log_keys := recent_log_keys || jsonb_build_object(dedupe_key, true);
953
+ END IF;
954
+ line_context := NULL;
955
+ IF item ? 'lineContexts' THEN
956
+ line_context := item->'lineContexts'->((line_ordinal - 1)::int);
957
+ IF jsonb_typeof(line_context) = 'null' THEN
958
+ line_context := NULL;
959
+ ELSIF jsonb_typeof(line_context) IS DISTINCT FROM 'object' THEN
960
+ RAISE EXCEPTION 'Search projection log context must be an object';
961
+ ELSIF (SELECT count(*) FROM jsonb_object_keys(line_context)) > 8
962
+ -- jsonb::text adds at most one formatting space per key
963
+ -- separator; leave that small allowance above the compact
964
+ -- 2 KiB runtime JSON encoding limit.
965
+ OR octet_length(line_context::text) > 2064 THEN
966
+ RAISE EXCEPTION 'Search projection log context exceeds its object limits';
967
+ END IF;
968
+ IF line_context IS NOT NULL AND EXISTS (
969
+ SELECT 1 FROM jsonb_each(line_context) AS field(key, value)
970
+ WHERE field.key !~ '^[A-Za-z_][A-Za-z0-9_]{0,63}$'
971
+ OR field.key IN ('__proto__', 'constructor', 'prototype')
972
+ OR jsonb_typeof(field.value) NOT IN ('string', 'number', 'boolean')
973
+ OR (jsonb_typeof(field.value) = 'string'
974
+ AND octet_length(field.value #>> '{}') > 256)
975
+ OR (jsonb_typeof(field.value) = 'number'
976
+ AND octet_length(field.value #>> '{}') > 32)
977
+ ) THEN
978
+ RAISE EXCEPTION 'Search projection log context has an unsupported key or value';
979
+ END IF;
980
+ END IF;
981
+ line_doc := CASE WHEN line_context IS NULL THEN doc
982
+ ELSE doc || jsonb_build_object('context', line_context) END;
983
+ eid := CASE WHEN (item->>'channelOffset') ~ '^[0-9]{1,12}$'
984
+ AND (item->>'producerAttempt') ~ '^[0-9]{1,6}$'
985
+ THEN 'log:' || observation.run_id || ':' || (item->>'producerAttempt') || ':' ||
986
+ ((item->>'channelOffset')::bigint + line_ordinal - 1)::text
987
+ ELSE observation.event_id || ':' || ordinal || ':' || line_ordinal END;
988
+ digest := CASE WHEN line_context IS NULL
989
+ THEN encode(sha256(convert_to(line #>> '{}', 'UTF8')), 'hex')
990
+ ELSE encode(sha256(convert_to(jsonb_build_array(line #>> '{}', line_context)::text, 'UTF8')), 'hex') END;
991
+ IF (item->>'channelOffset') ~ '^[0-9]{1,12}$'
992
+ AND (item->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$' THEN
993
+ -- These transports retain different buffers. Their offsets
994
+ -- are immutable within a channel, never interchangeable.
995
+ log_channel := CASE item->>'source'
996
+ WHEN 'coordinator' THEN 'replay' WHEN 'terminal' THEN 'envelope'
997
+ ELSE 'worker' END;
998
+ log_position := (item->>'channelOffset')::bigint + line_ordinal - 1;
999
+ legacy_position_prefix := 'log:' || observation.run_id || ':' || (item->>'producerAttempt') || ':';
1000
+ eid := CASE WHEN log_channel = 'worker' THEN legacy_position_prefix
1001
+ ELSE 'log-' || log_channel || ':' || observation.run_id || ':' || (item->>'producerAttempt') || ':' END
1002
+ || log_position::text;
1003
+ -- Older coordinator replays used the worker-shaped primary
1004
+ -- key. Do not overwrite it when a delayed worker owns a
1005
+ -- different line at that number; aliases remain authoritative.
1006
+ IF log_channel = 'worker' AND EXISTS (
1007
+ SELECT 1 FROM runtime_observability.events
1008
+ WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org AND event_id = eid
1009
+ AND search_doc->>'source' = 'coordinator'
1010
+ AND NOT (coalesce(search_doc->'logPositions', '{}'::jsonb) ? 'worker')
1011
+ ) THEN
1012
+ eid := 'log-worker:' || observation.run_id || ':' || (item->>'producerAttempt') || ':' || log_position::text;
1013
+ END IF;
1014
+ log_alias := jsonb_build_object('producerAttempt', (item->>'producerAttempt')::integer,
1015
+ 'logPositions', jsonb_build_object(log_channel, log_position));
1016
+ matched_event_id := NULL;
1017
+ -- A channel position names one canonical occurrence. Sorting
1018
+ -- here makes cached plans scan the primary key on a miss.
1019
+ SELECT event.event_id, event.source_digest, event.search_doc
1020
+ INTO matched_event_id, existing_digest, matched_doc
1021
+ FROM runtime_observability.events AS event
1022
+ WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
1023
+ AND event.run_id = observation.run_id AND event.kind = 'log.appended'
1024
+ AND event.search_doc @> log_alias;
1025
+ IF matched_event_id IS NOT NULL THEN
1026
+ IF existing_digest IS DISTINCT FROM digest THEN
1027
+ RAISE EXCEPTION 'Positional log event identity changed its content';
1028
+ END IF;
1029
+ CONTINUE;
1030
+ END IF;
1031
+ -- Bind one equal occurrence from another channel. Aliases
1032
+ -- are checked first, so retries cannot consume another copy.
1033
+ -- Full digests include context; repeated text is not a set.
1034
+ SELECT event.event_id, event.search_doc
1035
+ INTO matched_event_id, matched_doc
1036
+ FROM runtime_observability.events AS event
1037
+ WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
1038
+ AND event.run_id = observation.run_id AND event.kind = 'log.appended'
1039
+ AND event.search_doc @> jsonb_build_object(
1040
+ 'producerAttempt', (item->>'producerAttempt')::integer, 'logContentDigest', digest)
1041
+ AND event.source_digest = digest
1042
+ AND NOT (coalesce(event.search_doc->'logPositions', '{}'::jsonb) ? log_channel)
1043
+ AND (event.search_doc ? 'logPositions' OR log_channel <>
1044
+ CASE WHEN event.search_doc->>'source' = 'coordinator' THEN 'replay' ELSE 'worker' END)
1045
+ ORDER BY event.log_seq NULLS LAST, event.event_id LIMIT 1;
1046
+ IF matched_event_id IS NOT NULL THEN
1047
+ UPDATE runtime_observability.events SET search_doc = matched_doc ||
1048
+ jsonb_build_object('logContentDigest', digest, 'logPositions',
1049
+ coalesce(matched_doc->'logPositions', '{}'::jsonb) || jsonb_build_object(log_channel, log_position))
1050
+ WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org AND event_id = matched_event_id;
1051
+ CONTINUE;
1052
+ END IF;
1053
+ line_doc := line_doc || log_alias || jsonb_build_object('logContentDigest', digest);
1054
+ END IF;
1055
+ -- Serialize with terminal acceptance on the log mutex. A late
1056
+ -- retry of an accepted position is harmless; a new position
1057
+ -- after terminal closure must never recreate expired history.
1058
+ IF compact_logs AND observation.event_type = 'run.events'
1059
+ AND run_status IN ('completed', 'failed', 'cancelled')
1060
+ AND NOT EXISTS (SELECT 1 FROM runtime_observability.events
1061
+ WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org
1062
+ AND event_id = eid) THEN
1063
+ RAISE EXCEPTION 'Run log ingress is closed after terminal acceptance';
1064
+ END IF;
1065
+ INSERT INTO runtime_observability.events (
1066
+ runtime_scope, org_id, event_id, occurred_at, run_id, play_name,
1067
+ kind, level, message, log_body, source_digest, log_text_digest, search_doc
1068
+ ) VALUES (
1069
+ ${quoteLiteral(scope)}, run_org, eid, at_time, observation.run_id, play,
1070
+ 'log.appended', NULL, CASE WHEN compact_logs THEN NULL ELSE left(line #>> '{}', 2048) END,
1071
+ line #>> '{}', digest,
1072
+ encode(sha256(convert_to(btrim(line #>> '{}'), 'UTF8')), 'hex'), line_doc
1073
+ ) ON CONFLICT DO NOTHING;
1074
+ IF FOUND THEN
1075
+ fresh_event_ids := array_append(fresh_event_ids, eid);
1076
+ ELSE
1077
+ SELECT source_digest INTO existing_digest FROM runtime_observability.events
1078
+ WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org
1079
+ AND event_id = eid;
1080
+ IF existing_digest IS DISTINCT FROM digest THEN
1081
+ RAISE EXCEPTION 'Positional log event identity changed its content';
1082
+ END IF;
1083
+ END IF;
1084
+ END LOOP;
1085
+ ELSE
1086
+ -- Final progress may arrive in the first terminal receipt and
1087
+ -- again in completion. One attempt/ordinal owns one immutable fact.
1088
+ terminal_progress_identity := observation.event_type IN ('runner.logs', 'run.completed', 'run.failed')
1089
+ AND coalesce((observation.payload_json->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$', false);
1090
+ eid := CASE WHEN terminal_progress_identity
1091
+ THEN 'terminal-progress:' || observation.run_id || ':' ||
1092
+ (observation.payload_json->>'producerAttempt') || ':' ||
1093
+ coalesce(item->>'_runtimeProgressOrdinal', ordinal::text)
1094
+ ELSE observation.event_id || ':' || ordinal END;
1095
+ digest := CASE WHEN terminal_progress_identity
1096
+ THEN encode(sha256(convert_to(item::text, 'UTF8')), 'hex') ELSE NULL END;
1097
+ INSERT INTO runtime_observability.events (
1098
+ runtime_scope, org_id, event_id, occurred_at, run_id, play_name,
1099
+ kind, source_digest, search_doc
1100
+ ) VALUES (
1101
+ ${quoteLiteral(scope)}, run_org, eid,
1102
+ at_time, observation.run_id, play, kind, digest, doc
1103
+ ) ON CONFLICT DO NOTHING;
1104
+ IF NOT FOUND AND terminal_progress_identity THEN
1105
+ SELECT source_digest INTO existing_digest FROM runtime_observability.events
1106
+ WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org AND event_id = eid;
1107
+ IF existing_digest IS DISTINCT FROM digest THEN
1108
+ RAISE EXCEPTION 'Progress event identity changed its content';
1109
+ END IF;
1110
+ END IF;
1111
+ END IF;
1112
+ END LOOP;
1113
+ END IF;
1114
+ IF observation.event_type IN (
1115
+ 'run.created', 'run.started', 'run.waiting', 'run.resumed',
1116
+ 'run.completed', 'run.failed', 'run.cancelled'
1117
+ ) AND NOT (observation.event_type IN ('run.waiting', 'run.resumed')
1118
+ AND observation.payload_json->>'kind' = 'detached_runner') THEN
1119
+ INSERT INTO runtime_observability.events (
1120
+ runtime_scope, org_id, event_id, occurred_at, run_id, play_name,
1121
+ kind, search_doc
1122
+ ) VALUES (
1123
+ ${quoteLiteral(scope)}, run_org, observation.event_id, observation.created_at,
1124
+ observation.run_id, play, observation.event_type, jsonb_build_object('kind', observation.event_type, 'source', 'system')
1125
+ ) ON CONFLICT DO NOTHING;
1126
+ END IF;
1127
+ IF observation.event_type IN ('run.completed', 'run.failed', 'runner.logs') THEN
1128
+ -- Detached runners deliver their retained ctx.log tail in the terminal
1129
+ -- outbox fact rather than a run.events batch. Project it from that
1130
+ -- authoritative write so search and the existing log stream agree.
1131
+ IF observation.payload_json ? 'terminalLogContexts'
1132
+ AND NOT (observation.payload_json ? 'terminalLogLines') THEN
1133
+ RAISE EXCEPTION 'Search projection requires terminalLogLines for terminalLogContexts';
1134
+ END IF;
1135
+ IF observation.event_type IN ('run.completed', 'run.failed', 'runner.logs')
1136
+ AND observation.payload_json ? 'terminalLogLines'
1137
+ AND NOT coalesce((envelope_attempt #>> '{}') ~ '^[1-9][0-9]{0,5}$', false) THEN
1138
+ IF jsonb_typeof(observation.payload_json->'terminalLogLines') IS DISTINCT FROM 'array' THEN
1139
+ RAISE EXCEPTION 'Search projection requires terminalLogLines array';
1140
+ END IF;
1141
+ IF observation.payload_json ? 'terminalLogContexts'
1142
+ AND (jsonb_typeof(observation.payload_json->'terminalLogContexts') IS DISTINCT FROM 'array'
1143
+ OR jsonb_array_length(observation.payload_json->'terminalLogContexts') <> jsonb_array_length(observation.payload_json->'terminalLogLines')) THEN
1144
+ RAISE EXCEPTION 'Search projection requires terminalLogContexts aligned to terminalLogLines';
1145
+ END IF;
1146
+ -- Count matching bodies already projected, including progress lines
1147
+ -- inserted earlier in this trigger (which do not have log_seq yet).
1148
+ -- Consume one count per overlap so repeated identical log lines in
1149
+ -- the terminal tail remain distinct events after the overlap ends.
1150
+ SELECT coalesce(
1151
+ jsonb_object_agg(counted.dedupe_key, to_jsonb(counted.occurrences)),
1152
+ '{}'::jsonb
1153
+ )
1154
+ INTO terminal_overlap_counts
1155
+ FROM (
1156
+ SELECT candidate.dedupe_key, count(*) AS occurrences
1157
+ FROM (
1158
+ SELECT coalesce(recent.log_text_digest, encode(sha256(convert_to(btrim(recent.log_body), 'UTF8')), 'hex')) AS dedupe_key
1159
+ FROM (
1160
+ SELECT event.log_body, event.log_text_digest
1161
+ FROM runtime_observability.events AS event
1162
+ WHERE event.runtime_scope = ${quoteLiteral(scope)}
1163
+ AND event.org_id = run_org AND event.run_id = observation.run_id
1164
+ AND event.kind = 'log.appended'
1165
+ AND (observation.payload_json->>'producerAttempt' IS NULL
1166
+ OR event.search_doc->>'producerAttempt' = observation.payload_json->>'producerAttempt')
1167
+ AND event.log_seq IS NOT NULL AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
1168
+ ORDER BY event.log_seq DESC
1169
+ LIMIT 256
1170
+ ) AS recent
1171
+ UNION ALL
1172
+ SELECT coalesce(event.log_text_digest, encode(sha256(convert_to(btrim(event.log_body), 'UTF8')), 'hex'))
1173
+ FROM runtime_observability.events AS event
1174
+ WHERE event.runtime_scope = ${quoteLiteral(scope)}
1175
+ AND event.org_id = run_org AND event.run_id = observation.run_id
1176
+ AND event.kind = 'log.appended' AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
1177
+ AND event.event_id = ANY(fresh_event_ids)
1178
+ AND (observation.payload_json->>'producerAttempt' IS NULL
1179
+ OR event.search_doc->>'producerAttempt' = observation.payload_json->>'producerAttempt')
1180
+ ) AS candidate
1181
+ GROUP BY candidate.dedupe_key
1182
+ ) AS counted;
1183
+ FOR line, line_ordinal IN
1184
+ SELECT value, ordinality
1185
+ FROM jsonb_array_elements(observation.payload_json->'terminalLogLines') WITH ORDINALITY
1186
+ LOOP
1187
+ IF jsonb_typeof(line) IS DISTINCT FROM 'string' THEN
1188
+ RAISE EXCEPTION 'Search projection requires terminal log strings';
1189
+ END IF;
1190
+ dedupe_key := encode(
1191
+ sha256(convert_to(btrim(line #>> '{}'), 'UTF8')), 'hex'
1192
+ );
1193
+ IF coalesce((terminal_overlap_counts->>dedupe_key)::bigint, 0) > 0 THEN
1194
+ terminal_overlap_counts := terminal_overlap_counts || jsonb_build_object(
1195
+ dedupe_key,
1196
+ (terminal_overlap_counts->>dedupe_key)::bigint - 1
1197
+ );
1198
+ CONTINUE;
1199
+ END IF;
1200
+ line_context := NULL;
1201
+ IF observation.payload_json ? 'terminalLogContexts' THEN
1202
+ line_context := observation.payload_json->'terminalLogContexts'->((line_ordinal - 1)::int);
1203
+ IF jsonb_typeof(line_context) = 'null' THEN
1204
+ line_context := NULL;
1205
+ ELSIF jsonb_typeof(line_context) IS DISTINCT FROM 'object' THEN
1206
+ RAISE EXCEPTION 'Search projection terminal log context must be an object';
1207
+ ELSIF (SELECT count(*) FROM jsonb_object_keys(line_context)) > 8
1208
+ -- Allow jsonb::text's bounded separator spaces above the
1209
+ -- runtime's compact 2 KiB context limit.
1210
+ OR octet_length(line_context::text) > 2064 THEN
1211
+ RAISE EXCEPTION 'Search projection terminal log context exceeds its object limits';
1212
+ END IF;
1213
+ IF line_context IS NOT NULL AND EXISTS (
1214
+ SELECT 1 FROM jsonb_each(line_context) AS field(key, value)
1215
+ WHERE field.key !~ '^[A-Za-z_][A-Za-z0-9_]{0,63}$'
1216
+ OR field.key IN ('__proto__', 'constructor', 'prototype')
1217
+ OR jsonb_typeof(field.value) NOT IN ('string', 'number', 'boolean')
1218
+ OR (jsonb_typeof(field.value) = 'string'
1219
+ AND octet_length(field.value #>> '{}') > 256)
1220
+ OR (jsonb_typeof(field.value) = 'number'
1221
+ AND octet_length(field.value #>> '{}') > 32)
1222
+ ) THEN
1223
+ RAISE EXCEPTION 'Search projection terminal log context has an unsupported key or value';
1224
+ END IF;
1225
+ END IF;
1226
+ line_doc := jsonb_strip_nulls(jsonb_build_object(
1227
+ 'kind', 'log.appended', 'source', 'terminal',
1228
+ 'producerAttempt', observation.payload_json->'producerAttempt'
1229
+ ));
1230
+ IF line_context IS NOT NULL THEN
1231
+ line_doc := line_doc || jsonb_build_object('context', line_context);
1232
+ END IF;
1233
+ INSERT INTO runtime_observability.events (
1234
+ runtime_scope, org_id, event_id, occurred_at, run_id, play_name,
1235
+ kind, level, message, log_body, source_digest, log_text_digest, search_doc
1236
+ ) VALUES (
1237
+ ${quoteLiteral(scope)}, run_org, observation.event_id || ':log:' || line_ordinal,
1238
+ observation.created_at, observation.run_id, play, 'log.appended', NULL,
1239
+ CASE WHEN compact_logs THEN NULL ELSE left(regexp_replace(line #>> '{}',
1240
+ '^' || chr(1) || 'prov:[^' || chr(1) || ']*' || chr(1), ''), 2048) END,
1241
+ line #>> '{}',
1242
+ CASE WHEN line_context IS NULL
1243
+ THEN encode(sha256(convert_to(line #>> '{}', 'UTF8')), 'hex')
1244
+ ELSE encode(sha256(convert_to(jsonb_build_array(line #>> '{}', line_context)::text, 'UTF8')), 'hex') END,
1245
+ encode(sha256(convert_to(btrim(line #>> '{}'), 'UTF8')), 'hex'),
1246
+ line_doc
1247
+ ) ON CONFLICT DO NOTHING;
1248
+ IF FOUND THEN
1249
+ fresh_event_ids := array_append(
1250
+ fresh_event_ids,
1251
+ observation.event_id || ':log:' || line_ordinal
1252
+ );
1253
+ END IF;
1254
+ END LOOP;
1255
+ END IF;
1256
+ END IF;
1257
+ line_count := cardinality(fresh_event_ids);
1258
+ IF line_count > 0 THEN
1259
+ INSERT INTO runtime_observability.run_log_cursors (
1260
+ runtime_scope, org_id, run_id, last_seq
1261
+ ) VALUES (${quoteLiteral(scope)}, run_org, observation.run_id, line_count)
1262
+ ON CONFLICT (runtime_scope, org_id, run_id)
1263
+ DO UPDATE SET last_seq = runtime_observability.run_log_cursors.last_seq + EXCLUDED.last_seq,
1264
+ updated_at = clock_timestamp()
1265
+ RETURNING last_seq INTO assigned_seq;
1266
+ UPDATE runtime_observability.events AS event
1267
+ SET log_seq = assigned_seq - line_count + fresh.ordinality
1268
+ FROM unnest(fresh_event_ids) WITH ORDINALITY AS fresh(event_id, ordinality)
1269
+ WHERE event.runtime_scope = ${quoteLiteral(scope)}
1270
+ AND event.org_id = run_org
1271
+ AND event.event_id = fresh.event_id;
1272
+ END IF;
1273
+ RETURN;
1274
+ END
1275
+ $function$
1276
+ `);
1277
+ await client.query(`
1278
+ CREATE OR REPLACE FUNCTION ${schema}.without_log_events(batch jsonb) RETURNS jsonb
1279
+ LANGUAGE sql IMMUTABLE SET search_path = pg_catalog AS $function$
1280
+ SELECT coalesce(jsonb_agg(item.value ORDER BY item.ordinality), '[]'::jsonb)
1281
+ FROM jsonb_array_elements(batch) WITH ORDINALITY AS item(value, ordinality)
1282
+ WHERE coalesce(item.value->>'type', item.value->>'key') IS DISTINCT FROM 'log.appended'
1283
+ $function$
1284
+ `);
1285
+ await client.query(`
1286
+ CREATE OR REPLACE FUNCTION ${schema}.without_runtime_log_tail(value jsonb) RETURNS jsonb
1287
+ LANGUAGE sql IMMUTABLE SET search_path = pg_catalog AS $function$
1288
+ SELECT CASE
1289
+ -- This reserved metadata shape is produced by the runtime. Authored
1290
+ -- output/checkpoint strings and unrelated metadata are left intact.
1291
+ WHEN jsonb_typeof(value#>'{_metadata,runLogTail,tail}') = 'array'
1292
+ AND jsonb_typeof(value#>'{_metadata,runLogTail,totalLogCount}') = 'number'
1293
+ THEN value #- '{_metadata,runLogTail}' ELSE value END
1294
+ $function$
1295
+ `);
1296
+ await client.query(`
1297
+ CREATE OR REPLACE FUNCTION ${schema}.compact_runner_log_envelope(value jsonb) RETURNS jsonb
1298
+ LANGUAGE sql IMMUTABLE SET search_path = pg_catalog AS $function$
1299
+ WITH envelope AS (
1300
+ SELECT CASE WHEN jsonb_typeof(value) = 'object' THEN
1301
+ (value - 'logContexts') || CASE WHEN value ? 'logs'
1302
+ THEN jsonb_build_object('logs', '[]'::jsonb) ELSE '{}'::jsonb END
1303
+ ELSE value END AS result
1304
+ )
1305
+ SELECT CASE WHEN result ? 'output' THEN jsonb_set(result, '{output}',
1306
+ ${schema}.without_runtime_log_tail(result->'output')) ELSE result END
1307
+ FROM envelope
1308
+ $function$
1309
+ `);
1310
+ await client.query(`
1311
+ CREATE OR REPLACE FUNCTION ${schema}.search_outbox_event() RETURNS trigger
1312
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
1313
+ DECLARE compact_logs boolean;
1314
+ BEGIN
1315
+ -- Project the original validated observation before replacing its
1316
+ -- transport body. Failure rolls back the outbox and canonical rows.
1317
+ PERFORM ${schema}.project_outbox_event(NEW);
1318
+ SELECT run_log_store = 'planet_scale' INTO compact_logs
1319
+ FROM ${schema}.runs WHERE run_id = NEW.run_id;
1320
+ IF compact_logs THEN
1321
+ NEW.payload_json := NEW.payload_json - 'terminalLogLines' - 'terminalLogContexts';
1322
+ IF NEW.event_type = 'run.events' THEN
1323
+ NEW.payload_json := jsonb_set(NEW.payload_json, '{events}',
1324
+ ${schema}.without_log_events(NEW.payload_json->'events'))
1325
+ || jsonb_build_object('logBodyStore', 'planet_scale');
1326
+ END IF;
1327
+ IF NEW.payload_json ? 'terminalProgressEvents' THEN
1328
+ NEW.payload_json := jsonb_set(NEW.payload_json, '{terminalProgressEvents}',
1329
+ ${schema}.without_log_events(NEW.payload_json->'terminalProgressEvents'));
1330
+ END IF;
1331
+ IF NEW.event_type = 'run.failed' AND NEW.payload_json ? 'result' THEN
1332
+ NEW.payload_json := jsonb_set(NEW.payload_json, '{result}',
1333
+ ${schema}.compact_runner_log_envelope(NEW.payload_json->'result'));
1334
+ END IF;
1335
+ IF NEW.event_type = 'run.completed' AND NEW.payload_json ? 'output' THEN
1336
+ NEW.payload_json := jsonb_set(NEW.payload_json, '{output}',
1337
+ ${schema}.without_runtime_log_tail(NEW.payload_json->'output'));
1338
+ END IF;
1339
+ END IF;
1340
+ RETURN NEW;
1341
+ END
1342
+ $function$
1343
+ `);
1344
+ await client.query(`
1345
+ CREATE OR REPLACE FUNCTION ${schema}.accept_runner_log_payload() RETURNS trigger
1346
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
1347
+ DECLARE observation ${schema}.outbox; compact_logs boolean;
1348
+ BEGIN
1349
+ SELECT run_log_store = 'planet_scale' INTO compact_logs
1350
+ FROM ${schema}.runs WHERE run_id = NEW.run_id;
1351
+ IF NOT coalesce(compact_logs, false) OR NEW.runner_log_payload_json IS NULL THEN
1352
+ RETURN NEW;
1353
+ END IF;
1354
+ IF OLD.runner_result_json IS NULL THEN
1355
+ NEW.runner_result_digest := encode(
1356
+ sha256(convert_to(NEW.runner_result_json::text, 'UTF8')), 'hex');
1357
+ observation.event_id := 'runner-terminal-logs:' || NEW.run_id || ':' || NEW.attempt;
1358
+ observation.run_id := NEW.run_id;
1359
+ observation.event_type := 'runner.logs';
1360
+ observation.created_at := coalesce(NEW.runner_terminal_received_at, clock_timestamp());
1361
+ observation.payload_json := jsonb_set(NEW.runner_log_payload_json,
1362
+ '{terminalProgressEvents}', (
1363
+ SELECT coalesce(jsonb_agg(
1364
+ CASE WHEN event->>'type' = 'log.appended' THEN event
1365
+ ELSE event || jsonb_build_object('_runtimeProgressOrdinal', position) END
1366
+ ORDER BY position), '[]'::jsonb)
1367
+ FROM jsonb_array_elements(CASE
1368
+ WHEN jsonb_typeof(NEW.runner_log_payload_json->'terminalProgressEvents') = 'array'
1369
+ THEN NEW.runner_log_payload_json->'terminalProgressEvents'
1370
+ ELSE '[]'::jsonb END) WITH ORDINALITY AS progress(event, position)
1371
+ ));
1372
+ PERFORM ${schema}.project_outbox_event(observation);
1373
+ NEW.runner_result_json := ${schema}.compact_runner_log_envelope(NEW.runner_result_json);
1374
+ IF jsonb_array_length(observation.payload_json->'recoveredRuntimeDiagnostics') > 0
1375
+ AND (NOT (NEW.runner_result_json ? 'runtimeDiagnostics')
1376
+ OR jsonb_typeof(NEW.runner_result_json->'runtimeDiagnostics') = 'array') THEN
1377
+ NEW.runner_result_json := jsonb_set(NEW.runner_result_json, '{runtimeDiagnostics}',
1378
+ coalesce(NEW.runner_result_json->'runtimeDiagnostics', '[]'::jsonb)
1379
+ || (observation.payload_json->'recoveredRuntimeDiagnostics'));
1380
+ END IF;
1381
+ -- Retain only the first accepted non-log facts. Gateway retries
1382
+ -- regenerate transport timestamps, so completion must reuse these
1383
+ -- facts rather than compare newly generated progress with the first.
1384
+ NEW.runner_log_payload_json := jsonb_build_object(
1385
+ 'terminalProgressEvents', ${schema}.without_log_events(
1386
+ observation.payload_json->'terminalProgressEvents'));
1387
+ ELSE
1388
+ NEW.runner_log_payload_json := OLD.runner_log_payload_json;
1389
+ END IF;
1390
+ RETURN NEW;
1391
+ END
1392
+ $function$
1393
+ `);
1394
+ await client.query(`
1395
+ CREATE OR REPLACE FUNCTION ${schema}.compact_run_log_result() RETURNS trigger
1396
+ LANGUAGE plpgsql SET search_path = pg_catalog AS $function$
1397
+ BEGIN
1398
+ IF NEW.run_log_store = 'planet_scale' THEN
1399
+ IF NEW.status = 'failed' THEN
1400
+ NEW.terminal_result_json := ${schema}.compact_runner_log_envelope(NEW.terminal_result_json);
1401
+ ELSIF NEW.status = 'completed' THEN
1402
+ NEW.terminal_result_json := ${schema}.without_runtime_log_tail(NEW.terminal_result_json);
1403
+ END IF;
1404
+ END IF;
1405
+ RETURN NEW;
1406
+ END
1407
+ $function$
1408
+ `);
1409
+ // Replacing AFTER with BEFORE is one DDL transaction, so live inserts cannot
1410
+ // observe an interval with no projector. All older unmarked runs stay full.
1411
+ await client.query(`DO $create$ BEGIN
1412
+ DROP TRIGGER IF EXISTS runtime_outbox_event_v1 ON ${schema}.outbox;
1413
+ CREATE TRIGGER runtime_outbox_event_v1 BEFORE INSERT ON ${schema}.outbox
1414
+ FOR EACH ROW EXECUTE FUNCTION ${schema}.search_outbox_event();
1415
+ IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.attempts'::regclass
1416
+ AND tgname = 'runtime_attempt_log_acceptance') THEN
1417
+ CREATE TRIGGER runtime_attempt_log_acceptance BEFORE UPDATE OF runner_result_json ON ${schema}.attempts
1418
+ FOR EACH ROW EXECUTE FUNCTION ${schema}.accept_runner_log_payload();
1419
+ END IF;
1420
+ IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.runs'::regclass
1421
+ AND tgname = 'runtime_run_log_result') THEN
1422
+ CREATE TRIGGER runtime_run_log_result BEFORE UPDATE OF terminal_result_json ON ${schema}.runs
1423
+ FOR EACH ROW EXECUTE FUNCTION ${schema}.compact_run_log_result();
1424
+ END IF;
1425
+ END $create$`);
1426
+ await client.query(`REVOKE ALL ON FUNCTION ${schema}.project_outbox_event(${schema}.outbox),
1427
+ ${schema}.accept_runner_log_payload(), ${schema}.search_outbox_event(),
1428
+ ${schema}.compact_run_log_result(), ${schema}.without_log_events(jsonb),
1429
+ ${schema}.compact_runner_log_envelope(jsonb), ${schema}.without_runtime_log_tail(jsonb)
1430
+ FROM PUBLIC`);
1431
+ // Publish readiness last, after all acceptance/compaction functions exist.
1432
+ await client.query(
1433
+ `COMMENT ON FUNCTION ${schema}.search_outbox_event() IS ${quoteLiteral(RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER)}`,
1434
+ );
1435
+ }
1436
+
1437
+ export async function hasRuntimeEventSearchShape(
1438
+ client: PostgresSchedulerQueryClient,
1439
+ runtimeScope: string,
1440
+ ): Promise<boolean> {
1441
+ const schema = quoteIdent(requiredScope(runtimeScope));
1442
+ const result = await client.query<{ ready: boolean }>(
1443
+ `SELECT to_regclass('runtime_observability.events') IS NOT NULL
1444
+ AND to_regclass('runtime_observability.run_log_cursors') IS NOT NULL
1445
+ AND EXISTS (SELECT 1 FROM pg_attribute
1446
+ WHERE attrelid = to_regclass('runtime_observability.events')
1447
+ AND attname = 'log_seq' AND NOT attisdropped)
1448
+ AND EXISTS (SELECT 1 FROM pg_attribute
1449
+ WHERE attrelid = to_regclass('runtime_observability.events')
1450
+ AND attname = 'log_body' AND NOT attisdropped)
1451
+ AND EXISTS (SELECT 1 FROM pg_attribute
1452
+ WHERE attrelid = to_regclass('runtime_observability.events')
1453
+ AND attname = 'log_text_digest' AND NOT attisdropped)
1454
+ AND EXISTS (SELECT 1 FROM pg_class
1455
+ WHERE oid = to_regclass('runtime_observability.events') AND relrowsecurity)
1456
+ AND EXISTS (SELECT 1 FROM pg_class
1457
+ WHERE oid = to_regclass('runtime_observability.run_log_cursors') AND relrowsecurity)
1458
+ AND EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.receipts'::regclass
1459
+ AND tgname = 'runtime_receipt_event_v1' AND NOT tgisinternal)
1460
+ AND EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.outbox'::regclass
1461
+ AND tgname = 'runtime_outbox_event_v1' AND NOT tgisinternal
1462
+ AND tgenabled <> 'D' AND (tgtype & 2) = 2)
1463
+ AND EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.attempts'::regclass
1464
+ AND tgname = 'runtime_attempt_log_acceptance' AND NOT tgisinternal AND tgenabled <> 'D')
1465
+ AND EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.runs'::regclass
1466
+ AND tgname = 'runtime_run_log_result' AND NOT tgisinternal AND tgenabled <> 'D')
1467
+ AND EXISTS (SELECT 1 FROM pg_proc AS proc
1468
+ JOIN pg_namespace AS namespace ON namespace.oid = proc.pronamespace
1469
+ WHERE namespace.nspname = $1 AND proc.proname = 'search_outbox_event'
1470
+ AND proc.pronargs = 0
1471
+ AND obj_description(proc.oid, 'pg_proc') = ${quoteLiteral(RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER)})
1472
+ AS ready`,
1473
+ [runtimeScope],
1474
+ );
1475
+ return result.rows[0]?.ready === true;
1476
+ }
1477
+
1478
+ const RUNTIME_EVENT_BODY_RETENTION_PREDICATE = `kind <> 'log.appended' OR log_body IS NOT NULL
1479
+ OR message IS NOT NULL OR search_doc ? 'context'`;
1480
+
1481
+ // pg_get_expr adds grouping and text casts. Preserve quoted literals exactly;
1482
+ // this known OR-only predicate rejects other operators, keys and literal values.
1483
+ function normalizedRetentionPredicate(value: string): string {
1484
+ return (value.match(/'(?:''|[^'])*'|::text\b|[()\s]+|./g) ?? [])
1485
+ .filter((token) => !/^(?:::text|[()\s]+)$/.test(token))
1486
+ .join('');
1487
+ }
1488
+
1489
+ /** Operator-only concurrent index build; never part of runtime bootstrap. */
1490
+ export async function prepareRuntimeEventRetentionIndex(
1491
+ client: PostgresSchedulerQueryClient,
1492
+ ): Promise<void> {
1493
+ type IndexShape = {
1494
+ valid: boolean;
1495
+ ready: boolean;
1496
+ is_unique: boolean;
1497
+ table_matches: boolean;
1498
+ key_count: number;
1499
+ attribute_count: number;
1500
+ method: string;
1501
+ key1: string | null;
1502
+ key2: string | null;
1503
+ predicate: string | null;
1504
+ };
1505
+ const inspect = () =>
1506
+ client.query<IndexShape>(
1507
+ `SELECT index.indisvalid AS valid, index.indisready AS ready, index.indisunique AS is_unique,
1508
+ index.indrelid = to_regclass('runtime_observability.events') AS table_matches,
1509
+ index.indnkeyatts AS key_count, index.indnatts AS attribute_count,
1510
+ method.amname AS method,
1511
+ pg_get_indexdef(index.indexrelid, 1, true) AS key1,
1512
+ pg_get_indexdef(index.indexrelid, 2, true) AS key2,
1513
+ pg_get_expr(index.indpred, index.indrelid) AS predicate
1514
+ FROM pg_index AS index
1515
+ JOIN pg_class AS relation ON relation.oid = index.indexrelid
1516
+ JOIN pg_am AS method ON method.oid = relation.relam
1517
+ WHERE index.indexrelid = to_regclass('runtime_observability.runtime_events_body_retention_idx')`,
1518
+ );
1519
+ const assertReady = (shape: IndexShape): void => {
1520
+ if (!shape.valid || !shape.ready) {
1521
+ throw new Error(
1522
+ 'Runtime body retention index is invalid or incomplete. Inspect the failed build; repair it explicitly before retrying.',
1523
+ );
1524
+ }
1525
+ if (
1526
+ !shape.table_matches ||
1527
+ shape.is_unique ||
1528
+ shape.key_count !== 2 ||
1529
+ shape.attribute_count !== 2 ||
1530
+ shape.method !== 'btree' ||
1531
+ shape.key1 !== 'ingested_at' ||
1532
+ shape.key2 !== 'event_id' ||
1533
+ shape.predicate === null ||
1534
+ normalizedRetentionPredicate(shape.predicate) !==
1535
+ normalizedRetentionPredicate(RUNTIME_EVENT_BODY_RETENTION_PREDICATE)
1536
+ ) {
1537
+ throw new Error(
1538
+ 'Runtime body retention index definition differs from its expected table, uniqueness, keys or predicate. Inspect and repair it explicitly.',
1539
+ );
1540
+ }
1541
+ };
1542
+ const existing = (await inspect()).rows[0];
1543
+ if (existing) assertReady(existing);
1544
+ await client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS runtime_events_body_retention_idx
1545
+ ON runtime_observability.events (ingested_at, event_id)
1546
+ WHERE ${RUNTIME_EVENT_BODY_RETENTION_PREDICATE}`);
1547
+ const built = (await inspect()).rows[0];
1548
+ if (!built)
1549
+ throw new Error(
1550
+ 'Runtime body retention index was not found after preparation.',
1551
+ );
1552
+ assertReady(built);
1553
+ }
1554
+
1555
+ /** One bounded operator-owned TTL pass. Never called by a customer request. */
1556
+ export async function pruneRuntimeEvents(
1557
+ client: PostgresSchedulerQueryClient,
1558
+ input: { before: Date; limit: number },
1559
+ ): Promise<number> {
1560
+ const before = input.before.getTime();
1561
+ if (!Number.isFinite(before) || before > Date.now() - 60 * 60 * 1000) {
1562
+ throw new RuntimeEventInputError(
1563
+ 'Runtime event cleanup cutoff must be at least one hour old.',
1564
+ );
1565
+ }
1566
+ if (
1567
+ !Number.isSafeInteger(input.limit) ||
1568
+ input.limit < 1 ||
1569
+ input.limit > 5_000
1570
+ ) {
1571
+ throw new RuntimeEventInputError(
1572
+ 'Runtime event cleanup limit must be between 1 and 5000.',
1573
+ );
1574
+ }
1575
+ const result = await client.query<{ event_id: string }>(
1576
+ `WITH candidates AS (
1577
+ SELECT ctid FROM runtime_observability.events
1578
+ WHERE ingested_at < $1::timestamptz
1579
+ AND (kind <> 'log.appended' OR log_body IS NOT NULL
1580
+ OR message IS NOT NULL OR search_doc ? 'context')
1581
+ ORDER BY ingested_at, event_id
1582
+ LIMIT $2::integer FOR UPDATE SKIP LOCKED
1583
+ ), scrubbed AS (
1584
+ UPDATE runtime_observability.events AS event
1585
+ SET log_text_digest = coalesce(event.log_text_digest,
1586
+ encode(sha256(convert_to(btrim(coalesce(event.log_body, event.message)), 'UTF8')), 'hex')),
1587
+ log_body = NULL, message = NULL,
1588
+ search_doc = jsonb_strip_nulls(jsonb_build_object(
1589
+ 'kind', 'log.appended', 'source', event.search_doc->'source',
1590
+ 'producerAttempt', event.search_doc->'producerAttempt',
1591
+ 'logPositions', event.search_doc->'logPositions',
1592
+ 'logContentDigest', event.search_doc->'logContentDigest'))
1593
+ FROM candidates WHERE event.ctid = candidates.ctid AND event.kind = 'log.appended'
1594
+ RETURNING event.event_id
1595
+ ), deleted AS (
1596
+ DELETE FROM runtime_observability.events AS event USING candidates
1597
+ WHERE event.ctid = candidates.ctid AND event.kind <> 'log.appended'
1598
+ RETURNING event.event_id
1599
+ )
1600
+ SELECT event_id FROM scrubbed UNION ALL SELECT event_id FROM deleted`,
1601
+ [input.before.toISOString(), input.limit],
1602
+ );
1603
+ // Keep identity/digests until fenced ingress and pending delivery drain. A
1604
+ // high-water mark alone cannot recognize a resent producer offset.
1605
+ return result.rows.length;
1606
+ }
1607
+
1608
+ /** Read-only bounded estimate for the same operator TTL cutoff. */
1609
+ export async function inspectRuntimeEventRetention(
1610
+ client: PostgresSchedulerQueryClient,
1611
+ input: { before: Date },
1612
+ ): Promise<{
1613
+ oldestIngestedAt: string | null;
1614
+ expiredCountAtLeast: number;
1615
+ countCapped: boolean;
1616
+ }> {
1617
+ const before = input.before.getTime();
1618
+ if (!Number.isFinite(before))
1619
+ throw new RuntimeEventInputError('Invalid runtime event retention cutoff.');
1620
+ const result = await client.query<{
1621
+ oldest: Date | string | null;
1622
+ count: number | string;
1623
+ }>(
1624
+ `WITH oldest AS (
1625
+ SELECT ingested_at FROM runtime_observability.events
1626
+ WHERE kind <> 'log.appended' OR log_body IS NOT NULL
1627
+ OR message IS NOT NULL OR search_doc ? 'context'
1628
+ ORDER BY ingested_at, event_id LIMIT 1
1629
+ ), bounded AS (
1630
+ SELECT 1 FROM runtime_observability.events
1631
+ WHERE ingested_at < $1::timestamptz
1632
+ AND (kind <> 'log.appended' OR log_body IS NOT NULL
1633
+ OR message IS NOT NULL OR search_doc ? 'context')
1634
+ ORDER BY ingested_at, event_id LIMIT 10001
1635
+ )
1636
+ SELECT (SELECT ingested_at FROM oldest) AS oldest,
1637
+ (SELECT count(*) FROM bounded) AS count`,
1638
+ [input.before.toISOString()],
1639
+ );
1640
+ const count = Number(result.rows[0]?.count ?? 0);
1641
+ return {
1642
+ oldestIngestedAt: result.rows[0]?.oldest
1643
+ ? iso(result.rows[0].oldest)
1644
+ : null,
1645
+ expiredCountAtLeast: Math.min(count, 10_000),
1646
+ countCapped: count > 10_000,
1647
+ };
1648
+ }