deepline 0.3.289 → 0.3.290

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
  3. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
  4. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
  5. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
  10. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
  17. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
  27. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
  28. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
  29. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
  31. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
  32. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
  33. package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
  35. package/dist/cli/index.js +3 -1
  36. package/dist/cli/index.mjs +3 -1
  37. package/dist/index.js +3 -1
  38. package/dist/index.mjs +3 -1
  39. package/dist/install-integrity.json +3 -0
  40. package/dist/release.d.mts +1 -1
  41. package/dist/release.d.ts +1 -1
  42. package/dist/release.js +1 -1
  43. package/dist/release.mjs +1 -1
  44. package/package.json +1 -1
@@ -1,8 +1,9 @@
1
1
  /**
2
2
  * The customer-visible, append-only search projection of durable Play facts.
3
3
  *
4
- * Outbox and receipt triggers write small rows in their authoritative SQL
5
- * transaction. They never deliver work, replay a Play, or copy receipt output.
4
+ * After reader-first cutover, triggers persist raw input in the existing
5
+ * outbox. The leased dispatcher projects and compacts it before delivery.
6
+ * Receipt output remains exclusively in the authoritative receipt store.
6
7
  * The API may read only through fixed, tenant-scoped queries in this module.
7
8
  */
8
9
  import { normalizeRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
@@ -56,7 +57,11 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
56
57
  // Migration readiness tracks this projection body independently of scheduler
57
58
  // wire and schema versions. Bump when a changed trigger must be reinstalled.
58
59
  export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
59
- 'runtime-event-search-log-context-v14-btree-position';
60
+ 'runtime-event-search-origin-batches-independent-progress';
61
+ /** Trusted trigger-authored state; absent on already-projected legacy rows. */
62
+ export const RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD =
63
+ '_runtimeSearchProjectionPending';
64
+ export const RUNTIME_EVENT_SEARCH_ONLY_FIELD = '_runtimeSearchOnly';
60
65
  export const RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER =
61
66
  'runtime-event-canonical-retention-v1';
62
67
  export const RUNTIME_EVENTS_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
@@ -950,11 +955,131 @@ export async function hasRuntimeEventReaderGrant(
950
955
  return result.rows[0]?.ready === true;
951
956
  }
952
957
 
958
+ /** Deployment-owned cutover; install readers everywhere before enabling. */
959
+ export async function setRuntimeEventDeferredProjection(
960
+ client: PostgresSchedulerQueryClient,
961
+ runtimeScope: string,
962
+ deferred: boolean,
963
+ independent?: boolean,
964
+ ): Promise<void> {
965
+ const schema = quoteIdent(requiredScope(runtimeScope));
966
+ if (!(await hasRuntimeEventSearchShape(client, runtimeScope))) {
967
+ throw new Error(
968
+ 'Install the deferred outbox reader before changing projection mode.',
969
+ );
970
+ }
971
+ if (deferred && !(await hasRuntimeEventLogPositionIndex(client))) {
972
+ throw new Error(
973
+ 'Prepare the log-position indexes before enabling deferred projection.',
974
+ );
975
+ }
976
+ await client.query('BEGIN');
977
+ try {
978
+ await client.query("SELECT set_config('lock_timeout', '1000', true)");
979
+ // Fence trigger execution while changing mode. This operator-only lock
980
+ // never spans projection; normal producer writes acquire no such lock.
981
+ await client.query(
982
+ `LOCK TABLE ${schema}.outbox IN SHARE ROW EXCLUSIVE MODE`,
983
+ );
984
+ if (!deferred) {
985
+ const pending = await client.query<{ event_id: string }>(
986
+ `SELECT event_id FROM ${schema}.outbox
987
+ WHERE delivered_at IS NULL
988
+ AND payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb
989
+ LIMIT 1`,
990
+ );
991
+ if (pending.rows.length > 0) {
992
+ throw new Error(
993
+ 'Drain deferred outbox rows before returning to synchronous projection.',
994
+ );
995
+ }
996
+ }
997
+ if (independent === false || !deferred) {
998
+ const raw = await client.query<{ event_id: string }>(
999
+ `SELECT event_id FROM ${schema}.outbox WHERE delivered_at IS NULL AND payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb LIMIT 1`,
1000
+ );
1001
+ if (raw.rows.length)
1002
+ throw new Error(
1003
+ 'Drain independent search rows before disabling independent projection.',
1004
+ );
1005
+ }
1006
+ await installRuntimeEventProjectionMode(client, schema, deferred);
1007
+ if (independent !== undefined || !deferred)
1008
+ await installIndependentProjectionMode(
1009
+ client,
1010
+ schema,
1011
+ deferred && independent === true,
1012
+ );
1013
+ await client.query('COMMIT');
1014
+ } catch (error) {
1015
+ await client.query('ROLLBACK');
1016
+ throw error;
1017
+ }
1018
+ }
1019
+
1020
+ /** Operator state uses the same durable pending set as the outbox projector. */
1021
+ export async function inspectRuntimeEventProjection(
1022
+ client: PostgresSchedulerQueryClient,
1023
+ runtimeScope: string,
1024
+ ): Promise<{
1025
+ deferred: boolean;
1026
+ independent: boolean;
1027
+ pending: number;
1028
+ blocked: number;
1029
+ oldestPendingAt: string | null;
1030
+ }> {
1031
+ const schema = quoteIdent(requiredScope(runtimeScope));
1032
+ const result = await client.query<{
1033
+ deferred: boolean;
1034
+ independent: boolean;
1035
+ pending: string | number;
1036
+ blocked: string | number;
1037
+ oldest: Date | string | null;
1038
+ }>(`SELECT ${schema}.defer_runtime_event_projection() AS deferred, ${schema}.independent_runtime_event_projection() AS independent,
1039
+ count(*) AS pending, count(*) FILTER (WHERE failed_at IS NOT NULL) AS blocked,
1040
+ min(created_at) AS oldest
1041
+ FROM ${schema}.outbox
1042
+ WHERE delivered_at IS NULL
1043
+ AND payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb`);
1044
+ const state = result.rows[0]!;
1045
+ return {
1046
+ deferred: state.deferred,
1047
+ independent: state.independent,
1048
+ pending: Number(state.pending),
1049
+ blocked: Number(state.blocked),
1050
+ oldestPendingAt: state.oldest === null ? null : iso(state.oldest),
1051
+ };
1052
+ }
1053
+
1054
+ async function installRuntimeEventProjectionMode(
1055
+ client: PostgresSchedulerQueryClient,
1056
+ schema: string,
1057
+ deferred: boolean,
1058
+ ): Promise<void> {
1059
+ await client.query(`CREATE OR REPLACE FUNCTION ${schema}.defer_runtime_event_projection()
1060
+ RETURNS boolean LANGUAGE sql VOLATILE SET search_path = pg_catalog
1061
+ AS 'SELECT ${deferred ? 'true' : 'false'}'`);
1062
+ await client.query(
1063
+ `REVOKE ALL ON FUNCTION ${schema}.defer_runtime_event_projection() FROM PUBLIC`,
1064
+ );
1065
+ }
1066
+
1067
+ async function installIndependentProjectionMode(
1068
+ client: PostgresSchedulerQueryClient,
1069
+ schema: string,
1070
+ enabled: boolean,
1071
+ ): Promise<void> {
1072
+ await client.query(`CREATE OR REPLACE FUNCTION ${schema}.independent_runtime_event_projection()
1073
+ RETURNS boolean LANGUAGE sql VOLATILE SET search_path = pg_catalog AS 'SELECT ${enabled ? 'true' : 'false'}'`);
1074
+ await client.query(
1075
+ `REVOKE ALL ON FUNCTION ${schema}.independent_runtime_event_projection() FROM PUBLIC`,
1076
+ );
1077
+ }
1078
+
953
1079
  /**
954
- * Triggers are attached to each scheduler namespace by the explicit migration.
955
- * Search writes are atomic with the already-durable outbox/receipt write. A
956
- * trigger failure is loud so no authoritative write can silently lose its
957
- * corresponding search history after enablement.
1080
+ * Migration installs readers with synchronous production unchanged. After
1081
+ * cutover, the producer persists raw input and only the leased dispatcher
1082
+ * projects and compacts it, atomically, before acknowledging its outbox row.
958
1083
  */
959
1084
  export async function attachRuntimeEventSearchTriggers(
960
1085
  client: PostgresSchedulerQueryClient,
@@ -962,6 +1087,33 @@ export async function attachRuntimeEventSearchTriggers(
962
1087
  ): Promise<void> {
963
1088
  const scope = requiredScope(runtimeScope);
964
1089
  const schema = quoteIdent(scope);
1090
+ // Refreshing the reader must preserve an already-enabled producer cutover.
1091
+ const mode = await client.query<{ installed: boolean }>(
1092
+ `SELECT to_regprocedure($1) IS NOT NULL AS installed`,
1093
+ [`${schema}.defer_runtime_event_projection()`],
1094
+ );
1095
+ if (!mode.rows[0]?.installed) {
1096
+ await installRuntimeEventProjectionMode(client, schema, false);
1097
+ }
1098
+ const independentMode = await client.query<{ installed: boolean }>(
1099
+ `SELECT to_regprocedure($1) IS NOT NULL AS installed`,
1100
+ [`${schema}.independent_runtime_event_projection()`],
1101
+ );
1102
+ if (!independentMode.rows[0]?.installed)
1103
+ await installIndependentProjectionMode(client, schema, false);
1104
+ // Stamp new admissions once. Existing launches never acquire a producer pin
1105
+ // on replay or worker upgrade. Ignore a caller-supplied pin before cutover.
1106
+ await client.query(`CREATE OR REPLACE FUNCTION ${schema}.pin_runtime_log_identity() RETURNS trigger
1107
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
1108
+ BEGIN
1109
+ NEW.launch_json := NEW.launch_json - 'runtimeLogIdentity';
1110
+ NEW.admission_json := NEW.admission_json - 'runtimeLogIdentity';
1111
+ IF ${schema}.independent_runtime_event_projection() THEN
1112
+ NEW.launch_json := NEW.launch_json || '{"runtimeLogIdentity":"origin"}'::jsonb;
1113
+ NEW.admission_json := NEW.admission_json || '{"runtimeLogIdentity":"origin"}'::jsonb;
1114
+ END IF;
1115
+ RETURN NEW;
1116
+ END $function$`);
965
1117
  await client.query(`
966
1118
  CREATE OR REPLACE FUNCTION ${schema}.search_receipt_event() RETURNS trigger
967
1119
  LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
@@ -978,6 +1130,23 @@ export async function attachRuntimeEventSearchTriggers(
978
1130
  IF NEW.run_id IS NULL OR NEW.org_id IS NULL THEN
979
1131
  RAISE EXCEPTION 'Terminal receipt search projection requires run and organization';
980
1132
  END IF;
1133
+ IF ${schema}.defer_runtime_event_projection() THEN
1134
+ IF NOT EXISTS (SELECT 1 FROM ${schema}.runs
1135
+ WHERE run_id = NEW.run_id AND org_id = NEW.org_id) THEN
1136
+ RAISE EXCEPTION 'Terminal receipt search projection has no matching run';
1137
+ END IF;
1138
+ ref := gen_random_uuid();
1139
+ INSERT INTO ${schema}.outbox (event_id, run_id, event_type, payload_json, created_at)
1140
+ VALUES ('receipt:' || ref::text, NEW.run_id,
1141
+ ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.receiptEvent},
1142
+ jsonb_strip_nulls(jsonb_build_object(
1143
+ 'orgId', NEW.org_id, 'receiptRef', ref, 'receiptKey', NEW.k,
1144
+ 'status', NEW.status, 'occurredAt', NEW.updated_at,
1145
+ 'producerAttempt', NEW.lease_owner_attempt)),
1146
+ GREATEST(clock_timestamp(), coalesce((SELECT max(created_at) + interval '1 microsecond'
1147
+ FROM ${schema}.outbox WHERE run_id = NEW.run_id), clock_timestamp())));
1148
+ RETURN NEW;
1149
+ END IF;
981
1150
  -- Serialize event ordering with the same per-run lock as outbox facts.
982
1151
  PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(NEW.run_id));
983
1152
  SELECT play_name, run_event_store = 'planet_scale'
@@ -1009,12 +1178,87 @@ export async function attachRuntimeEventSearchTriggers(
1009
1178
  $function$
1010
1179
  `);
1011
1180
  await client.query(`DO $create$ BEGIN
1181
+ IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.runs'::regclass
1182
+ AND tgname = 'runtime_run_log_identity') THEN
1183
+ CREATE TRIGGER runtime_run_log_identity BEFORE INSERT ON ${schema}.runs
1184
+ FOR EACH ROW EXECUTE FUNCTION ${schema}.pin_runtime_log_identity();
1185
+ END IF;
1012
1186
  IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.receipts'::regclass
1013
1187
  AND tgname = 'runtime_receipt_event_v1') THEN
1014
1188
  CREATE TRIGGER runtime_receipt_event_v1 AFTER INSERT OR UPDATE ON ${schema}.receipts
1015
1189
  FOR EACH ROW EXECUTE FUNCTION ${schema}.search_receipt_event();
1016
1190
  END IF;
1017
1191
  END $create$`);
1192
+ await client.query(`
1193
+ CREATE OR REPLACE FUNCTION ${schema}.project_origin_log_batch(observation ${schema}.outbox, item jsonb, run_org text, play text, compact_logs boolean, canonical_events boolean, at_time timestamptz, doc jsonb, closed boolean)
1194
+ RETURNS text[] LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
1195
+ DECLARE fresh text[]; mismatches bigint;
1196
+ BEGIN
1197
+ IF jsonb_typeof(item->'lineIds') IS DISTINCT FROM 'array'
1198
+ OR jsonb_array_length(item->'lineIds') <> jsonb_array_length(item->'lines')
1199
+ OR EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lineIds') id WHERE jsonb_typeof(id) <> 'string' OR (id #>> '{}') !~ '^[A-Za-z0-9:_-]{1,128}$')
1200
+ OR (SELECT count(DISTINCT id) FROM jsonb_array_elements(item->'lineIds') id) <> jsonb_array_length(item->'lineIds') THEN
1201
+ RAISE EXCEPTION 'Origin log identities must be distinct bounded strings aligned with lines';
1202
+ END IF;
1203
+ IF EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lines') line WHERE jsonb_typeof(line) <> 'string') THEN
1204
+ RAISE EXCEPTION 'Search projection requires log line strings';
1205
+ END IF;
1206
+ IF item ? 'lineContexts' AND EXISTS (
1207
+ SELECT 1 FROM jsonb_array_elements(item->'lineContexts') ctx
1208
+ WHERE jsonb_typeof(ctx) NOT IN ('object', 'null')
1209
+ ) THEN RAISE EXCEPTION 'Search projection log context must be an object'; END IF;
1210
+ IF item ? 'lineContexts' AND EXISTS (
1211
+ SELECT 1 FROM jsonb_array_elements(item->'lineContexts') ctx
1212
+ WHERE jsonb_typeof(ctx) = 'object' AND (
1213
+ (SELECT count(*) FROM jsonb_object_keys(ctx)) > 8 OR octet_length(ctx::text) > 2064
1214
+ OR EXISTS (SELECT 1 FROM jsonb_each(ctx) field WHERE
1215
+ field.key !~ '^[A-Za-z_][A-Za-z0-9_]{0,63}$' OR field.key IN ('__proto__', 'constructor', 'prototype')
1216
+ OR jsonb_typeof(field.value) NOT IN ('string', 'number', 'boolean')
1217
+ OR (jsonb_typeof(field.value) = 'string' AND octet_length(field.value #>> '{}') > 256)
1218
+ OR (jsonb_typeof(field.value) = 'number' AND octet_length(field.value #>> '{}') > 32))
1219
+ )
1220
+ ) THEN RAISE EXCEPTION 'Search projection log context has an unsupported key or value'; END IF;
1221
+ WITH entries AS MATERIALIZED (
1222
+ SELECT value #>> '{}' AS body, ordinality AS ordinal,
1223
+ 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinality-1)::int) AS eid,
1224
+ NULLIF(item->'lineContexts'->(ordinality-1)::int, 'null'::jsonb) AS ctx
1225
+ FROM jsonb_array_elements(item->'lines') WITH ORDINALITY
1226
+ ), inserted AS (
1227
+ INSERT INTO runtime_observability.events (
1228
+ runtime_scope, org_id, event_id, event_seq, event_payload, occurred_at,
1229
+ run_id, play_name, kind, level, message, log_body, source_digest, log_text_digest, search_doc
1230
+ ) SELECT ${quoteLiteral(scope)}, run_org, eid,
1231
+ CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
1232
+ CASE WHEN canonical_events THEN jsonb_strip_nulls((item - 'lines' - 'lineContexts' - 'lineIds') || jsonb_build_object(
1233
+ 'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
1234
+ 'occurredAt', floor(extract(epoch FROM at_time)*1000)::bigint, 'lineOrdinal', ordinal,
1235
+ 'entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx)) END,
1236
+ at_time, observation.run_id, play, ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, NULL,
1237
+ CASE WHEN compact_logs THEN NULL ELSE left(body, 2048) END, body,
1238
+ CASE WHEN ctx IS NULL THEN encode(sha256(convert_to(body,'UTF8')),'hex')
1239
+ ELSE encode(sha256(convert_to(jsonb_build_array(body,ctx)::text,'UTF8')),'hex') END,
1240
+ encode(sha256(convert_to(btrim(body),'UTF8')),'hex'),
1241
+ doc || jsonb_strip_nulls(jsonb_build_object('entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx))
1242
+ FROM entries WHERE NOT closed ORDER BY ordinal
1243
+ ON CONFLICT (runtime_scope, org_id, event_id) DO NOTHING RETURNING event_id
1244
+ ) SELECT coalesce(array_agg(inserted.event_id ORDER BY entries.ordinal), ARRAY[]::text[])
1245
+ INTO fresh FROM inserted JOIN entries ON entries.eid = inserted.event_id;
1246
+ -- A new READ COMMITTED statement sees a concurrent uniqueness winner.
1247
+ -- Tombstones retain digest/identity: retry never restores an expired body.
1248
+ SELECT count(*) INTO mismatches
1249
+ FROM jsonb_array_elements(item->'lines') WITH ORDINALITY AS entries(body, ordinal)
1250
+ LEFT JOIN runtime_observability.events event ON event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
1251
+ AND event.event_id = 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinal-1)::int)
1252
+ WHERE event.run_id IS DISTINCT FROM observation.run_id OR event.kind IS DISTINCT FROM ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1253
+ OR event.source_digest IS DISTINCT FROM CASE WHEN NULLIF(item->'lineContexts'->(ordinal-1)::int, 'null'::jsonb) IS NULL
1254
+ THEN encode(sha256(convert_to(body #>> '{}','UTF8')),'hex')
1255
+ ELSE encode(sha256(convert_to(jsonb_build_array(body #>> '{}',item->'lineContexts'->(ordinal-1)::int)::text,'UTF8')),'hex') END;
1256
+ IF mismatches > 0 THEN
1257
+ RAISE EXCEPTION 'Origin log identity changed its content or ingress is closed after terminal acceptance';
1258
+ END IF;
1259
+ RETURN fresh;
1260
+ END $function$
1261
+ `);
1018
1262
  await client.query(`
1019
1263
  CREATE OR REPLACE FUNCTION ${schema}.project_outbox_event(observation ${schema}.outbox) RETURNS void
1020
1264
  LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
@@ -1033,7 +1277,11 @@ export async function attachRuntimeEventSearchTriggers(
1033
1277
  -- Log ordering owns a transaction-scoped mutex, independent of the
1034
1278
  -- scheduler's Run/attempt row-lock order. In particular, old completion
1035
1279
  -- writers lock attempt before Run; this recorder must not invert that.
1036
- PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(observation.run_id));
1280
+ -- Stable batches are ordered by the leased background prefix and cursor row.
1281
+ -- The legacy alias path still needs its old transaction mutex.
1282
+ IF NOT ${schema}.defer_runtime_event_projection() THEN
1283
+ PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(observation.run_id));
1284
+ END IF;
1037
1285
  SELECT org_id, play_name, run_log_store = 'planet_scale', status,
1038
1286
  run_event_store = 'planet_scale'
1039
1287
  INTO run_org, play, compact_logs, run_status, canonical_events
@@ -1041,6 +1289,35 @@ export async function attachRuntimeEventSearchTriggers(
1041
1289
  IF run_org IS NULL OR play IS NULL THEN
1042
1290
  RAISE EXCEPTION 'Outbox search projection has no matching run';
1043
1291
  END IF;
1292
+ IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.receiptEvent} THEN
1293
+ IF observation.payload_json->>'orgId' IS DISTINCT FROM run_org
1294
+ OR NOT coalesce(observation.payload_json->>'status' IN ('completed', 'failed', 'skipped'), false)
1295
+ OR nullif(observation.payload_json->>'receiptKey', '') IS NULL
1296
+ OR observation.payload_json->>'receiptRef' IS NULL
1297
+ OR observation.payload_json->>'occurredAt' IS NULL THEN
1298
+ RAISE EXCEPTION 'Deferred receipt event has invalid identity or status';
1299
+ END IF;
1300
+ INSERT INTO runtime_observability.events (
1301
+ runtime_scope, org_id, event_id, event_seq, event_payload,
1302
+ occurred_at, run_id, play_name, kind, receipt_ref, receipt_key, search_doc
1303
+ ) VALUES (
1304
+ ${quoteLiteral(scope)}, run_org, observation.event_id,
1305
+ CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
1306
+ CASE WHEN canonical_events THEN jsonb_build_object(
1307
+ 'type', 'receipt.' || (observation.payload_json->>'status'), 'runId', observation.run_id,
1308
+ 'occurredAt', floor(extract(epoch FROM (observation.payload_json->>'occurredAt')::timestamptz) * 1000)::bigint,
1309
+ 'source', 'receipt', 'status', observation.payload_json->>'status') END,
1310
+ (observation.payload_json->>'occurredAt')::timestamptz,
1311
+ observation.run_id, play, 'receipt.' || (observation.payload_json->>'status'),
1312
+ (observation.payload_json->>'receiptRef')::uuid,
1313
+ observation.payload_json->>'receiptKey',
1314
+ jsonb_strip_nulls(jsonb_build_object(
1315
+ 'kind', 'receipt.' || (observation.payload_json->>'status'),
1316
+ 'status', observation.payload_json->>'status', 'source', 'receipt',
1317
+ 'producerAttempt', observation.payload_json->'producerAttempt'))
1318
+ ) ON CONFLICT DO NOTHING;
1319
+ RETURN;
1320
+ END IF;
1044
1321
  IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
1045
1322
  event_batch := observation.payload_json->'events';
1046
1323
  IF jsonb_typeof(event_batch) IS DISTINCT FROM 'array' THEN
@@ -1071,7 +1348,8 @@ export async function attachRuntimeEventSearchTriggers(
1071
1348
  'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id, 'source', 'terminal',
1072
1349
  'producerAttempt', envelope_attempt, 'channelOffset', 0,
1073
1350
  'lines', observation.payload_json->'terminalLogLines',
1074
- 'lineContexts', observation.payload_json->'terminalLogContexts'
1351
+ 'lineContexts', observation.payload_json->'terminalLogContexts',
1352
+ 'lineIds', NULLIF(observation.payload_json->'terminalLogIds', 'null'::jsonb)
1075
1353
  ))
1076
1354
  );
1077
1355
  END IF;
@@ -1117,11 +1395,14 @@ export async function attachRuntimeEventSearchTriggers(
1117
1395
  ));
1118
1396
  END IF;
1119
1397
  IF event_batch IS NOT NULL THEN
1398
+ IF ${schema}.defer_runtime_event_projection() AND EXISTS (SELECT 1 FROM jsonb_array_elements(event_batch) entry WHERE entry->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND (NOT (entry ? 'lineIds') OR jsonb_typeof(entry->'lineIds') IS DISTINCT FROM 'array' OR EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(entry->'lineIds') = 'array' THEN entry->'lineIds' ELSE '[]'::jsonb END) id WHERE id = 'null'::jsonb))) THEN
1399
+ PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(observation.run_id));
1400
+ END IF;
1120
1401
  -- Normalize old positional writers once per batch under the run mutex.
1121
1402
  -- Per-line lookups must remain digest-selective: an OR fallback for
1122
1403
  -- unmarked rows otherwise scans the whole attempt for every new line.
1123
1404
  IF EXISTS (SELECT 1 FROM jsonb_array_elements(event_batch) AS entry
1124
- WHERE entry->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}) THEN
1405
+ WHERE entry->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND (NOT (entry ? 'lineIds') OR EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(entry->'lineIds') = 'array' THEN entry->'lineIds' ELSE '[]'::jsonb END) id WHERE id = 'null'::jsonb))) THEN
1125
1406
  UPDATE runtime_observability.events AS event SET search_doc = event.search_doc ||
1126
1407
  jsonb_build_object('logContentDigest', event.source_digest) ||
1127
1408
  CASE WHEN NOT (event.search_doc ? 'logPositions')
@@ -1134,7 +1415,7 @@ export async function attachRuntimeEventSearchTriggers(
1134
1415
  ELSE '{}'::jsonb END
1135
1416
  WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
1136
1417
  AND event.run_id = observation.run_id AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1137
- AND event.source_digest IS NOT NULL AND NOT (event.search_doc ? 'logContentDigest')
1418
+ AND event.source_digest IS NOT NULL AND NOT (event.search_doc ? 'logContentDigest') AND NOT (event.search_doc ? 'entryId')
1138
1419
  AND (event.search_doc->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$';
1139
1420
  END IF;
1140
1421
  FOR item, ordinal IN SELECT value, ordinality FROM jsonb_array_elements(event_batch) WITH ORDINALITY LOOP
@@ -1172,12 +1453,23 @@ export async function attachRuntimeEventSearchTriggers(
1172
1453
  OR jsonb_array_length(item->'lineContexts') <> jsonb_array_length(item->'lines')) THEN
1173
1454
  RAISE EXCEPTION 'Search projection requires lineContexts aligned to log lines';
1174
1455
  END IF;
1456
+ IF kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND item ? 'lineIds' THEN
1457
+ IF jsonb_typeof(item->'lineIds') IS DISTINCT FROM 'array' OR jsonb_array_length(item->'lineIds') <> jsonb_array_length(item->'lines') THEN
1458
+ RAISE EXCEPTION 'Origin log identities must align with lines';
1459
+ END IF;
1460
+ IF EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lineIds') id WHERE id <> 'null'::jsonb AND (jsonb_typeof(id) <> 'string' OR (id #>> '{}') !~ '^[A-Za-z0-9:_-]{1,128}$')) OR (SELECT count(id) FROM jsonb_array_elements(item->'lineIds') id WHERE id <> 'null'::jsonb) <> (SELECT count(DISTINCT id) FROM jsonb_array_elements(item->'lineIds') id WHERE id <> 'null'::jsonb) THEN RAISE EXCEPTION 'Origin log identities must be distinct bounded strings'; END IF;
1461
+ IF NOT EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lineIds') id WHERE id = 'null'::jsonb) THEN
1462
+ fresh_event_ids := fresh_event_ids || ${schema}.project_origin_log_batch(observation, item, run_org, play, compact_logs, canonical_events, at_time, doc,
1463
+ compact_logs AND observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND coalesce((observation.payload_json->>'_runtimeSearchRunClosed')::boolean, run_status IN ('completed','failed','cancelled')));
1464
+ CONTINUE;
1465
+ END IF;
1466
+ END IF;
1175
1467
  IF kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} THEN
1176
1468
  FOR line, line_ordinal IN SELECT value, ordinality FROM jsonb_array_elements(item->'lines') WITH ORDINALITY LOOP
1177
1469
  IF jsonb_typeof(line) IS DISTINCT FROM 'string' THEN
1178
1470
  RAISE EXCEPTION 'Search projection requires log line strings';
1179
1471
  END IF;
1180
- IF (item->>'source' IS DISTINCT FROM 'coordinator')
1472
+ IF NULLIF(item->'lineIds'->((line_ordinal-1)::int), 'null'::jsonb) IS NULL AND (item->>'source' IS DISTINCT FROM 'coordinator')
1181
1473
  AND NOT coalesce((item->>'channelOffset') ~ '^[0-9]{1,12}$', false) THEN
1182
1474
  IF NOT recent_log_keys_loaded OR recent_log_attempt IS DISTINCT FROM item->>'producerAttempt' THEN
1183
1475
  SELECT coalesce(jsonb_object_agg(recent.dedupe_key, true), '{}'::jsonb)
@@ -1187,6 +1479,7 @@ export async function attachRuntimeEventSearchTriggers(
1187
1479
  FROM runtime_observability.events AS event
1188
1480
  WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
1189
1481
  AND event.run_id = observation.run_id AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1482
+ AND NOT (event.search_doc ? 'entryId')
1190
1483
  AND event.log_seq IS NOT NULL AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
1191
1484
  AND (item->>'producerAttempt' IS NULL
1192
1485
  OR event.search_doc->>'producerAttempt' = item->>'producerAttempt')
@@ -1241,7 +1534,11 @@ export async function attachRuntimeEventSearchTriggers(
1241
1534
  digest := CASE WHEN line_context IS NULL
1242
1535
  THEN encode(sha256(convert_to(line #>> '{}', 'UTF8')), 'hex')
1243
1536
  ELSE encode(sha256(convert_to(jsonb_build_array(line #>> '{}', line_context)::text, 'UTF8')), 'hex') END;
1244
- IF (item->>'channelOffset') ~ '^[0-9]{1,12}$'
1537
+ IF NULLIF(item->'lineIds'->((line_ordinal-1)::int), 'null'::jsonb) IS NOT NULL THEN
1538
+ IF jsonb_typeof(item->'lineIds'->((line_ordinal-1)::int)) <> 'string' OR (item->'lineIds'->>((line_ordinal-1)::int)) !~ '^[A-Za-z0-9:_-]{1,128}$' THEN RAISE EXCEPTION 'Origin log identity must be a bounded string'; END IF;
1539
+ eid := 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>((line_ordinal-1)::int));
1540
+ line_doc := line_doc || jsonb_build_object('entryId', item->'lineIds'->((line_ordinal-1)::int));
1541
+ ELSIF (item->>'channelOffset') ~ '^[0-9]{1,12}$'
1245
1542
  AND (item->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$' THEN
1246
1543
  -- These transports retain different buffers. Their offsets
1247
1544
  -- are immutable within a channel, never interchangeable.
@@ -1340,7 +1637,7 @@ export async function attachRuntimeEventSearchTriggers(
1340
1637
  AND event.run_id = observation.run_id AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1341
1638
  AND event.source_digest IS NOT NULL
1342
1639
  AND event.search_doc ? 'producerAttempt'
1343
- AND event.search_doc ? 'logContentDigest'
1640
+ AND event.search_doc ? 'logContentDigest' AND NOT (event.search_doc ? 'entryId')
1344
1641
  AND event.search_doc->>'producerAttempt' = ((item->>'producerAttempt')::integer)::text
1345
1642
  AND event.source_digest = digest
1346
1643
  AND event.search_doc @> jsonb_build_object('logContentDigest', digest)
@@ -1361,7 +1658,8 @@ export async function attachRuntimeEventSearchTriggers(
1361
1658
  -- retry of an accepted position is harmless; a new position
1362
1659
  -- after terminal closure must never recreate expired history.
1363
1660
  IF compact_logs AND observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
1364
- AND run_status IN ('completed', 'failed', 'cancelled')
1661
+ AND coalesce((observation.payload_json->>'_runtimeSearchRunClosed')::boolean,
1662
+ run_status IN ('completed', 'failed', 'cancelled'))
1365
1663
  AND NOT EXISTS (SELECT 1 FROM runtime_observability.events
1366
1664
  WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org
1367
1665
  AND event_id = eid) THEN
@@ -1375,7 +1673,7 @@ export async function attachRuntimeEventSearchTriggers(
1375
1673
  ${quoteLiteral(scope)}, run_org, eid,
1376
1674
  CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
1377
1675
  CASE WHEN canonical_events THEN jsonb_strip_nulls(
1378
- (item - 'lines' - 'lineContexts') || jsonb_build_object(
1676
+ (item - 'lines' - 'lineContexts' - 'lineIds') || jsonb_build_object(
1379
1677
  'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
1380
1678
  'occurredAt', floor(extract(epoch FROM at_time) * 1000)::bigint,
1381
1679
  'lineOrdinal', line_ordinal,
@@ -1494,6 +1792,7 @@ export async function attachRuntimeEventSearchTriggers(
1494
1792
  AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1495
1793
  AND (observation.payload_json->>'producerAttempt' IS NULL
1496
1794
  OR event.search_doc->>'producerAttempt' = observation.payload_json->>'producerAttempt')
1795
+ AND NOT (event.search_doc ? 'entryId')
1497
1796
  AND event.log_seq IS NOT NULL AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
1498
1797
  ORDER BY event.log_seq DESC
1499
1798
  LIMIT 256
@@ -1503,7 +1802,7 @@ export async function attachRuntimeEventSearchTriggers(
1503
1802
  FROM runtime_observability.events AS event
1504
1803
  WHERE event.runtime_scope = ${quoteLiteral(scope)}
1505
1804
  AND event.org_id = run_org AND event.run_id = observation.run_id
1506
- AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
1805
+ AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND NOT (event.search_doc ? 'entryId') AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
1507
1806
  AND event.event_id = ANY(fresh_event_ids)
1508
1807
  AND (observation.payload_json->>'producerAttempt' IS NULL
1509
1808
  OR event.search_doc->>'producerAttempt' = observation.payload_json->>'producerAttempt')
@@ -1720,7 +2019,7 @@ export async function attachRuntimeEventSearchTriggers(
1720
2019
  LANGUAGE sql IMMUTABLE SET search_path = pg_catalog AS $function$
1721
2020
  WITH envelope AS (
1722
2021
  SELECT CASE WHEN jsonb_typeof(value) = 'object' THEN
1723
- (value - 'logContexts') || CASE WHEN value ? 'logs'
2022
+ (value - 'logContexts' - 'logIds') || CASE WHEN value ? 'logs'
1724
2023
  THEN jsonb_build_object('logs', '[]'::jsonb) ELSE '{}'::jsonb END
1725
2024
  ELSE value END AS result
1726
2025
  )
@@ -1730,50 +2029,140 @@ export async function attachRuntimeEventSearchTriggers(
1730
2029
  $function$
1731
2030
  `);
1732
2031
  await client.query(`
1733
- CREATE OR REPLACE FUNCTION ${schema}.search_outbox_event() RETURNS trigger
2032
+ CREATE OR REPLACE FUNCTION ${schema}.compact_runtime_outbox_payload(observation ${schema}.outbox) RETURNS jsonb
1734
2033
  LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
1735
2034
  DECLARE compact_logs boolean; canonical_events boolean;
1736
2035
  BEGIN
1737
- -- Project the original validated observation before replacing its
1738
- -- transport body. Failure rolls back the outbox and canonical rows.
1739
- PERFORM ${schema}.project_outbox_event(NEW);
1740
2036
  SELECT run_log_store = 'planet_scale', run_event_store = 'planet_scale'
1741
2037
  INTO compact_logs, canonical_events
1742
- FROM ${schema}.runs WHERE run_id = NEW.run_id;
2038
+ FROM ${schema}.runs WHERE run_id = observation.run_id;
1743
2039
  IF compact_logs OR canonical_events THEN
1744
- NEW.payload_json := NEW.payload_json - 'terminalLogLines' - 'terminalLogContexts';
1745
- IF NEW.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
1746
- NEW.payload_json := jsonb_set(NEW.payload_json, '{events}',
1747
- ${schema}.without_log_events(NEW.payload_json->'events'))
2040
+ observation.payload_json := observation.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds';
2041
+ IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
2042
+ observation.payload_json := jsonb_set(observation.payload_json, '{events}',
2043
+ ${schema}.without_log_events(observation.payload_json->'events'))
1748
2044
  || jsonb_build_object('logBodyStore', 'planet_scale');
1749
2045
  END IF;
1750
- IF NEW.payload_json ? 'terminalProgressEvents' THEN
1751
- NEW.payload_json := jsonb_set(NEW.payload_json, '{terminalProgressEvents}',
1752
- ${schema}.without_log_events(NEW.payload_json->'terminalProgressEvents'));
2046
+ IF observation.payload_json ? 'terminalProgressEvents' THEN
2047
+ observation.payload_json := jsonb_set(observation.payload_json, '{terminalProgressEvents}',
2048
+ ${schema}.without_log_events(observation.payload_json->'terminalProgressEvents'));
1753
2049
  END IF;
1754
- IF NEW.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runFailed} AND NEW.payload_json ? 'result' THEN
1755
- NEW.payload_json := jsonb_set(NEW.payload_json, '{result}',
1756
- ${schema}.compact_runner_log_envelope(NEW.payload_json->'result'));
2050
+ IF observation.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runFailed} AND observation.payload_json ? 'result' THEN
2051
+ observation.payload_json := jsonb_set(observation.payload_json, '{result}',
2052
+ ${schema}.compact_runner_log_envelope(observation.payload_json->'result'));
1757
2053
  END IF;
1758
- IF NEW.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCompleted} AND NEW.payload_json ? 'output' THEN
1759
- NEW.payload_json := jsonb_set(NEW.payload_json, '{output}',
1760
- ${schema}.without_runtime_log_tail(NEW.payload_json->'output'));
2054
+ IF observation.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCompleted} AND observation.payload_json ? 'output' THEN
2055
+ observation.payload_json := jsonb_set(observation.payload_json, '{output}',
2056
+ ${schema}.without_runtime_log_tail(observation.payload_json->'output'));
1761
2057
  END IF;
1762
2058
  END IF;
1763
2059
  IF canonical_events THEN
2060
+ IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
2061
+ observation.payload_json := jsonb_set(observation.payload_json, '{events}',
2062
+ ${schema}.convex_projection_events(observation.payload_json->'events'));
2063
+ END IF;
2064
+ IF observation.payload_json ? 'terminalProgressEvents' THEN
2065
+ observation.payload_json := jsonb_set(observation.payload_json, '{terminalProgressEvents}',
2066
+ ${schema}.convex_projection_events(observation.payload_json->'terminalProgressEvents'));
2067
+ END IF;
2068
+ END IF;
2069
+ RETURN observation.payload_json - '${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' - '_runtimeSearchRunClosed';
2070
+ END
2071
+ $function$
2072
+ `);
2073
+ await client.query(`
2074
+ CREATE OR REPLACE FUNCTION ${schema}.search_outbox_event() RETURNS trigger
2075
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
2076
+ DECLARE run_closed boolean; search_payload jsonb;
2077
+ BEGIN
2078
+ IF NEW.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb THEN RETURN NEW; END IF;
2079
+ IF ${schema}.defer_runtime_event_projection()
2080
+ AND NEW.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runCreated} THEN
1764
2081
  IF NEW.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
1765
- NEW.payload_json := jsonb_set(NEW.payload_json, '{events}',
1766
- ${schema}.convex_projection_events(NEW.payload_json->'events'));
2082
+ -- One cheap Run fence captures whether ingress preceded terminal
2083
+ -- closure. No indexed log lookup or search mutex runs under it.
2084
+ SELECT status IN ('completed', 'failed', 'cancelled') INTO run_closed
2085
+ FROM ${schema}.runs WHERE run_id = NEW.run_id FOR SHARE;
2086
+ NEW.created_at := GREATEST(NEW.created_at, clock_timestamp(),
2087
+ coalesce((SELECT max(created_at) + interval '1 microsecond'
2088
+ FROM ${schema}.outbox WHERE run_id = NEW.run_id), clock_timestamp()));
2089
+ ELSE
2090
+ SELECT status IN ('completed', 'failed', 'cancelled') INTO run_closed
2091
+ FROM ${schema}.runs WHERE run_id = NEW.run_id;
1767
2092
  END IF;
1768
- IF NEW.payload_json ? 'terminalProgressEvents' THEN
1769
- NEW.payload_json := jsonb_set(NEW.payload_json, '{terminalProgressEvents}',
1770
- ${schema}.convex_projection_events(NEW.payload_json->'terminalProgressEvents'));
2093
+ NEW.payload_json := NEW.payload_json || jsonb_build_object(
2094
+ '${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}', true,
2095
+ '_runtimeSearchRunClosed', coalesce(run_closed, false));
2096
+ IF ${schema}.independent_runtime_event_projection() THEN
2097
+ IF NEW.event_type IN (${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.receiptEvent}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runnerLogs}) OR (NEW.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND jsonb_typeof(NEW.payload_json->'events') = 'array' AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(NEW.payload_json->'events') item WHERE item->>'type' IS DISTINCT FROM ${RUN_EVENT_KIND_SQL_LITERAL.logAppended})) THEN
2098
+ NEW.payload_json := NEW.payload_json || jsonb_build_object('${RUNTIME_EVENT_SEARCH_ONLY_FIELD}', true);
2099
+ ELSE
2100
+ search_payload := NEW.payload_json || jsonb_build_object('${RUNTIME_EVENT_SEARCH_ONLY_FIELD}', true, '_runtimeSearchEventId', NEW.event_id);
2101
+ INSERT INTO ${schema}.outbox(event_id, run_id, event_type, payload_json, created_at)
2102
+ VALUES ('search:' || NEW.event_id, NEW.run_id, NEW.event_type, search_payload, NEW.created_at)
2103
+ ON CONFLICT (event_id) DO NOTHING;
2104
+ NEW.payload_json := ${schema}.compact_runtime_outbox_payload(NEW);
2105
+ END IF;
1771
2106
  END IF;
2107
+ RETURN NEW;
1772
2108
  END IF;
2109
+ -- Legacy writers retain their atomic projection until explicit cutover.
2110
+ NEW.payload_json := NEW.payload_json - '${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' - '_runtimeSearchRunClosed';
2111
+ PERFORM ${schema}.project_outbox_event(NEW);
2112
+ NEW.payload_json := ${schema}.compact_runtime_outbox_payload(NEW);
1773
2113
  RETURN NEW;
1774
2114
  END
1775
2115
  $function$
1776
2116
  `);
2117
+ await client.query(`
2118
+ CREATE OR REPLACE FUNCTION ${schema}.guard_runtime_outbox_ack() RETURNS trigger
2119
+ LANGUAGE plpgsql SET search_path = pg_catalog AS $function$
2120
+ BEGIN
2121
+ IF OLD.delivered_at IS NULL AND NEW.delivered_at IS NOT NULL
2122
+ AND OLD.payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb THEN
2123
+ RAISE EXCEPTION 'Deferred outbox input must be projected before acknowledgement' USING ERRCODE = '55000';
2124
+ END IF;
2125
+ RETURN NEW;
2126
+ END
2127
+ $function$
2128
+ `);
2129
+ await client.query(`DO $create$ BEGIN
2130
+ IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.outbox'::regclass
2131
+ AND tgname = 'runtime_outbox_projection_ack_v1') THEN
2132
+ CREATE TRIGGER runtime_outbox_projection_ack_v1 BEFORE UPDATE OF delivered_at ON ${schema}.outbox
2133
+ FOR EACH ROW EXECUTE FUNCTION ${schema}.guard_runtime_outbox_ack();
2134
+ END IF;
2135
+ END $create$`);
2136
+ await client.query(`
2137
+ CREATE OR REPLACE FUNCTION ${schema}.materialize_runtime_outbox_event(eid text, owner text) RETURNS jsonb
2138
+ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
2139
+ DECLARE observation ${schema}.outbox; projection_observation ${schema}.outbox;
2140
+ BEGIN
2141
+ SELECT * INTO observation FROM ${schema}.outbox
2142
+ WHERE event_id = eid AND lease_owner = owner AND lease_expires_at > clock_timestamp()
2143
+ AND delivered_at IS NULL AND failed_at IS NULL;
2144
+ IF NOT FOUND THEN
2145
+ RAISE EXCEPTION 'Deferred event projection lease is not current' USING ERRCODE = '40001';
2146
+ END IF;
2147
+ IF observation.payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb THEN
2148
+ projection_observation := observation;
2149
+ projection_observation.event_id := coalesce(observation.payload_json->>'_runtimeSearchEventId', observation.event_id);
2150
+ PERFORM ${schema}.project_outbox_event(projection_observation);
2151
+ observation.payload_json := ${schema}.compact_runtime_outbox_payload(observation);
2152
+ -- Project against immutable input without locking the outbox row.
2153
+ -- Cancellation may transfer its lease while indexing is in flight.
2154
+ -- This final fence rolls the journal back if ownership changed.
2155
+ UPDATE ${schema}.outbox SET payload_json = observation.payload_json
2156
+ WHERE event_id = eid AND lease_owner = owner AND lease_expires_at > clock_timestamp()
2157
+ AND delivered_at IS NULL AND failed_at IS NULL;
2158
+ IF NOT FOUND THEN
2159
+ RAISE EXCEPTION 'Deferred event projection lease is not current' USING ERRCODE = '40001';
2160
+ END IF;
2161
+ END IF;
2162
+ RETURN observation.payload_json;
2163
+ END
2164
+ $function$
2165
+ `);
1777
2166
  await client.query(`
1778
2167
  CREATE OR REPLACE FUNCTION ${schema}.accept_runner_log_payload() RETURNS trigger
1779
2168
  LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
@@ -1803,7 +2192,16 @@ export async function attachRuntimeEventSearchTriggers(
1803
2192
  THEN NEW.runner_log_payload_json->'terminalProgressEvents'
1804
2193
  ELSE '[]'::jsonb END) WITH ORDINALITY AS progress(event, position)
1805
2194
  ));
1806
- PERFORM ${schema}.project_outbox_event(observation);
2195
+ IF ${schema}.defer_runtime_event_projection() THEN
2196
+ INSERT INTO ${schema}.outbox (event_id, run_id, event_type, payload_json, created_at)
2197
+ VALUES (observation.event_id, observation.run_id, observation.event_type,
2198
+ observation.payload_json,
2199
+ GREATEST(clock_timestamp(), coalesce((SELECT max(created_at) + interval '1 microsecond'
2200
+ FROM ${schema}.outbox WHERE run_id = NEW.run_id), clock_timestamp())))
2201
+ ON CONFLICT (event_id) DO NOTHING;
2202
+ ELSE
2203
+ PERFORM ${schema}.project_outbox_event(observation);
2204
+ END IF;
1807
2205
  NEW.runner_result_json := ${schema}.compact_runner_log_envelope(NEW.runner_result_json);
1808
2206
  IF jsonb_array_length(observation.payload_json->'recoveredRuntimeDiagnostics') > 0
1809
2207
  AND (NOT (NEW.runner_result_json ? 'runtimeDiagnostics')
@@ -1888,6 +2286,12 @@ export async function attachRuntimeEventSearchTriggers(
1888
2286
  END IF;
1889
2287
  END $create$`);
1890
2288
  await client.query(`REVOKE ALL ON FUNCTION ${schema}.project_outbox_event(${schema}.outbox),
2289
+ ${schema}.project_origin_log_batch(${schema}.outbox, jsonb, text, text, boolean, boolean, timestamptz, jsonb, boolean),
2290
+ ${schema}.independent_runtime_event_projection(),
2291
+ ${schema}.pin_runtime_log_identity(),
2292
+ ${schema}.materialize_runtime_outbox_event(text, text),
2293
+ ${schema}.guard_runtime_outbox_ack(),
2294
+ ${schema}.compact_runtime_outbox_payload(${schema}.outbox),
1891
2295
  ${schema}.accept_runner_log_payload(), ${schema}.search_outbox_event(),
1892
2296
  ${schema}.compact_run_log_result(), ${schema}.without_log_events(jsonb),
1893
2297
  ${schema}.convex_projection_event(jsonb), ${schema}.convex_projection_events(jsonb),
@@ -1905,7 +2309,11 @@ export async function hasRuntimeEventSearchShape(
1905
2309
  ): Promise<boolean> {
1906
2310
  const schema = quoteIdent(requiredScope(runtimeScope));
1907
2311
  const result = await client.query<{ ready: boolean }>(
1908
- `SELECT to_regclass('runtime_observability.events') IS NOT NULL
2312
+ `SELECT to_regprocedure('${schema}.materialize_runtime_outbox_event(text,text)') IS NOT NULL
2313
+ AND EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = to_regclass('${schema}.outbox')
2314
+ AND tgname = 'runtime_outbox_projection_ack_v1' AND tgenabled <> 'D')
2315
+ AND to_regprocedure('${schema}.defer_runtime_event_projection()') IS NOT NULL
2316
+ AND to_regclass('runtime_observability.events') IS NOT NULL
1909
2317
  AND to_regclass('runtime_observability.run_log_cursors') IS NOT NULL
1910
2318
  AND EXISTS (SELECT 1 FROM pg_attribute
1911
2319
  WHERE attrelid = to_regclass('runtime_observability.events')
@@ -2345,6 +2753,7 @@ export async function pruneRuntimeEvents(
2345
2753
  search_doc = jsonb_strip_nulls(jsonb_build_object(
2346
2754
  'kind', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'source', event.search_doc->'source',
2347
2755
  'producerAttempt', event.search_doc->'producerAttempt',
2756
+ 'entryId', event.search_doc->'entryId',
2348
2757
  'logPositions', event.search_doc->'logPositions',
2349
2758
  'logContentDigest', event.search_doc->'logContentDigest'))
2350
2759
  FROM candidates WHERE event.ctid = candidates.ctid AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}