deepline 0.3.288 → 0.3.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
- package/dist/cli/index.js +24 -11
- package/dist/cli/index.mjs +24 -11
- package/dist/index.js +3 -1
- package/dist/index.mjs +3 -1
- package/dist/install-integrity.json +3 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The customer-visible, append-only search projection of durable Play facts.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* After reader-first cutover, triggers persist raw input in the existing
|
|
5
|
+
* outbox. The leased dispatcher projects and compacts it before delivery.
|
|
6
|
+
* Receipt output remains exclusively in the authoritative receipt store.
|
|
6
7
|
* The API may read only through fixed, tenant-scoped queries in this module.
|
|
7
8
|
*/
|
|
8
9
|
import { normalizeRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
|
|
@@ -56,7 +57,11 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
|
|
|
56
57
|
// Migration readiness tracks this projection body independently of scheduler
|
|
57
58
|
// wire and schema versions. Bump when a changed trigger must be reinstalled.
|
|
58
59
|
export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
|
|
59
|
-
'runtime-event-search-
|
|
60
|
+
'runtime-event-search-origin-batches-independent-progress';
|
|
61
|
+
/** Trusted trigger-authored state; absent on already-projected legacy rows. */
|
|
62
|
+
export const RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD =
|
|
63
|
+
'_runtimeSearchProjectionPending';
|
|
64
|
+
export const RUNTIME_EVENT_SEARCH_ONLY_FIELD = '_runtimeSearchOnly';
|
|
60
65
|
export const RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER =
|
|
61
66
|
'runtime-event-canonical-retention-v1';
|
|
62
67
|
export const RUNTIME_EVENTS_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
@@ -950,11 +955,131 @@ export async function hasRuntimeEventReaderGrant(
|
|
|
950
955
|
return result.rows[0]?.ready === true;
|
|
951
956
|
}
|
|
952
957
|
|
|
958
|
+
/** Deployment-owned cutover; install readers everywhere before enabling. */
|
|
959
|
+
export async function setRuntimeEventDeferredProjection(
|
|
960
|
+
client: PostgresSchedulerQueryClient,
|
|
961
|
+
runtimeScope: string,
|
|
962
|
+
deferred: boolean,
|
|
963
|
+
independent?: boolean,
|
|
964
|
+
): Promise<void> {
|
|
965
|
+
const schema = quoteIdent(requiredScope(runtimeScope));
|
|
966
|
+
if (!(await hasRuntimeEventSearchShape(client, runtimeScope))) {
|
|
967
|
+
throw new Error(
|
|
968
|
+
'Install the deferred outbox reader before changing projection mode.',
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
if (deferred && !(await hasRuntimeEventLogPositionIndex(client))) {
|
|
972
|
+
throw new Error(
|
|
973
|
+
'Prepare the log-position indexes before enabling deferred projection.',
|
|
974
|
+
);
|
|
975
|
+
}
|
|
976
|
+
await client.query('BEGIN');
|
|
977
|
+
try {
|
|
978
|
+
await client.query("SELECT set_config('lock_timeout', '1000', true)");
|
|
979
|
+
// Fence trigger execution while changing mode. This operator-only lock
|
|
980
|
+
// never spans projection; normal producer writes acquire no such lock.
|
|
981
|
+
await client.query(
|
|
982
|
+
`LOCK TABLE ${schema}.outbox IN SHARE ROW EXCLUSIVE MODE`,
|
|
983
|
+
);
|
|
984
|
+
if (!deferred) {
|
|
985
|
+
const pending = await client.query<{ event_id: string }>(
|
|
986
|
+
`SELECT event_id FROM ${schema}.outbox
|
|
987
|
+
WHERE delivered_at IS NULL
|
|
988
|
+
AND payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb
|
|
989
|
+
LIMIT 1`,
|
|
990
|
+
);
|
|
991
|
+
if (pending.rows.length > 0) {
|
|
992
|
+
throw new Error(
|
|
993
|
+
'Drain deferred outbox rows before returning to synchronous projection.',
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
if (independent === false || !deferred) {
|
|
998
|
+
const raw = await client.query<{ event_id: string }>(
|
|
999
|
+
`SELECT event_id FROM ${schema}.outbox WHERE delivered_at IS NULL AND payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb LIMIT 1`,
|
|
1000
|
+
);
|
|
1001
|
+
if (raw.rows.length)
|
|
1002
|
+
throw new Error(
|
|
1003
|
+
'Drain independent search rows before disabling independent projection.',
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
1006
|
+
await installRuntimeEventProjectionMode(client, schema, deferred);
|
|
1007
|
+
if (independent !== undefined || !deferred)
|
|
1008
|
+
await installIndependentProjectionMode(
|
|
1009
|
+
client,
|
|
1010
|
+
schema,
|
|
1011
|
+
deferred && independent === true,
|
|
1012
|
+
);
|
|
1013
|
+
await client.query('COMMIT');
|
|
1014
|
+
} catch (error) {
|
|
1015
|
+
await client.query('ROLLBACK');
|
|
1016
|
+
throw error;
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/** Operator state uses the same durable pending set as the outbox projector. */
|
|
1021
|
+
export async function inspectRuntimeEventProjection(
|
|
1022
|
+
client: PostgresSchedulerQueryClient,
|
|
1023
|
+
runtimeScope: string,
|
|
1024
|
+
): Promise<{
|
|
1025
|
+
deferred: boolean;
|
|
1026
|
+
independent: boolean;
|
|
1027
|
+
pending: number;
|
|
1028
|
+
blocked: number;
|
|
1029
|
+
oldestPendingAt: string | null;
|
|
1030
|
+
}> {
|
|
1031
|
+
const schema = quoteIdent(requiredScope(runtimeScope));
|
|
1032
|
+
const result = await client.query<{
|
|
1033
|
+
deferred: boolean;
|
|
1034
|
+
independent: boolean;
|
|
1035
|
+
pending: string | number;
|
|
1036
|
+
blocked: string | number;
|
|
1037
|
+
oldest: Date | string | null;
|
|
1038
|
+
}>(`SELECT ${schema}.defer_runtime_event_projection() AS deferred, ${schema}.independent_runtime_event_projection() AS independent,
|
|
1039
|
+
count(*) AS pending, count(*) FILTER (WHERE failed_at IS NOT NULL) AS blocked,
|
|
1040
|
+
min(created_at) AS oldest
|
|
1041
|
+
FROM ${schema}.outbox
|
|
1042
|
+
WHERE delivered_at IS NULL
|
|
1043
|
+
AND payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb`);
|
|
1044
|
+
const state = result.rows[0]!;
|
|
1045
|
+
return {
|
|
1046
|
+
deferred: state.deferred,
|
|
1047
|
+
independent: state.independent,
|
|
1048
|
+
pending: Number(state.pending),
|
|
1049
|
+
blocked: Number(state.blocked),
|
|
1050
|
+
oldestPendingAt: state.oldest === null ? null : iso(state.oldest),
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
async function installRuntimeEventProjectionMode(
|
|
1055
|
+
client: PostgresSchedulerQueryClient,
|
|
1056
|
+
schema: string,
|
|
1057
|
+
deferred: boolean,
|
|
1058
|
+
): Promise<void> {
|
|
1059
|
+
await client.query(`CREATE OR REPLACE FUNCTION ${schema}.defer_runtime_event_projection()
|
|
1060
|
+
RETURNS boolean LANGUAGE sql VOLATILE SET search_path = pg_catalog
|
|
1061
|
+
AS 'SELECT ${deferred ? 'true' : 'false'}'`);
|
|
1062
|
+
await client.query(
|
|
1063
|
+
`REVOKE ALL ON FUNCTION ${schema}.defer_runtime_event_projection() FROM PUBLIC`,
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
async function installIndependentProjectionMode(
|
|
1068
|
+
client: PostgresSchedulerQueryClient,
|
|
1069
|
+
schema: string,
|
|
1070
|
+
enabled: boolean,
|
|
1071
|
+
): Promise<void> {
|
|
1072
|
+
await client.query(`CREATE OR REPLACE FUNCTION ${schema}.independent_runtime_event_projection()
|
|
1073
|
+
RETURNS boolean LANGUAGE sql VOLATILE SET search_path = pg_catalog AS 'SELECT ${enabled ? 'true' : 'false'}'`);
|
|
1074
|
+
await client.query(
|
|
1075
|
+
`REVOKE ALL ON FUNCTION ${schema}.independent_runtime_event_projection() FROM PUBLIC`,
|
|
1076
|
+
);
|
|
1077
|
+
}
|
|
1078
|
+
|
|
953
1079
|
/**
|
|
954
|
-
*
|
|
955
|
-
*
|
|
956
|
-
*
|
|
957
|
-
* corresponding search history after enablement.
|
|
1080
|
+
* Migration installs readers with synchronous production unchanged. After
|
|
1081
|
+
* cutover, the producer persists raw input and only the leased dispatcher
|
|
1082
|
+
* projects and compacts it, atomically, before acknowledging its outbox row.
|
|
958
1083
|
*/
|
|
959
1084
|
export async function attachRuntimeEventSearchTriggers(
|
|
960
1085
|
client: PostgresSchedulerQueryClient,
|
|
@@ -962,6 +1087,33 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
962
1087
|
): Promise<void> {
|
|
963
1088
|
const scope = requiredScope(runtimeScope);
|
|
964
1089
|
const schema = quoteIdent(scope);
|
|
1090
|
+
// Refreshing the reader must preserve an already-enabled producer cutover.
|
|
1091
|
+
const mode = await client.query<{ installed: boolean }>(
|
|
1092
|
+
`SELECT to_regprocedure($1) IS NOT NULL AS installed`,
|
|
1093
|
+
[`${schema}.defer_runtime_event_projection()`],
|
|
1094
|
+
);
|
|
1095
|
+
if (!mode.rows[0]?.installed) {
|
|
1096
|
+
await installRuntimeEventProjectionMode(client, schema, false);
|
|
1097
|
+
}
|
|
1098
|
+
const independentMode = await client.query<{ installed: boolean }>(
|
|
1099
|
+
`SELECT to_regprocedure($1) IS NOT NULL AS installed`,
|
|
1100
|
+
[`${schema}.independent_runtime_event_projection()`],
|
|
1101
|
+
);
|
|
1102
|
+
if (!independentMode.rows[0]?.installed)
|
|
1103
|
+
await installIndependentProjectionMode(client, schema, false);
|
|
1104
|
+
// Stamp new admissions once. Existing launches never acquire a producer pin
|
|
1105
|
+
// on replay or worker upgrade. Ignore a caller-supplied pin before cutover.
|
|
1106
|
+
await client.query(`CREATE OR REPLACE FUNCTION ${schema}.pin_runtime_log_identity() RETURNS trigger
|
|
1107
|
+
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
1108
|
+
BEGIN
|
|
1109
|
+
NEW.launch_json := NEW.launch_json - 'runtimeLogIdentity';
|
|
1110
|
+
NEW.admission_json := NEW.admission_json - 'runtimeLogIdentity';
|
|
1111
|
+
IF ${schema}.independent_runtime_event_projection() THEN
|
|
1112
|
+
NEW.launch_json := NEW.launch_json || '{"runtimeLogIdentity":"origin"}'::jsonb;
|
|
1113
|
+
NEW.admission_json := NEW.admission_json || '{"runtimeLogIdentity":"origin"}'::jsonb;
|
|
1114
|
+
END IF;
|
|
1115
|
+
RETURN NEW;
|
|
1116
|
+
END $function$`);
|
|
965
1117
|
await client.query(`
|
|
966
1118
|
CREATE OR REPLACE FUNCTION ${schema}.search_receipt_event() RETURNS trigger
|
|
967
1119
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
@@ -978,6 +1130,23 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
978
1130
|
IF NEW.run_id IS NULL OR NEW.org_id IS NULL THEN
|
|
979
1131
|
RAISE EXCEPTION 'Terminal receipt search projection requires run and organization';
|
|
980
1132
|
END IF;
|
|
1133
|
+
IF ${schema}.defer_runtime_event_projection() THEN
|
|
1134
|
+
IF NOT EXISTS (SELECT 1 FROM ${schema}.runs
|
|
1135
|
+
WHERE run_id = NEW.run_id AND org_id = NEW.org_id) THEN
|
|
1136
|
+
RAISE EXCEPTION 'Terminal receipt search projection has no matching run';
|
|
1137
|
+
END IF;
|
|
1138
|
+
ref := gen_random_uuid();
|
|
1139
|
+
INSERT INTO ${schema}.outbox (event_id, run_id, event_type, payload_json, created_at)
|
|
1140
|
+
VALUES ('receipt:' || ref::text, NEW.run_id,
|
|
1141
|
+
${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.receiptEvent},
|
|
1142
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
1143
|
+
'orgId', NEW.org_id, 'receiptRef', ref, 'receiptKey', NEW.k,
|
|
1144
|
+
'status', NEW.status, 'occurredAt', NEW.updated_at,
|
|
1145
|
+
'producerAttempt', NEW.lease_owner_attempt)),
|
|
1146
|
+
GREATEST(clock_timestamp(), coalesce((SELECT max(created_at) + interval '1 microsecond'
|
|
1147
|
+
FROM ${schema}.outbox WHERE run_id = NEW.run_id), clock_timestamp())));
|
|
1148
|
+
RETURN NEW;
|
|
1149
|
+
END IF;
|
|
981
1150
|
-- Serialize event ordering with the same per-run lock as outbox facts.
|
|
982
1151
|
PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(NEW.run_id));
|
|
983
1152
|
SELECT play_name, run_event_store = 'planet_scale'
|
|
@@ -1009,12 +1178,87 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1009
1178
|
$function$
|
|
1010
1179
|
`);
|
|
1011
1180
|
await client.query(`DO $create$ BEGIN
|
|
1181
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.runs'::regclass
|
|
1182
|
+
AND tgname = 'runtime_run_log_identity') THEN
|
|
1183
|
+
CREATE TRIGGER runtime_run_log_identity BEFORE INSERT ON ${schema}.runs
|
|
1184
|
+
FOR EACH ROW EXECUTE FUNCTION ${schema}.pin_runtime_log_identity();
|
|
1185
|
+
END IF;
|
|
1012
1186
|
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.receipts'::regclass
|
|
1013
1187
|
AND tgname = 'runtime_receipt_event_v1') THEN
|
|
1014
1188
|
CREATE TRIGGER runtime_receipt_event_v1 AFTER INSERT OR UPDATE ON ${schema}.receipts
|
|
1015
1189
|
FOR EACH ROW EXECUTE FUNCTION ${schema}.search_receipt_event();
|
|
1016
1190
|
END IF;
|
|
1017
1191
|
END $create$`);
|
|
1192
|
+
await client.query(`
|
|
1193
|
+
CREATE OR REPLACE FUNCTION ${schema}.project_origin_log_batch(observation ${schema}.outbox, item jsonb, run_org text, play text, compact_logs boolean, canonical_events boolean, at_time timestamptz, doc jsonb, closed boolean)
|
|
1194
|
+
RETURNS text[] LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
1195
|
+
DECLARE fresh text[]; mismatches bigint;
|
|
1196
|
+
BEGIN
|
|
1197
|
+
IF jsonb_typeof(item->'lineIds') IS DISTINCT FROM 'array'
|
|
1198
|
+
OR jsonb_array_length(item->'lineIds') <> jsonb_array_length(item->'lines')
|
|
1199
|
+
OR EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lineIds') id WHERE jsonb_typeof(id) <> 'string' OR (id #>> '{}') !~ '^[A-Za-z0-9:_-]{1,128}$')
|
|
1200
|
+
OR (SELECT count(DISTINCT id) FROM jsonb_array_elements(item->'lineIds') id) <> jsonb_array_length(item->'lineIds') THEN
|
|
1201
|
+
RAISE EXCEPTION 'Origin log identities must be distinct bounded strings aligned with lines';
|
|
1202
|
+
END IF;
|
|
1203
|
+
IF EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lines') line WHERE jsonb_typeof(line) <> 'string') THEN
|
|
1204
|
+
RAISE EXCEPTION 'Search projection requires log line strings';
|
|
1205
|
+
END IF;
|
|
1206
|
+
IF item ? 'lineContexts' AND EXISTS (
|
|
1207
|
+
SELECT 1 FROM jsonb_array_elements(item->'lineContexts') ctx
|
|
1208
|
+
WHERE jsonb_typeof(ctx) NOT IN ('object', 'null')
|
|
1209
|
+
) THEN RAISE EXCEPTION 'Search projection log context must be an object'; END IF;
|
|
1210
|
+
IF item ? 'lineContexts' AND EXISTS (
|
|
1211
|
+
SELECT 1 FROM jsonb_array_elements(item->'lineContexts') ctx
|
|
1212
|
+
WHERE jsonb_typeof(ctx) = 'object' AND (
|
|
1213
|
+
(SELECT count(*) FROM jsonb_object_keys(ctx)) > 8 OR octet_length(ctx::text) > 2064
|
|
1214
|
+
OR EXISTS (SELECT 1 FROM jsonb_each(ctx) field WHERE
|
|
1215
|
+
field.key !~ '^[A-Za-z_][A-Za-z0-9_]{0,63}$' OR field.key IN ('__proto__', 'constructor', 'prototype')
|
|
1216
|
+
OR jsonb_typeof(field.value) NOT IN ('string', 'number', 'boolean')
|
|
1217
|
+
OR (jsonb_typeof(field.value) = 'string' AND octet_length(field.value #>> '{}') > 256)
|
|
1218
|
+
OR (jsonb_typeof(field.value) = 'number' AND octet_length(field.value #>> '{}') > 32))
|
|
1219
|
+
)
|
|
1220
|
+
) THEN RAISE EXCEPTION 'Search projection log context has an unsupported key or value'; END IF;
|
|
1221
|
+
WITH entries AS MATERIALIZED (
|
|
1222
|
+
SELECT value #>> '{}' AS body, ordinality AS ordinal,
|
|
1223
|
+
'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinality-1)::int) AS eid,
|
|
1224
|
+
NULLIF(item->'lineContexts'->(ordinality-1)::int, 'null'::jsonb) AS ctx
|
|
1225
|
+
FROM jsonb_array_elements(item->'lines') WITH ORDINALITY
|
|
1226
|
+
), inserted AS (
|
|
1227
|
+
INSERT INTO runtime_observability.events (
|
|
1228
|
+
runtime_scope, org_id, event_id, event_seq, event_payload, occurred_at,
|
|
1229
|
+
run_id, play_name, kind, level, message, log_body, source_digest, log_text_digest, search_doc
|
|
1230
|
+
) SELECT ${quoteLiteral(scope)}, run_org, eid,
|
|
1231
|
+
CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
|
|
1232
|
+
CASE WHEN canonical_events THEN jsonb_strip_nulls((item - 'lines' - 'lineContexts' - 'lineIds') || jsonb_build_object(
|
|
1233
|
+
'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
|
|
1234
|
+
'occurredAt', floor(extract(epoch FROM at_time)*1000)::bigint, 'lineOrdinal', ordinal,
|
|
1235
|
+
'entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx)) END,
|
|
1236
|
+
at_time, observation.run_id, play, ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, NULL,
|
|
1237
|
+
CASE WHEN compact_logs THEN NULL ELSE left(body, 2048) END, body,
|
|
1238
|
+
CASE WHEN ctx IS NULL THEN encode(sha256(convert_to(body,'UTF8')),'hex')
|
|
1239
|
+
ELSE encode(sha256(convert_to(jsonb_build_array(body,ctx)::text,'UTF8')),'hex') END,
|
|
1240
|
+
encode(sha256(convert_to(btrim(body),'UTF8')),'hex'),
|
|
1241
|
+
doc || jsonb_strip_nulls(jsonb_build_object('entryId', item->'lineIds'->(ordinal-1)::int, 'context', ctx))
|
|
1242
|
+
FROM entries WHERE NOT closed ORDER BY ordinal
|
|
1243
|
+
ON CONFLICT (runtime_scope, org_id, event_id) DO NOTHING RETURNING event_id
|
|
1244
|
+
) SELECT coalesce(array_agg(inserted.event_id ORDER BY entries.ordinal), ARRAY[]::text[])
|
|
1245
|
+
INTO fresh FROM inserted JOIN entries ON entries.eid = inserted.event_id;
|
|
1246
|
+
-- A new READ COMMITTED statement sees a concurrent uniqueness winner.
|
|
1247
|
+
-- Tombstones retain digest/identity: retry never restores an expired body.
|
|
1248
|
+
SELECT count(*) INTO mismatches
|
|
1249
|
+
FROM jsonb_array_elements(item->'lines') WITH ORDINALITY AS entries(body, ordinal)
|
|
1250
|
+
LEFT JOIN runtime_observability.events event ON event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
|
|
1251
|
+
AND event.event_id = 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinal-1)::int)
|
|
1252
|
+
WHERE event.run_id IS DISTINCT FROM observation.run_id OR event.kind IS DISTINCT FROM ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
1253
|
+
OR event.source_digest IS DISTINCT FROM CASE WHEN NULLIF(item->'lineContexts'->(ordinal-1)::int, 'null'::jsonb) IS NULL
|
|
1254
|
+
THEN encode(sha256(convert_to(body #>> '{}','UTF8')),'hex')
|
|
1255
|
+
ELSE encode(sha256(convert_to(jsonb_build_array(body #>> '{}',item->'lineContexts'->(ordinal-1)::int)::text,'UTF8')),'hex') END;
|
|
1256
|
+
IF mismatches > 0 THEN
|
|
1257
|
+
RAISE EXCEPTION 'Origin log identity changed its content or ingress is closed after terminal acceptance';
|
|
1258
|
+
END IF;
|
|
1259
|
+
RETURN fresh;
|
|
1260
|
+
END $function$
|
|
1261
|
+
`);
|
|
1018
1262
|
await client.query(`
|
|
1019
1263
|
CREATE OR REPLACE FUNCTION ${schema}.project_outbox_event(observation ${schema}.outbox) RETURNS void
|
|
1020
1264
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
@@ -1033,7 +1277,11 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1033
1277
|
-- Log ordering owns a transaction-scoped mutex, independent of the
|
|
1034
1278
|
-- scheduler's Run/attempt row-lock order. In particular, old completion
|
|
1035
1279
|
-- writers lock attempt before Run; this recorder must not invert that.
|
|
1036
|
-
|
|
1280
|
+
-- Stable batches are ordered by the leased background prefix and cursor row.
|
|
1281
|
+
-- The legacy alias path still needs its old transaction mutex.
|
|
1282
|
+
IF NOT ${schema}.defer_runtime_event_projection() THEN
|
|
1283
|
+
PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(observation.run_id));
|
|
1284
|
+
END IF;
|
|
1037
1285
|
SELECT org_id, play_name, run_log_store = 'planet_scale', status,
|
|
1038
1286
|
run_event_store = 'planet_scale'
|
|
1039
1287
|
INTO run_org, play, compact_logs, run_status, canonical_events
|
|
@@ -1041,6 +1289,35 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1041
1289
|
IF run_org IS NULL OR play IS NULL THEN
|
|
1042
1290
|
RAISE EXCEPTION 'Outbox search projection has no matching run';
|
|
1043
1291
|
END IF;
|
|
1292
|
+
IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.receiptEvent} THEN
|
|
1293
|
+
IF observation.payload_json->>'orgId' IS DISTINCT FROM run_org
|
|
1294
|
+
OR NOT coalesce(observation.payload_json->>'status' IN ('completed', 'failed', 'skipped'), false)
|
|
1295
|
+
OR nullif(observation.payload_json->>'receiptKey', '') IS NULL
|
|
1296
|
+
OR observation.payload_json->>'receiptRef' IS NULL
|
|
1297
|
+
OR observation.payload_json->>'occurredAt' IS NULL THEN
|
|
1298
|
+
RAISE EXCEPTION 'Deferred receipt event has invalid identity or status';
|
|
1299
|
+
END IF;
|
|
1300
|
+
INSERT INTO runtime_observability.events (
|
|
1301
|
+
runtime_scope, org_id, event_id, event_seq, event_payload,
|
|
1302
|
+
occurred_at, run_id, play_name, kind, receipt_ref, receipt_key, search_doc
|
|
1303
|
+
) VALUES (
|
|
1304
|
+
${quoteLiteral(scope)}, run_org, observation.event_id,
|
|
1305
|
+
CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
|
|
1306
|
+
CASE WHEN canonical_events THEN jsonb_build_object(
|
|
1307
|
+
'type', 'receipt.' || (observation.payload_json->>'status'), 'runId', observation.run_id,
|
|
1308
|
+
'occurredAt', floor(extract(epoch FROM (observation.payload_json->>'occurredAt')::timestamptz) * 1000)::bigint,
|
|
1309
|
+
'source', 'receipt', 'status', observation.payload_json->>'status') END,
|
|
1310
|
+
(observation.payload_json->>'occurredAt')::timestamptz,
|
|
1311
|
+
observation.run_id, play, 'receipt.' || (observation.payload_json->>'status'),
|
|
1312
|
+
(observation.payload_json->>'receiptRef')::uuid,
|
|
1313
|
+
observation.payload_json->>'receiptKey',
|
|
1314
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
1315
|
+
'kind', 'receipt.' || (observation.payload_json->>'status'),
|
|
1316
|
+
'status', observation.payload_json->>'status', 'source', 'receipt',
|
|
1317
|
+
'producerAttempt', observation.payload_json->'producerAttempt'))
|
|
1318
|
+
) ON CONFLICT DO NOTHING;
|
|
1319
|
+
RETURN;
|
|
1320
|
+
END IF;
|
|
1044
1321
|
IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
|
|
1045
1322
|
event_batch := observation.payload_json->'events';
|
|
1046
1323
|
IF jsonb_typeof(event_batch) IS DISTINCT FROM 'array' THEN
|
|
@@ -1071,7 +1348,8 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1071
1348
|
'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id, 'source', 'terminal',
|
|
1072
1349
|
'producerAttempt', envelope_attempt, 'channelOffset', 0,
|
|
1073
1350
|
'lines', observation.payload_json->'terminalLogLines',
|
|
1074
|
-
'lineContexts', observation.payload_json->'terminalLogContexts'
|
|
1351
|
+
'lineContexts', observation.payload_json->'terminalLogContexts',
|
|
1352
|
+
'lineIds', NULLIF(observation.payload_json->'terminalLogIds', 'null'::jsonb)
|
|
1075
1353
|
))
|
|
1076
1354
|
);
|
|
1077
1355
|
END IF;
|
|
@@ -1117,11 +1395,14 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1117
1395
|
));
|
|
1118
1396
|
END IF;
|
|
1119
1397
|
IF event_batch IS NOT NULL THEN
|
|
1398
|
+
IF ${schema}.defer_runtime_event_projection() AND EXISTS (SELECT 1 FROM jsonb_array_elements(event_batch) entry WHERE entry->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND (NOT (entry ? 'lineIds') OR jsonb_typeof(entry->'lineIds') IS DISTINCT FROM 'array' OR EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(entry->'lineIds') = 'array' THEN entry->'lineIds' ELSE '[]'::jsonb END) id WHERE id = 'null'::jsonb))) THEN
|
|
1399
|
+
PERFORM pg_advisory_xact_lock(hashtext(${quoteLiteral(scope)}), hashtext(observation.run_id));
|
|
1400
|
+
END IF;
|
|
1120
1401
|
-- Normalize old positional writers once per batch under the run mutex.
|
|
1121
1402
|
-- Per-line lookups must remain digest-selective: an OR fallback for
|
|
1122
1403
|
-- unmarked rows otherwise scans the whole attempt for every new line.
|
|
1123
1404
|
IF EXISTS (SELECT 1 FROM jsonb_array_elements(event_batch) AS entry
|
|
1124
|
-
WHERE entry->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}) THEN
|
|
1405
|
+
WHERE entry->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND (NOT (entry ? 'lineIds') OR EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(entry->'lineIds') = 'array' THEN entry->'lineIds' ELSE '[]'::jsonb END) id WHERE id = 'null'::jsonb))) THEN
|
|
1125
1406
|
UPDATE runtime_observability.events AS event SET search_doc = event.search_doc ||
|
|
1126
1407
|
jsonb_build_object('logContentDigest', event.source_digest) ||
|
|
1127
1408
|
CASE WHEN NOT (event.search_doc ? 'logPositions')
|
|
@@ -1134,7 +1415,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1134
1415
|
ELSE '{}'::jsonb END
|
|
1135
1416
|
WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
|
|
1136
1417
|
AND event.run_id = observation.run_id AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
1137
|
-
AND event.source_digest IS NOT NULL AND NOT (event.search_doc ? 'logContentDigest')
|
|
1418
|
+
AND event.source_digest IS NOT NULL AND NOT (event.search_doc ? 'logContentDigest') AND NOT (event.search_doc ? 'entryId')
|
|
1138
1419
|
AND (event.search_doc->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$';
|
|
1139
1420
|
END IF;
|
|
1140
1421
|
FOR item, ordinal IN SELECT value, ordinality FROM jsonb_array_elements(event_batch) WITH ORDINALITY LOOP
|
|
@@ -1172,12 +1453,23 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1172
1453
|
OR jsonb_array_length(item->'lineContexts') <> jsonb_array_length(item->'lines')) THEN
|
|
1173
1454
|
RAISE EXCEPTION 'Search projection requires lineContexts aligned to log lines';
|
|
1174
1455
|
END IF;
|
|
1456
|
+
IF kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND item ? 'lineIds' THEN
|
|
1457
|
+
IF jsonb_typeof(item->'lineIds') IS DISTINCT FROM 'array' OR jsonb_array_length(item->'lineIds') <> jsonb_array_length(item->'lines') THEN
|
|
1458
|
+
RAISE EXCEPTION 'Origin log identities must align with lines';
|
|
1459
|
+
END IF;
|
|
1460
|
+
IF EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lineIds') id WHERE id <> 'null'::jsonb AND (jsonb_typeof(id) <> 'string' OR (id #>> '{}') !~ '^[A-Za-z0-9:_-]{1,128}$')) OR (SELECT count(id) FROM jsonb_array_elements(item->'lineIds') id WHERE id <> 'null'::jsonb) <> (SELECT count(DISTINCT id) FROM jsonb_array_elements(item->'lineIds') id WHERE id <> 'null'::jsonb) THEN RAISE EXCEPTION 'Origin log identities must be distinct bounded strings'; END IF;
|
|
1461
|
+
IF NOT EXISTS (SELECT 1 FROM jsonb_array_elements(item->'lineIds') id WHERE id = 'null'::jsonb) THEN
|
|
1462
|
+
fresh_event_ids := fresh_event_ids || ${schema}.project_origin_log_batch(observation, item, run_org, play, compact_logs, canonical_events, at_time, doc,
|
|
1463
|
+
compact_logs AND observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND coalesce((observation.payload_json->>'_runtimeSearchRunClosed')::boolean, run_status IN ('completed','failed','cancelled')));
|
|
1464
|
+
CONTINUE;
|
|
1465
|
+
END IF;
|
|
1466
|
+
END IF;
|
|
1175
1467
|
IF kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} THEN
|
|
1176
1468
|
FOR line, line_ordinal IN SELECT value, ordinality FROM jsonb_array_elements(item->'lines') WITH ORDINALITY LOOP
|
|
1177
1469
|
IF jsonb_typeof(line) IS DISTINCT FROM 'string' THEN
|
|
1178
1470
|
RAISE EXCEPTION 'Search projection requires log line strings';
|
|
1179
1471
|
END IF;
|
|
1180
|
-
IF (item->>'source' IS DISTINCT FROM 'coordinator')
|
|
1472
|
+
IF NULLIF(item->'lineIds'->((line_ordinal-1)::int), 'null'::jsonb) IS NULL AND (item->>'source' IS DISTINCT FROM 'coordinator')
|
|
1181
1473
|
AND NOT coalesce((item->>'channelOffset') ~ '^[0-9]{1,12}$', false) THEN
|
|
1182
1474
|
IF NOT recent_log_keys_loaded OR recent_log_attempt IS DISTINCT FROM item->>'producerAttempt' THEN
|
|
1183
1475
|
SELECT coalesce(jsonb_object_agg(recent.dedupe_key, true), '{}'::jsonb)
|
|
@@ -1187,6 +1479,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1187
1479
|
FROM runtime_observability.events AS event
|
|
1188
1480
|
WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
|
|
1189
1481
|
AND event.run_id = observation.run_id AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
1482
|
+
AND NOT (event.search_doc ? 'entryId')
|
|
1190
1483
|
AND event.log_seq IS NOT NULL AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
|
|
1191
1484
|
AND (item->>'producerAttempt' IS NULL
|
|
1192
1485
|
OR event.search_doc->>'producerAttempt' = item->>'producerAttempt')
|
|
@@ -1241,7 +1534,11 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1241
1534
|
digest := CASE WHEN line_context IS NULL
|
|
1242
1535
|
THEN encode(sha256(convert_to(line #>> '{}', 'UTF8')), 'hex')
|
|
1243
1536
|
ELSE encode(sha256(convert_to(jsonb_build_array(line #>> '{}', line_context)::text, 'UTF8')), 'hex') END;
|
|
1244
|
-
IF (item
|
|
1537
|
+
IF NULLIF(item->'lineIds'->((line_ordinal-1)::int), 'null'::jsonb) IS NOT NULL THEN
|
|
1538
|
+
IF jsonb_typeof(item->'lineIds'->((line_ordinal-1)::int)) <> 'string' OR (item->'lineIds'->>((line_ordinal-1)::int)) !~ '^[A-Za-z0-9:_-]{1,128}$' THEN RAISE EXCEPTION 'Origin log identity must be a bounded string'; END IF;
|
|
1539
|
+
eid := 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>((line_ordinal-1)::int));
|
|
1540
|
+
line_doc := line_doc || jsonb_build_object('entryId', item->'lineIds'->((line_ordinal-1)::int));
|
|
1541
|
+
ELSIF (item->>'channelOffset') ~ '^[0-9]{1,12}$'
|
|
1245
1542
|
AND (item->>'producerAttempt') ~ '^[1-9][0-9]{0,5}$' THEN
|
|
1246
1543
|
-- These transports retain different buffers. Their offsets
|
|
1247
1544
|
-- are immutable within a channel, never interchangeable.
|
|
@@ -1340,7 +1637,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1340
1637
|
AND event.run_id = observation.run_id AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
1341
1638
|
AND event.source_digest IS NOT NULL
|
|
1342
1639
|
AND event.search_doc ? 'producerAttempt'
|
|
1343
|
-
AND event.search_doc ? 'logContentDigest'
|
|
1640
|
+
AND event.search_doc ? 'logContentDigest' AND NOT (event.search_doc ? 'entryId')
|
|
1344
1641
|
AND event.search_doc->>'producerAttempt' = ((item->>'producerAttempt')::integer)::text
|
|
1345
1642
|
AND event.source_digest = digest
|
|
1346
1643
|
AND event.search_doc @> jsonb_build_object('logContentDigest', digest)
|
|
@@ -1361,7 +1658,8 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1361
1658
|
-- retry of an accepted position is harmless; a new position
|
|
1362
1659
|
-- after terminal closure must never recreate expired history.
|
|
1363
1660
|
IF compact_logs AND observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
1364
|
-
AND
|
|
1661
|
+
AND coalesce((observation.payload_json->>'_runtimeSearchRunClosed')::boolean,
|
|
1662
|
+
run_status IN ('completed', 'failed', 'cancelled'))
|
|
1365
1663
|
AND NOT EXISTS (SELECT 1 FROM runtime_observability.events
|
|
1366
1664
|
WHERE runtime_scope = ${quoteLiteral(scope)} AND org_id = run_org
|
|
1367
1665
|
AND event_id = eid) THEN
|
|
@@ -1375,7 +1673,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1375
1673
|
${quoteLiteral(scope)}, run_org, eid,
|
|
1376
1674
|
CASE WHEN canonical_events THEN nextval('runtime_observability.runtime_event_order_seq') END,
|
|
1377
1675
|
CASE WHEN canonical_events THEN jsonb_strip_nulls(
|
|
1378
|
-
(item - 'lines' - 'lineContexts') || jsonb_build_object(
|
|
1676
|
+
(item - 'lines' - 'lineContexts' - 'lineIds') || jsonb_build_object(
|
|
1379
1677
|
'type', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'runId', observation.run_id,
|
|
1380
1678
|
'occurredAt', floor(extract(epoch FROM at_time) * 1000)::bigint,
|
|
1381
1679
|
'lineOrdinal', line_ordinal,
|
|
@@ -1494,6 +1792,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1494
1792
|
AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
1495
1793
|
AND (observation.payload_json->>'producerAttempt' IS NULL
|
|
1496
1794
|
OR event.search_doc->>'producerAttempt' = observation.payload_json->>'producerAttempt')
|
|
1795
|
+
AND NOT (event.search_doc ? 'entryId')
|
|
1497
1796
|
AND event.log_seq IS NOT NULL AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
|
|
1498
1797
|
ORDER BY event.log_seq DESC
|
|
1499
1798
|
LIMIT 256
|
|
@@ -1503,7 +1802,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1503
1802
|
FROM runtime_observability.events AS event
|
|
1504
1803
|
WHERE event.runtime_scope = ${quoteLiteral(scope)}
|
|
1505
1804
|
AND event.org_id = run_org AND event.run_id = observation.run_id
|
|
1506
|
-
AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
|
|
1805
|
+
AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended} AND NOT (event.search_doc ? 'entryId') AND (event.log_text_digest IS NOT NULL OR event.log_body IS NOT NULL)
|
|
1507
1806
|
AND event.event_id = ANY(fresh_event_ids)
|
|
1508
1807
|
AND (observation.payload_json->>'producerAttempt' IS NULL
|
|
1509
1808
|
OR event.search_doc->>'producerAttempt' = observation.payload_json->>'producerAttempt')
|
|
@@ -1720,7 +2019,7 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1720
2019
|
LANGUAGE sql IMMUTABLE SET search_path = pg_catalog AS $function$
|
|
1721
2020
|
WITH envelope AS (
|
|
1722
2021
|
SELECT CASE WHEN jsonb_typeof(value) = 'object' THEN
|
|
1723
|
-
(value - 'logContexts') || CASE WHEN value ? 'logs'
|
|
2022
|
+
(value - 'logContexts' - 'logIds') || CASE WHEN value ? 'logs'
|
|
1724
2023
|
THEN jsonb_build_object('logs', '[]'::jsonb) ELSE '{}'::jsonb END
|
|
1725
2024
|
ELSE value END AS result
|
|
1726
2025
|
)
|
|
@@ -1730,50 +2029,140 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1730
2029
|
$function$
|
|
1731
2030
|
`);
|
|
1732
2031
|
await client.query(`
|
|
1733
|
-
CREATE OR REPLACE FUNCTION ${schema}.
|
|
2032
|
+
CREATE OR REPLACE FUNCTION ${schema}.compact_runtime_outbox_payload(observation ${schema}.outbox) RETURNS jsonb
|
|
1734
2033
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
1735
2034
|
DECLARE compact_logs boolean; canonical_events boolean;
|
|
1736
2035
|
BEGIN
|
|
1737
|
-
-- Project the original validated observation before replacing its
|
|
1738
|
-
-- transport body. Failure rolls back the outbox and canonical rows.
|
|
1739
|
-
PERFORM ${schema}.project_outbox_event(NEW);
|
|
1740
2036
|
SELECT run_log_store = 'planet_scale', run_event_store = 'planet_scale'
|
|
1741
2037
|
INTO compact_logs, canonical_events
|
|
1742
|
-
FROM ${schema}.runs WHERE run_id =
|
|
2038
|
+
FROM ${schema}.runs WHERE run_id = observation.run_id;
|
|
1743
2039
|
IF compact_logs OR canonical_events THEN
|
|
1744
|
-
|
|
1745
|
-
IF
|
|
1746
|
-
|
|
1747
|
-
${schema}.without_log_events(
|
|
2040
|
+
observation.payload_json := observation.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds';
|
|
2041
|
+
IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
|
|
2042
|
+
observation.payload_json := jsonb_set(observation.payload_json, '{events}',
|
|
2043
|
+
${schema}.without_log_events(observation.payload_json->'events'))
|
|
1748
2044
|
|| jsonb_build_object('logBodyStore', 'planet_scale');
|
|
1749
2045
|
END IF;
|
|
1750
|
-
IF
|
|
1751
|
-
|
|
1752
|
-
${schema}.without_log_events(
|
|
2046
|
+
IF observation.payload_json ? 'terminalProgressEvents' THEN
|
|
2047
|
+
observation.payload_json := jsonb_set(observation.payload_json, '{terminalProgressEvents}',
|
|
2048
|
+
${schema}.without_log_events(observation.payload_json->'terminalProgressEvents'));
|
|
1753
2049
|
END IF;
|
|
1754
|
-
IF
|
|
1755
|
-
|
|
1756
|
-
${schema}.compact_runner_log_envelope(
|
|
2050
|
+
IF observation.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runFailed} AND observation.payload_json ? 'result' THEN
|
|
2051
|
+
observation.payload_json := jsonb_set(observation.payload_json, '{result}',
|
|
2052
|
+
${schema}.compact_runner_log_envelope(observation.payload_json->'result'));
|
|
1757
2053
|
END IF;
|
|
1758
|
-
IF
|
|
1759
|
-
|
|
1760
|
-
${schema}.without_runtime_log_tail(
|
|
2054
|
+
IF observation.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCompleted} AND observation.payload_json ? 'output' THEN
|
|
2055
|
+
observation.payload_json := jsonb_set(observation.payload_json, '{output}',
|
|
2056
|
+
${schema}.without_runtime_log_tail(observation.payload_json->'output'));
|
|
1761
2057
|
END IF;
|
|
1762
2058
|
END IF;
|
|
1763
2059
|
IF canonical_events THEN
|
|
2060
|
+
IF observation.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
|
|
2061
|
+
observation.payload_json := jsonb_set(observation.payload_json, '{events}',
|
|
2062
|
+
${schema}.convex_projection_events(observation.payload_json->'events'));
|
|
2063
|
+
END IF;
|
|
2064
|
+
IF observation.payload_json ? 'terminalProgressEvents' THEN
|
|
2065
|
+
observation.payload_json := jsonb_set(observation.payload_json, '{terminalProgressEvents}',
|
|
2066
|
+
${schema}.convex_projection_events(observation.payload_json->'terminalProgressEvents'));
|
|
2067
|
+
END IF;
|
|
2068
|
+
END IF;
|
|
2069
|
+
RETURN observation.payload_json - '${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' - '_runtimeSearchRunClosed';
|
|
2070
|
+
END
|
|
2071
|
+
$function$
|
|
2072
|
+
`);
|
|
2073
|
+
await client.query(`
|
|
2074
|
+
CREATE OR REPLACE FUNCTION ${schema}.search_outbox_event() RETURNS trigger
|
|
2075
|
+
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
2076
|
+
DECLARE run_closed boolean; search_payload jsonb;
|
|
2077
|
+
BEGIN
|
|
2078
|
+
IF NEW.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb THEN RETURN NEW; END IF;
|
|
2079
|
+
IF ${schema}.defer_runtime_event_projection()
|
|
2080
|
+
AND NEW.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runCreated} THEN
|
|
1764
2081
|
IF NEW.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} THEN
|
|
1765
|
-
|
|
1766
|
-
|
|
2082
|
+
-- One cheap Run fence captures whether ingress preceded terminal
|
|
2083
|
+
-- closure. No indexed log lookup or search mutex runs under it.
|
|
2084
|
+
SELECT status IN ('completed', 'failed', 'cancelled') INTO run_closed
|
|
2085
|
+
FROM ${schema}.runs WHERE run_id = NEW.run_id FOR SHARE;
|
|
2086
|
+
NEW.created_at := GREATEST(NEW.created_at, clock_timestamp(),
|
|
2087
|
+
coalesce((SELECT max(created_at) + interval '1 microsecond'
|
|
2088
|
+
FROM ${schema}.outbox WHERE run_id = NEW.run_id), clock_timestamp()));
|
|
2089
|
+
ELSE
|
|
2090
|
+
SELECT status IN ('completed', 'failed', 'cancelled') INTO run_closed
|
|
2091
|
+
FROM ${schema}.runs WHERE run_id = NEW.run_id;
|
|
1767
2092
|
END IF;
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
2093
|
+
NEW.payload_json := NEW.payload_json || jsonb_build_object(
|
|
2094
|
+
'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}', true,
|
|
2095
|
+
'_runtimeSearchRunClosed', coalesce(run_closed, false));
|
|
2096
|
+
IF ${schema}.independent_runtime_event_projection() THEN
|
|
2097
|
+
IF NEW.event_type IN (${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.receiptEvent}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runnerLogs}) OR (NEW.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND jsonb_typeof(NEW.payload_json->'events') = 'array' AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(NEW.payload_json->'events') item WHERE item->>'type' IS DISTINCT FROM ${RUN_EVENT_KIND_SQL_LITERAL.logAppended})) THEN
|
|
2098
|
+
NEW.payload_json := NEW.payload_json || jsonb_build_object('${RUNTIME_EVENT_SEARCH_ONLY_FIELD}', true);
|
|
2099
|
+
ELSE
|
|
2100
|
+
search_payload := NEW.payload_json || jsonb_build_object('${RUNTIME_EVENT_SEARCH_ONLY_FIELD}', true, '_runtimeSearchEventId', NEW.event_id);
|
|
2101
|
+
INSERT INTO ${schema}.outbox(event_id, run_id, event_type, payload_json, created_at)
|
|
2102
|
+
VALUES ('search:' || NEW.event_id, NEW.run_id, NEW.event_type, search_payload, NEW.created_at)
|
|
2103
|
+
ON CONFLICT (event_id) DO NOTHING;
|
|
2104
|
+
NEW.payload_json := ${schema}.compact_runtime_outbox_payload(NEW);
|
|
2105
|
+
END IF;
|
|
1771
2106
|
END IF;
|
|
2107
|
+
RETURN NEW;
|
|
1772
2108
|
END IF;
|
|
2109
|
+
-- Legacy writers retain their atomic projection until explicit cutover.
|
|
2110
|
+
NEW.payload_json := NEW.payload_json - '${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' - '_runtimeSearchRunClosed';
|
|
2111
|
+
PERFORM ${schema}.project_outbox_event(NEW);
|
|
2112
|
+
NEW.payload_json := ${schema}.compact_runtime_outbox_payload(NEW);
|
|
1773
2113
|
RETURN NEW;
|
|
1774
2114
|
END
|
|
1775
2115
|
$function$
|
|
1776
2116
|
`);
|
|
2117
|
+
await client.query(`
|
|
2118
|
+
CREATE OR REPLACE FUNCTION ${schema}.guard_runtime_outbox_ack() RETURNS trigger
|
|
2119
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $function$
|
|
2120
|
+
BEGIN
|
|
2121
|
+
IF OLD.delivered_at IS NULL AND NEW.delivered_at IS NOT NULL
|
|
2122
|
+
AND OLD.payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb THEN
|
|
2123
|
+
RAISE EXCEPTION 'Deferred outbox input must be projected before acknowledgement' USING ERRCODE = '55000';
|
|
2124
|
+
END IF;
|
|
2125
|
+
RETURN NEW;
|
|
2126
|
+
END
|
|
2127
|
+
$function$
|
|
2128
|
+
`);
|
|
2129
|
+
await client.query(`DO $create$ BEGIN
|
|
2130
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = '${schema}.outbox'::regclass
|
|
2131
|
+
AND tgname = 'runtime_outbox_projection_ack_v1') THEN
|
|
2132
|
+
CREATE TRIGGER runtime_outbox_projection_ack_v1 BEFORE UPDATE OF delivered_at ON ${schema}.outbox
|
|
2133
|
+
FOR EACH ROW EXECUTE FUNCTION ${schema}.guard_runtime_outbox_ack();
|
|
2134
|
+
END IF;
|
|
2135
|
+
END $create$`);
|
|
2136
|
+
await client.query(`
|
|
2137
|
+
CREATE OR REPLACE FUNCTION ${schema}.materialize_runtime_outbox_event(eid text, owner text) RETURNS jsonb
|
|
2138
|
+
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
2139
|
+
DECLARE observation ${schema}.outbox; projection_observation ${schema}.outbox;
|
|
2140
|
+
BEGIN
|
|
2141
|
+
SELECT * INTO observation FROM ${schema}.outbox
|
|
2142
|
+
WHERE event_id = eid AND lease_owner = owner AND lease_expires_at > clock_timestamp()
|
|
2143
|
+
AND delivered_at IS NULL AND failed_at IS NULL;
|
|
2144
|
+
IF NOT FOUND THEN
|
|
2145
|
+
RAISE EXCEPTION 'Deferred event projection lease is not current' USING ERRCODE = '40001';
|
|
2146
|
+
END IF;
|
|
2147
|
+
IF observation.payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' = 'true'::jsonb THEN
|
|
2148
|
+
projection_observation := observation;
|
|
2149
|
+
projection_observation.event_id := coalesce(observation.payload_json->>'_runtimeSearchEventId', observation.event_id);
|
|
2150
|
+
PERFORM ${schema}.project_outbox_event(projection_observation);
|
|
2151
|
+
observation.payload_json := ${schema}.compact_runtime_outbox_payload(observation);
|
|
2152
|
+
-- Project against immutable input without locking the outbox row.
|
|
2153
|
+
-- Cancellation may transfer its lease while indexing is in flight.
|
|
2154
|
+
-- This final fence rolls the journal back if ownership changed.
|
|
2155
|
+
UPDATE ${schema}.outbox SET payload_json = observation.payload_json
|
|
2156
|
+
WHERE event_id = eid AND lease_owner = owner AND lease_expires_at > clock_timestamp()
|
|
2157
|
+
AND delivered_at IS NULL AND failed_at IS NULL;
|
|
2158
|
+
IF NOT FOUND THEN
|
|
2159
|
+
RAISE EXCEPTION 'Deferred event projection lease is not current' USING ERRCODE = '40001';
|
|
2160
|
+
END IF;
|
|
2161
|
+
END IF;
|
|
2162
|
+
RETURN observation.payload_json;
|
|
2163
|
+
END
|
|
2164
|
+
$function$
|
|
2165
|
+
`);
|
|
1777
2166
|
await client.query(`
|
|
1778
2167
|
CREATE OR REPLACE FUNCTION ${schema}.accept_runner_log_payload() RETURNS trigger
|
|
1779
2168
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $function$
|
|
@@ -1803,7 +2192,16 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1803
2192
|
THEN NEW.runner_log_payload_json->'terminalProgressEvents'
|
|
1804
2193
|
ELSE '[]'::jsonb END) WITH ORDINALITY AS progress(event, position)
|
|
1805
2194
|
));
|
|
1806
|
-
|
|
2195
|
+
IF ${schema}.defer_runtime_event_projection() THEN
|
|
2196
|
+
INSERT INTO ${schema}.outbox (event_id, run_id, event_type, payload_json, created_at)
|
|
2197
|
+
VALUES (observation.event_id, observation.run_id, observation.event_type,
|
|
2198
|
+
observation.payload_json,
|
|
2199
|
+
GREATEST(clock_timestamp(), coalesce((SELECT max(created_at) + interval '1 microsecond'
|
|
2200
|
+
FROM ${schema}.outbox WHERE run_id = NEW.run_id), clock_timestamp())))
|
|
2201
|
+
ON CONFLICT (event_id) DO NOTHING;
|
|
2202
|
+
ELSE
|
|
2203
|
+
PERFORM ${schema}.project_outbox_event(observation);
|
|
2204
|
+
END IF;
|
|
1807
2205
|
NEW.runner_result_json := ${schema}.compact_runner_log_envelope(NEW.runner_result_json);
|
|
1808
2206
|
IF jsonb_array_length(observation.payload_json->'recoveredRuntimeDiagnostics') > 0
|
|
1809
2207
|
AND (NOT (NEW.runner_result_json ? 'runtimeDiagnostics')
|
|
@@ -1888,6 +2286,12 @@ export async function attachRuntimeEventSearchTriggers(
|
|
|
1888
2286
|
END IF;
|
|
1889
2287
|
END $create$`);
|
|
1890
2288
|
await client.query(`REVOKE ALL ON FUNCTION ${schema}.project_outbox_event(${schema}.outbox),
|
|
2289
|
+
${schema}.project_origin_log_batch(${schema}.outbox, jsonb, text, text, boolean, boolean, timestamptz, jsonb, boolean),
|
|
2290
|
+
${schema}.independent_runtime_event_projection(),
|
|
2291
|
+
${schema}.pin_runtime_log_identity(),
|
|
2292
|
+
${schema}.materialize_runtime_outbox_event(text, text),
|
|
2293
|
+
${schema}.guard_runtime_outbox_ack(),
|
|
2294
|
+
${schema}.compact_runtime_outbox_payload(${schema}.outbox),
|
|
1891
2295
|
${schema}.accept_runner_log_payload(), ${schema}.search_outbox_event(),
|
|
1892
2296
|
${schema}.compact_run_log_result(), ${schema}.without_log_events(jsonb),
|
|
1893
2297
|
${schema}.convex_projection_event(jsonb), ${schema}.convex_projection_events(jsonb),
|
|
@@ -1905,7 +2309,11 @@ export async function hasRuntimeEventSearchShape(
|
|
|
1905
2309
|
): Promise<boolean> {
|
|
1906
2310
|
const schema = quoteIdent(requiredScope(runtimeScope));
|
|
1907
2311
|
const result = await client.query<{ ready: boolean }>(
|
|
1908
|
-
`SELECT
|
|
2312
|
+
`SELECT to_regprocedure('${schema}.materialize_runtime_outbox_event(text,text)') IS NOT NULL
|
|
2313
|
+
AND EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = to_regclass('${schema}.outbox')
|
|
2314
|
+
AND tgname = 'runtime_outbox_projection_ack_v1' AND tgenabled <> 'D')
|
|
2315
|
+
AND to_regprocedure('${schema}.defer_runtime_event_projection()') IS NOT NULL
|
|
2316
|
+
AND to_regclass('runtime_observability.events') IS NOT NULL
|
|
1909
2317
|
AND to_regclass('runtime_observability.run_log_cursors') IS NOT NULL
|
|
1910
2318
|
AND EXISTS (SELECT 1 FROM pg_attribute
|
|
1911
2319
|
WHERE attrelid = to_regclass('runtime_observability.events')
|
|
@@ -2345,6 +2753,7 @@ export async function pruneRuntimeEvents(
|
|
|
2345
2753
|
search_doc = jsonb_strip_nulls(jsonb_build_object(
|
|
2346
2754
|
'kind', ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}, 'source', event.search_doc->'source',
|
|
2347
2755
|
'producerAttempt', event.search_doc->'producerAttempt',
|
|
2756
|
+
'entryId', event.search_doc->'entryId',
|
|
2348
2757
|
'logPositions', event.search_doc->'logPositions',
|
|
2349
2758
|
'logContentDigest', event.search_doc->'logContentDigest'))
|
|
2350
2759
|
FROM candidates WHERE event.ctid = candidates.ctid AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|