@opengeni/db 0.7.3 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-B22X3IEZ.js → chunk-4LG5NBTC.js} +512 -32
- package/dist/chunk-4LG5NBTC.js.map +1 -0
- package/dist/{chunk-YFQ7SGE4.js → chunk-BMFDXFPA.js} +23 -1
- package/dist/chunk-BMFDXFPA.js.map +1 -0
- package/dist/chunk-KW526IJA.js +127 -0
- package/dist/chunk-KW526IJA.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +6068 -2090
- package/dist/index.js.map +1 -1
- package/dist/migrate.d.ts +29 -4
- package/dist/migrate.js +3 -1
- package/dist/provision-roles.d.ts +721 -46
- package/dist/provision-roles.js +1 -1
- package/dist/{schema-BN5mB9xZ.d.ts → schema-CdPGTHlD.d.ts} +2118 -119
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +17 -1
- package/drizzle/0064_rotation_strategy_sharded_backfill.sql +15 -0
- package/drizzle/0065_session_attempt_quiescence.sql +26 -0
- package/drizzle/0066_session_interruption_attempt_lookup.sql +4 -0
- package/drizzle/0067_session_event_payload_bounds.sql +209 -0
- package/drizzle/0068_workspace_control_event_bounds.sql +134 -0
- package/drizzle/0069_session_event_history_backfill.sql +32 -0
- package/drizzle/0070_session_event_type_sequence_lookup.sql +4 -0
- package/drizzle/0071_session_event_monitoring_tail.sql +10 -0
- package/drizzle/0072_sessions_workspace_created_id_idx.sql +4 -0
- package/drizzle/0073_sessions_workspace_updated_id_idx.sql +4 -0
- package/drizzle/0074_session_activity_revisions.sql +72 -0
- package/drizzle/0075_sessions_workspace_activity_revision_idx.sql +4 -0
- package/drizzle/0076_session_workflow_wake_acl.sql +13 -0
- package/drizzle/0077_session_attempt_latest_lookup.sql +4 -0
- package/drizzle/0094_quarantine_credential_bearing_catalog_urls.sql +51 -0
- package/drizzle/0095_github_existing_installations.sql +84 -0
- package/drizzle/0096_session_turn_initiators.sql +97 -0
- package/drizzle/0097_host_export_outbox.sql +1220 -0
- package/drizzle/0098_usage_events_workspace_session_idx.sql +4 -0
- package/drizzle/0099_session_human_input_attempt_owner_index.sql +6 -0
- package/drizzle/0100_session_human_input_requests.sql +89 -0
- package/drizzle/0101_session_mcp_connection_refs.sql +30 -0
- package/drizzle/0102_session_command_receipt_service_actor.sql +16 -0
- package/drizzle/0103_host_export_root_session.sql +166 -0
- package/drizzle/0104_host_export_root_session_backfill.sql +27 -0
- package/drizzle/0105_session_turn_instructions.sql +9 -0
- package/package.json +4 -4
- package/src/connection-token-resolver.ts +287 -1
- package/src/event-payload-sanitizer.ts +57 -19
- package/src/index.ts +5377 -1125
- package/src/memory-domain.ts +1 -1
- package/src/migrate.ts +86 -23
- package/src/persistence-errors.ts +252 -0
- package/src/provision-roles.ts +42 -0
- package/src/schema.ts +552 -34
- package/src/session-control.ts +518 -38
- package/src/session-queue-commands.ts +308 -57
- package/src/session-tool-call-settlement.ts +58 -7
- package/src/turn-initiator.ts +155 -0
- package/dist/chunk-7LDU7F5P.js +0 -80
- package/dist/chunk-7LDU7F5P.js.map +0 -1
- package/dist/chunk-B22X3IEZ.js.map +0 -1
- package/dist/chunk-YFQ7SGE4.js.map +0 -1
|
@@ -0,0 +1,1220 @@
|
|
|
1
|
+
-- deployment-mode: rolling
|
|
2
|
+
-- Generic durable host export: source-atomic bounded snapshots, post-commit
|
|
3
|
+
-- cursors, and named at-least-once consumer checkpoints. Standalone remains
|
|
4
|
+
-- disabled until a host registers a consumer.
|
|
5
|
+
|
|
6
|
+
SET LOCAL lock_timeout = '5s';
|
|
7
|
+
SET LOCAL statement_timeout = '10min';
|
|
8
|
+
|
|
9
|
+
ALTER TABLE "usage_events"
|
|
10
|
+
ADD COLUMN IF NOT EXISTS "session_id" uuid,
|
|
11
|
+
ADD COLUMN IF NOT EXISTS "turn_id" uuid,
|
|
12
|
+
ADD COLUMN IF NOT EXISTS "turn_attempt_id" uuid,
|
|
13
|
+
ADD COLUMN IF NOT EXISTS "initiator_kind" text,
|
|
14
|
+
ADD COLUMN IF NOT EXISTS "initiator_subject_id" text,
|
|
15
|
+
ADD COLUMN IF NOT EXISTS "initiator_context" jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
16
|
+
ADD COLUMN IF NOT EXISTS "origin" text;
|
|
17
|
+
|
|
18
|
+
DO $$
|
|
19
|
+
BEGIN
|
|
20
|
+
IF NOT EXISTS (
|
|
21
|
+
SELECT 1 FROM pg_constraint
|
|
22
|
+
WHERE conname = 'usage_events_context_hierarchy_check'
|
|
23
|
+
AND conrelid = 'usage_events'::regclass
|
|
24
|
+
) THEN
|
|
25
|
+
ALTER TABLE "usage_events" ADD CONSTRAINT "usage_events_context_hierarchy_check"
|
|
26
|
+
CHECK (("turn_id" IS NULL OR "session_id" IS NOT NULL)
|
|
27
|
+
AND ("turn_attempt_id" IS NULL OR "turn_id" IS NOT NULL)) NOT VALID;
|
|
28
|
+
END IF;
|
|
29
|
+
IF NOT EXISTS (
|
|
30
|
+
SELECT 1 FROM pg_constraint
|
|
31
|
+
WHERE conname = 'usage_events_initiator_check'
|
|
32
|
+
AND conrelid = 'usage_events'::regclass
|
|
33
|
+
) THEN
|
|
34
|
+
ALTER TABLE "usage_events" ADD CONSTRAINT "usage_events_initiator_check"
|
|
35
|
+
CHECK (("initiator_kind" IS NULL AND "initiator_subject_id" IS NULL)
|
|
36
|
+
OR ("initiator_kind" IN ('subject', 'service')
|
|
37
|
+
AND "initiator_subject_id" IS NOT NULL
|
|
38
|
+
AND octet_length("initiator_subject_id") BETWEEN 1 AND 1024)) NOT VALID;
|
|
39
|
+
END IF;
|
|
40
|
+
IF NOT EXISTS (
|
|
41
|
+
SELECT 1 FROM pg_constraint
|
|
42
|
+
WHERE conname = 'usage_events_initiator_context_bytes_check'
|
|
43
|
+
AND conrelid = 'usage_events'::regclass
|
|
44
|
+
) THEN
|
|
45
|
+
ALTER TABLE "usage_events" ADD CONSTRAINT "usage_events_initiator_context_bytes_check"
|
|
46
|
+
CHECK (octet_length("initiator_context"::text) <= 4096) NOT VALID;
|
|
47
|
+
END IF;
|
|
48
|
+
IF NOT EXISTS (
|
|
49
|
+
SELECT 1 FROM pg_constraint
|
|
50
|
+
WHERE conname = 'usage_events_origin_check'
|
|
51
|
+
AND conrelid = 'usage_events'::regclass
|
|
52
|
+
) THEN
|
|
53
|
+
ALTER TABLE "usage_events" ADD CONSTRAINT "usage_events_origin_check"
|
|
54
|
+
CHECK ("origin" IS NULL OR "origin" IN (
|
|
55
|
+
'user', 'scheduled_task', 'api', 'goal', 'system', 'compaction'
|
|
56
|
+
)) NOT VALID;
|
|
57
|
+
END IF;
|
|
58
|
+
END $$;
|
|
59
|
+
|
|
60
|
+
ALTER TABLE "usage_events" VALIDATE CONSTRAINT "usage_events_context_hierarchy_check";
|
|
61
|
+
ALTER TABLE "usage_events" VALIDATE CONSTRAINT "usage_events_initiator_check";
|
|
62
|
+
ALTER TABLE "usage_events" VALIDATE CONSTRAINT "usage_events_initiator_context_bytes_check";
|
|
63
|
+
ALTER TABLE "usage_events" VALIDATE CONSTRAINT "usage_events_origin_check";
|
|
64
|
+
|
|
65
|
+
DO $migration$
|
|
66
|
+
DECLARE target_schema text := current_schema();
|
|
67
|
+
BEGIN
|
|
68
|
+
EXECUTE format($create$
|
|
69
|
+
CREATE OR REPLACE FUNCTION opengeni_private.validate_usage_event_execution_context()
|
|
70
|
+
RETURNS trigger
|
|
71
|
+
LANGUAGE plpgsql
|
|
72
|
+
SET search_path = pg_catalog
|
|
73
|
+
AS $function$
|
|
74
|
+
BEGIN
|
|
75
|
+
-- Usage is an immutable billing/audit fact. Validate newly supplied
|
|
76
|
+
-- execution identity under row locks, but keep those UUIDs as soft
|
|
77
|
+
-- references after the source session or turn is deleted. An unchanged
|
|
78
|
+
-- idempotent retry after deletion must also remain valid.
|
|
79
|
+
IF TG_OP = 'INSERT'
|
|
80
|
+
OR NEW.workspace_id IS DISTINCT FROM OLD.workspace_id
|
|
81
|
+
OR NEW.session_id IS DISTINCT FROM OLD.session_id
|
|
82
|
+
OR NEW.turn_id IS DISTINCT FROM OLD.turn_id
|
|
83
|
+
OR NEW.turn_attempt_id IS DISTINCT FROM OLD.turn_attempt_id THEN
|
|
84
|
+
IF NEW.session_id IS NOT NULL THEN
|
|
85
|
+
PERFORM 1 FROM %1$I.sessions s
|
|
86
|
+
WHERE s.workspace_id = NEW.workspace_id AND s.id = NEW.session_id
|
|
87
|
+
FOR KEY SHARE;
|
|
88
|
+
IF NOT FOUND THEN
|
|
89
|
+
RAISE EXCEPTION 'usage event session does not exist in its workspace'
|
|
90
|
+
USING ERRCODE = '23514';
|
|
91
|
+
END IF;
|
|
92
|
+
END IF;
|
|
93
|
+
IF NEW.turn_id IS NOT NULL THEN
|
|
94
|
+
PERFORM 1 FROM %1$I.session_turns t
|
|
95
|
+
WHERE t.workspace_id = NEW.workspace_id
|
|
96
|
+
AND t.session_id = NEW.session_id
|
|
97
|
+
AND t.id = NEW.turn_id
|
|
98
|
+
FOR KEY SHARE;
|
|
99
|
+
IF NOT FOUND THEN
|
|
100
|
+
RAISE EXCEPTION 'usage event turn does not belong to its session'
|
|
101
|
+
USING ERRCODE = '23514';
|
|
102
|
+
END IF;
|
|
103
|
+
END IF;
|
|
104
|
+
IF NEW.turn_attempt_id IS NOT NULL THEN
|
|
105
|
+
PERFORM 1 FROM %1$I.session_turn_attempts a
|
|
106
|
+
WHERE a.workspace_id = NEW.workspace_id
|
|
107
|
+
AND a.session_id = NEW.session_id
|
|
108
|
+
AND a.turn_id = NEW.turn_id
|
|
109
|
+
AND a.id = NEW.turn_attempt_id
|
|
110
|
+
FOR KEY SHARE;
|
|
111
|
+
IF NOT FOUND THEN
|
|
112
|
+
RAISE EXCEPTION 'usage event attempt does not belong to its turn'
|
|
113
|
+
USING ERRCODE = '23514';
|
|
114
|
+
END IF;
|
|
115
|
+
END IF;
|
|
116
|
+
END IF;
|
|
117
|
+
RETURN NEW;
|
|
118
|
+
END $function$;
|
|
119
|
+
$create$, target_schema);
|
|
120
|
+
END $migration$;
|
|
121
|
+
|
|
122
|
+
DROP TRIGGER IF EXISTS usage_events_execution_context_guard ON "usage_events";
|
|
123
|
+
CREATE TRIGGER usage_events_execution_context_guard
|
|
124
|
+
BEFORE INSERT OR UPDATE OF "workspace_id", "session_id", "turn_id", "turn_attempt_id"
|
|
125
|
+
ON "usage_events"
|
|
126
|
+
FOR EACH ROW EXECUTE FUNCTION opengeni_private.validate_usage_event_execution_context();
|
|
127
|
+
|
|
128
|
+
CREATE SCHEMA IF NOT EXISTS opengeni_host_export;
|
|
129
|
+
REVOKE ALL ON SCHEMA opengeni_host_export FROM PUBLIC;
|
|
130
|
+
|
|
131
|
+
CREATE TABLE IF NOT EXISTS "host_export_config" (
|
|
132
|
+
"id" integer PRIMARY KEY DEFAULT 1,
|
|
133
|
+
"session_events_enabled" boolean NOT NULL DEFAULT false,
|
|
134
|
+
"usage_events_enabled" boolean NOT NULL DEFAULT false,
|
|
135
|
+
"updated_at" timestamptz NOT NULL DEFAULT now(),
|
|
136
|
+
CONSTRAINT "host_export_config_singleton_check" CHECK ("id" = 1)
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
INSERT INTO "host_export_config" ("id") VALUES (1)
|
|
140
|
+
ON CONFLICT ("id") DO NOTHING;
|
|
141
|
+
|
|
142
|
+
CREATE TABLE IF NOT EXISTS "host_export_cursor_state" (
|
|
143
|
+
"export_kind" text PRIMARY KEY,
|
|
144
|
+
"next_cursor" bigint NOT NULL DEFAULT 1,
|
|
145
|
+
"pruned_through" bigint NOT NULL DEFAULT 0,
|
|
146
|
+
"updated_at" timestamptz NOT NULL DEFAULT now(),
|
|
147
|
+
CONSTRAINT "host_export_cursor_state_kind_check"
|
|
148
|
+
CHECK ("export_kind" IN ('session_event', 'usage_event')),
|
|
149
|
+
CONSTRAINT "host_export_cursor_state_next_check"
|
|
150
|
+
CHECK ("next_cursor" > 0 AND "pruned_through" >= 0
|
|
151
|
+
AND "pruned_through" < "next_cursor")
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
ALTER TABLE "host_export_cursor_state"
|
|
155
|
+
ADD COLUMN IF NOT EXISTS "pruned_through" bigint NOT NULL DEFAULT 0;
|
|
156
|
+
|
|
157
|
+
INSERT INTO "host_export_cursor_state" ("export_kind")
|
|
158
|
+
VALUES ('session_event'), ('usage_event')
|
|
159
|
+
ON CONFLICT ("export_kind") DO NOTHING;
|
|
160
|
+
|
|
161
|
+
CREATE TABLE IF NOT EXISTS "host_export_outbox" (
|
|
162
|
+
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
163
|
+
"export_kind" text NOT NULL,
|
|
164
|
+
"export_cursor" bigint,
|
|
165
|
+
"source_id" uuid NOT NULL,
|
|
166
|
+
"account_id" uuid NOT NULL,
|
|
167
|
+
"workspace_id" uuid NOT NULL,
|
|
168
|
+
"session_id" uuid,
|
|
169
|
+
"turn_id" uuid,
|
|
170
|
+
"turn_generation" integer,
|
|
171
|
+
"turn_attempt_id" uuid,
|
|
172
|
+
"session_sequence" integer,
|
|
173
|
+
"client_event_id" text,
|
|
174
|
+
"turn_association" text,
|
|
175
|
+
"duplicate_of_event_id" uuid,
|
|
176
|
+
"duplicate_reason" text,
|
|
177
|
+
"event_type" text NOT NULL,
|
|
178
|
+
"idempotency_key" text NOT NULL,
|
|
179
|
+
"initiator" jsonb,
|
|
180
|
+
"initiator_context" jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
181
|
+
"origin" text,
|
|
182
|
+
"payload" jsonb NOT NULL,
|
|
183
|
+
"envelope_bytes" integer NOT NULL,
|
|
184
|
+
"occurred_at" timestamptz NOT NULL,
|
|
185
|
+
"source_recorded_at" timestamptz NOT NULL,
|
|
186
|
+
"enqueued_at" timestamptz NOT NULL,
|
|
187
|
+
CONSTRAINT "host_export_outbox_kind_check"
|
|
188
|
+
CHECK ("export_kind" IN ('session_event', 'usage_event')),
|
|
189
|
+
CONSTRAINT "host_export_outbox_cursor_check"
|
|
190
|
+
CHECK ("export_cursor" IS NULL OR "export_cursor" > 0),
|
|
191
|
+
CONSTRAINT "host_export_outbox_sequence_check"
|
|
192
|
+
CHECK ("session_sequence" IS NULL OR "session_sequence" > 0),
|
|
193
|
+
CONSTRAINT "host_export_outbox_envelope_bytes_check"
|
|
194
|
+
CHECK ("envelope_bytes" > 0 AND "envelope_bytes" <= 98304),
|
|
195
|
+
CONSTRAINT "host_export_outbox_payload_bytes_check"
|
|
196
|
+
CHECK (pg_column_size("payload") <= 73728),
|
|
197
|
+
CONSTRAINT "host_export_outbox_initiator_check"
|
|
198
|
+
CHECK (
|
|
199
|
+
"initiator" IS NULL OR (
|
|
200
|
+
jsonb_typeof("initiator") = 'object'
|
|
201
|
+
AND "initiator" ->> 'kind' IN ('subject', 'service')
|
|
202
|
+
AND length("initiator" ->> 'subjectId') > 0
|
|
203
|
+
)
|
|
204
|
+
),
|
|
205
|
+
CONSTRAINT "host_export_outbox_context_bytes_check"
|
|
206
|
+
CHECK (octet_length("initiator_context"::text) <= 4096),
|
|
207
|
+
CONSTRAINT "host_export_outbox_origin_check"
|
|
208
|
+
CHECK ("origin" IS NULL OR "origin" IN (
|
|
209
|
+
'user', 'scheduled_task', 'api', 'goal', 'system', 'compaction'
|
|
210
|
+
))
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
CREATE UNIQUE INDEX IF NOT EXISTS "host_export_outbox_source_uq"
|
|
214
|
+
ON "host_export_outbox" ("export_kind", "source_id");
|
|
215
|
+
CREATE UNIQUE INDEX IF NOT EXISTS "host_export_outbox_cursor_uq"
|
|
216
|
+
ON "host_export_outbox" ("export_kind", "export_cursor")
|
|
217
|
+
WHERE "export_cursor" IS NOT NULL;
|
|
218
|
+
CREATE INDEX IF NOT EXISTS "host_export_outbox_unassigned_idx"
|
|
219
|
+
ON "host_export_outbox" ("export_kind", "enqueued_at", "id")
|
|
220
|
+
WHERE "export_cursor" IS NULL;
|
|
221
|
+
CREATE INDEX IF NOT EXISTS "host_export_outbox_unassigned_session_idx"
|
|
222
|
+
ON "host_export_outbox" ("export_kind", "session_id", "session_sequence")
|
|
223
|
+
WHERE "export_cursor" IS NULL AND "session_id" IS NOT NULL;
|
|
224
|
+
CREATE INDEX IF NOT EXISTS "host_export_outbox_cursor_read_idx"
|
|
225
|
+
ON "host_export_outbox" ("export_kind", "export_cursor")
|
|
226
|
+
INCLUDE ("envelope_bytes") WHERE "export_cursor" IS NOT NULL;
|
|
227
|
+
|
|
228
|
+
CREATE TABLE IF NOT EXISTS "host_export_consumers" (
|
|
229
|
+
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
230
|
+
"consumer_id" text NOT NULL,
|
|
231
|
+
"export_kind" text NOT NULL,
|
|
232
|
+
"checkpoint" bigint NOT NULL DEFAULT 0,
|
|
233
|
+
"enabled" boolean NOT NULL DEFAULT true,
|
|
234
|
+
"lease_token" uuid,
|
|
235
|
+
"lease_holder_id" text,
|
|
236
|
+
"lease_expires_at" timestamptz,
|
|
237
|
+
"lease_from" bigint,
|
|
238
|
+
"lease_through" bigint,
|
|
239
|
+
"consecutive_failures" integer NOT NULL DEFAULT 0,
|
|
240
|
+
"next_attempt_at" timestamptz NOT NULL DEFAULT now(),
|
|
241
|
+
"last_error" text,
|
|
242
|
+
"last_error_at" timestamptz,
|
|
243
|
+
"blocked_at" timestamptz,
|
|
244
|
+
"created_at" timestamptz NOT NULL DEFAULT now(),
|
|
245
|
+
"updated_at" timestamptz NOT NULL DEFAULT now(),
|
|
246
|
+
CONSTRAINT "host_export_consumers_kind_check"
|
|
247
|
+
CHECK ("export_kind" IN ('session_event', 'usage_event')),
|
|
248
|
+
CONSTRAINT "host_export_consumers_checkpoint_check" CHECK ("checkpoint" >= 0),
|
|
249
|
+
CONSTRAINT "host_export_consumers_id_check"
|
|
250
|
+
CHECK (
|
|
251
|
+
length("consumer_id") BETWEEN 1 AND 128
|
|
252
|
+
AND "consumer_id" ~ '^[A-Za-z0-9][A-Za-z0-9._:-]*$'
|
|
253
|
+
),
|
|
254
|
+
CONSTRAINT "host_export_consumers_lease_check"
|
|
255
|
+
CHECK (
|
|
256
|
+
("lease_token" IS NULL AND "lease_holder_id" IS NULL
|
|
257
|
+
AND "lease_expires_at" IS NULL AND "lease_from" IS NULL
|
|
258
|
+
AND "lease_through" IS NULL)
|
|
259
|
+
OR
|
|
260
|
+
("lease_token" IS NOT NULL AND "lease_holder_id" IS NOT NULL
|
|
261
|
+
AND "lease_expires_at" IS NOT NULL AND "lease_from" IS NOT NULL
|
|
262
|
+
AND "lease_through" IS NOT NULL AND "lease_through" > "lease_from")
|
|
263
|
+
)
|
|
264
|
+
);
|
|
265
|
+
|
|
266
|
+
CREATE UNIQUE INDEX IF NOT EXISTS "host_export_consumers_kind_id_uq"
|
|
267
|
+
ON "host_export_consumers" ("export_kind", "consumer_id");
|
|
268
|
+
CREATE INDEX IF NOT EXISTS "host_export_consumers_due_idx"
|
|
269
|
+
ON "host_export_consumers" ("enabled", "blocked_at", "next_attempt_at");
|
|
270
|
+
|
|
271
|
+
CREATE TABLE IF NOT EXISTS "host_export_dead_letters" (
|
|
272
|
+
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
273
|
+
"consumer_id" text NOT NULL,
|
|
274
|
+
"export_kind" text NOT NULL,
|
|
275
|
+
"export_cursor" bigint NOT NULL,
|
|
276
|
+
"source_id" uuid NOT NULL,
|
|
277
|
+
"reason" text NOT NULL,
|
|
278
|
+
"envelope" jsonb NOT NULL,
|
|
279
|
+
"created_at" timestamptz NOT NULL DEFAULT now(),
|
|
280
|
+
CONSTRAINT "host_export_dead_letters_kind_check"
|
|
281
|
+
CHECK ("export_kind" IN ('session_event', 'usage_event')),
|
|
282
|
+
CONSTRAINT "host_export_dead_letters_reason_check"
|
|
283
|
+
CHECK (length("reason") BETWEEN 1 AND 500),
|
|
284
|
+
CONSTRAINT "host_export_dead_letters_envelope_bytes_check"
|
|
285
|
+
CHECK (pg_column_size("envelope") <= 114688)
|
|
286
|
+
);
|
|
287
|
+
|
|
288
|
+
CREATE UNIQUE INDEX IF NOT EXISTS "host_export_dead_letters_consumer_cursor_uq"
|
|
289
|
+
ON "host_export_dead_letters" ("export_kind", "consumer_id", "export_cursor");
|
|
290
|
+
|
|
291
|
+
ALTER TABLE "host_export_config" ENABLE ROW LEVEL SECURITY;
|
|
292
|
+
ALTER TABLE "host_export_config" FORCE ROW LEVEL SECURITY;
|
|
293
|
+
ALTER TABLE "host_export_cursor_state" ENABLE ROW LEVEL SECURITY;
|
|
294
|
+
ALTER TABLE "host_export_cursor_state" FORCE ROW LEVEL SECURITY;
|
|
295
|
+
ALTER TABLE "host_export_outbox" ENABLE ROW LEVEL SECURITY;
|
|
296
|
+
ALTER TABLE "host_export_outbox" FORCE ROW LEVEL SECURITY;
|
|
297
|
+
ALTER TABLE "host_export_consumers" ENABLE ROW LEVEL SECURITY;
|
|
298
|
+
ALTER TABLE "host_export_consumers" FORCE ROW LEVEL SECURITY;
|
|
299
|
+
ALTER TABLE "host_export_dead_letters" ENABLE ROW LEVEL SECURITY;
|
|
300
|
+
ALTER TABLE "host_export_dead_letters" FORCE ROW LEVEL SECURITY;
|
|
301
|
+
|
|
302
|
+
DROP POLICY IF EXISTS host_export_system_only ON "host_export_config";
|
|
303
|
+
CREATE POLICY host_export_system_only ON "host_export_config" USING (false) WITH CHECK (false);
|
|
304
|
+
DROP POLICY IF EXISTS host_export_system_only ON "host_export_cursor_state";
|
|
305
|
+
CREATE POLICY host_export_system_only ON "host_export_cursor_state" USING (false) WITH CHECK (false);
|
|
306
|
+
DROP POLICY IF EXISTS host_export_system_only ON "host_export_outbox";
|
|
307
|
+
CREATE POLICY host_export_system_only ON "host_export_outbox" USING (false) WITH CHECK (false);
|
|
308
|
+
DROP POLICY IF EXISTS host_export_system_only ON "host_export_consumers";
|
|
309
|
+
CREATE POLICY host_export_system_only ON "host_export_consumers" USING (false) WITH CHECK (false);
|
|
310
|
+
DROP POLICY IF EXISTS host_export_system_only ON "host_export_dead_letters";
|
|
311
|
+
CREATE POLICY host_export_system_only ON "host_export_dead_letters" USING (false) WITH CHECK (false);
|
|
312
|
+
|
|
313
|
+
-- FORCE RLS still applies to the table owner. Permit only the exact migration
|
|
314
|
+
-- owner (also the SECURITY DEFINER owner below); the runtime app role remains
|
|
315
|
+
-- denied even if a later provisioning step grants broad table privileges.
|
|
316
|
+
DO $policies$
|
|
317
|
+
DECLARE owner_role text := current_user;
|
|
318
|
+
BEGIN
|
|
319
|
+
EXECUTE format('DROP POLICY IF EXISTS host_export_owner ON %I.host_export_config', current_schema());
|
|
320
|
+
EXECUTE format(
|
|
321
|
+
'CREATE POLICY host_export_owner ON %I.host_export_config USING (current_user = %L) WITH CHECK (current_user = %L)',
|
|
322
|
+
current_schema(), owner_role, owner_role
|
|
323
|
+
);
|
|
324
|
+
EXECUTE format('DROP POLICY IF EXISTS host_export_owner ON %I.host_export_cursor_state', current_schema());
|
|
325
|
+
EXECUTE format(
|
|
326
|
+
'CREATE POLICY host_export_owner ON %I.host_export_cursor_state USING (current_user = %L) WITH CHECK (current_user = %L)',
|
|
327
|
+
current_schema(), owner_role, owner_role
|
|
328
|
+
);
|
|
329
|
+
EXECUTE format('DROP POLICY IF EXISTS host_export_owner ON %I.host_export_outbox', current_schema());
|
|
330
|
+
EXECUTE format(
|
|
331
|
+
'CREATE POLICY host_export_owner ON %I.host_export_outbox USING (current_user = %L) WITH CHECK (current_user = %L)',
|
|
332
|
+
current_schema(), owner_role, owner_role
|
|
333
|
+
);
|
|
334
|
+
EXECUTE format('DROP POLICY IF EXISTS host_export_owner ON %I.host_export_consumers', current_schema());
|
|
335
|
+
EXECUTE format(
|
|
336
|
+
'CREATE POLICY host_export_owner ON %I.host_export_consumers USING (current_user = %L) WITH CHECK (current_user = %L)',
|
|
337
|
+
current_schema(), owner_role, owner_role
|
|
338
|
+
);
|
|
339
|
+
EXECUTE format('DROP POLICY IF EXISTS host_export_owner ON %I.host_export_dead_letters', current_schema());
|
|
340
|
+
EXECUTE format(
|
|
341
|
+
'CREATE POLICY host_export_owner ON %I.host_export_dead_letters USING (current_user = %L) WITH CHECK (current_user = %L)',
|
|
342
|
+
current_schema(), owner_role, owner_role
|
|
343
|
+
);
|
|
344
|
+
END $policies$;
|
|
345
|
+
|
|
346
|
+
DO $migration$
|
|
347
|
+
DECLARE target_schema text := current_schema();
|
|
348
|
+
BEGIN
|
|
349
|
+
EXECUTE format($create$
|
|
350
|
+
CREATE OR REPLACE FUNCTION opengeni_private.enqueue_host_session_event_export()
|
|
351
|
+
RETURNS trigger
|
|
352
|
+
LANGUAGE plpgsql
|
|
353
|
+
SECURITY DEFINER
|
|
354
|
+
SET search_path = pg_catalog
|
|
355
|
+
AS $function$
|
|
356
|
+
DECLARE
|
|
357
|
+
v_enabled boolean;
|
|
358
|
+
v_initiator jsonb;
|
|
359
|
+
v_context jsonb := '{}'::jsonb;
|
|
360
|
+
v_origin text;
|
|
361
|
+
v_payload_bytes integer;
|
|
362
|
+
BEGIN
|
|
363
|
+
IF NEW.type IN (
|
|
364
|
+
'agent.message.delta', 'agent.reasoning.delta',
|
|
365
|
+
'sandbox.command.output.delta', 'terminal.pty.output.delta'
|
|
366
|
+
) THEN
|
|
367
|
+
RETURN NEW;
|
|
368
|
+
END IF;
|
|
369
|
+
|
|
370
|
+
SELECT c.session_events_enabled INTO v_enabled
|
|
371
|
+
FROM %1$I.host_export_config c WHERE c.id = 1;
|
|
372
|
+
IF coalesce(v_enabled, false) = false THEN
|
|
373
|
+
RETURN NEW;
|
|
374
|
+
END IF;
|
|
375
|
+
|
|
376
|
+
IF NEW.turn_id IS NOT NULL THEN
|
|
377
|
+
SELECT
|
|
378
|
+
CASE WHEN octet_length(t.initiator_subject_id) <= 1024 THEN
|
|
379
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
380
|
+
'kind', t.initiator_kind,
|
|
381
|
+
'subjectId', t.initiator_subject_id,
|
|
382
|
+
'label', CASE
|
|
383
|
+
WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
|
|
384
|
+
THEN left(t.initiator_context ->> 'label', 256)
|
|
385
|
+
ELSE NULL
|
|
386
|
+
END
|
|
387
|
+
))
|
|
388
|
+
ELSE NULL END,
|
|
389
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
390
|
+
'label', CASE
|
|
391
|
+
WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
|
|
392
|
+
THEN left(t.initiator_context ->> 'label', 256)
|
|
393
|
+
ELSE NULL
|
|
394
|
+
END,
|
|
395
|
+
'backfill', CASE
|
|
396
|
+
WHEN jsonb_typeof(t.initiator_context -> 'backfill') = 'boolean'
|
|
397
|
+
THEN t.initiator_context -> 'backfill'
|
|
398
|
+
ELSE NULL
|
|
399
|
+
END,
|
|
400
|
+
'attributionOmitted', CASE
|
|
401
|
+
WHEN octet_length(t.initiator_subject_id) > 1024 THEN 'subject_id_too_large'
|
|
402
|
+
ELSE NULL
|
|
403
|
+
END
|
|
404
|
+
)),
|
|
405
|
+
t.source
|
|
406
|
+
INTO v_initiator, v_context, v_origin
|
|
407
|
+
FROM %1$I.session_turns t
|
|
408
|
+
WHERE t.workspace_id = NEW.workspace_id AND t.id = NEW.turn_id;
|
|
409
|
+
ELSIF NEW.type = 'session.created' THEN
|
|
410
|
+
SELECT
|
|
411
|
+
CASE WHEN octet_length(s.created_by_subject_id) <= 1024 THEN
|
|
412
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
413
|
+
'kind', s.created_by_kind,
|
|
414
|
+
'subjectId', s.created_by_subject_id,
|
|
415
|
+
'label', CASE
|
|
416
|
+
WHEN jsonb_typeof(s.created_by_context -> 'label') = 'string'
|
|
417
|
+
THEN left(s.created_by_context ->> 'label', 256)
|
|
418
|
+
ELSE NULL
|
|
419
|
+
END
|
|
420
|
+
))
|
|
421
|
+
ELSE NULL END,
|
|
422
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
423
|
+
'label', CASE
|
|
424
|
+
WHEN jsonb_typeof(s.created_by_context -> 'label') = 'string'
|
|
425
|
+
THEN left(s.created_by_context ->> 'label', 256)
|
|
426
|
+
ELSE NULL
|
|
427
|
+
END,
|
|
428
|
+
'backfill', CASE
|
|
429
|
+
WHEN jsonb_typeof(s.created_by_context -> 'backfill') = 'boolean'
|
|
430
|
+
THEN s.created_by_context -> 'backfill'
|
|
431
|
+
ELSE NULL
|
|
432
|
+
END,
|
|
433
|
+
'attributionOmitted', CASE
|
|
434
|
+
WHEN octet_length(s.created_by_subject_id) > 1024 THEN 'subject_id_too_large'
|
|
435
|
+
ELSE NULL
|
|
436
|
+
END
|
|
437
|
+
)),
|
|
438
|
+
NULL
|
|
439
|
+
INTO v_initiator, v_context, v_origin
|
|
440
|
+
FROM %1$I.sessions s
|
|
441
|
+
WHERE s.workspace_id = NEW.workspace_id AND s.id = NEW.session_id;
|
|
442
|
+
END IF;
|
|
443
|
+
|
|
444
|
+
v_payload_bytes := octet_length(NEW.payload::text)
|
|
445
|
+
+ octet_length(NEW.type)
|
|
446
|
+
+ coalesce(octet_length(NEW.client_event_id), 0)
|
|
447
|
+
+ coalesce(octet_length(NEW.turn_association), 0)
|
|
448
|
+
+ coalesce(octet_length(NEW.duplicate_reason), 0)
|
|
449
|
+
+ octet_length(coalesce(v_initiator, 'null'::jsonb)::text)
|
|
450
|
+
+ octet_length(v_context::text)
|
|
451
|
+
+ 768;
|
|
452
|
+
INSERT INTO %1$I.host_export_outbox (
|
|
453
|
+
export_kind, source_id, account_id, workspace_id, session_id,
|
|
454
|
+
turn_id, turn_generation, turn_attempt_id, session_sequence,
|
|
455
|
+
client_event_id, turn_association, duplicate_of_event_id, duplicate_reason,
|
|
456
|
+
event_type, idempotency_key, initiator, initiator_context, origin,
|
|
457
|
+
payload, envelope_bytes, occurred_at, source_recorded_at, enqueued_at
|
|
458
|
+
) VALUES (
|
|
459
|
+
'session_event', NEW.id, NEW.account_id, NEW.workspace_id, NEW.session_id,
|
|
460
|
+
NEW.turn_id, NEW.turn_generation, NEW.turn_attempt_id, NEW.sequence,
|
|
461
|
+
NEW.client_event_id, NEW.turn_association, NEW.duplicate_of_event_id,
|
|
462
|
+
NEW.duplicate_reason,
|
|
463
|
+
NEW.type, 'session_event:' || NEW.id::text, v_initiator,
|
|
464
|
+
coalesce(v_context, '{}'::jsonb), v_origin, NEW.payload,
|
|
465
|
+
greatest(1, v_payload_bytes), NEW.occurred_at,
|
|
466
|
+
NEW.created_at, clock_timestamp()
|
|
467
|
+
)
|
|
468
|
+
ON CONFLICT (export_kind, source_id) DO NOTHING;
|
|
469
|
+
RETURN NEW;
|
|
470
|
+
END $function$;
|
|
471
|
+
$create$, target_schema);
|
|
472
|
+
|
|
473
|
+
EXECUTE format($create$
|
|
474
|
+
CREATE OR REPLACE FUNCTION opengeni_private.enqueue_host_usage_event_export()
|
|
475
|
+
RETURNS trigger
|
|
476
|
+
LANGUAGE plpgsql
|
|
477
|
+
SECURITY DEFINER
|
|
478
|
+
SET search_path = pg_catalog
|
|
479
|
+
AS $function$
|
|
480
|
+
DECLARE
|
|
481
|
+
v_enabled boolean;
|
|
482
|
+
v_initiator jsonb := CASE
|
|
483
|
+
WHEN NEW.initiator_kind IS NOT NULL THEN jsonb_strip_nulls(jsonb_build_object(
|
|
484
|
+
'kind', NEW.initiator_kind,
|
|
485
|
+
'subjectId', NEW.initiator_subject_id,
|
|
486
|
+
'label', CASE
|
|
487
|
+
WHEN jsonb_typeof(NEW.initiator_context -> 'label') = 'string'
|
|
488
|
+
THEN left(NEW.initiator_context ->> 'label', 256)
|
|
489
|
+
ELSE NULL
|
|
490
|
+
END
|
|
491
|
+
))
|
|
492
|
+
ELSE NULL
|
|
493
|
+
END;
|
|
494
|
+
v_context jsonb := jsonb_strip_nulls(jsonb_build_object(
|
|
495
|
+
'label', CASE
|
|
496
|
+
WHEN jsonb_typeof(NEW.initiator_context -> 'label') = 'string'
|
|
497
|
+
THEN left(NEW.initiator_context ->> 'label', 256)
|
|
498
|
+
ELSE NULL
|
|
499
|
+
END,
|
|
500
|
+
'backfill', CASE
|
|
501
|
+
WHEN jsonb_typeof(NEW.initiator_context -> 'backfill') = 'boolean'
|
|
502
|
+
THEN NEW.initiator_context -> 'backfill'
|
|
503
|
+
ELSE NULL
|
|
504
|
+
END
|
|
505
|
+
));
|
|
506
|
+
v_origin text := NEW.origin;
|
|
507
|
+
v_payload jsonb;
|
|
508
|
+
v_payload_bytes integer;
|
|
509
|
+
BEGIN
|
|
510
|
+
SELECT c.usage_events_enabled INTO v_enabled
|
|
511
|
+
FROM %1$I.host_export_config c WHERE c.id = 1;
|
|
512
|
+
IF coalesce(v_enabled, false) = false THEN
|
|
513
|
+
RETURN NEW;
|
|
514
|
+
END IF;
|
|
515
|
+
|
|
516
|
+
-- Export-specific bounds apply only while the optional host stream is
|
|
517
|
+
-- enabled. Standalone/custom usage writers retain their historical
|
|
518
|
+
-- behavior; an embedded writer that cannot be represented fails its
|
|
519
|
+
-- source transaction visibly instead of committing a poison outbox row.
|
|
520
|
+
IF octet_length(NEW.event_type) NOT BETWEEN 1 AND 256
|
|
521
|
+
OR octet_length(NEW.unit) NOT BETWEEN 1 AND 128
|
|
522
|
+
OR (NEW.subject_id IS NOT NULL AND octet_length(NEW.subject_id) > 1024)
|
|
523
|
+
OR (NEW.source_resource_type IS NOT NULL
|
|
524
|
+
AND octet_length(NEW.source_resource_type) > 256)
|
|
525
|
+
OR (NEW.source_resource_id IS NOT NULL
|
|
526
|
+
AND octet_length(NEW.source_resource_id) > 2048)
|
|
527
|
+
OR octet_length(NEW.idempotency_key) NOT BETWEEN 1 AND 2048
|
|
528
|
+
OR (NEW.billing_provider_event_id IS NOT NULL
|
|
529
|
+
AND octet_length(NEW.billing_provider_event_id) > 2048) THEN
|
|
530
|
+
RAISE EXCEPTION 'usage event exceeds the enabled host-export wire bounds'
|
|
531
|
+
USING ERRCODE = '22001';
|
|
532
|
+
END IF;
|
|
533
|
+
|
|
534
|
+
IF NEW.turn_id IS NOT NULL THEN
|
|
535
|
+
SELECT
|
|
536
|
+
CASE WHEN octet_length(t.initiator_subject_id) <= 1024 THEN
|
|
537
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
538
|
+
'kind', t.initiator_kind,
|
|
539
|
+
'subjectId', t.initiator_subject_id,
|
|
540
|
+
'label', CASE
|
|
541
|
+
WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
|
|
542
|
+
THEN left(t.initiator_context ->> 'label', 256)
|
|
543
|
+
ELSE NULL
|
|
544
|
+
END
|
|
545
|
+
))
|
|
546
|
+
ELSE NULL END,
|
|
547
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
548
|
+
'label', CASE
|
|
549
|
+
WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
|
|
550
|
+
THEN left(t.initiator_context ->> 'label', 256)
|
|
551
|
+
ELSE NULL
|
|
552
|
+
END,
|
|
553
|
+
'backfill', CASE
|
|
554
|
+
WHEN jsonb_typeof(t.initiator_context -> 'backfill') = 'boolean'
|
|
555
|
+
THEN t.initiator_context -> 'backfill'
|
|
556
|
+
ELSE NULL
|
|
557
|
+
END,
|
|
558
|
+
'attributionOmitted', CASE
|
|
559
|
+
WHEN octet_length(t.initiator_subject_id) > 1024 THEN 'subject_id_too_large'
|
|
560
|
+
ELSE NULL
|
|
561
|
+
END
|
|
562
|
+
)),
|
|
563
|
+
t.source
|
|
564
|
+
INTO v_initiator, v_context, v_origin
|
|
565
|
+
FROM %1$I.session_turns t
|
|
566
|
+
WHERE t.workspace_id = NEW.workspace_id AND t.id = NEW.turn_id;
|
|
567
|
+
ELSIF v_initiator IS NULL AND NEW.subject_id IS NOT NULL
|
|
568
|
+
AND octet_length(NEW.subject_id) <= 1024 THEN
|
|
569
|
+
v_initiator := jsonb_build_object('kind', 'subject', 'subjectId', NEW.subject_id);
|
|
570
|
+
END IF;
|
|
571
|
+
|
|
572
|
+
v_payload := jsonb_build_object(
|
|
573
|
+
'id', NEW.id,
|
|
574
|
+
'workspaceId', NEW.workspace_id,
|
|
575
|
+
'accountId', NEW.account_id,
|
|
576
|
+
'subjectId', NEW.subject_id,
|
|
577
|
+
'eventType', NEW.event_type,
|
|
578
|
+
'quantity', NEW.quantity,
|
|
579
|
+
'unit', NEW.unit,
|
|
580
|
+
'sourceResourceType', NEW.source_resource_type,
|
|
581
|
+
'sourceResourceId', NEW.source_resource_id,
|
|
582
|
+
'idempotencyKey', NEW.idempotency_key,
|
|
583
|
+
'occurredAt', NEW.occurred_at,
|
|
584
|
+
'recordedAt', NEW.recorded_at,
|
|
585
|
+
'exportedToBillingAt', NEW.exported_to_billing_at,
|
|
586
|
+
'billingProviderEventId', NEW.billing_provider_event_id
|
|
587
|
+
);
|
|
588
|
+
v_payload_bytes := octet_length(v_payload::text)
|
|
589
|
+
+ octet_length(coalesce(v_initiator, 'null'::jsonb)::text)
|
|
590
|
+
+ octet_length(v_context::text)
|
|
591
|
+
+ 768;
|
|
592
|
+
INSERT INTO %1$I.host_export_outbox (
|
|
593
|
+
export_kind, source_id, account_id, workspace_id, session_id,
|
|
594
|
+
turn_id, turn_attempt_id, event_type, idempotency_key, initiator,
|
|
595
|
+
initiator_context, origin, payload, envelope_bytes, occurred_at,
|
|
596
|
+
source_recorded_at, enqueued_at
|
|
597
|
+
) VALUES (
|
|
598
|
+
'usage_event', NEW.id, NEW.account_id, NEW.workspace_id, NEW.session_id,
|
|
599
|
+
NEW.turn_id, NEW.turn_attempt_id, NEW.event_type, NEW.idempotency_key,
|
|
600
|
+
v_initiator, coalesce(v_context, '{}'::jsonb), v_origin, v_payload,
|
|
601
|
+
greatest(1, v_payload_bytes), NEW.occurred_at,
|
|
602
|
+
NEW.recorded_at, clock_timestamp()
|
|
603
|
+
)
|
|
604
|
+
ON CONFLICT (export_kind, source_id) DO NOTHING;
|
|
605
|
+
RETURN NEW;
|
|
606
|
+
END $function$;
|
|
607
|
+
$create$, target_schema);
|
|
608
|
+
END $migration$;
|
|
609
|
+
|
|
610
|
+
DROP TRIGGER IF EXISTS session_events_host_export ON "session_events";
|
|
611
|
+
CREATE CONSTRAINT TRIGGER session_events_host_export
|
|
612
|
+
AFTER INSERT ON "session_events"
|
|
613
|
+
DEFERRABLE INITIALLY DEFERRED
|
|
614
|
+
FOR EACH ROW EXECUTE FUNCTION opengeni_private.enqueue_host_session_event_export();
|
|
615
|
+
|
|
616
|
+
DROP TRIGGER IF EXISTS usage_events_host_export ON "usage_events";
|
|
617
|
+
CREATE CONSTRAINT TRIGGER usage_events_host_export
|
|
618
|
+
AFTER INSERT ON "usage_events"
|
|
619
|
+
DEFERRABLE INITIALLY DEFERRED
|
|
620
|
+
FOR EACH ROW EXECUTE FUNCTION opengeni_private.enqueue_host_usage_event_export();
|
|
621
|
+
|
|
622
|
+
DO $migration$
|
|
623
|
+
DECLARE target_schema text := current_schema();
|
|
624
|
+
BEGIN
|
|
625
|
+
EXECUTE format($create$
|
|
626
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.register_host_export_consumer(
|
|
627
|
+
p_export_kind text, p_consumer_id text
|
|
628
|
+
) RETURNS void
|
|
629
|
+
LANGUAGE plpgsql
|
|
630
|
+
SECURITY DEFINER
|
|
631
|
+
SET search_path = pg_catalog
|
|
632
|
+
AS $function$
|
|
633
|
+
DECLARE v_checkpoint bigint;
|
|
634
|
+
BEGIN
|
|
635
|
+
IF p_export_kind NOT IN ('session_event', 'usage_event') THEN
|
|
636
|
+
RAISE EXCEPTION 'invalid host export kind' USING ERRCODE = '22023';
|
|
637
|
+
END IF;
|
|
638
|
+
IF p_consumer_id IS NULL OR length(p_consumer_id) NOT BETWEEN 1 AND 128
|
|
639
|
+
OR p_consumer_id !~ '^[A-Za-z0-9][A-Za-z0-9._:-]*$' THEN
|
|
640
|
+
RAISE EXCEPTION 'invalid host export consumer id' USING ERRCODE = '22023';
|
|
641
|
+
END IF;
|
|
642
|
+
|
|
643
|
+
SELECT s.pruned_through INTO v_checkpoint
|
|
644
|
+
FROM %1$I.host_export_cursor_state s
|
|
645
|
+
WHERE s.export_kind = p_export_kind
|
|
646
|
+
FOR UPDATE;
|
|
647
|
+
IF NOT FOUND THEN
|
|
648
|
+
RAISE EXCEPTION 'unknown host export kind' USING ERRCODE = '22023';
|
|
649
|
+
END IF;
|
|
650
|
+
|
|
651
|
+
INSERT INTO %1$I.host_export_consumers (consumer_id, export_kind, checkpoint)
|
|
652
|
+
VALUES (p_consumer_id, p_export_kind, v_checkpoint)
|
|
653
|
+
ON CONFLICT (export_kind, consumer_id) DO UPDATE
|
|
654
|
+
SET enabled = true, updated_at = now();
|
|
655
|
+
|
|
656
|
+
UPDATE %1$I.host_export_config
|
|
657
|
+
SET session_events_enabled = CASE WHEN p_export_kind = 'session_event' THEN true ELSE session_events_enabled END,
|
|
658
|
+
usage_events_enabled = CASE WHEN p_export_kind = 'usage_event' THEN true ELSE usage_events_enabled END,
|
|
659
|
+
updated_at = now()
|
|
660
|
+
WHERE id = 1;
|
|
661
|
+
END $function$;
|
|
662
|
+
$create$, target_schema);
|
|
663
|
+
|
|
664
|
+
EXECUTE format($create$
|
|
665
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.allocate_host_export_cursors(
|
|
666
|
+
p_export_kind text, p_limit integer
|
|
667
|
+
) RETURNS integer
|
|
668
|
+
LANGUAGE plpgsql
|
|
669
|
+
SECURITY DEFINER
|
|
670
|
+
SET search_path = pg_catalog
|
|
671
|
+
AS $function$
|
|
672
|
+
DECLARE
|
|
673
|
+
v_id uuid;
|
|
674
|
+
v_next bigint;
|
|
675
|
+
v_count integer := 0;
|
|
676
|
+
v_limit integer := greatest(1, least(coalesce(p_limit, 512), 4096));
|
|
677
|
+
BEGIN
|
|
678
|
+
SELECT s.next_cursor INTO v_next
|
|
679
|
+
FROM %1$I.host_export_cursor_state s
|
|
680
|
+
WHERE s.export_kind = p_export_kind
|
|
681
|
+
FOR UPDATE;
|
|
682
|
+
IF v_next IS NULL THEN
|
|
683
|
+
RAISE EXCEPTION 'unknown host export kind' USING ERRCODE = '22023';
|
|
684
|
+
END IF;
|
|
685
|
+
|
|
686
|
+
WHILE v_count < v_limit LOOP
|
|
687
|
+
SELECT o.id INTO v_id
|
|
688
|
+
FROM %1$I.host_export_outbox o
|
|
689
|
+
WHERE o.export_kind = p_export_kind
|
|
690
|
+
AND o.export_cursor IS NULL
|
|
691
|
+
AND (
|
|
692
|
+
o.session_id IS NULL OR o.session_sequence IS NULL OR NOT EXISTS (
|
|
693
|
+
SELECT 1 FROM %1$I.host_export_outbox earlier
|
|
694
|
+
WHERE earlier.export_kind = o.export_kind
|
|
695
|
+
AND earlier.session_id = o.session_id
|
|
696
|
+
AND earlier.session_sequence IS NOT NULL
|
|
697
|
+
AND earlier.session_sequence < o.session_sequence
|
|
698
|
+
AND earlier.export_cursor IS NULL
|
|
699
|
+
)
|
|
700
|
+
)
|
|
701
|
+
ORDER BY o.enqueued_at, o.id
|
|
702
|
+
FOR UPDATE SKIP LOCKED
|
|
703
|
+
LIMIT 1;
|
|
704
|
+
EXIT WHEN v_id IS NULL;
|
|
705
|
+
|
|
706
|
+
UPDATE %1$I.host_export_outbox
|
|
707
|
+
SET export_cursor = v_next
|
|
708
|
+
WHERE id = v_id AND export_cursor IS NULL;
|
|
709
|
+
v_next := v_next + 1;
|
|
710
|
+
v_count := v_count + 1;
|
|
711
|
+
v_id := NULL;
|
|
712
|
+
END LOOP;
|
|
713
|
+
|
|
714
|
+
UPDATE %1$I.host_export_cursor_state
|
|
715
|
+
SET next_cursor = v_next, updated_at = now()
|
|
716
|
+
WHERE export_kind = p_export_kind;
|
|
717
|
+
RETURN v_count;
|
|
718
|
+
END $function$;
|
|
719
|
+
$create$, target_schema);
|
|
720
|
+
|
|
721
|
+
EXECUTE format($create$
|
|
722
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.claim_host_export_batch(
|
|
723
|
+
p_export_kind text,
|
|
724
|
+
p_consumer_id text,
|
|
725
|
+
p_lease_token uuid,
|
|
726
|
+
p_lease_holder_id text,
|
|
727
|
+
p_lease_seconds integer,
|
|
728
|
+
p_limit integer,
|
|
729
|
+
p_max_bytes integer
|
|
730
|
+
) RETURNS TABLE (
|
|
731
|
+
consumer_id text, export_kind text, checkpoint bigint,
|
|
732
|
+
lease_token uuid, lease_through bigint, export_cursor bigint,
|
|
733
|
+
source_id uuid, account_id uuid, workspace_id uuid, session_id uuid,
|
|
734
|
+
turn_id uuid, turn_generation integer, turn_attempt_id uuid,
|
|
735
|
+
session_sequence integer, client_event_id text, turn_association text,
|
|
736
|
+
duplicate_of_event_id uuid, duplicate_reason text,
|
|
737
|
+
event_type text, idempotency_key text,
|
|
738
|
+
initiator jsonb, initiator_context jsonb, origin text, payload jsonb,
|
|
739
|
+
occurred_at timestamptz, source_recorded_at timestamptz
|
|
740
|
+
)
|
|
741
|
+
LANGUAGE plpgsql
|
|
742
|
+
SECURITY DEFINER
|
|
743
|
+
SET search_path = pg_catalog
|
|
744
|
+
AS $function$
|
|
745
|
+
DECLARE
|
|
746
|
+
v_consumer %1$I.host_export_consumers%%ROWTYPE;
|
|
747
|
+
v_through bigint;
|
|
748
|
+
v_limit integer := greatest(1, least(coalesce(p_limit, 100), 256));
|
|
749
|
+
v_max_bytes integer := greatest(98304, least(coalesce(p_max_bytes, 1048576), 4194304));
|
|
750
|
+
v_lease_seconds integer := greatest(5, least(coalesce(p_lease_seconds, 60), 300));
|
|
751
|
+
BEGIN
|
|
752
|
+
IF p_lease_token IS NULL THEN
|
|
753
|
+
RAISE EXCEPTION 'host export lease token is required' USING ERRCODE = '22023';
|
|
754
|
+
END IF;
|
|
755
|
+
IF p_lease_holder_id IS NULL OR length(p_lease_holder_id) NOT BETWEEN 1 AND 128 THEN
|
|
756
|
+
RAISE EXCEPTION 'host export lease holder id is invalid' USING ERRCODE = '22023';
|
|
757
|
+
END IF;
|
|
758
|
+
PERFORM opengeni_host_export.allocate_host_export_cursors(
|
|
759
|
+
p_export_kind, greatest(v_limit, 512)
|
|
760
|
+
);
|
|
761
|
+
|
|
762
|
+
SELECT * INTO v_consumer
|
|
763
|
+
FROM %1$I.host_export_consumers c
|
|
764
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id
|
|
765
|
+
FOR UPDATE;
|
|
766
|
+
IF NOT FOUND OR NOT v_consumer.enabled OR v_consumer.blocked_at IS NOT NULL
|
|
767
|
+
OR v_consumer.next_attempt_at > now()
|
|
768
|
+
OR (v_consumer.lease_expires_at IS NOT NULL AND v_consumer.lease_expires_at > now()) THEN
|
|
769
|
+
RETURN;
|
|
770
|
+
END IF;
|
|
771
|
+
|
|
772
|
+
WITH candidates AS MATERIALIZED (
|
|
773
|
+
SELECT o.export_cursor, o.envelope_bytes
|
|
774
|
+
FROM %1$I.host_export_outbox o
|
|
775
|
+
WHERE o.export_kind = p_export_kind
|
|
776
|
+
AND o.export_cursor > v_consumer.checkpoint
|
|
777
|
+
ORDER BY o.export_cursor
|
|
778
|
+
LIMIT v_limit
|
|
779
|
+
), ranked AS (
|
|
780
|
+
SELECT o.export_cursor,
|
|
781
|
+
row_number() OVER (ORDER BY o.export_cursor) AS row_number,
|
|
782
|
+
sum(o.envelope_bytes) OVER (ORDER BY o.export_cursor) AS running_bytes
|
|
783
|
+
FROM candidates o
|
|
784
|
+
), selected AS (
|
|
785
|
+
SELECT r.export_cursor FROM ranked r
|
|
786
|
+
WHERE r.running_bytes <= v_max_bytes OR r.row_number = 1
|
|
787
|
+
)
|
|
788
|
+
SELECT max(s.export_cursor) INTO v_through FROM selected s;
|
|
789
|
+
IF v_through IS NULL THEN
|
|
790
|
+
RETURN;
|
|
791
|
+
END IF;
|
|
792
|
+
|
|
793
|
+
UPDATE %1$I.host_export_consumers c
|
|
794
|
+
SET lease_token = p_lease_token,
|
|
795
|
+
lease_holder_id = left(p_lease_holder_id, 128),
|
|
796
|
+
lease_expires_at = now() + make_interval(secs => v_lease_seconds),
|
|
797
|
+
lease_from = v_consumer.checkpoint,
|
|
798
|
+
lease_through = v_through,
|
|
799
|
+
updated_at = now()
|
|
800
|
+
WHERE c.id = v_consumer.id;
|
|
801
|
+
|
|
802
|
+
RETURN QUERY
|
|
803
|
+
SELECT p_consumer_id, p_export_kind, v_consumer.checkpoint,
|
|
804
|
+
p_lease_token, v_through, o.export_cursor, o.source_id,
|
|
805
|
+
o.account_id, o.workspace_id, o.session_id, o.turn_id,
|
|
806
|
+
o.turn_generation, o.turn_attempt_id, o.session_sequence,
|
|
807
|
+
o.client_event_id, o.turn_association, o.duplicate_of_event_id,
|
|
808
|
+
o.duplicate_reason,
|
|
809
|
+
o.event_type, o.idempotency_key, o.initiator,
|
|
810
|
+
o.initiator_context, o.origin, o.payload, o.occurred_at,
|
|
811
|
+
o.source_recorded_at
|
|
812
|
+
FROM %1$I.host_export_outbox o
|
|
813
|
+
WHERE o.export_kind = p_export_kind
|
|
814
|
+
AND o.export_cursor > v_consumer.checkpoint
|
|
815
|
+
AND o.export_cursor <= v_through
|
|
816
|
+
ORDER BY o.export_cursor;
|
|
817
|
+
END $function$;
|
|
818
|
+
$create$, target_schema);
|
|
819
|
+
END $migration$;
|
|
820
|
+
|
|
821
|
+
DO $migration$
|
|
822
|
+
DECLARE target_schema text := current_schema();
|
|
823
|
+
BEGIN
|
|
824
|
+
EXECUTE format($create$
|
|
825
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.ack_host_export_batch(
|
|
826
|
+
p_export_kind text, p_consumer_id text, p_lease_token uuid
|
|
827
|
+
) RETURNS bigint
|
|
828
|
+
LANGUAGE plpgsql
|
|
829
|
+
SECURITY DEFINER
|
|
830
|
+
SET search_path = pg_catalog
|
|
831
|
+
AS $function$
|
|
832
|
+
DECLARE v_checkpoint bigint;
|
|
833
|
+
BEGIN
|
|
834
|
+
UPDATE %1$I.host_export_consumers c
|
|
835
|
+
SET checkpoint = c.lease_through,
|
|
836
|
+
lease_token = NULL, lease_holder_id = NULL, lease_expires_at = NULL,
|
|
837
|
+
lease_from = NULL, lease_through = NULL,
|
|
838
|
+
consecutive_failures = 0, next_attempt_at = now(),
|
|
839
|
+
last_error = NULL, last_error_at = NULL, updated_at = now()
|
|
840
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id
|
|
841
|
+
AND c.lease_token = p_lease_token AND c.lease_through IS NOT NULL
|
|
842
|
+
RETURNING c.checkpoint INTO v_checkpoint;
|
|
843
|
+
IF v_checkpoint IS NULL THEN
|
|
844
|
+
RAISE EXCEPTION 'host export lease is stale' USING ERRCODE = '40001';
|
|
845
|
+
END IF;
|
|
846
|
+
RETURN v_checkpoint;
|
|
847
|
+
END $function$;
|
|
848
|
+
$create$, target_schema);
|
|
849
|
+
|
|
850
|
+
EXECUTE format($create$
|
|
851
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.fail_host_export_batch(
|
|
852
|
+
p_export_kind text, p_consumer_id text, p_lease_token uuid,
|
|
853
|
+
p_error text, p_max_failures integer
|
|
854
|
+
) RETURNS integer
|
|
855
|
+
LANGUAGE plpgsql
|
|
856
|
+
SECURITY DEFINER
|
|
857
|
+
SET search_path = pg_catalog
|
|
858
|
+
AS $function$
|
|
859
|
+
DECLARE v_failures integer;
|
|
860
|
+
BEGIN
|
|
861
|
+
UPDATE %1$I.host_export_consumers c
|
|
862
|
+
SET consecutive_failures = c.consecutive_failures + 1,
|
|
863
|
+
lease_token = NULL, lease_holder_id = NULL, lease_expires_at = NULL,
|
|
864
|
+
lease_from = NULL, lease_through = NULL,
|
|
865
|
+
next_attempt_at = now() + make_interval(
|
|
866
|
+
secs => least(300, greatest(1, power(2, least(c.consecutive_failures, 8))::integer))
|
|
867
|
+
),
|
|
868
|
+
last_error = left(coalesce(p_error, 'host sink failed'), 500),
|
|
869
|
+
last_error_at = now(),
|
|
870
|
+
blocked_at = CASE
|
|
871
|
+
WHEN c.consecutive_failures + 1 >= greatest(1, least(coalesce(p_max_failures, 20), 1000))
|
|
872
|
+
THEN now() ELSE NULL END,
|
|
873
|
+
updated_at = now()
|
|
874
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id
|
|
875
|
+
AND c.lease_token = p_lease_token
|
|
876
|
+
RETURNING c.consecutive_failures INTO v_failures;
|
|
877
|
+
IF v_failures IS NULL THEN
|
|
878
|
+
RAISE EXCEPTION 'host export lease is stale' USING ERRCODE = '40001';
|
|
879
|
+
END IF;
|
|
880
|
+
RETURN v_failures;
|
|
881
|
+
END $function$;
|
|
882
|
+
$create$, target_schema);
|
|
883
|
+
|
|
884
|
+
EXECUTE format($create$
|
|
885
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.resume_host_export_consumer(
|
|
886
|
+
p_export_kind text, p_consumer_id text
|
|
887
|
+
) RETURNS void
|
|
888
|
+
LANGUAGE plpgsql
|
|
889
|
+
SECURITY DEFINER
|
|
890
|
+
SET search_path = pg_catalog
|
|
891
|
+
AS $function$
|
|
892
|
+
BEGIN
|
|
893
|
+
UPDATE %1$I.host_export_consumers c
|
|
894
|
+
SET blocked_at = NULL, consecutive_failures = 0, next_attempt_at = now(),
|
|
895
|
+
lease_token = NULL, lease_holder_id = NULL, lease_expires_at = NULL,
|
|
896
|
+
lease_from = NULL, lease_through = NULL, updated_at = now()
|
|
897
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id;
|
|
898
|
+
IF NOT FOUND THEN RAISE EXCEPTION 'host export consumer not found' USING ERRCODE = 'P0002'; END IF;
|
|
899
|
+
END $function$;
|
|
900
|
+
$create$, target_schema);
|
|
901
|
+
|
|
902
|
+
EXECUTE format($create$
|
|
903
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.rewind_host_export_consumer(
|
|
904
|
+
p_export_kind text, p_consumer_id text, p_checkpoint bigint
|
|
905
|
+
) RETURNS void
|
|
906
|
+
LANGUAGE plpgsql
|
|
907
|
+
SECURITY DEFINER
|
|
908
|
+
SET search_path = pg_catalog
|
|
909
|
+
AS $function$
|
|
910
|
+
DECLARE v_pruned_through bigint; v_max bigint;
|
|
911
|
+
BEGIN
|
|
912
|
+
IF p_checkpoint < 0 THEN RAISE EXCEPTION 'checkpoint must be nonnegative' USING ERRCODE = '22023'; END IF;
|
|
913
|
+
SELECT s.pruned_through, s.next_cursor - 1
|
|
914
|
+
INTO v_pruned_through, v_max
|
|
915
|
+
FROM %1$I.host_export_cursor_state s
|
|
916
|
+
WHERE s.export_kind = p_export_kind
|
|
917
|
+
FOR UPDATE;
|
|
918
|
+
IF NOT FOUND THEN
|
|
919
|
+
RAISE EXCEPTION 'unknown host export kind' USING ERRCODE = '22023';
|
|
920
|
+
END IF;
|
|
921
|
+
IF p_checkpoint < v_pruned_through THEN
|
|
922
|
+
RAISE EXCEPTION 'requested host export cursor is no longer retained' USING ERRCODE = '22023';
|
|
923
|
+
END IF;
|
|
924
|
+
IF p_checkpoint > v_max THEN
|
|
925
|
+
RAISE EXCEPTION 'requested host export cursor has not been allocated' USING ERRCODE = '22023';
|
|
926
|
+
END IF;
|
|
927
|
+
UPDATE %1$I.host_export_consumers c
|
|
928
|
+
SET checkpoint = p_checkpoint, blocked_at = NULL, consecutive_failures = 0,
|
|
929
|
+
next_attempt_at = now(), lease_token = NULL, lease_holder_id = NULL,
|
|
930
|
+
lease_expires_at = NULL, lease_from = NULL, lease_through = NULL,
|
|
931
|
+
updated_at = now()
|
|
932
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id;
|
|
933
|
+
IF NOT FOUND THEN RAISE EXCEPTION 'host export consumer not found' USING ERRCODE = 'P0002'; END IF;
|
|
934
|
+
END $function$;
|
|
935
|
+
$create$, target_schema);
|
|
936
|
+
|
|
937
|
+
EXECUTE format($create$
|
|
938
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.prune_host_export_outbox(
|
|
939
|
+
p_export_kind text, p_grace_seconds integer, p_limit integer
|
|
940
|
+
) RETURNS integer
|
|
941
|
+
LANGUAGE plpgsql
|
|
942
|
+
SECURITY DEFINER
|
|
943
|
+
SET search_path = pg_catalog
|
|
944
|
+
AS $function$
|
|
945
|
+
DECLARE v_checkpoint bigint; v_deleted integer; v_pruned_through bigint;
|
|
946
|
+
BEGIN
|
|
947
|
+
PERFORM 1 FROM %1$I.host_export_cursor_state s
|
|
948
|
+
WHERE s.export_kind = p_export_kind
|
|
949
|
+
FOR UPDATE;
|
|
950
|
+
IF NOT FOUND THEN
|
|
951
|
+
RAISE EXCEPTION 'unknown host export kind' USING ERRCODE = '22023';
|
|
952
|
+
END IF;
|
|
953
|
+
SELECT min(c.checkpoint) INTO v_checkpoint FROM %1$I.host_export_consumers c
|
|
954
|
+
WHERE c.export_kind = p_export_kind;
|
|
955
|
+
IF v_checkpoint IS NULL OR v_checkpoint = 0 THEN RETURN 0; END IF;
|
|
956
|
+
WITH doomed AS (
|
|
957
|
+
SELECT o.id FROM %1$I.host_export_outbox o
|
|
958
|
+
WHERE o.export_kind = p_export_kind
|
|
959
|
+
AND o.export_cursor <= v_checkpoint
|
|
960
|
+
AND o.enqueued_at < now() - make_interval(
|
|
961
|
+
secs => greatest(0, least(coalesce(p_grace_seconds, 3600), 604800))
|
|
962
|
+
)
|
|
963
|
+
ORDER BY o.export_cursor
|
|
964
|
+
LIMIT greatest(1, least(coalesce(p_limit, 1000), 10000))
|
|
965
|
+
FOR UPDATE SKIP LOCKED
|
|
966
|
+
), deleted AS (
|
|
967
|
+
DELETE FROM %1$I.host_export_outbox o
|
|
968
|
+
USING doomed d WHERE o.id = d.id
|
|
969
|
+
RETURNING o.export_cursor
|
|
970
|
+
)
|
|
971
|
+
SELECT count(*)::integer, max(d.export_cursor)
|
|
972
|
+
INTO v_deleted, v_pruned_through
|
|
973
|
+
FROM deleted d;
|
|
974
|
+
IF v_pruned_through IS NOT NULL THEN
|
|
975
|
+
UPDATE %1$I.host_export_cursor_state s
|
|
976
|
+
SET pruned_through = greatest(s.pruned_through, v_pruned_through),
|
|
977
|
+
updated_at = now()
|
|
978
|
+
WHERE s.export_kind = p_export_kind;
|
|
979
|
+
END IF;
|
|
980
|
+
RETURN v_deleted;
|
|
981
|
+
END $function$;
|
|
982
|
+
$create$, target_schema);
|
|
983
|
+
|
|
984
|
+
EXECUTE format($create$
|
|
985
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.disable_host_export_consumer(
|
|
986
|
+
p_export_kind text, p_consumer_id text
|
|
987
|
+
) RETURNS void
|
|
988
|
+
LANGUAGE plpgsql
|
|
989
|
+
SECURITY DEFINER
|
|
990
|
+
SET search_path = pg_catalog
|
|
991
|
+
AS $function$
|
|
992
|
+
BEGIN
|
|
993
|
+
PERFORM 1 FROM %1$I.host_export_cursor_state s
|
|
994
|
+
WHERE s.export_kind = p_export_kind
|
|
995
|
+
FOR UPDATE;
|
|
996
|
+
IF NOT FOUND THEN
|
|
997
|
+
RAISE EXCEPTION 'unknown host export kind' USING ERRCODE = '22023';
|
|
998
|
+
END IF;
|
|
999
|
+
|
|
1000
|
+
UPDATE %1$I.host_export_consumers c
|
|
1001
|
+
SET enabled = false, updated_at = now()
|
|
1002
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id;
|
|
1003
|
+
IF NOT FOUND THEN
|
|
1004
|
+
RAISE EXCEPTION 'host export consumer not found' USING ERRCODE = 'P0002';
|
|
1005
|
+
END IF;
|
|
1006
|
+
|
|
1007
|
+
IF NOT EXISTS (
|
|
1008
|
+
SELECT 1 FROM %1$I.host_export_consumers c
|
|
1009
|
+
WHERE c.export_kind = p_export_kind AND c.enabled
|
|
1010
|
+
) THEN
|
|
1011
|
+
UPDATE %1$I.host_export_config
|
|
1012
|
+
SET session_events_enabled = CASE
|
|
1013
|
+
WHEN p_export_kind = 'session_event' THEN false
|
|
1014
|
+
ELSE session_events_enabled
|
|
1015
|
+
END,
|
|
1016
|
+
usage_events_enabled = CASE
|
|
1017
|
+
WHEN p_export_kind = 'usage_event' THEN false
|
|
1018
|
+
ELSE usage_events_enabled
|
|
1019
|
+
END,
|
|
1020
|
+
updated_at = now()
|
|
1021
|
+
WHERE id = 1;
|
|
1022
|
+
END IF;
|
|
1023
|
+
END $function$;
|
|
1024
|
+
$create$, target_schema);
|
|
1025
|
+
|
|
1026
|
+
EXECUTE format($create$
|
|
1027
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.retire_host_export_consumer(
|
|
1028
|
+
p_export_kind text, p_consumer_id text
|
|
1029
|
+
) RETURNS void
|
|
1030
|
+
LANGUAGE plpgsql
|
|
1031
|
+
SECURITY DEFINER
|
|
1032
|
+
SET search_path = pg_catalog
|
|
1033
|
+
AS $function$
|
|
1034
|
+
BEGIN
|
|
1035
|
+
PERFORM 1 FROM %1$I.host_export_cursor_state s
|
|
1036
|
+
WHERE s.export_kind = p_export_kind
|
|
1037
|
+
FOR UPDATE;
|
|
1038
|
+
IF NOT FOUND THEN
|
|
1039
|
+
RAISE EXCEPTION 'unknown host export kind' USING ERRCODE = '22023';
|
|
1040
|
+
END IF;
|
|
1041
|
+
|
|
1042
|
+
DELETE FROM %1$I.host_export_consumers c
|
|
1043
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id;
|
|
1044
|
+
IF NOT FOUND THEN
|
|
1045
|
+
RAISE EXCEPTION 'host export consumer not found' USING ERRCODE = 'P0002';
|
|
1046
|
+
END IF;
|
|
1047
|
+
|
|
1048
|
+
IF NOT EXISTS (
|
|
1049
|
+
SELECT 1 FROM %1$I.host_export_consumers c
|
|
1050
|
+
WHERE c.export_kind = p_export_kind AND c.enabled
|
|
1051
|
+
) THEN
|
|
1052
|
+
UPDATE %1$I.host_export_config
|
|
1053
|
+
SET session_events_enabled = CASE
|
|
1054
|
+
WHEN p_export_kind = 'session_event' THEN false
|
|
1055
|
+
ELSE session_events_enabled
|
|
1056
|
+
END,
|
|
1057
|
+
usage_events_enabled = CASE
|
|
1058
|
+
WHEN p_export_kind = 'usage_event' THEN false
|
|
1059
|
+
ELSE usage_events_enabled
|
|
1060
|
+
END,
|
|
1061
|
+
updated_at = now()
|
|
1062
|
+
WHERE id = 1;
|
|
1063
|
+
END IF;
|
|
1064
|
+
END $function$;
|
|
1065
|
+
$create$, target_schema);
|
|
1066
|
+
|
|
1067
|
+
EXECUTE format($create$
|
|
1068
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.dead_letter_host_export_head(
|
|
1069
|
+
p_export_kind text, p_consumer_id text, p_lease_token uuid,
|
|
1070
|
+
p_export_cursor bigint, p_reason text
|
|
1071
|
+
) RETURNS bigint
|
|
1072
|
+
LANGUAGE plpgsql
|
|
1073
|
+
SECURITY DEFINER
|
|
1074
|
+
SET search_path = pg_catalog
|
|
1075
|
+
AS $function$
|
|
1076
|
+
DECLARE
|
|
1077
|
+
v_consumer %1$I.host_export_consumers%%ROWTYPE;
|
|
1078
|
+
v_row %1$I.host_export_outbox%%ROWTYPE;
|
|
1079
|
+
v_envelope jsonb;
|
|
1080
|
+
BEGIN
|
|
1081
|
+
IF p_reason IS NULL OR length(p_reason) NOT BETWEEN 1 AND 500 THEN
|
|
1082
|
+
RAISE EXCEPTION 'dead-letter reason must contain 1 to 500 characters'
|
|
1083
|
+
USING ERRCODE = '22023';
|
|
1084
|
+
END IF;
|
|
1085
|
+
SELECT * INTO v_consumer
|
|
1086
|
+
FROM %1$I.host_export_consumers c
|
|
1087
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id
|
|
1088
|
+
FOR UPDATE;
|
|
1089
|
+
IF NOT FOUND OR v_consumer.lease_token IS DISTINCT FROM p_lease_token
|
|
1090
|
+
OR v_consumer.lease_through IS NULL THEN
|
|
1091
|
+
RAISE EXCEPTION 'host export lease is stale' USING ERRCODE = '40001';
|
|
1092
|
+
END IF;
|
|
1093
|
+
IF p_export_cursor IS NULL OR p_export_cursor <> v_consumer.checkpoint + 1
|
|
1094
|
+
OR p_export_cursor > v_consumer.lease_through THEN
|
|
1095
|
+
RAISE EXCEPTION 'only the leased head event may be dead-lettered'
|
|
1096
|
+
USING ERRCODE = '22023';
|
|
1097
|
+
END IF;
|
|
1098
|
+
|
|
1099
|
+
SELECT * INTO STRICT v_row FROM %1$I.host_export_outbox o
|
|
1100
|
+
WHERE o.export_kind = p_export_kind AND o.export_cursor = p_export_cursor;
|
|
1101
|
+
IF p_export_kind = 'session_event' THEN
|
|
1102
|
+
v_envelope := jsonb_build_object(
|
|
1103
|
+
'schemaRevision', '2026-07-host-export-v1',
|
|
1104
|
+
'cursor', v_row.export_cursor::text,
|
|
1105
|
+
'idempotencyKey', v_row.idempotency_key,
|
|
1106
|
+
'accountId', v_row.account_id,
|
|
1107
|
+
'workspaceId', v_row.workspace_id,
|
|
1108
|
+
'initiator', v_row.initiator,
|
|
1109
|
+
'initiatorContext', v_row.initiator_context,
|
|
1110
|
+
'origin', v_row.origin,
|
|
1111
|
+
'event', jsonb_strip_nulls(jsonb_build_object(
|
|
1112
|
+
'id', v_row.source_id,
|
|
1113
|
+
'workspaceId', v_row.workspace_id,
|
|
1114
|
+
'sessionId', v_row.session_id,
|
|
1115
|
+
'sequence', v_row.session_sequence,
|
|
1116
|
+
'type', v_row.event_type,
|
|
1117
|
+
'payload', v_row.payload,
|
|
1118
|
+
'occurredAt', v_row.occurred_at,
|
|
1119
|
+
'clientEventId', v_row.client_event_id,
|
|
1120
|
+
'turnId', v_row.turn_id,
|
|
1121
|
+
'turnGeneration', v_row.turn_generation,
|
|
1122
|
+
'turnAttemptId', v_row.turn_attempt_id,
|
|
1123
|
+
'turnAssociation', v_row.turn_association,
|
|
1124
|
+
'duplicateOfEventId', v_row.duplicate_of_event_id,
|
|
1125
|
+
'duplicateReason', v_row.duplicate_reason
|
|
1126
|
+
))
|
|
1127
|
+
);
|
|
1128
|
+
ELSE
|
|
1129
|
+
v_envelope := jsonb_build_object(
|
|
1130
|
+
'schemaRevision', '2026-07-host-export-v1',
|
|
1131
|
+
'cursor', v_row.export_cursor::text,
|
|
1132
|
+
'accountId', v_row.account_id,
|
|
1133
|
+
'workspaceId', v_row.workspace_id,
|
|
1134
|
+
'sessionId', v_row.session_id,
|
|
1135
|
+
'turnId', v_row.turn_id,
|
|
1136
|
+
'turnAttemptId', v_row.turn_attempt_id,
|
|
1137
|
+
'initiator', v_row.initiator,
|
|
1138
|
+
'initiatorContext', v_row.initiator_context,
|
|
1139
|
+
'origin', v_row.origin,
|
|
1140
|
+
'usage', v_row.payload
|
|
1141
|
+
);
|
|
1142
|
+
END IF;
|
|
1143
|
+
|
|
1144
|
+
INSERT INTO %1$I.host_export_dead_letters (
|
|
1145
|
+
consumer_id, export_kind, export_cursor, source_id, reason, envelope
|
|
1146
|
+
) VALUES (
|
|
1147
|
+
p_consumer_id, p_export_kind, p_export_cursor, v_row.source_id,
|
|
1148
|
+
p_reason, v_envelope
|
|
1149
|
+
) ON CONFLICT (export_kind, consumer_id, export_cursor) DO NOTHING;
|
|
1150
|
+
|
|
1151
|
+
UPDATE %1$I.host_export_consumers c
|
|
1152
|
+
SET checkpoint = p_export_cursor,
|
|
1153
|
+
lease_token = NULL, lease_holder_id = NULL, lease_expires_at = NULL,
|
|
1154
|
+
lease_from = NULL, lease_through = NULL,
|
|
1155
|
+
consecutive_failures = 0, next_attempt_at = now(),
|
|
1156
|
+
last_error = left('dead-lettered: ' || p_reason, 500),
|
|
1157
|
+
last_error_at = now(), blocked_at = NULL, updated_at = now()
|
|
1158
|
+
WHERE c.id = v_consumer.id;
|
|
1159
|
+
RETURN p_export_cursor;
|
|
1160
|
+
END $function$;
|
|
1161
|
+
$create$, target_schema);
|
|
1162
|
+
|
|
1163
|
+
EXECUTE format($create$
|
|
1164
|
+
CREATE OR REPLACE FUNCTION opengeni_host_export.host_export_consumer_status(
|
|
1165
|
+
p_export_kind text, p_consumer_id text
|
|
1166
|
+
) RETURNS TABLE (
|
|
1167
|
+
export_kind text, consumer_id text, checkpoint bigint, enabled boolean,
|
|
1168
|
+
consecutive_failures integer, next_attempt_at timestamptz,
|
|
1169
|
+
last_error text, last_error_at timestamptz, blocked_at timestamptz,
|
|
1170
|
+
lease_expires_at timestamptz, max_cursor bigint, pending_count bigint,
|
|
1171
|
+
unassigned_count bigint, pruned_through bigint
|
|
1172
|
+
)
|
|
1173
|
+
LANGUAGE sql
|
|
1174
|
+
SECURITY DEFINER
|
|
1175
|
+
SET search_path = pg_catalog
|
|
1176
|
+
AS $function$
|
|
1177
|
+
SELECT c.export_kind, c.consumer_id, c.checkpoint, c.enabled,
|
|
1178
|
+
c.consecutive_failures, c.next_attempt_at, c.last_error,
|
|
1179
|
+
c.last_error_at, c.blocked_at, c.lease_expires_at,
|
|
1180
|
+
greatest(s.next_cursor - 1, c.checkpoint),
|
|
1181
|
+
(
|
|
1182
|
+
SELECT count(*) FROM %1$I.host_export_outbox assigned
|
|
1183
|
+
WHERE assigned.export_kind = c.export_kind
|
|
1184
|
+
AND assigned.export_cursor > c.checkpoint
|
|
1185
|
+
) + (
|
|
1186
|
+
SELECT count(*) FROM %1$I.host_export_outbox unassigned
|
|
1187
|
+
WHERE unassigned.export_kind = c.export_kind
|
|
1188
|
+
AND unassigned.export_cursor IS NULL
|
|
1189
|
+
),
|
|
1190
|
+
(
|
|
1191
|
+
SELECT count(*) FROM %1$I.host_export_outbox unassigned
|
|
1192
|
+
WHERE unassigned.export_kind = c.export_kind
|
|
1193
|
+
AND unassigned.export_cursor IS NULL
|
|
1194
|
+
),
|
|
1195
|
+
s.pruned_through
|
|
1196
|
+
FROM %1$I.host_export_consumers c
|
|
1197
|
+
JOIN %1$I.host_export_cursor_state s ON s.export_kind = c.export_kind
|
|
1198
|
+
WHERE c.export_kind = p_export_kind AND c.consumer_id = p_consumer_id;
|
|
1199
|
+
$function$;
|
|
1200
|
+
$create$, target_schema);
|
|
1201
|
+
END $migration$;
|
|
1202
|
+
|
|
1203
|
+
REVOKE ALL ON FUNCTION opengeni_private.enqueue_host_session_event_export() FROM PUBLIC;
|
|
1204
|
+
REVOKE ALL ON FUNCTION opengeni_private.enqueue_host_usage_event_export() FROM PUBLIC;
|
|
1205
|
+
REVOKE ALL ON FUNCTION opengeni_private.validate_usage_event_execution_context() FROM PUBLIC;
|
|
1206
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.register_host_export_consumer(text, text) FROM PUBLIC;
|
|
1207
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.allocate_host_export_cursors(text, integer) FROM PUBLIC;
|
|
1208
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.claim_host_export_batch(text, text, uuid, text, integer, integer, integer) FROM PUBLIC;
|
|
1209
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.ack_host_export_batch(text, text, uuid) FROM PUBLIC;
|
|
1210
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.fail_host_export_batch(text, text, uuid, text, integer) FROM PUBLIC;
|
|
1211
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.resume_host_export_consumer(text, text) FROM PUBLIC;
|
|
1212
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.rewind_host_export_consumer(text, text, bigint) FROM PUBLIC;
|
|
1213
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.prune_host_export_outbox(text, integer, integer) FROM PUBLIC;
|
|
1214
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.disable_host_export_consumer(text, text) FROM PUBLIC;
|
|
1215
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.retire_host_export_consumer(text, text) FROM PUBLIC;
|
|
1216
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.dead_letter_host_export_head(text, text, uuid, bigint, text) FROM PUBLIC;
|
|
1217
|
+
REVOKE ALL ON FUNCTION opengeni_host_export.host_export_consumer_status(text, text) FROM PUBLIC;
|
|
1218
|
+
|
|
1219
|
+
-- Intentionally no app-role grant. Cross-workspace host projection is an
|
|
1220
|
+
-- operator capability provisioned onto a distinct optional exporter role.
|