@axinom/mosaic-db-common 0.35.0-rc.9 → 0.36.0-rc.0

package/migrations/define/define-functions.sql

@@ -1151,3 +1151,172 @@ BEGIN
     RETURN counter;
 END;
 $$;
+
+/*-snippet
+{
+  "body": [
+    "SELECT ax_define.create_trx_table(",
+    "  '${1:app_hidden}',",
+    "  '${2|inbox,outbox|}',",
+    "  '{${3::DATABASE_ENV_OWNER},${4::DATABASE_GQL_ROLE}}',",
+    "  '${5|sequential,parallel|}');"
+  ],
+  "description": [
+   "Create a new table for transaction inbox/outbox handling. \n"
+  ]
+}
+snippet-*/
+CREATE OR REPLACE FUNCTION ax_define.create_trx_table (
+  trx_schema TEXT,
+  trx_table_name TEXT,
+  additional_roles_to_grant TEXT[],
+  concurrency TEXT
+) RETURNS VOID
+  LANGUAGE plpgsql
+  AS $$
+DECLARE 
+  role_ TEXT;
+BEGIN
+  EXECUTE 'DROP TABLE IF EXISTS ' || trx_schema || '.'|| trx_table_name ||' CASCADE;';
+  EXECUTE 'CREATE TABLE ' || trx_schema || '.'|| trx_table_name ||' (
+            id uuid PRIMARY KEY,
+            aggregate_type TEXT NOT NULL,
+            aggregate_id TEXT NOT NULL, 
+            message_type TEXT NOT NULL,
+            payload JSONB NOT NULL,
+            metadata JSONB,
+            created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
+            processed_at TIMESTAMPTZ,
+            started_attempts smallint NOT NULL DEFAULT 0,
+            finished_attempts smallint NOT NULL DEFAULT 0,
+            segment TEXT,
+            locked_until TIMESTAMPTZ NOT NULL DEFAULT to_timestamp(0),
+            concurrency TEXT NOT NULL DEFAULT ''' || concurrency || ''',
+            abandoned_at TIMESTAMPTZ
+          );';
+  EXECUTE 'ALTER TABLE ' || trx_schema || '.'|| trx_table_name ||' ADD CONSTRAINT '|| trx_table_name ||'_concurrency_check
+            CHECK (concurrency IN (''sequential'', ''parallel''));';
+  
+  -- The owner role has full access - no grants needed
+  -- Grants for additional roles are given here
+  FOREACH role_ IN ARRAY additional_roles_to_grant LOOP
+    EXECUTE 'GRANT SELECT, INSERT, DELETE ON ' || trx_schema || '.'|| trx_table_name ||' TO ' || role_ || ';';
+    EXECUTE 'GRANT UPDATE (locked_until, processed_at, abandoned_at, started_attempts, finished_attempts) ON ' || trx_schema || '.'|| trx_table_name ||' TO ' || role_ || ';';
+  END LOOP;  
+
+  EXECUTE 'SELECT ax_define.define_index(''segment'', '''|| trx_table_name ||''', '''||trx_schema||''');';
+  EXECUTE 'SELECT ax_define.define_index(''created_at'', '''|| trx_table_name ||''', ''' || trx_schema || ''');';
+  EXECUTE 'SELECT ax_define.define_index(''processed_at'', '''|| trx_table_name ||''', ''' || trx_schema || ''');';
+  EXECUTE 'SELECT ax_define.define_index(''abandoned_at'', '''|| trx_table_name ||''', ''' || trx_schema || ''');';
+  EXECUTE 'SELECT ax_define.define_index(''locked_until'', '''|| trx_table_name ||''', ''' || trx_schema || ''');';
+END;
+$$;
+
+/*-snippet
+{
+  "body": [
+    "SELECT ax_define.create_trx_next_messages_function(",
+    "  '${1:app_hidden}',",
+    "  '${2:app_hidden}',",
+    "  '${3|inbox,outbox|}');"
+  ],
+  "description": [
+   "Create next_messages function that is used to poll the inbox/outbox tables. \n"
+  ]
+}
+snippet-*/
+CREATE OR REPLACE FUNCTION ax_define.create_trx_next_messages_function(
+  trx_next_message_function_schema TEXT,
+  trx_table_schema TEXT,
+  trx_table_name TEXT
+) RETURNS VOID
+  LANGUAGE plpgsql
+  AS $$
+BEGIN
+  EXECUTE 'DROP FUNCTION IF EXISTS ' || trx_next_message_function_schema || '.next_'|| trx_table_name ||'_messages(integer, integer);';
+  EXECUTE 'CREATE OR REPLACE FUNCTION ' || trx_next_message_function_schema || '.next_'|| trx_table_name ||'_messages(
+  max_size integer, lock_ms integer)
+    RETURNS SETOF ' || trx_table_schema || '.'|| trx_table_name ||' 
+    LANGUAGE ''plpgsql''
+
+AS $BODY$
+DECLARE 
+  loop_row ' || trx_table_schema || '.'|| trx_table_name ||'%ROWTYPE;
+  message_row ' || trx_table_schema || '.'|| trx_table_name ||'%ROWTYPE;
+  ids uuid[] := ''{}'';
+BEGIN
+
+  IF max_size < 1 THEN
+    RAISE EXCEPTION ''The max_size for the next messages batch must be at least one.'' using errcode = ''MAXNR'';
+  END IF;
+
+  -- get (only) the oldest message of every segment but only return it if it is not locked
+  FOR loop_row IN
+    SELECT * FROM ' || trx_table_schema || '.'|| trx_table_name ||' m WHERE m.id in (SELECT DISTINCT ON (segment) id
+      FROM ' || trx_table_schema || '.'|| trx_table_name ||'
+      WHERE processed_at IS NULL AND abandoned_at IS NULL
+      ORDER BY segment, created_at) order by created_at
+  LOOP
+    BEGIN
+      EXIT WHEN cardinality(ids) >= max_size;
+    
+      SELECT *
+        INTO message_row
+        FROM ' || trx_table_schema || '.'|| trx_table_name ||'
+        WHERE id = loop_row.id
+        FOR NO KEY UPDATE NOWAIT; -- throw/catch error when locked
+      
+      IF message_row.locked_until > NOW() THEN
+        CONTINUE;
+      END IF;
+      
+      ids := array_append(ids, message_row.id);
+    EXCEPTION 
+      WHEN lock_not_available THEN
+        CONTINUE;
+      WHEN serialization_failure THEN
+        CONTINUE;
+    END;
+  END LOOP;
+  
+  -- if max_size not reached: get the oldest parallelizable message independent of segment
+  IF cardinality(ids) < max_size THEN
+    FOR loop_row IN
+      SELECT * FROM ' || trx_table_schema || '.'|| trx_table_name ||'
+        WHERE concurrency = ''parallel'' AND processed_at IS NULL AND abandoned_at IS NULL AND locked_until < NOW() 
+          AND id NOT IN (SELECT UNNEST(ids))
+        order by created_at
+    LOOP
+      BEGIN
+        EXIT WHEN cardinality(ids) >= max_size;
+
+        SELECT *
+          INTO message_row
+          FROM ' || trx_table_schema || '.'|| trx_table_name ||'
+          WHERE id = loop_row.id
+          FOR NO KEY UPDATE NOWAIT; -- throw/catch error when locked
+
+        ids := array_append(ids, message_row.id);
+      EXCEPTION 
+        WHEN lock_not_available THEN
+          CONTINUE;
+        WHEN serialization_failure THEN
+          CONTINUE;
+      END;
+    END LOOP;
+  END IF;
+  
+  -- set a short lock value so the the workers can each process a message
+  IF cardinality(ids) > 0 THEN
+
+    RETURN QUERY 
+      UPDATE ' || trx_table_schema || '.'|| trx_table_name ||'
+        SET locked_until = clock_timestamp() + (lock_ms || '' milliseconds'')::INTERVAL, started_attempts = started_attempts + 1
+        WHERE ID = ANY(ids)
+        RETURNING *;
+
+  END IF;
+END;
+$BODY$;';
+END;
+$$;

package/migrations/snippets/custom.code-snippets

@@ -251,5 +251,41 @@
       "Make sure the dropped value is not used in other tables.\n",
       "This snippet has a simplified migration sample that might require adjustments."
     ]
+  },
+  "Create Transaction Inbox Outbox Tables and Functions (Ax Custom)": {
+    "scope": "sql",
+    "prefix": "ax-add-trx-inbox-outbox-tables-and-functions",
+    "body": [
+      "-- Create Transaction Inbox Table",
+      "SELECT ax_define.create_trx_table(",
+      "'${1:app_hidden}',",
+      "'${2:inbox}',",
+      "'{${3::DATABASE_ENV_OWNER},${4::DATABASE_GQL_ROLE}}',",
+      "'${5|sequential,parallel|}');",
+      "",
+      "-- Create Transaction Outbox Table",
+      "SELECT ax_define.create_trx_table(",
+      "'${1:app_hidden}',",
+      "'${6:outbox}',",
+      "'{${3::DATABASE_ENV_OWNER},${4::DATABASE_GQL_ROLE}}',",
+      "'${7|sequential,parallel|}');",
+      "",
+      "-- Create Transaction Inbox Next Message function",
+      "SELECT ax_define.create_trx_next_messages_function(",
+      "'${8:app_hidden}',",
+      "'${1:app_hidden}',",
+      "'${2:inbox}');",
+      "",
+      "-- Create Transaction Outbox Next Message function",
+      "SELECT ax_define.create_trx_next_messages_function(",
+      "'${8:app_hidden}',",
+      "'${1:app_hidden}',",
+      "'${6:outbox}');"
+    ],
+    "description": [
+      "Drops a value from the enum table.\n",
+      "Make sure the dropped value is not used in other tables.\n",
+      "This snippet has a simplified migration sample that might require adjustments."
+    ]
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axinom/mosaic-db-common",
-  "version": "0.35.0-rc.9",
+  "version": "0.36.0-rc.0",
   "description": "This library encapsulates database-related functionality to develop Mosaic based services.",
   "author": "Axinom",
   "license": "PROPRIETARY",
@@ -54,5 +54,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "f4c4a0b3a69aca4e15ec878adc9c3dd046b8447b"
+  "gitHead": "a853ae03afed074c98b13b05d781e144c041955d"
 }