@vrplatform/kysely 1.3.45-10 → 1.3.45-21

package/build/migrations-v2/0000000000019-asynchronous-operations.ts

@@ -0,0 +1,736 @@
+import type { Kysely } from 'kysely';
+import { sql } from 'kysely';
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await sql`
+    create table core.operation (
+      id uuid primary key default gen_random_uuid(),
+      tenant_id uuid not null,
+      initiated_by_user_id uuid,
+      type text not null,
+      status text not null default 'queued',
+      failure_code text,
+      failure_message text,
+      created_at timestamptz not null default now(),
+      started_at timestamptz,
+      finished_at timestamptz,
+      updated_at timestamptz not null default now(),
+      source_version bigint not null default 1,
+      retain_until timestamptz,
+      constraint operation_type_check check (type in (
+        'ach-verification',
+        'bank-rule-run',
+        'calendar-block',
+        'calendar-unblock',
+        'connection-archive',
+        'connection-connect',
+        'connection-extract',
+        'connection-sync',
+        'csv-import',
+        'csv-import-preview',
+        'export',
+        'flow-run',
+        'historical-gl-import',
+        'historical-statement-import',
+        'listing-journal-refresh',
+        'plaid-initial-sync',
+        'pms-cutover',
+        'pms-cutover-preview',
+        'provider-payment',
+        'provider-payment-recovery',
+        'recurring-transaction-run',
+        'reservation-journal-refresh',
+        'team-demo-data',
+        'team-duplicate',
+        'team-initialize',
+        'transaction-journal-refresh',
+        'vri-statement-import',
+        'vri-team-migration',
+        'webhook-test'
+      )),
+      constraint operation_status_check
+        check (status in ('queued', 'running', 'completed', 'failed')),
+      constraint operation_failure_check check (
+        (status = 'failed' and failure_code is not null and failure_message is not null)
+        or (status != 'failed' and failure_code is null and failure_message is null)
+      ),
+      constraint operation_finished_check check (
+        (status in ('completed', 'failed') and finished_at is not null)
+        or (status in ('queued', 'running') and finished_at is null)
+      ),
+      constraint operation_source_version_check check (source_version > 0)
+    )
+  `.execute(db);
+  await sql`
+    create index operation_tenant_history_idx
+      on core.operation (tenant_id, created_at desc, id desc)
+  `.execute(db);
+  await sql`
+    create index operation_tenant_status_history_idx
+      on core.operation (tenant_id, status, created_at desc, id desc)
+  `.execute(db);
+  await sql`
+    create index operation_tenant_type_history_idx
+      on core.operation (tenant_id, type, created_at desc, id desc)
+  `.execute(db);
+
+  await sql`
+    create table core.operation_source (
+      operation_id uuid not null references core.operation(id) on delete cascade,
+      source_type text not null,
+      source_id uuid not null,
+      required boolean not null default true,
+      status text not null default 'queued',
+      started_at timestamptz,
+      finished_at timestamptz,
+      source_version bigint not null default 1,
+      primary key (operation_id, source_type, source_id),
+      constraint operation_source_type_check check (source_type in (
+        'audit-action',
+        'export-job',
+        'sync',
+        'webhook-delivery'
+      )),
+      constraint operation_source_status_check
+        check (status in ('queued', 'running', 'completed', 'failed')),
+      constraint operation_source_finished_check check (
+        (status in ('completed', 'failed') and finished_at is not null)
+        or (status in ('queued', 'running') and finished_at is null)
+      ),
+      constraint operation_source_version_check check (source_version > 0),
+      constraint operation_source_unique unique (source_type, source_id)
+    )
+  `.execute(db);
+  await sql`
+    create index operation_source_operation_idx
+      on core.operation_source (operation_id, required, status)
+  `.execute(db);
+
+  await sql`
+    create table core.operation_resource (
+      operation_id uuid not null references core.operation(id) on delete cascade,
+      resource_type text not null,
+      resource_id uuid not null,
+      role text not null,
+      is_primary boolean not null default false,
+      created_at timestamptz not null default now(),
+      primary key (operation_id, resource_type, resource_id, role),
+      constraint operation_resource_type_check check (resource_type in (
+        'bank-rule',
+        'connection',
+        'contact',
+        'file',
+        'flow',
+        'listing',
+        'recurring-transaction-template',
+        'statement',
+        'team',
+        'transaction',
+        'webhook'
+      )),
+      constraint operation_resource_role_check
+        check (role in ('source', 'target', 'result'))
+    )
+  `.execute(db);
+  await sql`
+    create unique index operation_resource_primary_idx
+      on core.operation_resource (operation_id)
+      where is_primary
+  `.execute(db);
+  await sql`
+    create index operation_resource_lookup_idx
+      on core.operation_resource (
+        resource_type,
+        resource_id,
+        operation_id,
+        role
+      )
+  `.execute(db);
+
+  await sql`
+    create table core.operation_projection_outbox (
+      id uuid primary key default gen_random_uuid(),
+      operation_id uuid not null references core.operation(id) on delete cascade,
+      source_version bigint not null,
+      attempts integer not null default 0,
+      available_at timestamptz not null default now(),
+      published_at timestamptz,
+      last_error text,
+      created_at timestamptz not null default now(),
+      constraint operation_projection_outbox_version_check
+        check (source_version > 0),
+      constraint operation_projection_outbox_attempts_check
+        check (attempts >= 0),
+      constraint operation_projection_outbox_unique
+        unique (operation_id, source_version)
+    )
+  `.execute(db);
+  await sql`
+    create index operation_projection_outbox_pending_idx
+      on core.operation_projection_outbox (available_at, created_at, id)
+      where published_at is null
+  `.execute(db);
+
+  await sql`
+    create table core.csv_operation_result (
+      sync_id uuid primary key references core.sync(id) on delete cascade,
+      tenant_id uuid not null,
+      connection_id uuid not null,
+      kind text not null,
+      status text not null default 'queued',
+      result jsonb,
+      error_message text,
+      created_at timestamptz not null default now(),
+      updated_at timestamptz not null default now(),
+      constraint csv_operation_result_kind_check
+        check (kind in ('import', 'preview')),
+      constraint csv_operation_result_status_check
+        check (status in ('queued', 'processing', 'completed', 'failed')),
+      constraint csv_operation_result_terminal_check check (
+        (status = 'completed' and result is not null and error_message is null)
+        or (status = 'failed' and error_message is not null)
+        or (status in ('queued', 'processing') and result is null and error_message is null)
+      )
+    )
+  `.execute(db);
+  await sql`
+    create index csv_operation_result_tenant_connection_idx
+      on core.csv_operation_result (tenant_id, connection_id, created_at desc)
+  `.execute(db);
+
+  await sql`
+    create table core.async_operation_result (
+      operation_id uuid primary key references core.operation(id) on delete cascade,
+      tenant_id uuid not null,
+      type text not null,
+      status text not null default 'queued',
+      result jsonb,
+      error_message text,
+      created_at timestamptz not null default now(),
+      updated_at timestamptz not null default now(),
+      constraint async_operation_result_status_check
+        check (status in ('queued', 'completed', 'failed')),
+      constraint async_operation_result_terminal_check check (
+        (status = 'completed' and result is not null and error_message is null)
+        or (status = 'failed' and error_message is not null)
+        or (status = 'queued' and result is null and error_message is null)
+      )
+    )
+  `.execute(db);
+  await sql`
+    create index async_operation_result_tenant_type_idx
+      on core.async_operation_result (tenant_id, type, created_at desc)
+  `.execute(db);
+
+  await sql`
+    create table core.export_job (
+      id uuid primary key,
+      tenant_id uuid not null,
+      task_id text not null,
+      payload jsonb not null,
+      filename text not null,
+      format text not null,
+      file_id uuid not null,
+      status text not null default 'queued',
+      error_message text,
+      created_at timestamptz not null default now(),
+      started_at timestamptz,
+      finished_at timestamptz,
+      updated_at timestamptz not null default now(),
+      constraint export_job_format_check check (format in ('csv', 'pdf', 'zip')),
+      constraint export_job_status_check
+        check (status in ('queued', 'running', 'completed', 'failed')),
+      constraint export_job_terminal_check check (
+        (status = 'completed' and finished_at is not null and error_message is null)
+        or (status = 'failed' and finished_at is not null and error_message is not null)
+        or (status in ('queued', 'running') and finished_at is null and error_message is null)
+      )
+    )
+  `.execute(db);
+  await sql`
+    create index export_job_tenant_history_idx
+      on core.export_job (tenant_id, created_at desc, id desc)
+  `.execute(db);
+
+  await sql`
+    create function core.refresh_operation_state(target_operation_id uuid)
+    returns void
+    language plpgsql
+    as $$
+    declare
+      next_status text;
+      next_started_at timestamptz;
+      next_finished_at timestamptz;
+    begin
+      select
+        case
+          when count(*) filter (where required) = 0 then 'queued'
+          when bool_and(status = 'completed') filter (where required)
+            then 'completed'
+          when bool_and(status in ('completed', 'failed')) filter (where required)
+            then 'failed'
+          when bool_or(status = 'running') filter (where required)
+            or bool_or(status in ('completed', 'failed')) filter (where required)
+            then 'running'
+          else 'queued'
+        end,
+        min(started_at) filter (where required),
+        max(finished_at) filter (where required)
+      into next_status, next_started_at, next_finished_at
+      from core.operation_source
+      where operation_id = target_operation_id;
+
+      update core.operation
+      set
+        status = next_status,
+        started_at = coalesce(core.operation.started_at, next_started_at),
+        finished_at = case
+          when next_status in ('completed', 'failed')
+            then coalesce(next_finished_at, now())
+          else null
+        end,
+        failure_code = case when next_status = 'failed' then 'operation_failed' end,
+        failure_message = case when next_status = 'failed' then 'Operation failed' end,
+        updated_at = now(),
+        source_version = source_version + 1
+      where id = target_operation_id
+        and (
+          status is distinct from next_status
+          or started_at is distinct from coalesce(started_at, next_started_at)
+          or finished_at is distinct from case
+            when next_status in ('completed', 'failed')
+              then coalesce(next_finished_at, finished_at, now())
+            else null
+          end
+        );
+    end;
+    $$
+  `.execute(db);
+
+  await sql`
+    create function core.refresh_audit_action_source(target_action_id uuid)
+    returns void
+    language plpgsql
+    as $$
+    declare
+      target_operation_id uuid;
+    begin
+      update core.operation_source as source
+      set
+        status = state.status,
+        started_at = state.started_at,
+        finished_at = state.finished_at,
+        source_version = source.source_version + 1
+      from (
+        select
+          case
+            when count(*) = 0 then 'completed'
+            when bool_and(effect.status = 'completed') then 'completed'
+            when bool_and(
+              effect.status in ('completed', 'dead_letter')
+              or (
+                effect.status = 'failed'
+                and effect.retry_count >= effect.max_retries
+              )
+            ) then 'failed'
+            when bool_or(effect.status in ('claimed', 'running'))
+              or bool_or(
+                effect.status in ('completed', 'dead_letter')
+                or (
+                  effect.status = 'failed'
+                  and effect.retry_count >= effect.max_retries
+                )
+              ) then 'running'
+            else 'queued'
+          end as status,
+          case
+            when count(*) = 0 then now()
+            else min(coalesce(effect.last_attempt_at, effect.claimed_at))
+          end as started_at,
+          case
+            when count(*) = 0 then now()
+            when bool_and(
+              effect.status in ('completed', 'dead_letter')
+              or (
+                effect.status = 'failed'
+                and effect.retry_count >= effect.max_retries
+              )
+            ) then coalesce(
+              max(coalesce(effect.completed_at, effect.last_attempt_at)),
+              now()
+            )
+          end as finished_at
+        from audit.effect
+        where action_id = target_action_id
+      ) as state
+      where source.source_type = 'audit-action'
+        and source.source_id = target_action_id
+        and (
+          source.status is distinct from state.status
+          or source.started_at is distinct from state.started_at
+          or source.finished_at is distinct from state.finished_at
+        );
+
+      for target_operation_id in
+        select operation_id
+        from core.operation_source
+        where source_type = 'audit-action'
+          and source_id = target_action_id
+      loop
+        perform core.refresh_operation_state(target_operation_id);
+      end loop;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create function core.refresh_audit_action_operation()
+    returns trigger
+    language plpgsql
+    as $$
+    begin
+      perform core.refresh_audit_action_source(
+        coalesce(new.action_id, old.action_id)
+      );
+      return coalesce(new, old);
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger refresh_audit_action_operation
+    after insert or update of status, retry_count, claimed_at, completed_at, last_attempt_at
+    or delete on audit.effect
+    for each row execute function core.refresh_audit_action_operation()
+  `.execute(db);
+
+  await sql`
+    create function core.refresh_sync_operation()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      target_operation_id uuid;
+    begin
+      update core.operation_source as source
+      set
+        status = case
+          when new.status = 'started' then 'running'
+          when new.status = 'completed' then 'completed'
+          when new.status in ('failed', 'canceled') then 'failed'
+          else 'queued'
+        end,
+        started_at = case
+          when new.status = 'started'
+            then coalesce(source.started_at, new.updated_at)
+          else source.started_at
+        end,
+        finished_at = case
+          when new.status in ('completed', 'failed', 'canceled')
+            then coalesce(new.synced_at, new.updated_at, now())
+        end,
+        source_version = source.source_version + 1
+      where source.source_type = 'sync'
+        and source.source_id = new.id;
+
+      for target_operation_id in
+        select operation_id
+        from core.operation_source
+        where source_type = 'sync' and source_id = new.id
+      loop
+        perform core.refresh_operation_state(target_operation_id);
+      end loop;
+      return new;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger refresh_sync_operation
+    after insert or update of status, synced_at on core.sync
+    for each row execute function core.refresh_sync_operation()
+  `.execute(db);
+
+  await sql`
+    create function core.refresh_export_job_operation()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      target_operation_id uuid;
+    begin
+      update core.operation_source as source
+      set
+        status = new.status,
+        started_at = coalesce(source.started_at, new.started_at),
+        finished_at = new.finished_at,
+        source_version = source.source_version + 1
+      where source.source_type = 'export-job'
+        and source.source_id = new.id;
+
+      for target_operation_id in
+        select operation_id
+        from core.operation_source
+        where source_type = 'export-job' and source_id = new.id
+      loop
+        perform core.refresh_operation_state(target_operation_id);
+      end loop;
+      return new;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger refresh_export_job_operation
+    after insert or update of status, started_at, finished_at
+    on core.export_job
+    for each row execute function core.refresh_export_job_operation()
+  `.execute(db);
+
+  await sql`
+    create function core.refresh_webhook_delivery_operation()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      target_operation_id uuid;
+    begin
+      update core.operation_source as source
+      set
+        status = case
+          when new.status in ('queued', 'retrying') then 'queued'
+          when new.status = 'delivering' then 'running'
+          when new.status = 'succeeded' then 'completed'
+          else 'failed'
+        end,
+        started_at = case
+          when new.status = 'delivering'
+            then coalesce(source.started_at, new.claimed_at, new.updated_at)
+          else source.started_at
+        end,
+        finished_at = case
+          when new.status = 'succeeded'
+            then coalesce(new.delivered_at, new.updated_at, now())
+          when new.status in ('failed', 'canceled')
+            then coalesce(new.failed_at, new.updated_at, now())
+        end,
+        source_version = source.source_version + 1
+      where source.source_type = 'webhook-delivery'
+        and source.source_id = new.id;
+
+      for target_operation_id in
+        select operation_id
+        from core.operation_source
+        where source_type = 'webhook-delivery' and source_id = new.id
+      loop
+        perform core.refresh_operation_state(target_operation_id);
+      end loop;
+      return new;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger refresh_webhook_delivery_operation
+    after insert or update of status, claimed_at, delivered_at, failed_at
+    on webhook.delivery
+    for each row execute function core.refresh_webhook_delivery_operation()
+  `.execute(db);
+
+  await sql`
+    create function core.queue_operation_projection()
+    returns trigger
+    language plpgsql
+    as $$
+    begin
+      if exists (
+        select 1
+        from core.operation_resource
+        where operation_id = new.id and resource_type = 'team'
+      ) then
+        insert into core.operation_projection_outbox (
+          operation_id,
+          source_version
+        ) values (new.id, new.source_version)
+        on conflict (operation_id, source_version) do nothing;
+      end if;
+      return new;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger queue_operation_projection
+    after insert or update of source_version on core.operation
+    for each row execute function core.queue_operation_projection()
+  `.execute(db);
+
+  await sql`
+    create function core.operation_resource_changed()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      target_operation_id uuid;
+    begin
+      target_operation_id := coalesce(new.operation_id, old.operation_id);
+      update core.operation
+      set updated_at = now(), source_version = source_version + 1
+      where id = target_operation_id;
+      return coalesce(new, old);
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger operation_resource_changed
+    after insert or update or delete on core.operation_resource
+    for each row execute function core.operation_resource_changed()
+  `.execute(db);
+
+  await sql`
+    insert into core.operation (
+      id,
+      tenant_id,
+      initiated_by_user_id,
+      type,
+      status,
+      created_at,
+      updated_at
+    )
+    select
+      action.id,
+      action.tenant_id,
+      action.actor_id,
+      'vri-statement-import',
+      'queued',
+      action.created_at,
+      action.created_at
+    from audit.action as action
+    where action.action_type = 'partner.vri_to_vrt.statements.create'
+      and exists (
+        select 1 from audit.effect where action_id = action.id
+      )
+    on conflict (id) do nothing
+  `.execute(db);
+  await sql`
+    insert into core.operation_source (
+      operation_id,
+      source_type,
+      source_id
+    )
+    select id, 'audit-action', id
+    from core.operation
+    where type = 'vri-statement-import'
+    on conflict (source_type, source_id) do nothing
+  `.execute(db);
+  await sql`
+    insert into core.operation_resource (
+      operation_id,
+      resource_type,
+      resource_id,
+      role,
+      is_primary
+    )
+    select
+      operation.id,
+      'team',
+      coalesce(action.root_entity_id, effect.entity_id),
+      'target',
+      true
+    from core.operation as operation
+    inner join audit.action as action on action.id = operation.id
+    left join lateral (
+      select entity_id
+      from audit.effect
+      where action_id = action.id and entity_id is not null
+      order by created_at, id
+      limit 1
+    ) as effect on true
+    where operation.type = 'vri-statement-import'
+      and coalesce(action.root_entity_id, effect.entity_id) is not null
+    on conflict do nothing
+  `.execute(db);
+  await sql`
+    do $$
+    declare operation_row record;
+    begin
+      for operation_row in
+        select id from core.operation where type = 'vri-statement-import'
+      loop
+        update audit.effect set status = status
+        where action_id = operation_row.id
+          and id = (
+            select id from audit.effect
+            where action_id = operation_row.id
+            order by created_at, id
+            limit 1
+          );
+      end loop;
+    end;
+    $$
+  `.execute(db);
+
+  await sql`
+    do $$
+    begin
+      if exists (select 1 from pg_roles where rolname = 'application') then
+        grant usage on schema core to application;
+        grant select, insert, update, delete
+          on core.operation,
+             core.operation_source,
+             core.operation_resource,
+             core.operation_projection_outbox,
+             core.csv_operation_result,
+             core.async_operation_result,
+             core.export_job
+          to application;
+        grant execute on function core.refresh_operation_state(uuid),
+                                  core.refresh_audit_action_source(uuid)
+          to application;
+      end if;
+    end
+    $$
+  `.execute(db);
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await sql`drop trigger if exists operation_resource_changed on core.operation_resource`.execute(
+    db
+  );
+  await sql`drop function if exists core.operation_resource_changed`.execute(
+    db
+  );
+  await sql`drop trigger if exists queue_operation_projection on core.operation`.execute(
+    db
+  );
+  await sql`drop function if exists core.queue_operation_projection`.execute(
+    db
+  );
+  await sql`drop trigger if exists refresh_webhook_delivery_operation on webhook.delivery`.execute(
+    db
+  );
+  await sql`drop function if exists core.refresh_webhook_delivery_operation`.execute(
+    db
+  );
+  await sql`drop trigger if exists refresh_export_job_operation on core.export_job`.execute(
+    db
+  );
+  await sql`drop function if exists core.refresh_export_job_operation`.execute(
+    db
+  );
+  await sql`drop trigger if exists refresh_sync_operation on core.sync`.execute(
+    db
+  );
+  await sql`drop function if exists core.refresh_sync_operation`.execute(db);
+  await sql`drop trigger if exists refresh_audit_action_operation on audit.effect`.execute(
+    db
+  );
+  await sql`drop function if exists core.refresh_audit_action_operation`.execute(
+    db
+  );
+  await sql`drop function if exists core.refresh_audit_action_source`.execute(
+    db
+  );
+  await sql`drop function if exists core.refresh_operation_state`.execute(db);
+  await sql`drop table if exists core.async_operation_result`.execute(db);
+  await sql`drop table if exists core.export_job`.execute(db);
+  await sql`drop table if exists core.csv_operation_result`.execute(db);
+  await sql`drop table if exists core.operation_projection_outbox`.execute(db);
+  await sql`drop table if exists core.operation_resource`.execute(db);
+  await sql`drop table if exists core.operation_source`.execute(db);
+  await sql`drop table if exists core.operation`.execute(db);
+}