envio 2.8.1 → 2.9.0

package/src/db/EntityHistory.res

@@ -0,0 +1,335 @@
+open Table
+
+module RowAction = {
+  type t = SET | DELETE
+  let variants = [SET, DELETE]
+  let name = "ENTITY_HISTORY_ROW_ACTION"
+  let enum = Enum.make(~name, ~variants)
+}
+
+type historyFieldsGeneral<'a> = {
+  chain_id: 'a,
+  block_timestamp: 'a,
+  block_number: 'a,
+  log_index: 'a,
+}
+
+type historyFields = historyFieldsGeneral<int>
+
+type entityIdOnly = {id: string}
+let entityIdOnlySchema = S.schema(s => {id: s.matches(S.string)})
+type entityData<'entity> = Delete(entityIdOnly) | Set('entity)
+
+type historyRow<'entity> = {
+  current: historyFields,
+  previous: option<historyFields>,
+  entityData: entityData<'entity>,
+}
+
+type previousHistoryFields = historyFieldsGeneral<option<int>>
+
+//For flattening the optional previous fields into their own individual nullable fields
+let previousHistoryFieldsSchema = S.object(s => {
+  chain_id: s.field("previous_entity_history_chain_id", S.null(S.int)),
+  block_timestamp: s.field("previous_entity_history_block_timestamp", S.null(S.int)),
+  block_number: s.field("previous_entity_history_block_number", S.null(S.int)),
+  log_index: s.field("previous_entity_history_log_index", S.null(S.int)),
+})
+
+let currentHistoryFieldsSchema = S.object(s => {
+  chain_id: s.field("entity_history_chain_id", S.int),
+  block_timestamp: s.field("entity_history_block_timestamp", S.int),
+  block_number: s.field("entity_history_block_number", S.int),
+  log_index: s.field("entity_history_log_index", S.int),
+})
+
+let makeHistoryRowSchema: S.t<'entity> => S.t<historyRow<'entity>> = entitySchema => {
+  //Maps a schema object for the given entity with all fields nullable except for the id field
+  //Keeps any original nullable fields
+  let nullableEntitySchema: S.t<Js.Dict.t<unknown>> = S.schema(s =>
+    switch entitySchema->S.classify {
+    | Object({items}) =>
+      let nulldict = Js.Dict.empty()
+      items->Belt.Array.forEach(({location, schema}) => {
+        let nullableFieldSchema = switch (location, schema->S.classify) {
+        | ("id", _)
+        | (_, Null(_)) => schema //TODO double check this works for array types
+        | _ => S.null(schema)->S.toUnknown
+        }
+
+        nulldict->Js.Dict.set(location, s.matches(nullableFieldSchema))
+      })
+      nulldict
+    | _ =>
+      Js.Exn.raiseError(
+        "Failed creating nullableEntitySchema. Expected an object schema for entity",
+      )
+    }
+  )
+
+  let previousWithNullFields = {
+    chain_id: None,
+    block_timestamp: None,
+    block_number: None,
+    log_index: None,
+  }
+
+  S.object(s => {
+    {
+      "current": s.flatten(currentHistoryFieldsSchema),
+      "previous": s.flatten(previousHistoryFieldsSchema),
+      "entityData": s.flatten(nullableEntitySchema),
+      "action": s.field("action", RowAction.enum.schema),
+    }
+  })->S.transform(s => {
+    parser: v => {
+      current: v["current"],
+      previous: switch v["previous"] {
+      | {
+          chain_id: Some(chain_id),
+          block_timestamp: Some(block_timestamp),
+          block_number: Some(block_number),
+          log_index: Some(log_index),
+        } =>
+        Some({
+          chain_id,
+          block_timestamp,
+          block_number,
+          log_index,
+        })
+      | {chain_id: None, block_timestamp: None, block_number: None, log_index: None} => None
+      | _ => s.fail("Unexpected mix of null and non-null values in previous history fields")
+      },
+      entityData: switch v["action"] {
+      | SET => v["entityData"]->(Utils.magic: Js.Dict.t<unknown> => 'entity)->Set
+      | DELETE =>
+        let {id} = v["entityData"]->(Utils.magic: Js.Dict.t<unknown> => entityIdOnly)
+        Delete({id: id})
+      },
+    },
+    serializer: v => {
+      let (entityData, action) = switch v.entityData {
+      | Set(entityData) => (entityData->(Utils.magic: 'entity => Js.Dict.t<unknown>), RowAction.SET)
+      | Delete(entityIdOnly) => (
+          entityIdOnly->(Utils.magic: entityIdOnly => Js.Dict.t<unknown>),
+          DELETE,
+        )
+      }
+
+      {
+        "current": v.current,
+        "entityData": entityData,
+        "action": action,
+        "previous": switch v.previous {
+        | Some(historyFields) =>
+          historyFields->(Utils.magic: historyFields => previousHistoryFields) //Cast to previousHistoryFields (with "Some" field values)
+        | None => previousWithNullFields
+        },
+      }
+    },
+  })
+}
+
+type t<'entity> = {
+  table: table,
+  createInsertFnQuery: string,
+  schema: S.t<historyRow<'entity>>,
+  schemaRows: S.t<array<historyRow<'entity>>>,
+  insertFn: (Postgres.sql, Js.Json.t, ~shouldCopyCurrentEntity: bool) => promise<unit>,
+}
+
+let insertRow = (
+  self: t<'entity>,
+  ~sql,
+  ~historyRow: historyRow<'entity>,
+  ~shouldCopyCurrentEntity,
+) => {
+  let row = historyRow->S.serializeOrRaiseWith(self.schema)
+  self.insertFn(sql, row, ~shouldCopyCurrentEntity)
+}
+
+let batchInsertRows = (
+  self: t<'entity>,
+  ~sql,
+  ~rows: array<historyRow<'entity>>,
+  ~shouldCopyCurrentEntity,
+) => {
+  let rows =
+    rows->S.serializeOrRaiseWith(self.schemaRows)->(Utils.magic: Js.Json.t => array<Js.Json.t>)
+  rows
+  ->Belt.Array.map(row => {
+    self.insertFn(sql, row, ~shouldCopyCurrentEntity)
+  })
+  ->Promise.all
+  ->Promise.thenResolve(_ => ())
+}
+
+type entityInternal
+
+external castInternal: t<'entity> => t<entityInternal> = "%identity"
+
+let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
+  let entity_history_block_timestamp = "entity_history_block_timestamp"
+  let entity_history_chain_id = "entity_history_chain_id"
+  let entity_history_block_number = "entity_history_block_number"
+  let entity_history_log_index = "entity_history_log_index"
+
+  //NB: Ordered by hirarchy of event ordering
+  let currentChangeFieldNames = [
+    entity_history_block_timestamp,
+    entity_history_chain_id,
+    entity_history_block_number,
+    entity_history_log_index,
+  ]
+
+  let currentHistoryFields =
+    currentChangeFieldNames->Belt.Array.map(fieldName =>
+      mkField(fieldName, Integer, ~isPrimaryKey=true)
+    )
+
+  let previousChangeFieldNames =
+    currentChangeFieldNames->Belt.Array.map(fieldName => "previous_" ++ fieldName)
+
+  let previousHistoryFields =
+    previousChangeFieldNames->Belt.Array.map(fieldName =>
+      mkField(fieldName, Integer, ~isNullable=true)
+    )
+
+  let id = "id"
+
+  let dataFields = table.fields->Belt.Array.keepMap(field =>
+    switch field {
+    | Field(field) =>
+      switch field.fieldName {
+      //id is not nullable and should be part of the pk
+      | "id" => {...field, fieldName: id, isPrimaryKey: true}->Field->Some
+      //db_write_timestamp can be removed for this. TODO: remove this when we depracate
+      //automatic db_write_timestamp creation
+      | "db_write_timestamp" => None
+      | _ =>
+        {
+          ...field,
+          isNullable: true, //All entity fields are nullable in the case
+          isIndex: false, //No need to index any additional entity data fields in entity history
+        }
+        ->Field
+        ->Some
+      }
+
+    | DerivedFrom(_) => None
+    }
+  )
+
+  let actionFieldName = "action"
+
+  let actionField = mkField(actionFieldName, Custom(RowAction.enum.name))
+
+  let serialField = mkField("serial", Serial, ~isNullable=true, ~isIndex=true)
+
+  let dataFieldNames = dataFields->Belt.Array.map(field => field->getFieldName)
+
+  let originTableName = table.tableName
+  let historyTableName = originTableName ++ "_history"
+  //ignore composite indices
+  let table = mkTable(
+    historyTableName,
+    ~fields=Belt.Array.concatMany([
+      currentHistoryFields,
+      previousHistoryFields,
+      dataFields,
+      [actionField, serialField],
+    ]),
+  )
+
+  let insertFnName = `"insert_${table.tableName}"`
+  let historyRowArg = "history_row"
+  let historyTablePath = `"public"."${historyTableName}"`
+  let originTablePath = `"public"."${originTableName}"`
+
+  let previousHistoryFieldsAreNullStr =
+    previousChangeFieldNames
+    ->Belt.Array.map(fieldName => `${historyRowArg}.${fieldName} IS NULL`)
+    ->Js.Array2.joinWith(" OR ")
+
+  let currentChangeFieldNamesCommaSeparated = currentChangeFieldNames->Js.Array2.joinWith(", ")
+
+  let dataFieldNamesDoubleQuoted = dataFieldNames->Belt.Array.map(fieldName => `"${fieldName}"`)
+  let dataFieldNamesCommaSeparated = dataFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")
+
+  let allFieldNamesDoubleQuoted =
+    Belt.Array.concatMany([
+      currentChangeFieldNames,
+      previousChangeFieldNames,
+      dataFieldNames,
+      [actionFieldName],
+    ])->Belt.Array.map(fieldName => `"${fieldName}"`)
+
+  let createInsertFnQuery = {
+    `CREATE OR REPLACE FUNCTION ${insertFnName}(${historyRowArg} ${historyTablePath}, should_copy_current_entity BOOLEAN)
+      RETURNS void AS $$
+      DECLARE
+        v_previous_record RECORD;
+        v_origin_record RECORD;
+      BEGIN
+        -- Check if previous values are not provided
+        IF ${previousHistoryFieldsAreNullStr} THEN
+          -- Find the most recent record for the same id
+          SELECT ${currentChangeFieldNamesCommaSeparated} INTO v_previous_record
+          FROM ${historyTablePath}
+          WHERE ${id} = ${historyRowArg}.${id}
+          ORDER BY ${currentChangeFieldNames
+      ->Belt.Array.map(fieldName => fieldName ++ " DESC")
+      ->Js.Array2.joinWith(", ")}
+          LIMIT 1;
+
+          -- If a previous record exists, use its values
+          IF FOUND THEN
+            ${Belt.Array.zip(currentChangeFieldNames, previousChangeFieldNames)
+      ->Belt.Array.map(((currentFieldName, previousFieldName)) => {
+        `${historyRowArg}.${previousFieldName} := v_previous_record.${currentFieldName};`
+      })
+      ->Js.Array2.joinWith(" ")}
+            ElSIF should_copy_current_entity THEN
+            -- Check if a value for the id exists in the origin table and if so, insert a history row for it.
+            SELECT ${dataFieldNamesCommaSeparated} FROM ${originTablePath} WHERE id = ${historyRowArg}.${id} INTO v_origin_record;
+            IF FOUND THEN
+              INSERT INTO ${historyTablePath} (${currentChangeFieldNamesCommaSeparated}, ${dataFieldNamesCommaSeparated}, "${actionFieldName}")
+              -- SET the current change data fields to 0 since we don't know what they were
+              -- and it doesn't matter provided they are less than any new values
+              VALUES (${currentChangeFieldNames
+      ->Belt.Array.map(_ => "0")
+      ->Js.Array2.joinWith(", ")}, ${dataFieldNames
+      ->Belt.Array.map(fieldName => `v_origin_record."${fieldName}"`)
+      ->Js.Array2.joinWith(", ")}, 'SET');
+
+              ${previousChangeFieldNames
+      ->Belt.Array.map(previousFieldName => {
+        `${historyRowArg}.${previousFieldName} := 0;`
+      })
+      ->Js.Array2.joinWith(" ")}
+            END IF;
+          END IF;
+        END IF;
+
+        INSERT INTO ${historyTablePath} (${allFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")})
+        VALUES (${allFieldNamesDoubleQuoted
+      ->Belt.Array.map(fieldName => `${historyRowArg}.${fieldName}`)
+      ->Js.Array2.joinWith(", ")});
+      END;
+      $$ LANGUAGE plpgsql;
+      `
+  }
+
+  let insertFnString = `(sql, rowArgs, shouldCopyCurrentEntity) =>
+      sql\`select ${insertFnName}(ROW(${allFieldNamesDoubleQuoted
+    ->Belt.Array.map(fieldNameDoubleQuoted => `\${rowArgs[${fieldNameDoubleQuoted}]\}`)
+    ->Js.Array2.joinWith(", ")}, NULL),  --NULL argument for SERIAL field
+    \${shouldCopyCurrentEntity});\``
+
+  let insertFn: (Postgres.sql, Js.Json.t, ~shouldCopyCurrentEntity: bool) => promise<unit> =
+    insertFnString->Table.PostgresInterop.eval
+
+  let schema = makeHistoryRowSchema(schema)
+
+  {table, createInsertFnQuery, schema, schemaRows: S.array(schema), insertFn}
+}

package/src/db/Schema.res

@@ -0,0 +1,18 @@
+open Belt
+type t = dict<Table.table>
+
+let make = (tables: array<Table.table>) => {
+  tables->Array.map(table => (table.tableName, table))->Js.Dict.fromArray
+}
+
+exception UndefinedEntity(Table.derivedFromField)
+exception UndefinedFieldInEntity(Table.derivedFromField)
+let getDerivedFromFieldName = (schema: t, derivedFromField: Table.derivedFromField) =>
+  switch schema->Utils.Dict.dangerouslyGetNonOption(derivedFromField.derivedFromEntity) {
+  | Some(entity) =>
+    switch entity->Table.getFieldByName(derivedFromField.derivedFromField) {
+    | Some(field) => field->Table.getFieldName->Ok
+    | None => Error(UndefinedFieldInEntity(derivedFromField)) //Unexpected, schema should be parsed on codegen
+    }
+  | None => Error(UndefinedEntity(derivedFromField)) //Unexpected, schema should be parsed on codegen
+  }

package/src/db/Table.res

@@ -0,0 +1,251 @@
+open Belt
+
+type primitive
+type derived
+@unboxed
+type fieldType =
+  | @as("INTEGER") Integer
+  | @as("BOOLEAN") Boolean
+  | @as("NUMERIC") Numeric
+  | @as("DOUBLE PRECISION") DoublePrecision
+  | @as("TEXT") Text
+  | @as("SERIAL") Serial
+  | @as("JSONB") JsonB
+  | @as("TIMESTAMP WITH TIME ZONE") Timestamp
+  | @as("TIMESTAMP") TimestampWithoutTimezone
+  | @as("TIMESTAMP WITH TIME ZONE NULL") TimestampWithNullTimezone
+  | Custom(string)
+
+type field = {
+  fieldName: string,
+  fieldType: fieldType,
+  isArray: bool,
+  isNullable: bool,
+  isPrimaryKey: bool,
+  isIndex: bool,
+  linkedEntity: option<string>,
+  defaultValue: option<string>,
+}
+
+type derivedFromField = {
+  fieldName: string,
+  derivedFromEntity: string,
+  derivedFromField: string,
+}
+
+type fieldOrDerived = Field(field) | DerivedFrom(derivedFromField)
+
+let mkField = (
+  ~default=?,
+  ~isArray=false,
+  ~isNullable=false,
+  ~isPrimaryKey=false,
+  ~isIndex=false,
+  ~linkedEntity=?,
+  fieldName,
+  fieldType,
+) =>
+  {
+    fieldName,
+    fieldType,
+    isArray,
+    isNullable,
+    isPrimaryKey,
+    isIndex,
+    linkedEntity,
+    defaultValue: default,
+  }->Field
+
+let mkDerivedFromField = (fieldName, ~derivedFromEntity, ~derivedFromField) =>
+  {
+    fieldName,
+    derivedFromField,
+    derivedFromEntity,
+  }->DerivedFrom
+
+let getUserDefinedFieldName = fieldOrDerived =>
+  switch fieldOrDerived {
+  | Field({fieldName})
+  | DerivedFrom({fieldName}) => fieldName
+  }
+
+let isLinkedEntityField = field => field.linkedEntity->Option.isSome
+
+let getDbFieldName = field =>
+  field->isLinkedEntityField ? field.fieldName ++ "_id" : field.fieldName
+
+let getFieldName = fieldOrDerived =>
+  switch fieldOrDerived {
+  | Field(field) => field->getDbFieldName
+  | DerivedFrom({fieldName}) => fieldName
+  }
+
+type table = {
+  tableName: string,
+  fields: array<fieldOrDerived>,
+  compositeIndices: array<array<string>>,
+}
+
+let mkTable: 'b. (
+  ~compositeIndices: array<array<string>>=?,
+  ~fields: array<fieldOrDerived>,
+  string,
+) => 'c = (~compositeIndices=[], ~fields, tableName) => {
+  tableName,
+  fields,
+  compositeIndices,
+}
+
+let getPrimaryKeyFieldNames = table =>
+  table.fields->Array.keepMap(field =>
+    switch field {
+    | Field({isPrimaryKey: true, fieldName}) => Some(fieldName)
+    | _ => None
+    }
+  )
+
+let getFields = table =>
+  table.fields->Array.keepMap(field =>
+    switch field {
+    | Field(field) => Some(field)
+    | DerivedFrom(_) => None
+    }
+  )
+
+let getNonDefaultFields = table =>
+  table.fields->Array.keepMap(field =>
+    switch field {
+    | Field(field) if field.defaultValue->Option.isNone => Some(field)
+    | _ => None
+    }
+  )
+
+let getLinkedEntityFields = table =>
+  table.fields->Array.keepMap(field =>
+    switch field {
+    | Field({linkedEntity: Some(linkedEntityName)} as field) => Some((field, linkedEntityName))
+    | Field({linkedEntity: None})
+    | DerivedFrom(_) =>
+      None
+    }
+  )
+
+let getDerivedFromFields = table =>
+  table.fields->Array.keepMap(field =>
+    switch field {
+    | DerivedFrom(field) => Some(field)
+    | Field(_) => None
+    }
+  )
+
+let getFieldNames = table => {
+  table->getFields->Array.map(getDbFieldName)
+}
+
+let getNonDefaultFieldNames = table => {
+  table->getNonDefaultFields->Array.map(getDbFieldName)
+}
+
+let getFieldByName = (table, fieldNameSearch) =>
+  table.fields->Js.Array2.find(field => field->getUserDefinedFieldName == fieldNameSearch)
+
+exception NonExistingTableField(string)
+
+/*
+Gets all composite indicies (whether they are single indices or not)
+And maps the fields defined to their actual db name (some have _id suffix)
+*/
+let getUnfilteredCompositeIndicesUnsafe = (table): array<array<string>> => {
+  table.compositeIndices->Array.map(compositeIndex =>
+    compositeIndex->Array.map(userDefinedFieldName =>
+      switch table->getFieldByName(userDefinedFieldName) {
+      | Some(field) => field->getFieldName
+      | None => raise(NonExistingTableField(userDefinedFieldName)) //Unexpected should be validated in schema parser
+      }
+    )
+  )
+}
+
+/*
+Gets all single indicies
+And maps the fields defined to their actual db name (some have _id suffix)
+*/
+let getSingleIndices = (table): array<string> => {
+  let indexFields = table.fields->Array.keepMap(field =>
+    switch field {
+    | Field(field) if field.isIndex => Some(field->getDbFieldName)
+    | _ => None
+    }
+  )
+
+  table
+  ->getUnfilteredCompositeIndicesUnsafe
+  //get all composite indices with only 1 field defined
+  //this is still a single index
+  ->Array.keep(cidx => cidx->Array.length == 1)
+  ->Array.concat([indexFields])
+  ->Array.concatMany
+  ->Set.String.fromArray
+  ->Set.String.toArray
+  ->Js.Array2.sortInPlace
+}
+
+/*
+Gets all composite indicies
+And maps the fields defined to their actual db name (some have _id suffix)
+*/
+let getCompositeIndices = (table): array<array<string>> => {
+  table
+  ->getUnfilteredCompositeIndicesUnsafe
+  ->Array.keep(ind => ind->Array.length > 1)
+}
+
+module PostgresInterop = {
+  type pgFn<'payload, 'return> = (Postgres.sql, 'payload) => promise<'return>
+  type batchSetFn<'a> = (Postgres.sql, array<'a>) => promise<unit>
+  external eval: string => 'a = "eval"
+
+  let makeBatchSetFnString = (table: table) => {
+    let fieldNamesInQuotes =
+      table->getNonDefaultFieldNames->Array.map(fieldName => `"${fieldName}"`)
+    `(sql, rows) => {
+      return sql\`
+        INSERT INTO "public"."${table.tableName}"
+        \${sql(rows, ${fieldNamesInQuotes->Js.Array2.joinWith(", ")})}
+        ON CONFLICT(${table->getPrimaryKeyFieldNames->Js.Array2.joinWith(", ")}) DO UPDATE
+        SET
+        ${fieldNamesInQuotes
+      ->Array.map(fieldNameInQuotes => `${fieldNameInQuotes} = EXCLUDED.${fieldNameInQuotes}`)
+      ->Js.Array2.joinWith(", ")};\`
+    }`
+  }
+
+  let chunkBatchQuery = (
+    sql,
+    entityDataArray: array<'entity>,
+    queryToExecute: pgFn<array<'entity>, 'return>,
+    ~maxItemsPerQuery=500,
+  ): promise<array<'return>> => {
+    let responses = []
+    let i = ref(0)
+    let shouldContinue = () => i.contents < entityDataArray->Array.length
+    // Split entityDataArray into chunks of maxItemsPerQuery
+    while shouldContinue() {
+      let chunk =
+        entityDataArray->Js.Array2.slice(~start=i.contents, ~end_=i.contents + maxItemsPerQuery)
+      let response = queryToExecute(sql, chunk)
+      responses->Js.Array2.push(response)->ignore
+      i := i.contents + maxItemsPerQuery
+    }
+    Promise.all(responses)
+  }
+
+  let makeBatchSetFn = (~table, ~rowsSchema: S.t<array<'a>>): batchSetFn<'a> => {
+    let batchSetFn: pgFn<array<Js.Json.t>, unit> = table->makeBatchSetFnString->eval
+    async (sql, rows) => {
+      let rowsJson =
+        rows->S.serializeOrRaiseWith(rowsSchema)->(Utils.magic: Js.Json.t => array<Js.Json.t>)
+      let _res = await chunkBatchQuery(sql, rowsJson, batchSetFn)
+    }
+  }
+}