@mojaloop/bulk-api-adapter 17.1.6

package/docker/bulk-api-adapter/default.json

@@ -0,0 +1,147 @@
+{
+  "HUB_PARTICIPANT": {
+    "ID": 1,
+    "NAME": "Hub"
+  },
+  "PORT": 3003,
+  "HOSTNAME": "http://bulk-api-adapter",
+  "ENDPOINT_SOURCE_URL": "http://central-ledger:3001/participants/{{fsp}}/endpoints",
+  "ENDPOINT_HEALTH_URL": "http://central-ledger:3001/health",
+  "ENDPOINT_CACHE_CONFIG": {
+    "expiresIn": 180000,
+    "generateTimeout": 30000
+  },
+  "MONGODB": {
+    "HOST": "objstore",
+    "PORT": 27017,
+    "USER": "",
+    "PASSWORD": "",
+    "DATABASE": "mlos"
+  },
+  "ENDPOINT_SECURITY":{
+    "TLS": {
+      "rejectUnauthorized": true
+    },
+    "JWS": {
+      "JWS_SIGN": false,
+      "JWS_SIGNING_KEY_PATH": "secrets/jwsSigningKey.key"
+    }
+  },
+  "MAX_FULFIL_TIMEOUT_DURATION_SECONDS": 300,
+  "AMOUNT": {
+    "PRECISION": 18,
+    "SCALE": 4
+  },
+  "HANDLERS": {
+    "DISABLED": false,
+    "API": {
+      "DISABLED": false
+    }
+  },
+  "INSTRUMENTATION": {
+    "METRICS": {
+      "DISABLED": false,
+      "config": {
+        "timeout": 5000,
+        "prefix": "moja_bl_",
+        "defaultLabels": {
+          "serviceName": "bulk-service"
+        }
+      }
+    }
+  },
+  "KAFKA": {
+    "TOPIC_TEMPLATES": {
+      "GENERAL_TOPIC_TEMPLATE": {
+        "TEMPLATE": "topic-{{functionality}}-{{action}}",
+        "REGEX": "topic-(.*)-(.*)"
+      }
+    },
+    "CONSUMER": {
+      "NOTIFICATION": {
+        "EVENT": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "bulk-con-notification-event",
+              "group.id": "bulk-group-notification-event",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        }
+      }
+    },
+    "PRODUCER": {
+      "BULK": {
+        "PREPARE": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "bulk-prod-bulk-prepare",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        },
+        "FULFIL": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "bulk-prod-bulk-fulfil",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        },
+        "GET": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "bulk-prod-bulk-get",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/docker/central-ledger/default.json

@@ -0,0 +1,460 @@
+{
+  "PORT": 3001,
+  "HOSTNAME": "http://central-ledger",
+  "DATABASE": {
+    "DIALECT": "mysql",
+    "HOST": "mysql",
+    "PORT": 3306,
+    "USER": "central_ledger",
+    "PASSWORD": "password",
+    "SCHEMA": "central_ledger",
+    "POOL_MIN_SIZE": 10,
+    "POOL_MAX_SIZE": 10,
+    "ACQUIRE_TIMEOUT_MILLIS": 30000,
+    "CREATE_TIMEOUT_MILLIS": 30000,
+    "DESTROY_TIMEOUT_MILLIS": 5000,
+    "IDLE_TIMEOUT_MILLIS": 30000,
+    "REAP_INTERVAL_MILLIS": 1000,
+    "CREATE_RETRY_INTERVAL_MILLIS": 200,
+    "DEBUG": false
+  },
+  "MIGRATIONS": {
+    "DISABLED": false,
+    "RUN_DATA_MIGRATIONS": true
+  },
+  "AMOUNT": {
+    "PRECISION": 10,
+    "SCALE": 2
+  },
+  "MONGODB": {
+    "DISABLED": true,
+    "URI": "mongodb://objstore:27017/mlos"
+  },
+  "ERROR_HANDLING": {
+    "includeCauseExtension": false,
+    "truncateCause": true
+  },
+  "HANDLERS": {
+    "DISABLED": false,
+    "API": {
+      "DISABLED": false
+    },
+    "TIMEOUT": {
+      "DISABLED": false,
+      "TIMEXP": "*/15 * * * * *",
+      "TIMEZONE": "UTC"
+    }
+  },
+  "INSTRUMENTATION": {
+    "METRICS": {
+      "DISABLED": false,
+      "labels": {
+        "fspId": "*"
+      },
+      "config": {
+        "timeout": 5000,
+        "prefix": "moja_cl_",
+        "defaultLabels": {
+          "serviceName": "central-service"
+        }
+      }
+    }
+  },
+  "PARTICIPANT_INITIAL_POSITION": 0,
+  "HUB_PARTICIPANT": {
+    "ID": 1,
+    "NAME": "Hub",
+    "ACCOUNTS": [
+      "HUB_RECONCILIATION",
+      "HUB_MULTILATERAL_SETTLEMENT",
+      "HUB_FEE"
+    ]
+  },
+  "INTERNAL_TRANSFER_VALIDITY_SECONDS": "432000",
+  "ENABLE_ON_US_TRANSFERS": false,
+  "CACHE": {
+    "CACHE_ENABLED": false,
+    "MAX_BYTE_SIZE": 10000000,
+    "EXPIRES_IN_MS": 1000
+  },
+  "KAFKA": {
+    "TOPIC_TEMPLATES": {
+      "PARTICIPANT_TOPIC_TEMPLATE": {
+        "TEMPLATE": "topic-{{participantName}}-{{functionality}}-{{action}}",
+        "REGEX": "topic-(.*)-(.*)-(.*)"
+      },
+      "GENERAL_TOPIC_TEMPLATE": {
+        "TEMPLATE": "topic-{{functionality}}-{{action}}",
+        "REGEX": "topic-(.*)-(.*)"
+      }
+    },
+    "CONSUMER": {
+      "BULK": {
+        "PREPARE": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-prepare",
+              "group.id": "cl-group-bulk-prepare",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "PROCESSING": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-processing",
+              "group.id": "cl-group-bulk-processing",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "FULFIL": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-fulfil",
+              "group.id": "cl-group-bulk-fulfil",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "GET": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-get",
+              "group.id": "cl-group-bulk-get",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        }
+      },
+      "TRANSFER": {
+        "PREPARE": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-transfer-prepare",
+              "group.id": "cl-group-transfer-prepare",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "GET": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-transfer-get",
+              "group.id": "cl-group-transfer-get",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "FULFIL": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-transfer-fulfil",
+              "group.id": "cl-group-transfer-fulfil",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "POSITION": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-transfer-position",
+              "group.id": "cl-group-transfer-position",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        }
+      },
+      "ADMIN": {
+        "TRANSFER": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-transfer-admin",
+              "group.id": "cl-group-transfer-admin",
+              "metadata.broker.list": "kafka:29092",
+              "allow.auto.create.topics": true,
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        }
+      },
+      "NOTIFICATION": {
+        "EVENT": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "TESTONLY",
+              "group.id": "TESTONLY",
+              "metadata.broker.list": "kafka:29092",
+              "socket.keepalive.enable": true,
+              "allow.auto.create.topics": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        }
+      }
+    },
+    "PRODUCER": {
+      "BULK": {
+        "PROCESSING": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "cl-prod-bulk-processing",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        }
+      },
+      "TRANSFER": {
+        "PREPARE": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "cl-prod-transfer-prepare",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        },
+        "FULFIL": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "cl-prod-transfer-fulfil",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        },
+        "POSITION": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "cl-prod-transfer-position",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        }
+      },
+      "NOTIFICATION": {
+        "EVENT": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "cl-prod-notification-event",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        }
+      },
+      "ADMIN": {
+        "TRANSFER": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092",
+              "client.id": "cl-prod-transfer-admin",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/docker/kafka/scripts/provision.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+KAFKAHOST="${1:-kafka}"
+KAFKAPORT="${2:-29092}"
+
+# blocks until kafka is reachable
+echo -e "------------------------------------------"
+echo -e "Listing current kafka topics"
+echo -e "__________________________________________"
+kafka-topics.sh --bootstrap-server $KAFKAHOST:$KAFKAPORT --list
+echo -e ""
+echo -e "------------------------------------------"
+echo -e "Creating kafka topics"
+echo -e "__________________________________________"
+
+
+# List of topics to create
+topics=(
+  "topic-transfer-prepare"
+  "topic-transfer-position"
+  "topic-transfer-fulfil"
+  "topic-notification-event"
+  "topic-transfer-get"
+  "topic-admin-transfer"
+  "topic-bulk-prepare"
+  "topic-bulk-fulfil"
+  "topic-bulk-processing"
+  "topic-bulk-get"
+)
+
+# Loop through the topics and create them using kafka-topics.sh
+for topic in "${topics[@]}"
+do
+  echo -e "--> Creating topic $topic..."
+  kafka-topics.sh --bootstrap-server $KAFKAHOST:$KAFKAPORT --create --if-not-exists --topic "$topic" --replication-factor 1 --partitions 1
+done
+
+echo -e ""
+echo -e "------------------------------------------"
+echo -e "Successfully created the following topics:"
+echo -e "__________________________________________"
+kafka-topics.sh --bootstrap-server $KAFKAHOST:$KAFKAPORT --list

package/docker/sql-init/01_permissions.sql

@@ -0,0 +1,2 @@
+ALTER USER 'central_ledger'@'%' identified WITH mysql_native_password by 'password';
+FLUSH PRIVILEGES;

package/docker/wait-for/wait-for-bulk-api-adapter.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "** STARTUP - Checking for Bulk-API-Adapter..."
+
+source /opt/wait-for/wait-for.env
+
+sh /opt/wait-for/wait-for-kafka.sh
+
+sh /opt/wait-for/wait-for-objstore.sh
+
+echo "** STARTUP - Bulk-API-Adapter successful!"

package/docker/wait-for/wait-for-central-ledger.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "** STARTUP - Checking for Central-Ledger..."
+
+source /opt/wait-for/wait-for.env
+
+sh /opt/wait-for/wait-for-mysql.sh
+
+sh /opt/wait-for/wait-for-kafka.sh
+
+sh /opt/wait-for/wait-for-objstore.sh
+
+echo "** STARTUP - Central-Ledger successful!"

package/docker/wait-for/wait-for-kafka.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo "** STARTUP - Checking for Broker connection..."
+
+source /opt/wait-for/wait-for.env
+
+sh /opt/wait-for/wait-for.sh $WAIT_FOR_DB_KAFKA_BROKER -t 240 -- echo "** STARTUP - Kafka connection successful!"