@rivetkit/engine 1.0.0

package/docker/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc

@@ -0,0 +1,62 @@
+{
+	"auth": {
+		"admin_token": "dev"
+	},
+	"guard": {
+		"port": 6420
+	},
+	"api_peer": {
+		"host": "0.0.0.0",
+		"port": 6421
+	},
+	"topology": {
+		"datacenter_label": 2,
+		"datacenters": [
+			{
+				"name": "dc-a",
+				"datacenter_label": 1,
+				"is_leader": true,
+				"peer_url": "http://rivet-engine-dc-a-0:6421",
+				"public_url": "http://rivet-engine-dc-a-0:6420"
+			},
+			{
+				"name": "dc-b",
+				"datacenter_label": 2,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-b-0:6421",
+				"public_url": "http://rivet-engine-dc-b-0:6420"
+			},
+			{
+				"name": "dc-c",
+				"datacenter_label": 3,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-c-0:6421",
+				"public_url": "http://rivet-engine-dc-c-0:6420"
+			}
+		]
+	},
+	"postgres": {
+		"url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine"
+	},
+	"cache": {
+		"driver": "in_memory"
+	},
+	"clickhouse": {
+		"http_url": "http://clickhouse:9300",
+		"native_url": "http://clickhouse:9301",
+		"username": "system",
+		"password": "default",
+		"provision_users": {
+			"vector": {
+				"username": "vector",
+				"password": "vector",
+				"role": "write"
+			}
+		},
+		"secure": false
+	},
+	"vector_http": {
+		"host": "vector-client-dc-b",
+		"port": 5022
+	}
+}

package/docker/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc

@@ -0,0 +1,62 @@
+{
+	"auth": {
+		"admin_token": "dev"
+	},
+	"guard": {
+		"port": 6420
+	},
+	"api_peer": {
+		"host": "0.0.0.0",
+		"port": 6421
+	},
+	"topology": {
+		"datacenter_label": 2,
+		"datacenters": [
+			{
+				"name": "dc-a",
+				"datacenter_label": 1,
+				"is_leader": true,
+				"peer_url": "http://rivet-engine-dc-a-0:6421",
+				"public_url": "http://rivet-engine-dc-a-0:6420"
+			},
+			{
+				"name": "dc-b",
+				"datacenter_label": 2,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-b-0:6421",
+				"public_url": "http://rivet-engine-dc-b-0:6420"
+			},
+			{
+				"name": "dc-c",
+				"datacenter_label": 3,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-c-0:6421",
+				"public_url": "http://rivet-engine-dc-c-0:6420"
+			}
+		]
+	},
+	"postgres": {
+		"url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine"
+	},
+	"cache": {
+		"driver": "in_memory"
+	},
+	"clickhouse": {
+		"http_url": "http://clickhouse:9300",
+		"native_url": "http://clickhouse:9301",
+		"username": "system",
+		"password": "default",
+		"provision_users": {
+			"vector": {
+				"username": "vector",
+				"password": "vector",
+				"role": "write"
+			}
+		},
+		"secure": false
+	},
+	"vector_http": {
+		"host": "vector-client-dc-b",
+		"port": 5022
+	}
+}

package/docker/dev-multidc-multinode/datacenters/dc-b/vector-client/vector.yaml

@@ -0,0 +1,31 @@
+# Vector Client Configuration for Datacenter: dc-b
+# This Vector instance runs on each server to collect local logs and forward them
+# to the central Vector server aggregator.
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  http_events:
+    type: http_server
+    address: 0.0.0.0:5022
+    encoding: ndjson
+transforms: {}
+sinks:
+  vector_server:
+    type: vector
+    inputs:
+      - http_events
+    address: vector-server-dc-b:6000
+    version: '2'
+    buffer:
+      type: disk
+      max_size: 2147483648
+      when_full: block
+  console_debug:
+    type: console
+    inputs:
+      - http_events
+    encoding:
+      codec: json
+ 

package/docker/dev-multidc-multinode/datacenters/dc-b/vector-server/vector.yaml

@@ -0,0 +1,61 @@
+# Vector Server (Aggregator) Configuration
+# This Vector instance acts as an aggregator that collects logs and metrics from all Vector clients
+# across different datacenters and forwards them to ClickHouse and other sinks.
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  vector:
+    type: vector
+    address: 0.0.0.0:6000
+    version: '2'
+  tcp_json:
+    type: socket
+    mode: tcp
+    address: 0.0.0.0:6100
+    decoding:
+      codec: json
+  http_json:
+    type: http_server
+    address: 0.0.0.0:6200
+    encoding: json
+  vector_metrics:
+    type: internal_metrics
+  vector_logs:
+    type: internal_logs
+transforms:
+  clickhouse_dynamic_events_filter:
+    type: filter
+    inputs:
+      - vector
+    condition:
+      type: vrl
+      source: .source == "clickhouse"
+  clickhouse_dynamic_events_transform:
+    type: remap
+    inputs:
+      - clickhouse_dynamic_events_filter
+    source: "# Extract and store metadata\n__database = .database\n__table = .table\n__columns = .columns\n\n# Create a new object with just the columns data\n. = {\n\t\"__database\": __database,\n\t\"__table\": __table,\n\t# By default insert namespace column since most tables include this\n\t\"namespace\": \"rivet\"\n}\n\t\n# Merge in the column data that should be inserted\n. = merge!(., __columns)\n"
+sinks:
+  clickhouse_dynamic_events:
+    type: clickhouse
+    inputs:
+      - clickhouse_dynamic_events_transform
+    endpoint: http://clickhouse:9300
+    database: '{{ __database }}'
+    table: '{{ __table }}'
+    compression: gzip
+    auth:
+      strategy: basic
+      user: system
+      password: default
+    batch:
+      max_events: 1000
+      timeout_secs: 10
+  console_vector_logs:
+    type: console
+    inputs:
+      - vector_logs
+    encoding:
+      codec: json

package/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-client/config.yaml

@@ -0,0 +1,39 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 10000
+exporters:
+  otlp/server:
+    endpoint: otel-collector-server-dc-c:4317
+    tls:
+      insecure: true
+service:
+  pipelines:
+    logs:
+      receivers:
+        - otlp
+      processors:
+        - batch
+      exporters:
+        - otlp/server
+    traces:
+      receivers:
+        - otlp
+      processors:
+        - batch
+      exporters:
+        - otlp/server
+    metrics:
+      receivers:
+        - otlp
+      processors:
+        - batch
+      exporters:
+        - otlp/server

package/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-server/config.yaml

@@ -0,0 +1,72 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+processors:
+  resource:
+    attributes:
+      - key: rivet.project
+        value: dev
+        action: upsert
+      - key: rivet.datacenter
+        value: dc-c
+        action: upsert
+  batch:
+    timeout: 5s
+    send_batch_size: 10000
+exporters:
+  clickhouse:
+    endpoint: http://clickhouse:9300
+    database: otel
+    username: default
+    password: ${env:CLICKHOUSE_PASSWORD}
+    async_insert: true
+    ttl: 72h
+    compress: lz4
+    create_schema: true
+    logs_table_name: otel_logs
+    traces_table_name: otel_traces
+    timeout: 5s
+    metrics_tables:
+      gauge:
+        name: otel_metrics_gauge
+      sum:
+        name: otel_metrics_sum
+      summary:
+        name: otel_metrics_summary
+      histogram:
+        name: otel_metrics_histogram
+      exponential_histogram:
+        name: otel_metrics_exp_histogram
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+service:
+  pipelines:
+    logs:
+      receivers:
+        - otlp
+      processors:
+        - resource
+        - batch
+      exporters:
+        - clickhouse
+    traces:
+      receivers:
+        - otlp
+      processors:
+        - resource
+        - batch
+      exporters:
+        - clickhouse
+    metrics:
+      receivers:
+        - otlp
+      processors:
+        - resource
+        - batch
+      exporters:
+        - clickhouse

package/docker/dev-multidc-multinode/datacenters/dc-c/postgres/init-db.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE DATABASE rivet_engine;
+EOSQL

package/docker/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc

@@ -0,0 +1,62 @@
+{
+	"auth": {
+		"admin_token": "dev"
+	},
+	"guard": {
+		"port": 6420
+	},
+	"api_peer": {
+		"host": "0.0.0.0",
+		"port": 6421
+	},
+	"topology": {
+		"datacenter_label": 3,
+		"datacenters": [
+			{
+				"name": "dc-a",
+				"datacenter_label": 1,
+				"is_leader": true,
+				"peer_url": "http://rivet-engine-dc-a-0:6421",
+				"public_url": "http://rivet-engine-dc-a-0:6420"
+			},
+			{
+				"name": "dc-b",
+				"datacenter_label": 2,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-b-0:6421",
+				"public_url": "http://rivet-engine-dc-b-0:6420"
+			},
+			{
+				"name": "dc-c",
+				"datacenter_label": 3,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-c-0:6421",
+				"public_url": "http://rivet-engine-dc-c-0:6420"
+			}
+		]
+	},
+	"postgres": {
+		"url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine"
+	},
+	"cache": {
+		"driver": "in_memory"
+	},
+	"clickhouse": {
+		"http_url": "http://clickhouse:9300",
+		"native_url": "http://clickhouse:9301",
+		"username": "system",
+		"password": "default",
+		"provision_users": {
+			"vector": {
+				"username": "vector",
+				"password": "vector",
+				"role": "write"
+			}
+		},
+		"secure": false
+	},
+	"vector_http": {
+		"host": "vector-client-dc-c",
+		"port": 5022
+	}
+}

package/docker/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc

@@ -0,0 +1,62 @@
+{
+	"auth": {
+		"admin_token": "dev"
+	},
+	"guard": {
+		"port": 6420
+	},
+	"api_peer": {
+		"host": "0.0.0.0",
+		"port": 6421
+	},
+	"topology": {
+		"datacenter_label": 3,
+		"datacenters": [
+			{
+				"name": "dc-a",
+				"datacenter_label": 1,
+				"is_leader": true,
+				"peer_url": "http://rivet-engine-dc-a-0:6421",
+				"public_url": "http://rivet-engine-dc-a-0:6420"
+			},
+			{
+				"name": "dc-b",
+				"datacenter_label": 2,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-b-0:6421",
+				"public_url": "http://rivet-engine-dc-b-0:6420"
+			},
+			{
+				"name": "dc-c",
+				"datacenter_label": 3,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-c-0:6421",
+				"public_url": "http://rivet-engine-dc-c-0:6420"
+			}
+		]
+	},
+	"postgres": {
+		"url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine"
+	},
+	"cache": {
+		"driver": "in_memory"
+	},
+	"clickhouse": {
+		"http_url": "http://clickhouse:9300",
+		"native_url": "http://clickhouse:9301",
+		"username": "system",
+		"password": "default",
+		"provision_users": {
+			"vector": {
+				"username": "vector",
+				"password": "vector",
+				"role": "write"
+			}
+		},
+		"secure": false
+	},
+	"vector_http": {
+		"host": "vector-client-dc-c",
+		"port": 5022
+	}
+}

package/docker/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc

@@ -0,0 +1,62 @@
+{
+	"auth": {
+		"admin_token": "dev"
+	},
+	"guard": {
+		"port": 6420
+	},
+	"api_peer": {
+		"host": "0.0.0.0",
+		"port": 6421
+	},
+	"topology": {
+		"datacenter_label": 3,
+		"datacenters": [
+			{
+				"name": "dc-a",
+				"datacenter_label": 1,
+				"is_leader": true,
+				"peer_url": "http://rivet-engine-dc-a-0:6421",
+				"public_url": "http://rivet-engine-dc-a-0:6420"
+			},
+			{
+				"name": "dc-b",
+				"datacenter_label": 2,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-b-0:6421",
+				"public_url": "http://rivet-engine-dc-b-0:6420"
+			},
+			{
+				"name": "dc-c",
+				"datacenter_label": 3,
+				"is_leader": false,
+				"peer_url": "http://rivet-engine-dc-c-0:6421",
+				"public_url": "http://rivet-engine-dc-c-0:6420"
+			}
+		]
+	},
+	"postgres": {
+		"url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine"
+	},
+	"cache": {
+		"driver": "in_memory"
+	},
+	"clickhouse": {
+		"http_url": "http://clickhouse:9300",
+		"native_url": "http://clickhouse:9301",
+		"username": "system",
+		"password": "default",
+		"provision_users": {
+			"vector": {
+				"username": "vector",
+				"password": "vector",
+				"role": "write"
+			}
+		},
+		"secure": false
+	},
+	"vector_http": {
+		"host": "vector-client-dc-c",
+		"port": 5022
+	}
+}

package/docker/dev-multidc-multinode/datacenters/dc-c/vector-client/vector.yaml

@@ -0,0 +1,31 @@
+# Vector Client Configuration for Datacenter: dc-c
+# This Vector instance runs on each server to collect local logs and forward them
+# to the central Vector server aggregator.
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  http_events:
+    type: http_server
+    address: 0.0.0.0:5022
+    encoding: ndjson
+transforms: {}
+sinks:
+  vector_server:
+    type: vector
+    inputs:
+      - http_events
+    address: vector-server-dc-c:6000
+    version: '2'
+    buffer:
+      type: disk
+      max_size: 2147483648
+      when_full: block
+  console_debug:
+    type: console
+    inputs:
+      - http_events
+    encoding:
+      codec: json
+ 

package/docker/dev-multidc-multinode/datacenters/dc-c/vector-server/vector.yaml

@@ -0,0 +1,61 @@
+# Vector Server (Aggregator) Configuration
+# This Vector instance acts as an aggregator that collects logs and metrics from all Vector clients
+# across different datacenters and forwards them to ClickHouse and other sinks.
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  vector:
+    type: vector
+    address: 0.0.0.0:6000
+    version: '2'
+  tcp_json:
+    type: socket
+    mode: tcp
+    address: 0.0.0.0:6100
+    decoding:
+      codec: json
+  http_json:
+    type: http_server
+    address: 0.0.0.0:6200
+    encoding: json
+  vector_metrics:
+    type: internal_metrics
+  vector_logs:
+    type: internal_logs
+transforms:
+  clickhouse_dynamic_events_filter:
+    type: filter
+    inputs:
+      - vector
+    condition:
+      type: vrl
+      source: .source == "clickhouse"
+  clickhouse_dynamic_events_transform:
+    type: remap
+    inputs:
+      - clickhouse_dynamic_events_filter
+    source: "# Extract and store metadata\n__database = .database\n__table = .table\n__columns = .columns\n\n# Create a new object with just the columns data\n. = {\n\t\"__database\": __database,\n\t\"__table\": __table,\n\t# By default insert namespace column since most tables include this\n\t\"namespace\": \"rivet\"\n}\n\t\n# Merge in the column data that should be inserted\n. = merge!(., __columns)\n"
+sinks:
+  clickhouse_dynamic_events:
+    type: clickhouse
+    inputs:
+      - clickhouse_dynamic_events_transform
+    endpoint: http://clickhouse:9300
+    database: '{{ __database }}'
+    table: '{{ __table }}'
+    compression: gzip
+    auth:
+      strategy: basic
+      user: system
+      password: default
+    batch:
+      max_events: 1000
+      timeout_secs: 10
+  console_vector_logs:
+    type: console
+    inputs:
+      - vector_logs
+    encoding:
+      codec: json