blokctl 1.3.0 → 1.4.0

package/dist/scaffold-repo/infra/metrics/docker-compose.yml

@@ -0,0 +1,193 @@
+services:
+  # docker-proxy:
+  #     image: tecnativa/docker-socket-proxy
+  #     container_name: docker-proxy
+  #     hostname: docker-proxy
+  #     restart: always
+  #     privileged: true
+  #     environment:
+  #       - CONTAINERS=1
+  #       - INFO=1
+  #       - SERVICES=1
+  #     volumes:
+  #       - /var/run/docker.sock:/var/run/docker.sock
+  #     ports:
+  #       - 2375:2375
+  # telegraf:
+  #   image: telegraf:1.34.3-alpine
+  #   container_name: telegraf
+  #   hostname: telegraf
+  #   user: root
+  #   privileged: true
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
+  #   ports:
+  #     - "9273:9273"
+  # cadvisor:
+  #   container_name: cadvisor
+  #   hostname: cadvisor
+  #   image: gcr.io/cadvisor/cadvisor:latest
+  #   restart: unless-stopped
+  #   volumes:
+  #     - "/:/rootfs"
+  #     - "/var/run:/var/run"
+  #     - "/sys:/sys"
+  #     - "/var/lib/docker/:/var/lib/docker"
+  #     - "/dev/disk/:/dev/disk"
+  #   privileged: true
+  #   devices:
+  #     - "/dev/kmsg"
+  #   networks:
+  #     - shared-network
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    hostname: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+      # OBS-05 A6 — alert rules dir, loaded via `rule_files` in prometheus.yml.
+      - "./rules/:/etc/prometheus/rules/"
+    privileged: true
+    # depends_on:
+    #   - cadvisor
+    restart: unless-stopped
+    networks:
+      - shared-network
+
+  # OBS-05 A6 — Alertmanager. Prometheus forwards firing alerts here; configure
+  # receivers (Slack / PagerDuty / email) in alertmanager.yml. Without this,
+  # rules fire in Prometheus but go nowhere. UI on http://localhost:9093.
+  alertmanager:
+    container_name: alertmanager
+    image: prom/alertmanager:latest
+    hostname: alertmanager
+    ports:
+      - "9093:9093"
+    volumes:
+      - "./alertmanager.yml:/etc/alertmanager/alertmanager.yml"
+    restart: unless-stopped
+    networks:
+      - shared-network
+
+  grafana:
+    container_name: grafana
+    hostname: grafana
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+      - DS_PROMETHEUS=prometheus
+    volumes:
+      - "grafana-data:/var/lib/grafana"
+      - "./datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml"
+      - "./dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+      - "./dashboards/:/var/lib/grafana/dashboards/blok/"
+      - "./default.yaml:/etc/grafana/provisioning/dashboards/default.yaml"
+    privileged: true
+    depends_on:
+      - prometheus
+      - loki
+      - tempo
+    restart: unless-stopped
+    networks:
+      - shared-network
+  tempo:
+    container_name: tempo
+    hostname: tempo
+    image: grafana/tempo:2.7.0
+    ports:
+      - "3201:3200"   # Tempo HTTP API (3200 taken by nginx)
+      - "4317:4317"   # OTLP gRPC receiver
+      - "4318:4318"   # OTLP HTTP receiver
+      - "9411:9411"   # Zipkin receiver
+    volumes:
+      - "./tempo.yaml:/etc/tempo/tempo.yaml"
+      - "tempo-data:/var/tempo"
+    command:
+      - -config.file=/etc/tempo/tempo.yaml
+    restart: unless-stopped
+    networks:
+      - shared-network
+
+  loki:
+    container_name: loki
+    hostname: loki
+    image: grafana/loki:3.5.0
+    ports:
+      - "3100:3100"
+    volumes:
+      - "./loki-config.yaml:/mnt/config/loki-config.yaml"
+    command:
+      - -config.file=/mnt/config/loki-config.yaml
+    restart: unless-stopped
+    networks:
+      - shared-network
+
+  nginx:
+    container_name: loki-proxy
+    image: nginx:alpine
+    ports:
+      - "3200:80"
+    volumes:
+      - "./nginx.conf:/etc/nginx/nginx.conf:ro"
+    depends_on:
+      - loki
+    restart: unless-stopped
+    networks:
+      - shared-network
+
+  # OBS-03 — Grafana Alloy: the log-shipper. Scrapes every container's stdout off
+  # the Docker socket, parses Blok's structured JSON (run_id/trace_id/span_id), and
+  # ships to Loki. Container-based + zero host plugins (vs. the old `logging:
+  # driver: loki` which needs `docker plugin install grafana/loki-docker-driver`).
+  alloy:
+    container_name: alloy
+    hostname: alloy
+    image: grafana/alloy:v1.10.0
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    ports:
+      - "12345:12345"   # Alloy UI / health
+    volumes:
+      - "./alloy-config.alloy:/etc/alloy/config.alloy:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+    depends_on:
+      - loki
+    restart: unless-stopped
+    networks:
+      - shared-network
+
+  # blok-http:
+  #   container_name: blok-http
+  #   hostname: blok-http
+  #   build:
+  #     context: ../../triggers/http
+  #     dockerfile: Dockerfile
+  #   ports:
+  #     - "4000:4000"
+  #   restart: unless-stopped
+  #   logging: # Log to Loki. Enable this to see logs in Grafana.
+  #     driver: loki
+  #     options:
+  #       loki-url: "http://localhost:3100/loki/api/v1/push"
+  #       loki-retries: 5
+  #       loki-batch-size: 400
+  #   depends_on:
+  #     - loki
+  #   networks:
+  #     - shared-network
+volumes:
+  grafana-data:
+  tempo-data:
+networks:
+  shared-network:
+    external: true
+
+# docker network create shared-network

package/dist/scaffold-repo/infra/metrics/loki-config.yaml

@@ -0,0 +1,50 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

package/dist/scaffold-repo/infra/metrics/nginx.conf

@@ -0,0 +1,25 @@
+events {}
+
+http {
+  server {
+    listen 80;
+
+    location /loki/ {
+      proxy_pass http://loki:3100;
+      proxy_set_header Host $host;
+
+      # CORS headers
+      add_header Access-Control-Allow-Origin *;
+      add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS';
+      add_header Access-Control-Allow-Headers 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+      add_header Access-Control-Expose-Headers 'Content-Length,Content-Range';
+
+      if ($request_method = 'OPTIONS') {
+        add_header Access-Control-Max-Age 1728000;
+        add_header Content-Type 'text/plain; charset=utf-8';
+        add_header Content-Length 0;
+        return 204;
+      }
+    }
+  }
+}

package/dist/scaffold-repo/infra/metrics/prometheus.yml

@@ -0,0 +1,51 @@
+global:
+  scrape_interval: 5s #Scrape interval to every 15 seconds.
+  evaluation_interval: 5s #Evaluate rules every 15 seconds.
+
+# OBS-05 A6 — alerting. Load Blok's alert rules and forward firing alerts to
+# Alertmanager (added to the metrics docker-compose). Mount the rules dir at
+# /etc/prometheus/rules. Without this block, metrics are recorded but nobody
+# is ever paged.
+rule_files:
+  - "/etc/prometheus/rules/*.yml"
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["prometheus:9090"]
+
+  - job_name: "cadvisor"
+    static_configs:
+      - targets: ["cadvisor:8080"]
+
+  - job_name: "blok-http"
+    static_configs:
+      - targets: ["blok-http:4000"]
+
+  - job_name: "blok-grpc"
+    static_configs:
+      - targets: ["blok-grpc:9464"]
+
+  # Other triggers expose metrics on BLOK_METRICS_PORT (default 9464)
+  - job_name: "blok-triggers"
+    static_configs:
+      - targets:
+          - "blok-websocket:9464"
+          - "blok-sse:9464"
+          - "blok-cron:9464"
+          - "blok-queue:9464"
+          - "blok-pubsub:9464"
+          - "blok-webhook:9464"
+          - "blok-worker:9464"
+
+  # Uncomment for file-based service discovery in dynamic environments
+  # - job_name: "blok-dynamic"
+  #   file_sd_configs:
+  #     - files:
+  #         - "/etc/prometheus/blok-targets.json"
+  #       refresh_interval: 30s

package/dist/scaffold-repo/infra/metrics/rules/blok-alerts.yml

@@ -0,0 +1,114 @@
+# Blok alerting rules (OBS-05 A6).
+#
+# These turn the metrics Blok already emits into pages. They are intentionally
+# conservative (warnings on rates over a sustained window) so they don't flap.
+# Tune the thresholds (`> 0.05`, `for: 10m`, etc.) to your SLOs.
+#
+# Crash / timeout alerts (run states `crashed` / `timedOut`) are distinguishable
+# via the `status` label on `blok_workflow_errors_total` (OBS-05 T2). The
+# `blok_unhandled_rejection_total` counter (OBS-06 T8) pages on process-level
+# fatal errors, and `blok_boot_error_total` (OBS-06 T9) pages on workflows that
+# fail to boot (config / middleware) before their run starts (BlokBootErrors).
+groups:
+  - name: blok-availability
+    rules:
+      - alert: BlokTargetDown
+        expr: up{job=~"blok-.*"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Blok target {{ $labels.job }} is down"
+          description: "Prometheus cannot scrape {{ $labels.instance }} ({{ $labels.job }}) for 2m. The process may have crashed or be unreachable."
+
+  - name: blok-errors
+    rules:
+      - alert: BlokWorkflowErrorRateHigh
+        expr: |
+          sum(rate(blok_workflow_errors_total[5m])) by (workflow_name)
+            /
+          clamp_min(sum(rate(blok_workflow_executions_total[5m])) by (workflow_name), 1)
+          > 0.05
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Workflow {{ $labels.workflow_name }} error rate > 5%"
+          description: "{{ $labels.workflow_name }} has been failing >5% of runs for 10m (current: {{ $value | humanizePercentage }})."
+
+      - alert: BlokWorkflowCrashes
+        expr: sum(rate(blok_workflow_errors_total{status="crashed"}[5m])) by (workflow_name) > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Workflow {{ $labels.workflow_name }} runs are crashing"
+          description: "{{ $labels.workflow_name }} has runs flipping to `crashed` (process-level fatal error) for 5m."
+
+      - alert: BlokWorkflowTimeouts
+        expr: sum(rate(blok_workflow_errors_total{status="timedOut"}[5m])) by (workflow_name) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Workflow {{ $labels.workflow_name }} runs are timing out"
+          description: "{{ $labels.workflow_name }} has runs flipping to `timedOut` (step maxDuration exceeded) for 10m."
+
+      - alert: BlokUnhandledRejections
+        expr: sum(rate(blok_unhandled_rejection_total[5m])) by (reason_class) > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Blok process hit unhandled errors ({{ $labels.reason_class }})"
+          description: "The crash handlers caught uncaughtException / unhandledRejection events ({{ $labels.reason_class }}) for 5m."
+
+      - alert: BlokBootErrors
+        expr: sum(rate(blok_boot_error_total[5m])) by (trigger_type, phase) > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Blok workflows failing to boot ({{ $labels.phase }})"
+          description: "Workflows are failing at the {{ $labels.phase }} phase before the run starts (config / middleware resolution) on the {{ $labels.trigger_type }} trigger for 5m — usually a bad deploy or config."
+
+      - alert: BlokRuntimeNodeErrors
+        expr: sum(rate(blok_runtime_node_errors_total[5m])) by (runtime_kind, node_name) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Cross-language node {{ $labels.node_name }} ({{ $labels.runtime_kind }}) is failing"
+          description: "The {{ $labels.runtime_kind }} runtime node {{ $labels.node_name }} has sustained gRPC execution failures for 10m."
+
+      - alert: BlokNodeErrorRateHigh
+        expr: sum(rate(blok_node_errors_total[5m])) by (node_name, workflow_name) > 0.1
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Node {{ $labels.node_name }} in {{ $labels.workflow_name }} is failing"
+          description: "{{ $labels.node_name }} has been erroring at >0.1/s for 10m."
+
+  - name: blok-saturation
+    rules:
+      - alert: BlokWorkflowLatencyHigh
+        expr: |
+          histogram_quantile(0.99,
+            sum(rate(blok_workflow_duration_seconds_bucket[5m])) by (le, workflow_name)
+          ) > 5
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Workflow {{ $labels.workflow_name }} p99 latency > 5s"
+          description: "p99 of {{ $labels.workflow_name }} has been above 5s for 10m (current: {{ $value | humanizeDuration }})."
+
+      - alert: BlokConcurrencyThrottling
+        expr: sum(rate(blok_concurrency_denied_total[5m])) by (mode) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Blok is throttling runs (mode={{ $labels.mode }})"
+          description: "Runs have been denied/queued by the concurrency gate for 10m — a tenant is over its limit or capacity is short."

package/dist/scaffold-repo/infra/metrics/telegraf.conf

@@ -0,0 +1,41 @@
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  hostname = ""
+  omit_hostname = true
+
+###############################################################################
+# OUTPUTS
+###############################################################################
+
+[[outputs.prometheus_client]]
+  listen = ":9273"
+  metric_version = 2
+  path = "/metrics"
+  expiration_interval = "60s"
+
+###############################################################################
+# INPUTS
+###############################################################################
+
+# [[inputs.docker]]
+#   endpoint = "tcp://docker-proxy:2375"
+#   gather_services = false
+#   container_names = []
+#   timeout = "5s"
+#   perdevice = true
+#   total = true
+#   docker_label_include = []
+#   docker_label_exclude = []
+#   tag_env = ["JAVA_HOME", "MY_CUSTOM_TAG"]
+
+# [[inputs.prometheus]]
+#   urls = [
+#     "http://blok-http:9091/metrics"
+#   ]

package/dist/scaffold-repo/infra/metrics/tempo.yaml

@@ -0,0 +1,81 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+        duration_slo: 5s
+        throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 100ms
+  metrics:
+    max_duration: 120h                # maximum duration of a metrics query, increase for local setups
+    query_backend_after: 5m
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+          endpoint: "tempo:14268"      # for a production deployment you should only enable the receivers you need!
+        grpc:
+          endpoint: "tempo:14250"
+        thrift_binary:
+          endpoint: "tempo:6832"
+        thrift_compact:
+          endpoint: "tempo:6831"
+    zipkin:
+      endpoint: "tempo:9411"
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "tempo:4317"
+        http:
+          endpoint: "tempo:4318"
+    opencensus:
+      endpoint: "tempo:55678"
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 24h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+  processor:
+    local_blocks:
+      filter_server_spans: false
+      flush_to_storage: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /var/tempo/wal             # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
+      generate_native_histograms: both

package/dist/scaffold-repo/infra/milvus/docker-compose.yml

@@ -0,0 +1,73 @@
+version: '3.5'
+
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.18
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      - shared-network
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    ports:
+      - "9001:9001"
+      - "9000:9000"
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      - shared-network
+
+  standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.5.9
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+    - seccomp:unconfined
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
+    networks:
+      - shared-network
+
+networks:
+  shared-network:
+    external: true
+
+# docker network create shared-network