@meshxdata/fops 0.1.36 → 0.1.38

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/dai-backend.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: dai-backend
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/dai/backend/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/dai-frontend.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: dai-frontend
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/dai/frontend/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-backend.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-backend
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/backend/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-frontend.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-frontend
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/frontend/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-hive.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-hive
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/hive/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-kafka.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-kafka
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/kafka/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-meltano.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-meltano
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/meltano/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-mlflow.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-mlflow
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/mlflow/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-opa.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-opa
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/opa/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-processor.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-processor
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/processor/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-scheduler.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-scheduler
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/scheduler/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-storage-engine.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-storage-engine
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/storage-engine/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-trino.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-trino
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/trino/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-watcher.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: foundation-watcher
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./apps/foundation/watcher/overlays/meshx/{{OVERLAY}}
+  prune: true

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/config/repository.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: foundation
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 1m0s
+  url: oci://meshxregistry.azurecr.io/helm
+  secretRef:
+    name: meshxregistry-helm-secret
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: foundation
+  namespace: velora
+spec:
+  type: oci
+  interval: 1m0s
+  url: oci://meshxregistry.azurecr.io/helm
+  secretRef:
+    name: meshxregistry-helm-secret
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: dai
+  namespace: velora
+spec:
+  type: oci
+  interval: 1m0s
+  url: oci://meshxregistry.azurecr.io/helm/dai
+  secretRef:
+    name: meshxregistry-helm-secret
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: hive-metastore
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 1m0s
+  url: oci://meshxregistry.azurecr.io/helm
+  secretRef:
+    name: meshxregistry-helm-secret
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: trinodb
+  namespace: flux-system
+spec:
+  interval: 30m
+  url: https://trinodb.github.io/charts
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: mlflow
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  url: https://community-charts.github.io/helm-charts

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/kustomization.yaml

@@ -0,0 +1,30 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - config/repository.yaml
+  - operator/acr-webhook-controller.yaml
+  - operator/kube-reflector.yaml
+  - operator/externalsecrets.yaml
+  - operator/istio.yaml
+  - operator/kafka.yaml
+  - operator/kubecost.yaml
+  - operator/spark.yaml
+  - operator/vertical-pod-autoscaler.yaml
+  - operator/prometheus-agent.yaml
+  - operator/tailscale.yaml
+  - operator/nats-server.yaml
+  - operator/reloader.yaml
+  - apps/foundation-backend.yaml
+  - apps/foundation-frontend.yaml
+  - apps/foundation-scheduler.yaml
+  - apps/foundation-processor.yaml
+  - apps/foundation-watcher.yaml
+  - apps/foundation-opa.yaml
+  - apps/foundation-meltano.yaml
+  - apps/foundation-kafka.yaml
+  - apps/foundation-hive.yaml
+  - apps/foundation-storage-engine.yaml
+  - apps/foundation-trino.yaml
+  - apps/foundation-mlflow.yaml
+  - apps/dai-backend.yaml
+  - apps/dai-frontend.yaml

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/acr-webhook-controller.yaml

@@ -0,0 +1,63 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: acr-webhook-controller
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/acr-webhook-controller
+  prune: false
+  patches:
+  - target:
+      kind: Secret
+      name: acr-pull-secret
+      namespace: acr-cache-system
+    patch: |-
+      - op: replace
+        path: /stringData/.dockerconfigjson
+        value: |
+          {
+            "auths": {
+              "meshxregistry.azurecr.io": {
+                "username": "{{ACR_USERNAME}}",
+                "password": "{{ACR_PASSWORD}}"
+              }
+            }
+          }
+  - target:
+      kind: Secret
+      name: meshxregistry-helm-secret
+      namespace: acr-cache-system
+    patch: |-
+      - op: replace
+        path: /stringData/username
+        value: "{{ACR_USERNAME}}"
+      - op: replace
+        path: /stringData/password
+        value: "{{ACR_PASSWORD}}"
+  - target:
+      kind: Deployment
+      name: acr-cache-webhook
+      namespace: acr-cache-system
+    patch: |
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: acr-cache-webhook
+      spec:
+        replicas: 3
+        template:
+          spec:
+            affinity:
+              podAntiAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchExpressions:
+                    - key: app
+                      operator: In
+                      values:
+                      - acr-cache-webhook
+                  topologyKey: kubernetes.io/hostname

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/externalsecrets.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: external-secrets
+  namespace: flux-system
+spec:
+  interval: 1h
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/external-secrets/manifests
+  prune: false
+  dependsOn:
+    - name: acr-webhook-controller
+      namespace: flux-system

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/istio.yaml

@@ -0,0 +1,42 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: istio-operator
+  namespace: flux-system
+spec:
+  interval: 1h
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/istio-operator
+  prune: false
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: istio-controlplane
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: istio-operator
+  interval: 10s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/istio-tenant/demo/azure/uaenorth
+  prune: false
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: jaeger
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: istio-operator
+  interval: 10m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/jaeger
+  prune: false

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/kafka.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kafka-operator
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/kafka-operator/velora
+  prune: false
+  dependsOn:
+    - name: acr-webhook-controller
+      namespace: flux-system

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/kube-reflector.yaml

@@ -0,0 +1,33 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kube-reflector
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/reflector
+  prune: false
+  targetNamespace: reflector
+  dependsOn:
+    - name: acr-webhook-controller
+      namespace: flux-system
+  patches:
+  - target:
+      kind: Secret
+      name: acr-pull-secret
+      namespace: reflector
+    patch: |-
+      - op: replace
+        path: /stringData/.dockerconfigjson
+        value: |
+          {
+            "auths": {
+              "meshxregistry.azurecr.io": {
+                "username": "{{ACR_USERNAME}}",
+                "password": "{{ACR_PASSWORD}}"
+              }
+            }
+          }

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/kubecost.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kubecost
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/kubecost/{{CLUSTER_NAME}}
+  prune: false

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/nats-server.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nats-server
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/nats-server/velora
+  prune: false
+  dependsOn:
+    - name: acr-webhook-controller
+      namespace: flux-system

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/prometheus-agent.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: prometheus-agent
+  namespace: flux-system
+spec:
+  interval: 1m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/prometheus-agent
+  prune: false
+  patches:
+    - target:
+        kind: ConfigMap
+        name: prometheus-agent-server
+      patch: |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: prometheus-agent-server
+          namespace: monitoring
+        data:
+          prometheus.yml: |
+            global:
+              scrape_interval: 15s
+              evaluation_interval: 30s
+            scrape_configs:
+              - job_name: "{{CLUSTER_NAME}}-nats-exporter"
+                static_configs:
+                  - targets: ["nats.nats.svc.cluster.local:7777"]
+            remote_write:
+              - url: "http://central-live-prometheus.privatelink.uaenorth.azmk8s.io/api/v1/write"

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/reloader.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: reloader
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/reloader
+  prune: false

package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/spark.yaml

@@ -0,0 +1,112 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: spark
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure/spark-operator/launchpad
+  prune: false
+  dependsOn:
+    - name: velora
+      namespace: flux-system
+    - name: acr-webhook-controller
+      namespace: flux-system
+  patches:
+    - target:
+        kind: HelmRelease
+        name: spark-operator
+        namespace: flux-system
+      patch: |-
+        apiVersion: helm.toolkit.fluxcd.io/v2
+        kind: HelmRelease
+        metadata:
+          name: spark-operator
+          namespace: flux-system
+        spec:
+          values:
+            image:
+              pullSecrets:
+                - name: ecr-credentials
+            spark:
+              jobNamespaces:
+                - velora
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: job-creator
+  namespace: velora
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["create", "delete", "get"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["sparkoperator.k8s.io"]
+  resources: ["sparkapplications"]
+  verbs: ["create", "delete", "get"]
+- apiGroups: ["sparkoperator.k8s.io"]
+  resources: ["scheduledsparkapplications"]
+  verbs: ["create", "delete", "get", "patch"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["create", "delete", "get", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: create-jobs
+  namespace: velora
+subjects:
+- kind: Group
+  name: system:serviceaccounts:foundation
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: job-creator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: spark
+  namespace: velora
+subjects:
+  - kind: Group
+    name: system:serviceaccounts:foundation
+roleRef:
+  kind: ClusterRole
+  name: job-creator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: allow-velora-sas-in-spark-jobs
+  namespace: spark-jobs
+subjects:
+- kind: Group
+  name: system:serviceaccounts:velora
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: job-creator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spark-jobs
+  labels:
+    name: spark-jobs