everything-gemini 1.8.0 → 2.0.0

package/skills/distributed-rate-limiting/SKILL.md

@@ -0,0 +1,138 @@
+---
+name: distributed-rate-limiting
+description: >-
+  Architects, implements, and tunes distributed rate limiting using Redis sliding window logs, token buckets, tiered API quotas, RFC 6585 headers, and circuit breakers.
+---
+
+# Distributed Rate Limiting & API Traffic Shaping
+
+This skill guides the design, implementation, and tuning of distributed rate limiting systems across microservice clusters using Redis, token buckets, and sliding window algorithms.
+
+---
+
+## ⚡ Algorithm Comparison Matrix
+
+| Algorithm | Precision | Memory Usage | Burst Handling | Best For |
+| :--- | :--- | :--- | :--- | :--- |
+| **`Sliding Window Log`** | 100% Exact | High ($O(N)$ requests) | Excellent | Low-to-medium traffic sensitive endpoints (auth, payments). |
+| **`Sliding Window Counter`**| ~99% Exact | Fixed ($O(1)$) | Smooth | High-scale public APIs and ingress gateways. |
+| **`Token Bucket`** | High | Very Low ($O(1)$) | Controlled bursts | Standard microservice backpressuring and rate smoothing. |
+| **`Fixed Window`** | Poor (2x burst at boundary) | Lowest | Prone to spikes | Simple non-critical background jobs only. |
+
+---
+
+## 🔒 Atomic Sliding Window Rate Limiter (Redis Lua)
+
+Atomic execution in Redis ensures no race conditions across distributed API instances:
+
+```lua
+-- sliding_window.lua
+-- KEYS[1]: Rate limit key (e.g., "ratelimit:tenant_123:orders")
+-- ARGV[1]: Current UNIX timestamp in milliseconds
+-- ARGV[2]: Window size in milliseconds (e.g., 60000 for 1 minute)
+-- ARGV[3]: Maximum allowed requests in window (e.g., 100)
+
+local key = KEYS[1]
+local now = tonumber(ARGV[1])
+local window = tonumber(ARGV[2])
+local limit = tonumber(ARGV[3])
+local clearBefore = now - window
+
+-- 1. Remove expired request timestamps outside the window
+redis.call('ZREMRANGEBYSCORE', key, 0, clearBefore)
+
+-- 2. Count requests currently in the sliding window
+local currentRequests = redis.call('ZCARD', key)
+
+if currentRequests < limit then
+    -- 3. Register current request with unique timestamp as score and member
+    redis.call('ZADD', key, now, now .. '-' .. math.random(1000, 9999))
+    redis.call('PEXPIRE', key, window)
+    return { 1, limit - currentRequests - 1, 0 } -- [Allowed: 1, Remaining, RetryAfter: 0]
+else
+    -- 4. Calculate time until oldest entry expires
+    local oldest = redis.call('ZRANGE', key, 0, 0, 'WITHSCORES')
+    local retryAfter = math.ceil((tonumber(oldest[2]) + window - now) / 1000)
+    return { 0, 0, math.max(1, retryAfter) }      -- [Allowed: 0, Remaining: 0, RetryAfter]
+end
+```
+
+---
+
+## 🌐 Express / Fastify / Node.js Middleware Implementation
+
+Wrap Redis Lua execution into production middleware with RFC 6585 compliance:
+
+```typescript
+import { Request, Response, NextFunction } from 'express';
+import Redis from 'ioredis';
+
+const redis = new Redis(process.env.REDIS_URL || 'redis://localhost:6379');
+
+export function createRateLimiter(options: { windowMs: number; max: number; keyGenerator?: (req: Request) => string }) {
+  return async (req: Request, res: Response, next: NextFunction) => {
+    const key = options.keyGenerator ? options.keyGenerator(req) : `ratelimit:${req.ip}`;
+    const now = Date.now();
+
+    try {
+      const [allowed, remaining, retryAfter] = await redis.eval(
+        SLIDING_WINDOW_LUA_SCRIPT,
+        1,
+        key,
+        now.toString(),
+        options.windowMs.toString(),
+        options.max.toString()
+      ) as [number, number, number];
+
+      // Set standard RFC 6585 RateLimit response headers
+      res.setHeader('X-RateLimit-Limit', options.max);
+      res.setHeader('X-RateLimit-Remaining', remaining);
+      res.setHeader('X-RateLimit-Reset', Math.ceil((now + options.windowMs) / 1000));
+
+      if (allowed === 1) {
+        return next();
+      }
+
+      res.setHeader('Retry-After', retryAfter);
+      return res.status(429).json({
+        error: 'Too Many Requests',
+        message: `Rate limit exceeded. Please retry after ${retryAfter} seconds.`,
+        retryAfter,
+      });
+    } catch (err) {
+      // Circuit Breaker: Fail-Open for high availability if Redis degrades
+      console.error('RateLimiter Redis failure, failing open:', err);
+      return next();
+    }
+  };
+}
+```
+
+---
+
+## 🎯 Tiered Multi-Dimensional Quotas
+
+Configure tiered rate limits based on authenticated API keys and endpoint criticality:
+
+```typescript
+export function getTierLimits(tier: 'free' | 'pro' | 'enterprise') {
+  switch (tier) {
+    case 'enterprise':
+      return { windowMs: 60_000, max: 10_000 };
+    case 'pro':
+      return { windowMs: 60_000, max: 1_200 };
+    case 'free':
+    default:
+      return { windowMs: 60_000, max: 60 };
+  }
+}
+```
+
+---
+
+## 📋 Architectural Rules & Best Practices
+
+1. **Atomic Scripts**: Never implement rate limiters with separated `GET` and `INCR` commands; always execute atomic Lua scripts or Redis Transactions (`MULTI/EXEC`) to prevent race conditions under load.
+2. **Fail-Open Strategy**: For user-facing critical flows, catch Redis connection timeouts and fail-open with alerting rather than causing total service outages.
+3. **Multi-Dimensional Keys**: Combine client identity with endpoint routes (e.g. `tenant:123:POST:/api/v1/checkout`) to prevent denial of service on high-cost endpoints.
+4. **Header Standards**: Always emit `Retry-After`, `X-RateLimit-Limit`, and `X-RateLimit-Remaining` to allow compliant clients to back off exponentially.

package/skills/keda-event-autoscaling/SKILL.md

@@ -0,0 +1,218 @@
+---
+name: keda-event-autoscaling
+description: Architects and configures Kubernetes Event-driven Autoscaling (KEDA) ScaledObjects, ScaledJobs, and event triggers for automated scale-to-zero and burst workloads.
+---
+
+# KEDA Event-Driven Autoscaling
+
+Configures Kubernetes Event-driven Autoscaling (KEDA) to dynamically scale workloads from 0 to N replicas based on external event queues, metrics streams, and consumer lag.
+
+---
+
+## 1. Core Architecture
+
+```
+ External Event Source          KEDA Operator              Kubernetes Runtime
+┌──────────────────────┐     ┌───────────────────┐     ┌──────────────────────┐
+│ Kafka / Redis / SQS  │────>│ keda-metrics-apiserver ──────>│ HorizontalPodAutoscaler │
+│ Prometheus / PubSub  │     │ (Metrics Provider)│     │ (Target Replicas)    │
+└──────────────────────┘     └─────────┬─────────┘     └──────────┬───────────┘
+                                       │                          │
+                             ┌─────────▼─────────┐     ┌──────────▼───────────┐
+                             │ ScaledObject CRD  │────>│ Scaled Deployment    │
+                             │ (0 <-> 1 Replica) │     │ (Min: 0, Max: 50)    │
+                             └───────────────────┘     └──────────────────────┘
+```
+
+- **Metrics Server**: Translates external trigger metrics into standard Kubernetes custom metrics for the native HPA.
+- **Scale to Zero (0 -> 1)**: Handled directly by KEDA controller activating idle deployments when events arrive.
+- **Scale Beyond One (1 -> N)**: Handled by native Kubernetes HPA driven by KEDA external metrics.
+
+---
+
+## 2. Kafka Consumer Lag ScaledObject
+
+Scales pod replicas based on consumer group topic lag with secure secret authentication:
+
+```yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: kafka-trigger-auth
+  namespace: production
+spec:
+  secretTargetRef:
+    - parameter: sasl
+      name: kafka-credentials
+      key: sasl-mechanism
+    - parameter: username
+      name: kafka-credentials
+      key: sasl-username
+    - parameter: password
+      name: kafka-credentials
+      key: sasl-password
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: order-processor-scaler
+  namespace: production
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: order-processor
+  pollingInterval: 15
+  cooldownPeriod: 120
+  idleReplicaCount: 0
+  minReplicaCount: 1
+  maxReplicaCount: 40
+  advanced:
+    horizontalPodAutoscalerConfig:
+      behavior:
+        scaleDown:
+          stabilizationWindowSeconds: 300
+          policies:
+            - type: Percent
+              value: 20
+              periodSeconds: 60
+        scaleUp:
+          stabilizationWindowSeconds: 0
+          policies:
+            - type: Percent
+              value: 100
+              periodSeconds: 15
+  triggers:
+    - type: kafka
+      metadata:
+        bootstrapServers: kafka-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092
+        consumerGroup: order-processor-group
+        topic: orders.v1
+        lagThreshold: "100"
+        activationLagThreshold: "1"
+      authenticationRef:
+        name: kafka-trigger-auth
+```
+
+---
+
+## 3. Redis Queue & Stream ScaledObject
+
+Scales Celery or BullMQ worker queues dynamically based on pending queue length:
+
+```yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: celery-worker-scaler
+  namespace: production
+spec:
+  scaleTargetRef:
+    name: celery-worker
+  minReplicaCount: 0
+  maxReplicaCount: 20
+  pollingInterval: 10
+  cooldownPeriod: 60
+  triggers:
+    - type: redis
+      metadata:
+        address: redis-cluster.cache.svc.cluster.local:6379
+        listName: celery
+        listLength: "50"
+        activationListLength: "1"
+        enableTLS: "true"
+      authenticationRef:
+        name: redis-trigger-auth
+```
+
+---
+
+## 4. ScaledJob for Finite Batch Workloads
+
+Spawns isolated Kubernetes Jobs for batch or run-to-completion work (1 job per event):
+
+```yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledJob
+metadata:
+  name: heavy-video-transcoder
+  namespace: batch
+spec:
+  jobTargetRef:
+    parallelism: 1
+    completions: 1
+    activeDeadlineSeconds: 1800
+    backoffLimit: 2
+    template:
+      spec:
+        restartPolicy: OnFailure
+        containers:
+          - name: transcoder
+            image: registry.example.com/video-transcoder:v2.0.0
+            resources:
+              requests:
+                cpu: "2"
+                memory: 4Gi
+  pollingInterval: 30
+  successfulJobsHistoryLimit: 5
+  failedJobsHistoryLimit: 10
+  maxReplicaCount: 25
+  scalingStrategy:
+    strategy: "custom"
+    customScalingQueueLengthDeduction: 1
+  triggers:
+    - type: aws-sqs-queue
+      metadata:
+        queueURL: https://sqs.us-east-1.amazonaws.com/123456789012/transcode-jobs
+        queueLength: "1"
+        awsRegion: us-east-1
+      authenticationRef:
+        name: keda-aws-auth
+```
+
+---
+
+## 5. Prometheus Custom Metric Scaler
+
+Scales HTTP microservices on rate of requests (RPS) or error ratios:
+
+```yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: api-service-scaler
+  namespace: production
+spec:
+  scaleTargetRef:
+    name: api-gateway
+  minReplicaCount: 3
+  maxReplicaCount: 50
+  triggers:
+    - type: prometheus
+      metadata:
+        serverAddress: http://prometheus-k8s.monitoring.svc:9090
+        metricName: http_requests_total_rate
+        query: sum(rate(http_requests_total{job="api-gateway", status=~"2.."}[2m]))
+        threshold: "500"
+```
+
+---
+
+## 6. Verification and Troubleshooting
+
+```bash
+# Check status and current active state of ScaledObjects
+kubectl get scaledobject -n production
+kubectl describe scaledobject order-processor-scaler -n production
+
+# Inspect generated HPA and custom metrics
+kubectl get hpa -n production
+kubectl describe hpa keda-hpa-order-processor-scaler -n production
+
+# View KEDA Operator and Metrics Server logs
+kubectl logs -n keda -l app=keda-operator -f
+kubectl logs -n keda -l app=keda-operator-metrics-apiserver -f
+
+# Verify custom metrics registered in K8s APIServer
+kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" | jq .
+```

package/skills/milvus-vector-clustering/SKILL.md

@@ -0,0 +1,134 @@
+---
+name: milvus-vector-clustering
+description: >-
+  Architects, scales, and optimizes Milvus distributed vector databases, HNSW/IVF-PQ indexing, hybrid dense-sparse search, dynamic schema partition keys, and scalar metadata filtering.
+---
+
+# Milvus Vector Clustering & Distributed Similarity Search
+
+This skill guides the design, indexing, and high-throughput querying of enterprise-scale vector collections using Milvus and Zilliz Cloud.
+
+---
+
+## ⚡ Indexing Selection Matrix
+
+| Index Type | Search Latency | Recall Rate | Memory Footprint | Best For |
+| :--- | :--- | :--- | :--- | :--- |
+| **`HNSW`** | Ultra-low (<5ms) | High (>98%) | High (in-memory graph) | Mission-critical search where latency is top priority. |
+| **`IVF_FLAT`** | Medium (10-30ms) | Tunable (nprobe) | Moderate | Standard baseline for medium datasets (<10M vectors). |
+| **`IVF_PQ`** | Low (5-15ms) | Moderate (85-92%) | Very Low (quantized) | Massive scale (100M+ vectors) with tight RAM budgets. |
+| **`SCANN`** | Very Low (<8ms) | High (>95%) | Moderate | High-dimensional CPU-accelerated similarity search. |
+| **`SPARSE_INVERTED_INDEX`**| Ultra-low | Exact / BM25 | Low | Sparse lexical vectors (SPLADE, BM25 hybrid ranking). |
+
+---
+
+## 🏗️ Production Collection Schema with Partition Keys
+
+Multi-tenant applications should leverage Milvus **Partition Keys** to isolate tenant data logically without exceeding physical partition limits:
+
+```python
+from pymilvus import (
+    connections, FieldSchema, CollectionSchema, DataType,
+    Collection, utility
+)
+
+# Connect to Milvus Cluster / Standalone
+connections.connect(alias="default", host="localhost", port="19530")
+
+fields = [
+    # Primary Key
+    FieldSchema(name="doc_id", dtype=DataType.VARCHAR, max_length=128, is_primary=True, auto_id=False),
+    
+    # Multi-tenant Partition Key (routes queries to physical partition clusters)
+    FieldSchema(name="tenant_id", dtype=DataType.VARCHAR, max_length=64, is_partition_key=True),
+    
+    # Scalar Metadata for Pre-filtering
+    FieldSchema(name="category", dtype=DataType.VARCHAR, max_length=64),
+    FieldSchema(name="created_at", dtype=DataType.INT64),
+    FieldSchema(name="is_archived", dtype=DataType.BOOL),
+    
+    # Dense Vector Embedding (e.g., text-embedding-004, 768 dimensions)
+    FieldSchema(name="dense_vector", dtype=DataType.FLOAT_VECTOR, dim=768),
+    
+    # Sparse Lexical Vector (e.g., SPLADE or BM25 tokens)
+    FieldSchema(name="sparse_vector", dtype=DataType.SPARSE_FLOAT_VECTOR),
+]
+
+schema = CollectionSchema(fields=fields, description="Enterprise Knowledge Base Vectors", enable_dynamic_field=True)
+collection = Collection(name="knowledge_embeddings", schema=schema)
+```
+
+---
+
+## 🔍 Building Vector Indexes
+
+Build indexes before loading the collection into memory:
+
+```python
+# 1. Dense Vector Index (HNSW for high recall & low latency)
+dense_index_params = {
+    "metric_type": "COSINE",
+    "index_type": "HNSW",
+    "params": {
+        "M": 16,               # Max outgoing links per node (16-64)
+        "efConstruction": 200  # Exploration factor during index build
+    }
+}
+collection.create_index(field_name="dense_vector", index_params=dense_index_params)
+
+# 2. Sparse Vector Index
+sparse_index_params = {
+    "metric_type": "IP",       # Inner Product for sparse lexical weights
+    "index_type": "SPARSE_INVERTED_INDEX",
+    "params": {"drop_ratio_build": 0.2}
+}
+collection.create_index(field_name="sparse_vector", index_params=sparse_index_params)
+
+# Load into query nodes
+collection.load()
+```
+
+---
+
+## 🚀 Hybrid Search with Reciprocal Rank Fusion (RRF)
+
+Combine dense semantic search with sparse lexical matching for optimal recall:
+
+```python
+from pymilvus import AnnSearchRequest, RRFRanker
+
+# Dense vector query request
+dense_req = AnnSearchRequest(
+    data=[query_dense_embedding],
+    anns_field="dense_vector",
+    param={"metric_type": "COSINE", "params": {"ef": 64}},
+    limit=50,
+    expr='tenant_id == "org_acme" and is_archived == false'
+)
+
+# Sparse vector query request
+sparse_req = AnnSearchRequest(
+    data=[query_sparse_dict],
+    anns_field="sparse_vector",
+    param={"metric_type": "IP", "params": {"drop_ratio_search": 0.1}},
+    limit=50,
+    expr='tenant_id == "org_acme" and is_archived == false'
+)
+
+# Execute hybrid search with RRF reranking
+results = collection.hybrid_search(
+    reqs=[dense_req, sparse_req],
+    rerank=RRFRanker(k=60),
+    limit=10,
+    output_fields=["doc_id", "category", "created_at"]
+)
+```
+
+---
+
+## 📋 Architectural Rules & Best Practices
+
+1. **Partition Key Limits**: Use `is_partition_key=True` for tenant routing; Milvus maps millions of logical tenants into ~64-256 physical partition clusters automatically.
+2. **Pre-Filtering Optimization**: Place high-selectivity scalar filters (`expr='category == "tech"'`) directly in the ANN search request to prune candidate sets before graph traversal.
+3. **Compaction Management**: Call `collection.compact()` after bulk insertions or deletions to merge small segment files and purge deleted entity tombstones.
+4. **Memory Sizing**: For HNSW, allocate approximately `(dim * 4 + M * 8) * num_vectors` bytes of RAM on QueryNodes.

package/skills/opa-gatekeeper-policy/SKILL.md

@@ -0,0 +1,200 @@
+---
+name: opa-gatekeeper-policy
+description: Authors and validates Open Policy Agent (OPA) Gatekeeper ConstraintTemplates and Rego admission policies for Kubernetes governance and security compliance.
+---
+
+# OPA Gatekeeper Policy
+
+Defines declarative policy enforcement for Kubernetes clusters using Open Policy Agent (OPA) Gatekeeper ConstraintTemplates, Rego logic, and shift-left CLI testing.
+
+---
+
+## 1. Admission Control Flow
+
+```
+ kubectl apply -f resource.yaml
+            │
+            ▼
+┌────────────────────────┐
+│ Kubernetes API Server  │
+└───────────┬────────────┘
+            │ ValidatingWebhookConfiguration
+            ▼
+┌────────────────────────┐      Matches Rules?
+│   Gatekeeper Pod       │────────────────────────┐
+│  (Admission Webhook)   │                        │
+└───────────┬────────────┘                        ▼
+            │ Rego Evaluation          ┌────────────────────┐
+            ▼                          │ Reject (403)       │
+ ┌──────────────────────┐              │ Error & Remediation│
+ │ Constraint Templates │              └────────────────────┘
+ └──────────┬───────────┘
+            │ Passed
+            ▼
+ ┌──────────────────────┐
+ │ Persist to etcd      │
+ └──────────────────────┘
+```
+
+---
+
+## 2. ConstraintTemplate (Rego Engine)
+
+Defines the reusable policy schema and Rego execution logic:
+
+```yaml
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: k8srequiredlabels
+  annotations:
+    description: "Requires all namespaces and workloads to specify standard governance labels."
+spec:
+  crd:
+    spec:
+      names:
+        kind: K8sRequiredLabels
+      validation:
+        openAPIV3Schema:
+          type: object
+          properties:
+            labels:
+              type: array
+              items:
+                type: string
+  targets:
+    - target: admission.k8s.gatekeeper.sh
+      rego: |
+        package k8srequiredlabels
+
+        violation[{"msg": msg, "details": {"missing_labels": missing}}] {
+          provided := {label | input.review.object.metadata.labels[label]}
+          required := {label | label := input.parameters.labels[_]}
+          missing := required - provided
+          count(missing) > 0
+          msg := sprintf("Resource %v '%v' violates policy: missing required labels: %v", [
+            input.review.object.kind,
+            input.review.object.metadata.name,
+            missing
+          ])
+        }
+```
+
+---
+
+## 3. Instantiating the Constraint
+
+Applies the template to specific resource kinds and namespaces:
+
+```yaml
+apiVersion: constraints.gatekeeper.sh/v1beta1
+kind: K8sRequiredLabels
+metadata:
+  name: require-team-and-cost-center
+spec:
+  enforcementAction: deny # Options: deny, dryrun, warn
+  match:
+    kinds:
+      - apiGroups: ["apps"]
+        kinds: ["Deployment", "StatefulSet"]
+      - apiGroups: [""]
+        kinds: ["Namespace"]
+    excludedNamespaces:
+      - kube-system
+      - gatekeeper-system
+  parameters:
+    labels:
+      - "app.kubernetes.io/managed-by"
+      - "cost-center"
+      - "owner-team"
+```
+
+---
+
+## 4. Container Security Constraint (Disallow Privileged)
+
+Enforces zero-trust rootless containers:
+
+```yaml
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: k8sdisallowprivileged
+spec:
+  crd:
+    spec:
+      names:
+        kind: K8sDisallowPrivileged
+  targets:
+    - target: admission.k8s.gatekeeper.sh
+      rego: |
+        package k8sdisallowprivileged
+
+        violation[{"msg": msg}] {
+          c := input_containers[_]
+          c.securityContext.privileged == true
+          msg := sprintf("Privileged container '%v' is prohibited in namespace '%v'.", [
+            c.name,
+            input.review.object.metadata.namespace
+          ])
+        }
+
+        input_containers[c] {
+          c := input.review.object.spec.template.spec.containers[_]
+        }
+        input_containers[c] {
+          c := input.review.object.spec.template.spec.initContainers[_]
+        }
+```
+
+---
+
+## 5. Shift-Left CI Validation with `gator` CLI
+
+Validates Kubernetes manifests against Gatekeeper templates inside CI pipelines before merge:
+
+```bash
+# Test manifests directly without a live Kubernetes cluster
+gator test \
+  --image-template=./templates/ \
+  --constraint=./constraints/ \
+  --manifests=./deploy/production/
+
+# Run unit tests on Rego source files
+gator verify ./policies/...
+```
+
+```yaml
+# Gator test suite definition (suite.yaml)
+kind: Suite
+apiVersion: test.gatekeeper.sh/v1alpha1
+metadata:
+  name: deployment-label-suite
+tests:
+  - name: test-missing-labels-denied
+    template: template.yaml
+    constraint: constraint.yaml
+    cases:
+      - name: deployment-without-cost-center
+        object: bad-deployment.yaml
+        assertions:
+          - violations: yes
+            message: "missing required labels: {\"cost-center\"}"
+      - name: compliant-deployment
+        object: good-deployment.yaml
+        assertions:
+          - violations: no
+```
+
+---
+
+## 6. Audit Cluster Violations
+
+```bash
+# Inspect non-compliant resources identified during continuous background audit
+kubectl get constraints -A
+
+# Detailed violation listing
+kubectl get k8srequiredlabels.constraints.gatekeeper.sh require-team-and-cost-center -o yaml \
+  | yq '.status.violations[] | {resource: .name, kind: .kind, message: .message}'
+```