everything-gemini 2.2.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BENCHMARK.md +33 -33
- package/CHANGELOG.md +63 -0
- package/README.md +57 -6
- package/agents/resilience-engineer.md +36 -0
- package/agents/swarm-coordinator.md +35 -0
- package/bin/everything-gemini +949 -12
- package/docs/LAUNCH_KIT_v2.4.0.md +164 -0
- package/docs/LAUNCH_KIT_v2.5.0.md +187 -0
- package/docs/index.html +13 -9
- package/examples/python-sdk/ephemeral_swarm_orchestration.py +169 -0
- package/examples/python-sdk/resilience_and_telemetry_pipeline.py +183 -0
- package/hooks/drift-detector/hooks.json +16 -0
- package/hooks/drift-detector/scripts/detect-drift.sh +50 -0
- package/hooks/thinking-budget/hooks.json +16 -0
- package/hooks/thinking-budget/scripts/budget-thinking.sh +56 -0
- package/package.json +1 -1
- package/skills/cost-oracle-finops/SKILL.md +142 -0
- package/skills/dynamic-thinking-allocation/SKILL.md +105 -0
- package/skills/ebpf-sandbox-enforcer/SKILL.md +101 -0
- package/skills/envoy-ai-gateway/SKILL.md +175 -0
- package/skills/ephemeral-swarm-orchestrator/SKILL.md +124 -0
- package/skills/gemini-live-multimodal/SKILL.md +149 -0
- package/skills/harness3-mutation-testing/SKILL.md +120 -0
- package/skills/opentelemetry-agent-observability/SKILL.md +139 -0
- package/skills/quantum-safe-cryptography/SKILL.md +84 -0
- package/skills/spanner-change-streams/SKILL.md +88 -0
- package/skills/vertex-ai-feature-store/SKILL.md +110 -0
- package/skills/webrtc-agent-gateway/SKILL.md +199 -0
- package/tests/cli.test.js +170 -14
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: opentelemetry-agent-observability
|
|
3
|
+
description: Implements OpenTelemetry GenAI semantic conventions, distributed multi-agent W3C trace propagation, tool call span attributes, and Google Cloud Trace export for autonomous agent workflows.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# OpenTelemetry Agent Observability (GenAI Semantic Conventions & Distributed Tracing)
|
|
7
|
+
|
|
8
|
+
Instruments autonomous multi-agent pipelines with OpenTelemetry (v1.30+), adhering strictly to OpenTelemetry GenAI Semantic Conventions (`gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`). Propagates W3C `traceparent` headers across subagent handoffs, captures internal thinking token events, and exports traces to Google Cloud Trace and Jaeger.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. Multi-Agent Distributed Tracing Architecture
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
16
|
+
│ OPENTELEMETRY TRACE TOPOLOGY │
|
|
17
|
+
│ │
|
|
18
|
+
│ Planner Span (trace_id: 4bf92f3577b34da6a3ce929d0e0e4736) │
|
|
19
|
+
│ ├── gen_ai.system: "gemini" │
|
|
20
|
+
│ ├── gen_ai.request.model: "gemini-3.8-thinking" │
|
|
21
|
+
│ │ │
|
|
22
|
+
│ ├── Subagent Handoff (W3C traceparent injected into message payload) │
|
|
23
|
+
│ │ └── Swarm Coordinator Span │
|
|
24
|
+
│ │ ├── Worker-1 Tool Execution Span (eBPF Sandboxed) │
|
|
25
|
+
│ │ └── Worker-2 Tool Execution Span (eBPF Sandboxed) │
|
|
26
|
+
│ │ │
|
|
27
|
+
│ └── Cloud Trace Exporter (OTLP gRPC -> Google Cloud Trace) │
|
|
28
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 2. Python OpenTelemetry GenAI Instrumentor
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
import os
|
|
37
|
+
from contextlib import contextmanager
|
|
38
|
+
from opentelemetry import trace
|
|
39
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
40
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
41
|
+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
42
|
+
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
43
|
+
from opentelemetry.trace import Status, StatusCode
|
|
44
|
+
|
|
45
|
+
# Initialize TracerProvider and OTLP Exporter
|
|
46
|
+
provider = TracerProvider()
|
|
47
|
+
otlp_exporter = OTLPSpanExporter(
|
|
48
|
+
endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "localhost:4317"),
|
|
49
|
+
insecure=True
|
|
50
|
+
)
|
|
51
|
+
provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
|
|
52
|
+
trace.set_tracer_provider(provider)
|
|
53
|
+
tracer = trace.get_tracer("everything-gemini.runtime", "2.5.0")
|
|
54
|
+
propagator = TraceContextTextMapPropagator()
|
|
55
|
+
|
|
56
|
+
class AgentTelemetryContext:
|
|
57
|
+
@staticmethod
|
|
58
|
+
@contextmanager
|
|
59
|
+
def start_llm_span(agent_name: str, model_name: str, operation_name: str = "chat"):
|
|
60
|
+
"""Starts a span compliant with OpenTelemetry GenAI Semantic Conventions."""
|
|
61
|
+
with tracer.start_as_current_span(
|
|
62
|
+
f"gen_ai.{operation_name} {agent_name}",
|
|
63
|
+
kind=trace.SpanKind.INTERNAL
|
|
64
|
+
) as span:
|
|
65
|
+
span.set_attribute("gen_ai.system", "gemini")
|
|
66
|
+
span.set_attribute("gen_ai.request.model", model_name)
|
|
67
|
+
span.set_attribute("gen_ai.agent.name", agent_name)
|
|
68
|
+
span.set_attribute("gen_ai.operation.name", operation_name)
|
|
69
|
+
try:
|
|
70
|
+
yield span
|
|
71
|
+
except Exception as exc:
|
|
72
|
+
span.record_exception(exc)
|
|
73
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
74
|
+
raise
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def record_usage(span: trace.Span, prompt_tokens: int, completion_tokens: int, thinking_tokens: int = 0):
|
|
78
|
+
"""Records standardized GenAI token metrics and thinking tokens."""
|
|
79
|
+
span.set_attribute("gen_ai.usage.input_tokens", prompt_tokens)
|
|
80
|
+
span.set_attribute("gen_ai.usage.output_tokens", completion_tokens)
|
|
81
|
+
span.set_attribute("gen_ai.usage.total_tokens", prompt_tokens + completion_tokens)
|
|
82
|
+
if thinking_tokens > 0:
|
|
83
|
+
span.set_attribute("gen_ai.usage.thinking_tokens", thinking_tokens)
|
|
84
|
+
span.add_event("thinking_budget_allocated", {"tokens": thinking_tokens})
|
|
85
|
+
|
|
86
|
+
@staticmethod
|
|
87
|
+
def inject_carrier() -> dict:
|
|
88
|
+
"""Injects W3C traceparent into a dictionary for cross-agent RPC or Pub/Sub."""
|
|
89
|
+
carrier = {}
|
|
90
|
+
propagator.inject(carrier)
|
|
91
|
+
return carrier
|
|
92
|
+
|
|
93
|
+
@staticmethod
|
|
94
|
+
def extract_context(carrier: dict):
|
|
95
|
+
"""Extracts remote trace context from carrier dictionary."""
|
|
96
|
+
return propagator.extract(carrier)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## 3. Subagent Handoff Trace Propagation Example
|
|
102
|
+
|
|
103
|
+
When delegating tasks across scoped subagents (e.g. `planner` to `swarm-coordinator`), inject W3C headers to preserve single continuous traces:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
# In Planner subagent:
|
|
107
|
+
with AgentTelemetryContext.start_llm_span("planner", "gemini-3.8-pro") as span:
|
|
108
|
+
carrier = AgentTelemetryContext.inject_carrier()
|
|
109
|
+
# Payload sent to swarm-coordinator includes trace context
|
|
110
|
+
dispatch_payload = {
|
|
111
|
+
"task": "refactor_auth_module",
|
|
112
|
+
"trace_context": carrier
|
|
113
|
+
}
|
|
114
|
+
AgentTelemetryContext.record_usage(span, prompt_tokens=1200, completion_tokens=340)
|
|
115
|
+
|
|
116
|
+
# In Swarm Coordinator:
|
|
117
|
+
ctx = AgentTelemetryContext.extract_context(dispatch_payload["trace_context"])
|
|
118
|
+
with tracer.start_as_current_span("swarm_coordinator.execute", context=ctx) as sub_span:
|
|
119
|
+
sub_span.set_attribute("swarm.workers.count", 4)
|
|
120
|
+
# Child spans automatically link to the parent Planner span
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 4. Google Cloud Trace Integration via Terraform
|
|
126
|
+
|
|
127
|
+
```hcl
|
|
128
|
+
resource "google_project_iam_member" "agent_trace_agent" {
|
|
129
|
+
project = var.gcp_project_id
|
|
130
|
+
role = "roles/cloudtrace.agent"
|
|
131
|
+
member = "serviceAccount:${google_service_account.agent_runtime.email}"
|
|
132
|
+
}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Export telemetry with standard environment variables:
|
|
136
|
+
```bash
|
|
137
|
+
export OTEL_EXPORTER_OTLP_ENDPOINT="https://cloudtrace.googleapis.com"
|
|
138
|
+
export OTEL_RESOURCE_ATTRIBUTES="service.name=everything-gemini,service.version=2.5.0"
|
|
139
|
+
```
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: quantum-safe-cryptography
|
|
3
|
+
description: Implements NIST FIPS 203/204 Post-Quantum Cryptography (PQC) algorithms, hybrid TLS 1.3 key exchange, and quantum-resistant API authentication.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Quantum-Safe Cryptography & Post-Quantum Cryptography (PQC)
|
|
7
|
+
|
|
8
|
+
Implements NIST Post-Quantum Cryptography standards (FIPS 203 ML-KEM / Kyber, FIPS 204 ML-DSA / Dilithium), hybrid classical-quantum TLS 1.3 key exchanges, and quantum-resistant authentication token signing for enterprise Gemini agent communications.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. Architecture: Post-Quantum Security for AI Agents
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
16
|
+
│ HYBRID POST-QUANTUM SECURITY PIPELINE │
|
|
17
|
+
│ │
|
|
18
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
19
|
+
│ │ 1. Agent Handshake & Mutual Authentication (mTLS 1.3) │ │
|
|
20
|
+
│ │ • Classical: X25519 or ECDH (P-256) │ │
|
|
21
|
+
│ │ • Post-Quantum: ML-KEM-768 (Module-Lattice KEM / Kyber) │ │
|
|
22
|
+
│ │ • Combined Shared Secret = KDF(Classical_SS || PQC_SS) │ │
|
|
23
|
+
│ └──────────────────────────────────┬──────────────────────────────────┘ │
|
|
24
|
+
│ │ │
|
|
25
|
+
│ ▼ │
|
|
26
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
27
|
+
│ │ 2. Quantum-Resistant Agent Token Signing (ML-DSA / Dilithium) │ │
|
|
28
|
+
│ │ • Ephemeral agent delegation tokens signed via FIPS 204 │ │
|
|
29
|
+
│ │ • Immune to Shor's algorithm and "Harvest Now, Decrypt Later" │ │
|
|
30
|
+
│ └──────────────────────────────────┬──────────────────────────────────┘ │
|
|
31
|
+
│ │ │
|
|
32
|
+
│ ▼ │
|
|
33
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
34
|
+
│ │ 3. Cloud KMS & Google Cloud Hybrid Integration │ │
|
|
35
|
+
│ │ • Google Cloud Certificate Authority Service (CAS) PQC roots │ │
|
|
36
|
+
│ │ • Vertex AI / Cloud Run encrypted ingress via X25519Kyber768 │ │
|
|
37
|
+
│ └─────────────────────────────────────────────────────────────────────┘ │
|
|
38
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## 2. NIST Standardized PQC Specifications
|
|
44
|
+
|
|
45
|
+
| Standard | Algorithm Name | Purpose | Classical Equivalent | Key/Sig Sizes |
|
|
46
|
+
| :--- | :--- | :--- | :--- | :--- |
|
|
47
|
+
| **FIPS 203** | **ML-KEM-768** (Kyber) | Key Encapsulation (KEM) | RSA 3072 / ECDH P-256 | Public Key: 1,184 B, Ciphertext: 1,088 B |
|
|
48
|
+
| **FIPS 204** | **ML-DSA-65** (Dilithium) | Digital Signatures | ECDSA P-256 / Ed25519 | Public Key: 1,952 B, Signature: 3,309 B |
|
|
49
|
+
| **FIPS 205** | **SLH-DSA-128s** (SPHINCS+) | Stateless Hash-Based Signatures | Fallback Digital Signatures | Public Key: 32 B, Signature: 7,856 B |
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## 3. Node.js & Web Cryptography Implementation
|
|
54
|
+
|
|
55
|
+
```javascript
|
|
56
|
+
import crypto from 'node:crypto';
|
|
57
|
+
|
|
58
|
+
// Verify runtime supports hybrid post-quantum cipher suites
|
|
59
|
+
export function getQuantumSafeCipherConfiguration() {
|
|
60
|
+
const ciphers = crypto.getCiphers();
|
|
61
|
+
const curves = crypto.getCurves ? crypto.getCurves() : [];
|
|
62
|
+
|
|
63
|
+
const supportsKyber = curves.includes('X25519Kyber768Draft00') || curves.includes('x25519_kyber768');
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
tlsVersion: 'TLSv1.3',
|
|
67
|
+
ecdhCurve: supportsKyber ? 'X25519Kyber768Draft00:P-256' : 'auto',
|
|
68
|
+
signatureAlgorithms: [
|
|
69
|
+
'mldsa65',
|
|
70
|
+
'ecdsa_secp256r1_sha256',
|
|
71
|
+
'ed25519'
|
|
72
|
+
],
|
|
73
|
+
pqcReady: supportsKyber
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## 4. Agent Migration Checklist
|
|
81
|
+
|
|
82
|
+
1. **Inventory Cryptographic Assets**: Audit all RSA 2048/4096 and ECDSA keys used for subagent authentication and webhook signing.
|
|
83
|
+
2. **Deploy Hybrid Key Exchange First**: Enable `X25519Kyber768` on all reverse proxies, load balancers, and Cloud Run agent endpoints. This protects against *Harvest Now, Decrypt Later* (HNDL) attacks with zero downtime.
|
|
84
|
+
3. **Upgrade Token Signatures**: Migrate internal agent JWTs to ML-DSA signatures with fallback validation for legacy clients.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: spanner-change-streams
|
|
3
|
+
description: Implements Google Cloud Spanner Change Streams for real-time transaction log Change Data Capture, Pub/Sub integration, and event-driven database synchronization.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Google Cloud Spanner Change Streams (Real-Time CDC)
|
|
7
|
+
|
|
8
|
+
Designs, provisions, and processes transaction-level Change Data Capture (CDC) events using Google Cloud Spanner Change Streams. Enables event-driven microservices, real-time cache invalidation, and continuous replication to BigQuery and Pub/Sub.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. Architecture: Spanner Change Streams & Data Mesh
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
16
|
+
│ SPANNER CHANGE STREAM PIPELINE │
|
|
17
|
+
│ │
|
|
18
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
19
|
+
│ │ Google Cloud Spanner Database │ │
|
|
20
|
+
│ │ (Transactions committing to primary tables) │ │
|
|
21
|
+
│ └──────────────────────────────────┬──────────────────────────────────┘ │
|
|
22
|
+
│ │ │
|
|
23
|
+
│ ▼ │
|
|
24
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
25
|
+
│ │ Spanner Change Stream (CREATE CHANGE STREAM) │ │
|
|
26
|
+
│ │ Captures INSERT, UPDATE, DELETE with externalized commit timestamps │ │
|
|
27
|
+
│ └──────────────────────────────────┬──────────────────────────────────┘ │
|
|
28
|
+
│ │ │
|
|
29
|
+
│ ▼ │
|
|
30
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
31
|
+
│ │ Dataflow CDC Consumer / Cloud Functions (Java / Python) │ │
|
|
32
|
+
│ └──────────────────┬────────────────────────────┬─────────────────────┘ │
|
|
33
|
+
│ │ │ │
|
|
34
|
+
│ ▼ Real-time Invalidation ▼ Continuous Analytics │
|
|
35
|
+
│ ┌─────────────────────────────┐ ┌─────────────────────────────┐ │
|
|
36
|
+
│ │ Redis / Valkey Cache │ │ BigQuery Real-Time Replica │ │
|
|
37
|
+
│ │ (Sub-second Invalidation) │ │ (Streaming Buffer Table) │ │
|
|
38
|
+
│ └─────────────────────────────┘ └─────────────────────────────┘ │
|
|
39
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## 2. Spanner DDL (`CREATE CHANGE STREAM`)
|
|
45
|
+
|
|
46
|
+
Provision a change stream watching specific tables and columns with retention policies:
|
|
47
|
+
|
|
48
|
+
```sql
|
|
49
|
+
-- 1. Create Change Stream tracking Accounts and Orders tables
|
|
50
|
+
CREATE CHANGE STREAM AgentAuditChangeStream
|
|
51
|
+
FOR Services, Dependencies
|
|
52
|
+
OPTIONS (
|
|
53
|
+
retention_period = '36h',
|
|
54
|
+
value_capture_type = 'OLD_AND_NEW_VALUES'
|
|
55
|
+
);
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
- **`retention_period`**: Retains change stream data for up to 7 days (default is 24 hours).
|
|
59
|
+
- **`value_capture_type`**: `OLD_AND_NEW_VALUES` captures before-and-after states, essential for audit logs and rollback verification.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 3. Dataflow Pipeline Template Deployment (`gcloud`)
|
|
64
|
+
|
|
65
|
+
Stream Spanner mutations directly into Google Cloud Pub/Sub:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
gcloud dataflow flex-template run spanner-to-pubsub-cdc \
|
|
69
|
+
--template-file-gcs-location="gs://dataflow-templates/latest/flex/Spanner_Change_Streams_to_PubSub" \
|
|
70
|
+
--region="us-central1" \
|
|
71
|
+
--parameters \
|
|
72
|
+
spannerInstanceId="production-instance",\
|
|
73
|
+
spannerDatabase="architecture-db",\
|
|
74
|
+
spannerMetadataInstanceId="production-instance",\
|
|
75
|
+
spannerMetadataDatabase="cdc-metadata",\
|
|
76
|
+
spannerChangeStreamName="AgentAuditChangeStream",\
|
|
77
|
+
pubsubTopic="projects/my_project/topics/spanner-cdc-events"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## 4. Change Stream Processing Best Practices
|
|
83
|
+
|
|
84
|
+
| Strategy | Recommendation | Anti-Pattern |
|
|
85
|
+
| :--- | :--- | :--- |
|
|
86
|
+
| **Metadata Database** | Maintain a dedicated Spanner metadata database for Dataflow partition tracking | Storing metadata in the production database increasing transaction contention |
|
|
87
|
+
| **Retention Window** | Set retention to at least 36 hours (`36h`) to survive Dataflow upgrades | 1-hour retention risking permanent data loss during pipeline restarts |
|
|
88
|
+
| **Payload Optimization** | Only track critical tables instead of `FOR ALL` database objects | Capturing ephemeral log tables overwhelming streaming workers |
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: vertex-ai-feature-store
|
|
3
|
+
description: Provisions and queries Google Cloud Vertex AI Feature Store, point-in-time correct historical joins, low-latency online serving, and Gemini embedding feature tables.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Vertex AI Feature Store & Embeddings Serving
|
|
7
|
+
|
|
8
|
+
Architects, provisions, and serves machine learning features and vector representations using Google Cloud Vertex AI Feature Store. Covers BigQuery-backed offline storage, point-in-time joins, low-latency online feature serving, and real-time agent memory retrieval.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. Architecture: Vertex AI Feature Store
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
16
|
+
│ VERTEX AI FEATURE STORE │
|
|
17
|
+
│ │
|
|
18
|
+
│ ┌─────────────────────┐ ┌───────────────────────────┐ │
|
|
19
|
+
│ │ BigQuery Lakehouse │ │ Streaming Ingestion │ │
|
|
20
|
+
│ │ (Offline Gold Data) │ │ (Pub/Sub + Dataflow) │ │
|
|
21
|
+
│ └──────────┬──────────┘ └─────────────┬─────────────┘ │
|
|
22
|
+
│ │ │ │
|
|
23
|
+
│ ▼ ▼ │
|
|
24
|
+
│ ┌─────────────────────────────────────────────────────────────────────┐ │
|
|
25
|
+
│ │ Feature Registry & View Definitions (BigQuery-backed Feature Store) │ │
|
|
26
|
+
│ │ • Entity: user_id, repo_id, session_id │ │
|
|
27
|
+
│ │ • Embeddings: text_embedding_004 (768-dim) │ │
|
|
28
|
+
│ └──────────────────┬────────────────────────────┬─────────────────────┘ │
|
|
29
|
+
│ │ │ │
|
|
30
|
+
│ ▼ Offline ▼ Online (<10ms) │
|
|
31
|
+
│ ┌─────────────────────────────┐ ┌─────────────────────────────┐ │
|
|
32
|
+
│ │ Point-in-Time Correct │ │ Optimized Online Serving │ │
|
|
33
|
+
│ │ Historical Training Joins │ │ (Bigtable / Cache Engine) │ │
|
|
34
|
+
│ └─────────────────────────────┘ └─────────────────────────────┘ │
|
|
35
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## 2. Infrastructure Provisioning (`gcloud`)
|
|
41
|
+
|
|
42
|
+
Set up a Feature Online Store optimized for sub-10ms feature retrieval:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
# 1. Create Feature Online Store with optimized serving
|
|
46
|
+
gcloud vertex-ai feature-online-stores create agent-memory-store \
|
|
47
|
+
--location=us-central1 \
|
|
48
|
+
--dedicated-serving-nodes-min=1 \
|
|
49
|
+
--dedicated-serving-nodes-max=5
|
|
50
|
+
|
|
51
|
+
# 2. Create Feature View backed by BigQuery table
|
|
52
|
+
gcloud vertex-ai feature-views create user-profile-features \
|
|
53
|
+
--feature-online-store=agent-memory-store \
|
|
54
|
+
--location=us-central1 \
|
|
55
|
+
--bigquery-table=projects/my_project/datasets/features/tables/user_profiles \
|
|
56
|
+
--entity-id-columns=user_id \
|
|
57
|
+
--cron-schedule="0 */4 * * *"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## 3. Real-Time Online Feature Retrieval (Python SDK)
|
|
63
|
+
|
|
64
|
+
Query real-time feature vectors inside an agent execution turn:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from google.cloud import aiplatform_v1beta1 as vertex_ai
|
|
68
|
+
|
|
69
|
+
client = vertex_ai.FeatureOnlineStoreServiceClient()
|
|
70
|
+
|
|
71
|
+
def fetch_agent_session_features(user_id: str) -> dict:
|
|
72
|
+
"""Fetches real-time developer context and historical preference vectors."""
|
|
73
|
+
feature_view_path = client.feature_view_path(
|
|
74
|
+
project="my_project",
|
|
75
|
+
location="us-central1",
|
|
76
|
+
feature_online_store="agent-memory-store",
|
|
77
|
+
feature_view="user-profile-features"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
response = client.fetch_feature_values(
|
|
81
|
+
request=vertex_ai.FetchFeatureValuesRequest(
|
|
82
|
+
feature_view=feature_view_path,
|
|
83
|
+
data_key=vertex_ai.FeatureViewDataKey(key=user_id)
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
field.name: field.value
|
|
89
|
+
for field in response.key_values.features
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## 4. Point-in-Time Correct Offline Joins
|
|
96
|
+
|
|
97
|
+
Avoid data leakage in offline agent evaluation benchmarks:
|
|
98
|
+
|
|
99
|
+
```sql
|
|
100
|
+
SELECT
|
|
101
|
+
observations.task_id,
|
|
102
|
+
observations.timestamp,
|
|
103
|
+
features.seniority_level,
|
|
104
|
+
features.preferred_framework,
|
|
105
|
+
features.embedding_vector
|
|
106
|
+
FROM `my_project.evals.agent_runs` AS observations
|
|
107
|
+
POINT_IN_TIME JOIN `my_project.features.user_profiles` AS features
|
|
108
|
+
ON observations.user_id = features.user_id
|
|
109
|
+
AND observations.timestamp >= features.feature_timestamp;
|
|
110
|
+
```
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: webrtc-agent-gateway
|
|
3
|
+
description: Architect and deploy ultra-low latency WebRTC audio duplex gateways, Opus audio transcoding, STUN/TURN ICE candidate negotiation, and real-time voice streaming for Gemini Live pair programming.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# WebRTC Agent Gateway (Ultra-Low Latency Duplex Voice Streaming)
|
|
7
|
+
|
|
8
|
+
Architects, secures, and deploys production-grade WebRTC media gateways that bridge browser/IDE client media streams directly into the Google Gemini Live API. Implements bidirectional Opus/PCM audio transcoding, interactive data channels, STUN/TURN NAT traversal, and jitter-buffer optimization for sub-200ms voice pair-programming.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. WebRTC Duplex Architecture
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
|
16
|
+
│ WEBRTC MEDIA GATEWAY TOPOLOGY │
|
|
17
|
+
│ │
|
|
18
|
+
│ ┌─────────────────────┐ WebRTC PeerConnection ┌────────────────┐ │
|
|
19
|
+
│ │ Developer Browser │◄──────────────────────────────►│ WebRTC Gateway │ │
|
|
20
|
+
│ │ / Antigravity IDE │ SRTP / Opus Audio Tracks │ (aiortc / Go) │ │
|
|
21
|
+
│ └─────────────────────┘ SCTP DataChannel (Events) └───────┬────────┘ │
|
|
22
|
+
│ │ │ │
|
|
23
|
+
│ │ SDP Offer / Answer (HTTPS / WSS) │ │
|
|
24
|
+
│ ▼ ▼ │
|
|
25
|
+
│ ┌─────────────────────┐ ┌────────────────────┐ │
|
|
26
|
+
│ │ Coturn STUN / TURN │ │ Google Gemini Live │ │
|
|
27
|
+
│ │ NAT Traversal Relay │ │ WebSocket Endpoint │ │
|
|
28
|
+
│ └─────────────────────┘ └────────────────────┘ │
|
|
29
|
+
└─────────────────────────────────────────────────────────────────────────────┘
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## 2. Python aiortc WebRTC Gateway Implementation
|
|
35
|
+
|
|
36
|
+
Below is a production-grade WebRTC audio bridge receiving developer microphone tracks, transcoding Opus to PCM 16kHz for Gemini Live, and streaming back synthesized agent speech:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import asyncio
|
|
40
|
+
import json
|
|
41
|
+
import fractions
|
|
42
|
+
import time
|
|
43
|
+
from aiortc import RTCPeerConnection, RTCSessionDescription, MediaStreamTrack
|
|
44
|
+
from aiortc.contrib.media import MediaBlackhole
|
|
45
|
+
from av import AudioResampler, AudioFrame
|
|
46
|
+
from google import genai
|
|
47
|
+
from google.genai import types
|
|
48
|
+
|
|
49
|
+
class GeminiLiveAudioTrack(MediaStreamTrack):
|
|
50
|
+
"""Generates outgoing WebRTC audio frames from Gemini Live synthesized speech."""
|
|
51
|
+
kind = "audio"
|
|
52
|
+
|
|
53
|
+
def __init__(self):
|
|
54
|
+
super().__init__()
|
|
55
|
+
self.queue = asyncio.Queue()
|
|
56
|
+
self._sample_rate = 48000
|
|
57
|
+
self._samples_per_frame = 960 # 20ms at 48kHz
|
|
58
|
+
self._pts = 0
|
|
59
|
+
|
|
60
|
+
def push_pcm_bytes(self, pcm_data: bytes):
|
|
61
|
+
"""Pushes raw 24kHz/16kHz PCM audio bytes from Gemini into the WebRTC audio queue."""
|
|
62
|
+
self.queue.put_nowait(pcm_data)
|
|
63
|
+
|
|
64
|
+
async def recv(self):
|
|
65
|
+
pcm_chunk = await self.queue.get()
|
|
66
|
+
frame = AudioFrame(format="s16", layout="mono", samples=self._samples_per_frame)
|
|
67
|
+
frame.planes[0].update(pcm_chunk[:frame.linesizes[0]])
|
|
68
|
+
frame.sample_rate = self._sample_rate
|
|
69
|
+
frame.time_base = fractions.Fraction(1, self._sample_rate)
|
|
70
|
+
frame.pts = self._pts
|
|
71
|
+
self._pts += self._samples_per_frame
|
|
72
|
+
return frame
|
|
73
|
+
|
|
74
|
+
class WebRTCAgentGateway:
|
|
75
|
+
def __init__(self, gemini_api_key: str):
|
|
76
|
+
self.client = genai.Client(api_key=gemini_api_key)
|
|
77
|
+
self.pcs = set()
|
|
78
|
+
|
|
79
|
+
async def handle_offer(self, sdp_offer: str, sdp_type: str = "offer") -> dict:
|
|
80
|
+
pc = RTCPeerConnection()
|
|
81
|
+
self.pcs.add(pc)
|
|
82
|
+
|
|
83
|
+
outgoing_audio = GeminiLiveAudioTrack()
|
|
84
|
+
pc.addTrack(outgoing_audio)
|
|
85
|
+
|
|
86
|
+
@pc.on("track")
|
|
87
|
+
def on_track(track):
|
|
88
|
+
if track.kind == "audio":
|
|
89
|
+
asyncio.create_task(self._relay_to_gemini(track, outgoing_audio))
|
|
90
|
+
|
|
91
|
+
@pc.on("connectionstatechange")
|
|
92
|
+
async def on_connectionstatechange():
|
|
93
|
+
if pc.connectionState in ["failed", "closed"]:
|
|
94
|
+
await pc.close()
|
|
95
|
+
self.pcs.discard(pc)
|
|
96
|
+
|
|
97
|
+
offer = RTCSessionDescription(sdp=sdp_offer, type=sdp_type)
|
|
98
|
+
await pc.setRemoteDescription(offer)
|
|
99
|
+
answer = await pc.createAnswer()
|
|
100
|
+
await pc.setLocalDescription(answer)
|
|
101
|
+
|
|
102
|
+
return {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
|
|
103
|
+
|
|
104
|
+
async def _relay_to_gemini(self, track: MediaStreamTrack, outgoing_audio: GeminiLiveAudioTrack):
|
|
105
|
+
resampler = AudioResampler(format="s16", layout="mono", rate=16000)
|
|
106
|
+
async with self.client.aio.live.connect(
|
|
107
|
+
model="gemini-2.0-flash-exp",
|
|
108
|
+
config=types.LiveConnectConfig(
|
|
109
|
+
response_modalities=[types.LiveModality.AUDIO],
|
|
110
|
+
speech_config=types.SpeechConfig(
|
|
111
|
+
voice_config=types.VoiceConfig(
|
|
112
|
+
prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name="Aoede")
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
)
|
|
116
|
+
) as session:
|
|
117
|
+
# Task to read synthesized audio from Gemini and queue to WebRTC
|
|
118
|
+
async def receive_from_gemini():
|
|
119
|
+
async for response in session.receive():
|
|
120
|
+
server_content = response.server_content
|
|
121
|
+
if server_content and server_content.model_turn:
|
|
122
|
+
for part in server_content.model_turn.parts:
|
|
123
|
+
if part.inline_data:
|
|
124
|
+
outgoing_audio.push_pcm_bytes(part.inline_data.data)
|
|
125
|
+
|
|
126
|
+
receiver_task = asyncio.create_task(receive_from_gemini())
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
while True:
|
|
130
|
+
frame = await track.recv()
|
|
131
|
+
resampled = resampler.resample(frame)
|
|
132
|
+
for r_frame in resampled:
|
|
133
|
+
pcm_bytes = r_frame.planes[0].to_bytes()
|
|
134
|
+
await session.send(input={"data": pcm_bytes, "mime_type": "audio/pcm;rate=16000"})
|
|
135
|
+
except asyncio.CancelledError:
|
|
136
|
+
receiver_task.cancel()
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## 3. Client WebRTC SDP Exchange & ICE Configuration
|
|
142
|
+
|
|
143
|
+
```typescript
|
|
144
|
+
export async function initializeVoiceSession(gatewayUrl: string): Promise<RTCPeerConnection> {
|
|
145
|
+
const pc = new RTCPeerConnection({
|
|
146
|
+
iceServers: [
|
|
147
|
+
{ urls: 'stun:stun.l.google.com:19302' },
|
|
148
|
+
{
|
|
149
|
+
urls: 'turn:turn.enterprise-antigravity.internal:3478',
|
|
150
|
+
username: 'agent-dev',
|
|
151
|
+
credential: 'ephemeral-turn-token'
|
|
152
|
+
}
|
|
153
|
+
]
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
// Capture developer microphone stream
|
|
157
|
+
const micStream = await navigator.mediaDevices.getUserMedia({
|
|
158
|
+
audio: {
|
|
159
|
+
channelCount: 1,
|
|
160
|
+
sampleRate: 48000,
|
|
161
|
+
echoCancellation: true,
|
|
162
|
+
noiseSuppression: true,
|
|
163
|
+
autoGainControl: true
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
micStream.getAudioTracks().forEach(track => pc.addTrack(track, micStream));
|
|
168
|
+
|
|
169
|
+
// Audio track speaker output
|
|
170
|
+
pc.ontrack = (event) => {
|
|
171
|
+
if (event.track.kind === 'audio') {
|
|
172
|
+
const audioEl = new Audio();
|
|
173
|
+
audioEl.srcObject = event.streams[0];
|
|
174
|
+
audioEl.play().catch(console.error);
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const offer = await pc.createOffer();
|
|
179
|
+
await pc.setLocalDescription(offer);
|
|
180
|
+
|
|
181
|
+
const response = await fetch(`${gatewayUrl}/api/webrtc/offer`, {
|
|
182
|
+
method: 'POST',
|
|
183
|
+
headers: { 'Content-Type': 'application/json' },
|
|
184
|
+
body: JSON.stringify({ sdp: offer.sdp, type: offer.type })
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
const answer = await response.json();
|
|
188
|
+
await pc.setRemoteDescription(new RTCSessionDescription(answer));
|
|
189
|
+
return pc;
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## 4. Operational Best Practices & SLA Targets
|
|
196
|
+
|
|
197
|
+
1. **Jitter-Buffer Tuning**: Target 40ms to 80ms playout delay. Avoid buffer bloating which degrades conversational turn-taking.
|
|
198
|
+
2. **STUN/TURN High Availability**: Deploy Coturn behind Network Load Balancers (NLB) across at least 2 GCP availability zones.
|
|
199
|
+
3. **Graceful Teardown**: Intercept `connectionstatechange` to immediately close Gemini WebSocket sessions on client hangup.
|