everything-gemini 1.1.1 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
  2. package/.github/ISSUE_TEMPLATE/feature_request.md +27 -0
  3. package/.github/ISSUE_TEMPLATE/subagent_proposal.md +29 -0
  4. package/.github/workflows/ci.yml +72 -1
  5. package/.github/workflows/release.yml +17 -2
  6. package/.vscode/extensions.json +8 -0
  7. package/.vscode/settings.json +3 -0
  8. package/CHANGELOG.md +167 -0
  9. package/CONTRIBUTING.md +85 -4
  10. package/README.md +139 -18
  11. package/SECURITY.md +38 -0
  12. package/action.yml +35 -0
  13. package/agents/browser-qa.md +1 -1
  14. package/agents/database-migrator.md +26 -0
  15. package/agents/documentation-writer.md +27 -0
  16. package/agents/incident-responder.md +34 -0
  17. package/agents/performance-tuner.md +27 -0
  18. package/agents/release-engineer.md +34 -0
  19. package/bin/everything-gemini +1550 -86
  20. package/docs/COMMUNITY_SUBMISSIONS.md +88 -0
  21. package/docs/LAUNCH_KIT.md +148 -22
  22. package/docs/architecture/progressive-disclosure.md +3 -3
  23. package/docs/index.html +401 -37
  24. package/docs/launch-kit.md +163 -0
  25. package/examples/cli-workflows/README.md +45 -0
  26. package/examples/cli-workflows/demo_workflow.sh +39 -0
  27. package/examples/python-sdk/README.md +28 -5
  28. package/examples/python-sdk/documentation_audit_pipeline.py +83 -0
  29. package/examples/python-sdk/incident_triage_pipeline.py +170 -0
  30. package/examples/python-sdk/multi_agent_pipeline.py +95 -0
  31. package/examples/python-sdk/release_readiness_audit.py +144 -0
  32. package/hooks/secret-scrubber/hooks.json +16 -0
  33. package/hooks/secret-scrubber/scripts/scrub-secrets.sh +62 -0
  34. package/mcp/README.md +20 -0
  35. package/mcp/mcp_config.example.json +35 -0
  36. package/package.json +3 -1
  37. package/plugins/core/rules/core-practices.md +63 -4
  38. package/plugins/python-ai/rules/ai-agent-patterns.md +67 -4
  39. package/plugins/security-shield/rules/zero-trust.md +60 -4
  40. package/plugins/web-fullstack/rules/frontend-standards.md +74 -4
  41. package/rules/paradigms/security-hardened.md +53 -10
  42. package/rules/paradigms/tdd.md +65 -8
  43. package/scripts/install.ps1 +14 -0
  44. package/scripts/install.sh +15 -1
  45. package/scripts/validate.sh +103 -12
  46. package/skills/agent-evals/SKILL.md +61 -0
  47. package/skills/apache-kafka-streams/SKILL.md +152 -0
  48. package/skills/argo-workflows/SKILL.md +151 -0
  49. package/skills/axum-web-service/SKILL.md +159 -0
  50. package/skills/cilium-ebpf-networking/SKILL.md +145 -0
  51. package/skills/clickhouse-analytics/SKILL.md +128 -0
  52. package/skills/dbt-transformation/SKILL.md +179 -0
  53. package/skills/debezium-cdc/SKILL.md +166 -0
  54. package/skills/docker-compose-orchestrator/SKILL.md +85 -0
  55. package/skills/duckdb-local-olap/SKILL.md +134 -0
  56. package/skills/fastapi-production/SKILL.md +157 -0
  57. package/skills/graphql-architect/SKILL.md +78 -0
  58. package/skills/grpc-connect-rpc/SKILL.md +176 -0
  59. package/skills/langgraph-orchestrator/SKILL.md +144 -0
  60. package/skills/litellm-proxy-gateway/SKILL.md +154 -0
  61. package/skills/nats-jetstream/SKILL.md +122 -0
  62. package/skills/open-telemetry-tracing/SKILL.md +151 -0
  63. package/skills/playwright-visual-regression/SKILL.md +124 -0
  64. package/skills/redis-cache-patterns/SKILL.md +159 -0
  65. package/skills/svelte-kit-architect/SKILL.md +190 -0
  66. package/skills/temporal-workflows/SKILL.md +158 -0
  67. package/skills/trpc-react-query/SKILL.md +169 -0
  68. package/skills/turborepo-monorepo/SKILL.md +199 -0
  69. package/skills/vllm-inference/SKILL.md +144 -0
  70. package/skills/wasm-rust-optimizer/SKILL.md +130 -0
  71. package/templates/full-power/hooks.json +14 -0
  72. package/templates/full-power/mcp_config.json +43 -0
  73. package/templates/recommended/hooks.json +14 -0
  74. package/tests/cli.test.js +383 -0
@@ -1,17 +1,60 @@
1
1
  # Security & Hardening Protocol
2
2
 
3
- Security rules enforced for all agent code generation and execution.
3
+ Security rules and defensive engineering standards enforced for all agent code generation and execution across projects.
4
+
5
+ ---
4
6
 
5
7
  ## 1. Zero Secret Leakage
6
- - Never print, log, or commit API keys, tokens, certificates, or credentials.
7
- - Always load secrets via environment variables (`process.env`, `os.environ`).
8
- - Ensure `.env` files and secret patterns are listed in `.gitignore`.
8
+ - **Never Commit Secrets**:
9
+ - Never print, log, or commit API keys, personal access tokens, private keys, or passwords.
10
+ - Always ingest sensitive configuration via environment variables (`process.env`, `os.environ`).
11
+ - **Git & Artifact Sanitation**:
12
+ - Ensure `.env`, `.env.local`, `.pem`, and certificate files are explicitly listed in `.gitignore`.
13
+ - When creating demonstration snippets or documentation, use standardized redaction placeholders (e.g. `GEMINI_API_KEY="YOUR_GEMINI_API_KEY"`).
14
+
15
+ ---
9
16
 
10
17
  ## 2. Command & Injection Defense
11
- - Never execute shell commands by concatenating unsanitized user strings.
12
- - Always use parameterized queries for SQL/NoSQL databases to prevent SQL injection.
13
- - Validate paths with `path.resolve` and verify they remain within the authorized workspace directory to prevent directory traversal.
18
+ - **Safe Process Spawning**:
19
+ - Never concatenate unvalidated user strings into shell commands.
20
+ - Avoid `shell: true` in Node.js `child_process.spawn`. Pass binary names and arguments as separate arrays:
21
+ ```javascript
22
+ // INSECURE: exec(`curl ${userInput}`)
23
+ // SECURE:
24
+ const { execFile } = require('child_process');
25
+ execFile('curl', ['--silent', sanitizedUrl]);
26
+ ```
27
+ - **SQL & Query Injection**:
28
+ - Always use parameterized queries or type-safe ORM query builders (Prisma, Drizzle, SQLAlchemy). Never interpolate raw string variables into SQL statements.
29
+ - **Path Traversal Guardrails**:
30
+ - Sanitize all file paths with `path.resolve` and verify that the target path begins with the authorized root directory:
31
+ ```javascript
32
+ const resolvedPath = path.resolve(authorizedDir, userFilename);
33
+ if (!resolvedPath.startsWith(path.resolve(authorizedDir))) {
34
+ throw new Error('Access denied: Path traversal attempt detected.');
35
+ }
36
+ ```
37
+
38
+ ---
39
+
40
+ ## 3. Cryptographic Hygiene
41
+ - **Constant-Time Comparisons**:
42
+ - When comparing webhook signatures, API tokens, or password hashes, always use constant-time comparison utilities (`crypto.timingSafeEqual`) to eliminate timing attack vectors.
43
+ - **Modern Cryptographic Primitives**:
44
+ - Use cryptographically secure pseudorandom number generators (`crypto.randomBytes` or `secrets.token_hex`). Never use `Math.random()` for security-critical values.
45
+
46
+ ---
47
+
48
+ ## 4. Prototype Pollution & Input Validation
49
+ - **Deep Object Cloning**:
50
+ - Prevent prototype pollution by sanitizing keys against `__proto__`, `constructor`, and `prototype`.
51
+ - **Strict Schema Parsing**:
52
+ - Validate all JSON payloads at HTTP route boundaries using schema validators (Zod, Pydantic, or TypeBox). Strip unknown properties before processing.
53
+
54
+ ---
14
55
 
15
- ## 3. Dependency Security
16
- - Review new packages before adding them to `package.json` or `pyproject.toml`.
17
- - Run automated vulnerability audits (`npm audit`, `pip-audit`, `govulncheck`) on dependencies.
56
+ ## 5. Dependency Audit & Supply Chain Security
57
+ - **Automated CVE Scans**:
58
+ - Run vulnerability audits (`npm audit`, `pip-audit`, `govulncheck`) on third-party dependencies before introducing new packages.
59
+ - **Lockfile Enforcement**:
60
+ - Commit deterministic lockfiles (`package-lock.json`, `poetry.lock`) to ensure repeatable, tamper-evident builds.
@@ -1,19 +1,76 @@
1
1
  # Test-Driven Development (TDD) Protocol
2
2
 
3
- Enforce this workflow when developing features, fixing bugs, or refactoring code.
3
+ Enforce this workflow whenever developing features, fixing bugs, or refactoring existing logic across the repository.
4
+
5
+ ---
4
6
 
5
7
  ## The TDD Iron Law
6
8
 
7
- > **Never write production code unless an existing test is currently failing because that code does not exist.**
9
+ > **Never write production code unless an existing test is currently failing because that code does not exist or does not yet behave as expected.**
10
+
11
+ ---
12
+
13
+ ## 1. The Three Disciplined Phases
8
14
 
9
15
  ### Phase 1: RED (Reproduce & Specify)
10
- 1. Write a minimal, focused unit or integration test describing the desired behavior or reproducing the reported bug.
11
- 2. Run the test suite and verify that the test fails for the expected reason (not due to syntax or import errors).
16
+ 1. **Write the Failing Test First**:
17
+ - Write a minimal, focused unit or integration test describing the desired behavior or reproducing the reported bug.
18
+ 2. **Execute to Confirm Failure**:
19
+ - Run the test suite and verify that the test fails for the expected reason (e.g. `AssertionError: expected false to be true` or `Function not defined`, not due to unrelated compilation or import syntax errors).
12
20
 
13
21
  ### Phase 2: GREEN (Minimal Implementation)
14
- 1. Write the absolute minimum production code required to make the failing test pass.
15
- 2. Run the test suite and ensure all tests pass. Do not write premature optimizations or unrelated code.
22
+ 1. **Surgical Implementation**:
23
+ - Write the absolute minimum production code required to make the failing test pass.
24
+ 2. **Resist Premature Generalization**:
25
+ - Do not write speculative helper functions, premature optimizations, or unrelated features during the Green phase. Get to green as quickly and simply as possible.
26
+ 3. **Verify All Tests Pass**:
27
+ - Confirm that the new test passes AND all existing regression tests remain green.
16
28
 
17
29
  ### Phase 3: REFACTOR (Clean & Harden)
18
- 1. Clean up duplicated logic, improve naming, extract helper functions, and enforce type safety.
19
- 2. Re-run tests after every micro-refactor to ensure zero regressions.
30
+ 1. **Eliminate Duplication**:
31
+ - Clean up repeated logic, extract reusable helpers, and clarify variable naming.
32
+ 2. **Enforce Type Safety**:
33
+ - Tighten type annotations, interfaces, and parameter docstrings.
34
+ 3. **Zero Regression Guarantee**:
35
+ - Re-run the automated test suite after every refactoring edit. Never finalize a refactor while any test is failing.
36
+
37
+ ---
38
+
39
+ ## 2. Standardized Test Structure: AAA Pattern
40
+
41
+ Every test case must follow the **Arrange-Act-Assert (AAA)** pattern:
42
+ ```typescript
43
+ describe('TokenCompactor', () => {
44
+ it('should truncate oldest turns when token budget is exceeded', () => {
45
+ // Arrange: Set up initial state and test inputs
46
+ const budget = 1000;
47
+ const history = createTranscriptFixture({ totalTokens: 1500 });
48
+
49
+ // Act: Invoke the unit under test
50
+ const compacted = compactHistory(history, budget);
51
+
52
+ // Assert: Verify expected output and invariants
53
+ expect(compacted.totalTokens).toBeLessThanOrEqual(budget);
54
+ expect(compacted.turns[0].isSummary).toBe(true);
55
+ });
56
+ });
57
+ ```
58
+
59
+ ---
60
+
61
+ ## 3. Mandatory Edge-Case Checklist
62
+
63
+ Every feature or bugfix test suite must assert against these boundary conditions:
64
+ - [ ] **Empty / Nil Inputs**: Null, undefined, empty strings `""`, empty arrays `[]`, empty objects `{}`.
65
+ - [ ] **Boundary Extremes**: Zero, negative integers, max integer bounds (`Number.MAX_SAFE_INTEGER`), very large strings (>1MB).
66
+ - [ ] **Malformed Payloads**: Invalid JSON, broken YAML frontmatter, truncated streams.
67
+ - [ ] **Network & Error Boundaries**: Simulated HTTP 429 (rate limits), 503 timeouts, socket hangups.
68
+ - [ ] **Idempotent Retries**: Running the same operation twice returns identical deterministic results without side effects.
69
+
70
+ ---
71
+
72
+ ## 4. Test Isolation & Determinism
73
+ - **No Shared Mutable State**:
74
+ - Each test must run independently. Clean up databases, mocks, and temporary directories in `afterEach` hooks.
75
+ - **Deterministic Clocks**:
76
+ - Avoid `setTimeout` loops in tests. Use mock timers (`vi.useFakeTimers()` or `jest.useFakeTimers()`) to eliminate flaky tests and speed up test execution.
@@ -8,9 +8,18 @@ param (
8
8
  Write-Host "🪐 Everything Gemini (everything-gemini) Installer" -ForegroundColor Cyan
9
9
  Write-Host "The premier agent harness & customization suite for Google Antigravity`n" -ForegroundColor Gray
10
10
 
11
+ $origDir = Get-Location
12
+ $tempDir = ""
13
+ $cleanupTemp = $false
14
+
15
+ if (-not $Global -and $Workspace -eq "") {
16
+ $Workspace = Join-Path $origDir ".agents"
17
+ }
18
+
11
19
  if (-not (Test-Path "bin\everything-gemini")) {
12
20
  Write-Host "Downloading latest everything-gemini release..." -ForegroundColor Yellow
13
21
  $tempDir = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
22
+ $cleanupTemp = $true
14
23
  git clone --depth 1 https://github.com/salmandev/everything-gemini.git $tempDir
15
24
  Set-Location $tempDir
16
25
  }
@@ -24,3 +33,8 @@ if ($Workspace -ne "") {
24
33
  }
25
34
 
26
35
  node bin\everything-gemini @cliArgs
36
+
37
+ if ($cleanupTemp -and (Test-Path $tempDir)) {
38
+ Set-Location $origDir
39
+ Remove-Item -Recurse -Force $tempDir -ErrorAction SilentlyContinue
40
+ }
@@ -15,9 +15,12 @@ echo " 🪐 Everything Gemini (everything-gemini) Installer"
15
15
  echo " The premier agent harness & customization suite for Google Antigravity"
16
16
  echo -e "${RESET}"
17
17
 
18
+ ORIG_DIR="$(pwd)"
18
19
  PROFILE="developer"
19
20
  INSTALL_GLOBAL=false
20
21
  TARGET_DIR=""
22
+ TMP_DIR=""
23
+ CLEANUP_TMP=false
21
24
 
22
25
  while [[ "$#" -gt 0 ]]; do
23
26
  case $1 in
@@ -29,10 +32,15 @@ while [[ "$#" -gt 0 ]]; do
29
32
  shift
30
33
  done
31
34
 
32
- # If run directly via curl pipe, clone into temporary directory or clone repo
35
+ if [ "$INSTALL_GLOBAL" = false ] && [ -z "$TARGET_DIR" ]; then
36
+ TARGET_DIR="$ORIG_DIR/.agents"
37
+ fi
38
+
39
+ # If run directly via curl pipe, clone into temporary directory
33
40
  if [ ! -f "bin/everything-gemini" ]; then
34
41
  echo -e "${YELLOW}Downloading latest everything-gemini release...${RESET}"
35
42
  TMP_DIR=$(mktemp -d)
43
+ CLEANUP_TMP=true
36
44
  git clone --depth 1 https://github.com/salmandev/everything-gemini.git "$TMP_DIR/everything-gemini" 2>/dev/null || {
37
45
  echo "Failed to clone repository. Please ensure git is installed."
38
46
  exit 1
@@ -50,3 +58,9 @@ if [ -n "$TARGET_DIR" ]; then
50
58
  fi
51
59
 
52
60
  ./bin/everything-gemini $CLI_ARGS
61
+
62
+ # Cleanup temp clone if created
63
+ if [ "$CLEANUP_TMP" = true ] && [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ]; then
64
+ cd "$ORIG_DIR"
65
+ node -e "try { require('fs').rmSync(process.argv[1], {recursive: true, force: true}); } catch(e){}" "$TMP_DIR" 2>/dev/null || true
66
+ fi
@@ -7,7 +7,7 @@ ERRORS=0
7
7
  echo "🔍 Validating Everything Gemini (everything-gemini) repository..."
8
8
 
9
9
  # 1. Validate SKILL.md Frontmatter
10
- echo " [1/4] Checking Skills & YAML frontmatter..."
10
+ echo " [1/6] Checking Skills & YAML frontmatter..."
11
11
  SKILL_FILES=$(find "$REPO_ROOT/skills" "$REPO_ROOT/plugins" -name "SKILL.md" 2>/dev/null || true)
12
12
 
13
13
  for skill_file in $SKILL_FILES; do
@@ -35,8 +35,29 @@ for skill_file in $SKILL_FILES; do
35
35
  echo " ✔ $rel_path"
36
36
  done
37
37
 
38
- # 2. Validate JSON Syntax
39
- echo " [2/4] Validating JSON files..."
38
+ # 2. Validate Scoped Subagents
39
+ echo " [2/6] Checking Subagents specifications..."
40
+ REQUIRED_AGENTS=("planner.md" "code-reviewer.md" "browser-qa.md" "build-repair.md" "security-auditor.md" "architect.md" "database-migrator.md" "performance-tuner.md" "documentation-writer.md" "release-engineer.md" "incident-responder.md")
41
+ for ag in "${REQUIRED_AGENTS[@]}"; do
42
+ if [ ! -f "$REPO_ROOT/agents/$ag" ]; then
43
+ echo " ❌ ERROR: Required subagent specification 'agents/$ag' is missing!"
44
+ ERRORS=$((ERRORS + 1))
45
+ fi
46
+ done
47
+
48
+ AGENT_FILES=$(find "$REPO_ROOT/agents" -name "*.md" 2>/dev/null || true)
49
+ for agent_file in $AGENT_FILES; do
50
+ rel_path="${agent_file#$REPO_ROOT/}"
51
+ if ! grep -q "^# " "$agent_file"; then
52
+ echo " ❌ ERROR: $rel_path is missing top-level '# ' title"
53
+ ERRORS=$((ERRORS + 1))
54
+ else
55
+ echo " ✔ $rel_path"
56
+ fi
57
+ done
58
+
59
+ # 3. Validate JSON & YAML Files
60
+ echo " [3/6] Validating JSON files & Action manifest..."
40
61
  JSON_FILES=$(find "$REPO_ROOT" -name "*.json" ! -path "*/node_modules/*" 2>/dev/null || true)
41
62
 
42
63
  for json_file in $JSON_FILES; do
@@ -49,21 +70,91 @@ for json_file in $JSON_FILES; do
49
70
  fi
50
71
  done
51
72
 
52
- # 3. Validate Shell Scripts Executability
53
- echo " [3/4] Verifying script permissions..."
54
- SH_FILES=$(find "$REPO_ROOT/scripts" "$REPO_ROOT/hooks" "$REPO_ROOT/skills" -name "*.sh" 2>/dev/null || true)
73
+ if [ -f "$REPO_ROOT/action.yml" ]; then
74
+ if grep -q "name:" "$REPO_ROOT/action.yml" && grep -q "runs:" "$REPO_ROOT/action.yml"; then
75
+ echo " ✔ action.yml (composite GitHub action valid)"
76
+ else
77
+ echo " ❌ ERROR: action.yml is missing required composite action fields"
78
+ ERRORS=$((ERRORS + 1))
79
+ fi
80
+ fi
81
+
82
+ # Validate Relative Documentation Links
83
+ if ! node -e "
84
+ const fs = require('fs');
85
+ const path = require('path');
86
+ const mdFiles = [
87
+ 'README.md', 'CONTRIBUTING.md', 'SECURITY.md', 'CHANGELOG.md',
88
+ ...fs.readdirSync('skills').map(s => 'skills/' + s + '/SKILL.md'),
89
+ ...fs.readdirSync('agents').map(a => 'agents/' + a)
90
+ ];
91
+ let broken = 0;
92
+ for (const file of mdFiles) {
93
+ if (!fs.existsSync(file)) continue;
94
+ let content = fs.readFileSync(file, 'utf8');
95
+ content = content.replace(/\`\`\`[\s\S]*?\`\`\`/g, '');
96
+ content = content.replace(/\`[^\`]*?\`/g, '');
97
+ const matches = content.matchAll(/\[([^\]]+)\]\(([^)]+)\)/g);
98
+ for (const m of matches) {
99
+ const link = m[2];
100
+ if (link.startsWith('http://') || link.startsWith('https://') || link.startsWith('#') || link.startsWith('mailto:') || link.startsWith('file://')) continue;
101
+ const cleanLink = link.split('#')[0];
102
+ if (!cleanLink) continue;
103
+ const resolved = path.resolve(path.dirname(file), cleanLink);
104
+ if (!fs.existsSync(resolved)) {
105
+ console.error('Broken link in ' + file + ' -> ' + link);
106
+ broken++;
107
+ }
108
+ }
109
+ }
110
+ if (broken > 0) process.exit(1);
111
+ " 2>/dev/null; then
112
+ echo " ❌ ERROR: Broken relative links detected in documentation"
113
+ ERRORS=$((ERRORS + 1))
114
+ else
115
+ echo " ✔ Documentation relative links verified"
116
+ fi
117
+
118
+ # 4. Validate Shell Scripts & CLI Executability
119
+ echo " [4/6] Verifying script permissions & code syntax..."
120
+ EXEC_FILES=$(find "$REPO_ROOT/scripts" "$REPO_ROOT/hooks" "$REPO_ROOT/skills" "$REPO_ROOT/examples" "$REPO_ROOT/bin" -type f \( -name "*.sh" -o -name "everything-gemini" \) 2>/dev/null || true)
55
121
 
56
- for sh_file in $SH_FILES; do
57
- rel_path="${sh_file#$REPO_ROOT/}"
58
- if [ ! -x "$sh_file" ]; then
122
+ for exec_file in $EXEC_FILES; do
123
+ rel_path="${exec_file#$REPO_ROOT/}"
124
+ if [ ! -x "$exec_file" ]; then
59
125
  echo " ⚠️ Fixing executable bit on: $rel_path"
60
- chmod +x "$sh_file"
126
+ chmod +x "$exec_file"
61
127
  fi
62
128
  echo " ✔ $rel_path (executable)"
63
129
  done
64
130
 
65
- # 4. Summary
66
- echo " [4/4] Validation Summary"
131
+ # Validate Python examples syntax
132
+ if command -v python3 >/dev/null 2>&1; then
133
+ PY_FILES=$(find "$REPO_ROOT/examples" -name "*.py" 2>/dev/null || true)
134
+ for py_file in $PY_FILES; do
135
+ rel_path="${py_file#$REPO_ROOT/}"
136
+ if ! python3 -m py_compile "$py_file" 2>/dev/null; then
137
+ echo " ❌ ERROR: Syntax error in $rel_path"
138
+ ERRORS=$((ERRORS + 1))
139
+ else
140
+ echo " ✔ $rel_path (valid Python syntax)"
141
+ fi
142
+ done
143
+ fi
144
+
145
+ # 5. CLI E2E Integration Suite
146
+ echo " [5/6] Running CLI E2E Integration Suite..."
147
+ if [ -f "$REPO_ROOT/tests/cli.test.js" ]; then
148
+ if ! node "$REPO_ROOT/tests/cli.test.js"; then
149
+ echo " ❌ ERROR: CLI test suite failed."
150
+ ERRORS=$((ERRORS + 1))
151
+ else
152
+ echo " ✔ CLI test suite passed (26 assertions)"
153
+ fi
154
+ fi
155
+
156
+ # 6. Summary
157
+ echo " [6/6] Validation Summary"
67
158
  if [ "$ERRORS" -gt 0 ]; then
68
159
  echo "❌ Validation failed with $ERRORS error(s)."
69
160
  exit 1
@@ -0,0 +1,61 @@
1
+ ---
2
+ name: agent-evals
3
+ description: Designs, executes, and benchmarks AI agent evaluation pipelines, LLM-as-a-judge scoring rubrics, golden test datasets, and prompt regression testing harnesses.
4
+ ---
5
+
6
+ # Agent Evals & LLM Benchmark Playbook
7
+
8
+ This skill guides the creation of automated evaluation harnesses, LLM-as-a-judge scoring pipelines, and regression testing suites for AI agents and LLM applications.
9
+
10
+ ---
11
+
12
+ ## 1. Core Principles
13
+
14
+ 1. **Deterministic Assertions First**:
15
+ - Prefer deterministic, unit-testable assertions (schema validation, regex matching, exit codes, latency limits) before falling back to subjective LLM judges.
16
+ 2. **Golden Datasets with Ground Truth**:
17
+ - Maintain versioned golden evaluation datasets (`evals/golden.jsonl`) pairing input prompts with expected outputs or behavioral criteria.
18
+ 3. **Regression Containment**:
19
+ - Any prompt revision or model switch must run against the baseline eval suite to measure accuracy deltas, token consumption, and latency shifts.
20
+
21
+ ---
22
+
23
+ ## 2. Evaluation Dimensions
24
+
25
+ When assessing agent output quality, evaluate across four standardized vectors:
26
+
27
+ | Dimension | Metric | Measurement Method |
28
+ | :--- | :--- | :--- |
29
+ | **Accuracy / Correctness** | Factual precision, task fulfillment | LLM-as-a-judge with binary pass/fail rubric |
30
+ | **Safety & Alignment** | Injection vulnerability, secret leakage | Automated static analysis & safety classifiers |
31
+ | **Schema Compliance** | Valid JSON, Pydantic type conformance | Deterministic parser validation (`json.loads`, Pydantic) |
32
+ | **Context Economics** | Token count, time-to-first-token (TTFT) | Profiler logs & Gemini metadata tokens |
33
+
34
+ ---
35
+
36
+ ## 3. LLM-as-a-Judge Scoring Template
37
+
38
+ When using Gemini as an automated evaluator, structure the prompt with strict zero-shot rubrics and JSON output:
39
+
40
+ ```python
41
+ EVAL_SYSTEM_PROMPT = """You are an impartial, expert evaluation judge.
42
+ Assess the candidate agent response against the provided Reference Criteria.
43
+ Provide a structured JSON evaluation adhering to the schema:
44
+ {
45
+ "score": <float 0.0 to 1.0>,
46
+ "reasoning": "<concise explanation of strengths and shortcomings>",
47
+ "violations": ["<list of any missed constraints>"]
48
+ }
49
+ Only output valid JSON.
50
+ """
51
+ ```
52
+
53
+ ---
54
+
55
+ ## 4. Eval Runner Execution Flow
56
+
57
+ 1. **Load Test Suite**: Ingest test cases from `evals/test_cases.jsonl`.
58
+ 2. **Execute Candidate Prompt**: Query the model with temperature=0.0 for reproducible benchmarking.
59
+ 3. **Run Deterministic Checks**: Validate JSON schema, required keys, and formatting constraints.
60
+ 4. **Run LLM Judge**: Pass model output + reference criteria to the evaluator.
61
+ 5. **Aggregate Metrics**: Calculate overall pass rate, p95 latency, and token cost per run.
@@ -0,0 +1,152 @@
1
+ ---
2
+ name: apache-kafka-streams
3
+ description: >-
4
+ Architects, deploys, and optimizes Apache Kafka event streaming architectures, stateful Kafka Streams / Faust topologies, exactly-once semantics (EOS), consumer group rebalancing, and Schema Registry serialization.
5
+ ---
6
+
7
+ # Apache Kafka Event Streaming & Stream Processing
8
+
9
+ This skill guides engineering resilient, high-throughput distributed event streaming systems with Apache Kafka, covering partition key strategies, transactional exactly-once processing (EOS), stateful sliding/tumbling window joins, and Schema Registry Avro/Protobuf governance.
10
+
11
+ ---
12
+
13
+ ## ⚡ Core Patterns
14
+
15
+ ### 1. Production Producer with Transactional Exactly-Once Semantics (EOS)
16
+
17
+ Configure idempotency, reliable acknowledgments, and transactional boundaries:
18
+
19
+ ```typescript
20
+ import { Kafka, CompressionTypes, logLevel } from 'kafkajs';
21
+
22
+ const kafka = new Kafka({
23
+ clientId: 'order-dispatch-service',
24
+ brokers: ['kafka-1.internal:9092', 'kafka-2.internal:9092', 'kafka-3.internal:9092'],
25
+ ssl: true,
26
+ logLevel: logLevel.WARN,
27
+ });
28
+
29
+ const producer = kafka.producer({
30
+ // Idempotent producer guarantees partition deduplication and in-flight order preservation
31
+ idempotent: true,
32
+ maxInFlightRequests: 1,
33
+ transactionalId: 'order-tx-producer-instance-01',
34
+ });
35
+
36
+ async function publishOrderEvent(order: { id: string; customerId: string; amount: number }) {
37
+ await producer.connect();
38
+ const transaction = await producer.transaction();
39
+
40
+ try {
41
+ await transaction.send({
42
+ topic: 'enterprise.commerce.orders.v1',
43
+ compression: CompressionTypes.GZIP,
44
+ messages: [
45
+ {
46
+ key: order.customerId, // Ensures hash-partition ordering per customer
47
+ value: JSON.stringify(order),
48
+ headers: {
49
+ 'correlation-id': crypto.randomUUID(),
50
+ 'event-type': 'OrderPlaced',
51
+ 'schema-version': '1.0.0',
52
+ },
53
+ },
54
+ ],
55
+ });
56
+
57
+ await transaction.commit();
58
+ } catch (err) {
59
+ await transaction.abort();
60
+ throw err;
61
+ }
62
+ }
63
+ ```
64
+
65
+ ---
66
+
67
+ ### 2. High-Throughput Consumer Group with Cooperative Sticky Rebalancing
68
+
69
+ Prevent stop-the-world partition revocations during scale-up or rolling deployments:
70
+
71
+ ```typescript
72
+ const consumer = kafka.consumer({
73
+ groupId: 'order-fulfillment-workers',
74
+ sessionTimeout: 30000,
75
+ heartbeatInterval: 3000,
76
+ rebalanceTimeout: 60000,
77
+ // CooperativeSticky assignor migrates only transferred partitions without halting healthy consumers
78
+ partitionAssigners: [
79
+ (config) => new (require('kafkajs').PartitionAssigners.cooperativeSticky)(config),
80
+ ],
81
+ });
82
+
83
+ await consumer.connect();
84
+ await consumer.subscribe({ topic: 'enterprise.commerce.orders.v1', fromBeginning: false });
85
+
86
+ await consumer.run({
87
+ autoCommit: false,
88
+ eachBatchAutoResolve: false,
89
+ eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
90
+ for (const message of batch.messages) {
91
+ if (!isRunning() || isStale()) break;
92
+
93
+ try {
94
+ const payload = JSON.parse(message.value.toString());
95
+ await processOrderFulfillment(payload);
96
+
97
+ // Mark individual offset for manual deterministic commit
98
+ resolveOffset(message.offset);
99
+ await heartbeat();
100
+ } catch (err) {
101
+ // Route unrecoverable poison pill to Dead-Letter Queue (DLQ)
102
+ await publishToDLQ('enterprise.commerce.orders.dlq', message, err);
103
+ resolveOffset(message.offset);
104
+ }
105
+ }
106
+ },
107
+ });
108
+ ```
109
+
110
+ ---
111
+
112
+ ### 3. Stateful Stream Processing (Kafka Streams / Java Topology)
113
+
114
+ Perform stateful 5-minute tumbling window aggregations with RocksDB state stores:
115
+
116
+ ```java
117
+ StreamsBuilder builder = new StreamsBuilder();
118
+ KStream<String, OrderEvent> orderStream = builder.stream(
119
+ "enterprise.commerce.orders.v1",
120
+ Consumed.with(Serdes.String(), new JsonSerde<>(OrderEvent.class))
121
+ );
122
+
123
+ KTable<Windowed<String>, Long> orderCountsPerStore = orderStream
124
+ .selectKey((key, order) -> order.getStoreId())
125
+ .groupByKey(Grouped.with(Serdes.String(), new JsonSerde<>(OrderEvent.class)))
126
+ .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMinutes(5)))
127
+ .count(
128
+ Materialized.<String, Long, WindowStore<Bytes, byte[]>>as("store-order-counts-store")
129
+ .withRetention(Duration.ofDays(1))
130
+ );
131
+
132
+ orderCountsPerStore.toStream()
133
+ .map((windowedKey, count) -> new KeyValue<>(windowedKey.key(), new StoreOrderSummary(windowedKey.window().start(), count)))
134
+ .to("enterprise.commerce.store-aggregates.v1", Produced.with(Serdes.String(), new JsonSerde<>(StoreOrderSummary.class)));
135
+ ```
136
+
137
+ ---
138
+
139
+ ## 🎯 Verification & Diagnostics
140
+
141
+ 1. **Check Consumer Group Lag & Partition Distribution**:
142
+ ```bash
143
+ kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group order-fulfillment-workers
144
+ ```
145
+ 2. **Inspect Topic Partition Watermarks**:
146
+ ```bash
147
+ kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic enterprise.commerce.orders.v1 --time -1
148
+ ```
149
+ 3. **Tail Stream Events with Header Decoding**:
150
+ ```bash
151
+ kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic enterprise.commerce.orders.v1 --from-beginning --property print.key=true --property print.headers=true
152
+ ```