@softspark/ai-toolkit 2.6.2 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -0
- package/CHANGELOG.md +29 -0
- package/README.md +14 -11
- package/app/.claude-plugin/plugin.json +1 -1
- package/app/ARCHITECTURE.md +47 -2
- package/app/output-styles/explanatory.md +64 -0
- package/app/output-styles/learning.md +70 -0
- package/app/skills/content-moderation-patterns/SKILL.md +141 -0
- package/app/skills/json-mode-patterns/SKILL.md +124 -0
- package/app/skills/mcp-builder/SKILL.md +139 -0
- package/app/skills/model-routing-patterns/SKILL.md +124 -0
- package/app/skills/prompt-caching-patterns/SKILL.md +114 -0
- package/app/skills/review/scripts/diff-analyzer.py +184 -46
- package/kb/reference/architecture-overview.md +2 -2
- package/kb/reference/skills-catalog.md +7 -2
- package/llms-full.txt +12 -2
- package/manifest.json +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: mcp-builder
|
|
3
|
+
description: "Build production-grade MCP (Model Context Protocol) servers from scratch using the 4-phase methodology: research, implement, test, evaluate. Use when creating new MCP integrations for external APIs, databases, or internal services."
|
|
4
|
+
effort: high
|
|
5
|
+
disable-model-invocation: true
|
|
6
|
+
argument-hint: "[service name or API description]"
|
|
7
|
+
allowed-tools: Read, Write, Edit, Bash, Grep, Glob
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# MCP Builder
|
|
11
|
+
|
|
12
|
+
$ARGUMENTS
|
|
13
|
+
|
|
14
|
+
Build a production-grade MCP server following Anthropic's 4-phase methodology.
|
|
15
|
+
|
|
16
|
+
## When to Use
|
|
17
|
+
|
|
18
|
+
- Wrapping a third-party REST API as MCP tools
|
|
19
|
+
- Exposing an internal database or service to Claude
|
|
20
|
+
- Creating reusable integrations for the team
|
|
21
|
+
- Migrating a custom tool into the MCP ecosystem
|
|
22
|
+
|
|
23
|
+
For MCP protocol theory, see `mcp-patterns` knowledge skill (auto-loaded).
|
|
24
|
+
|
|
25
|
+
## 4-Phase Workflow
|
|
26
|
+
|
|
27
|
+
### Phase 1 — Research & Planning
|
|
28
|
+
|
|
29
|
+
1. Read the target API's documentation (OpenAPI spec, README, changelog).
|
|
30
|
+
2. Identify the 5-15 most useful operations. Prefer workflow-oriented tools over 1:1 API mirror.
|
|
31
|
+
3. Decide transport: `stdio` for local dev tools, `streamable-http` for remote/shared.
|
|
32
|
+
4. Decide language: **TypeScript recommended** (best SDK), Python acceptable (`mcp` package).
|
|
33
|
+
5. List required secrets (API keys, tokens) and their env var names.
|
|
34
|
+
|
|
35
|
+
Output: `PLAN.md` with tool list, transport choice, auth model.
|
|
36
|
+
|
|
37
|
+
### Phase 2 — Implementation
|
|
38
|
+
|
|
39
|
+
Scaffold:
|
|
40
|
+
```
|
|
41
|
+
my-mcp/
|
|
42
|
+
├── package.json # or pyproject.toml
|
|
43
|
+
├── src/
|
|
44
|
+
│ ├── server.ts # entry point
|
|
45
|
+
│ ├── client.ts # API client (axios/httpx)
|
|
46
|
+
│ ├── tools/ # one file per tool
|
|
47
|
+
│ ├── schemas.ts # Zod/Pydantic schemas
|
|
48
|
+
│ └── errors.ts # typed errors
|
|
49
|
+
├── .env.example
|
|
50
|
+
└── README.md
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Per tool:
|
|
54
|
+
- Input/output schemas (Zod for TS, Pydantic for Python)
|
|
55
|
+
- Clear `name` with service prefix (e.g. `github_create_issue`)
|
|
56
|
+
- Description starts with a verb, mentions trigger keywords
|
|
57
|
+
- Annotations: `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`
|
|
58
|
+
- Pagination support via `cursor` or `page` parameters
|
|
59
|
+
- Focused responses — filter noise, don't dump raw API payloads
|
|
60
|
+
|
|
61
|
+
### Phase 3 — Review & Testing
|
|
62
|
+
|
|
63
|
+
- TypeScript: `npm run typecheck && npm run lint && npm test`
|
|
64
|
+
- Python: `ruff check . && mypy --strict src/ && pytest`
|
|
65
|
+
- MCP Inspector dry-run:
|
|
66
|
+
```bash
|
|
67
|
+
npx @modelcontextprotocol/inspector node dist/server.js
|
|
68
|
+
```
|
|
69
|
+
- Verify each tool's schema validates a real request and rejects malformed input.
|
|
70
|
+
|
|
71
|
+
### Phase 4 — Evaluation
|
|
72
|
+
|
|
73
|
+
Write 10 realistic end-user questions that an LLM should be able to answer using your server. Run them through Claude with the server attached. Grade: did the model call the right tool? Did the response give enough to answer? Fix the description, schema, or response format of any tool that failed.
|
|
74
|
+
|
|
75
|
+
Example eval questions for a `github-mcp`:
|
|
76
|
+
1. "What issues are open on repo X with label `bug`?"
|
|
77
|
+
2. "Create an issue titled Y in repo Z"
|
|
78
|
+
3. "Who has the most commits this month in repo X?"
|
|
79
|
+
|
|
80
|
+
## Tool Design Checklist
|
|
81
|
+
|
|
82
|
+
- [ ] Name has service prefix and is verb-led
|
|
83
|
+
- [ ] Description mentions when to use it and includes trigger keywords
|
|
84
|
+
- [ ] Input schema is strict, no free-form `object` with `additionalProperties: true`
|
|
85
|
+
- [ ] Output is focused — essential fields only, with pagination cursor if applicable
|
|
86
|
+
- [ ] Error responses are actionable ("API returned 403 — check `GITHUB_TOKEN` env var")
|
|
87
|
+
- [ ] Annotations set correctly (readonly/destructive/idempotent)
|
|
88
|
+
- [ ] No secrets logged or echoed in errors
|
|
89
|
+
- [ ] Rate limiting respects the upstream API
|
|
90
|
+
|
|
91
|
+
## Transport Cheat Sheet
|
|
92
|
+
|
|
93
|
+
| Scenario | Transport |
|
|
94
|
+
|----------|-----------|
|
|
95
|
+
| Local dev tool, 1 user | `stdio` |
|
|
96
|
+
| Remote server, multiple users | `streamable-http` with SSE |
|
|
97
|
+
| Internal company tool, auth required | `streamable-http` + OAuth proxy |
|
|
98
|
+
| Embedded in IDE/editor | `stdio` spawned by editor |
|
|
99
|
+
|
|
100
|
+
## Registration Cheat Sheet
|
|
101
|
+
|
|
102
|
+
Local Claude Code (`.mcp.json`):
|
|
103
|
+
```json
|
|
104
|
+
{
|
|
105
|
+
"mcpServers": {
|
|
106
|
+
"my-mcp": {
|
|
107
|
+
"command": "node",
|
|
108
|
+
"args": ["dist/server.js"],
|
|
109
|
+
"env": { "API_KEY": "$MY_API_KEY" }
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Global Claude Code (user-scope):
|
|
116
|
+
```bash
|
|
117
|
+
claude mcp add my-mcp --scope user -- node /path/to/server.js
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Claude Desktop: same JSON, placed in `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS).
|
|
121
|
+
|
|
122
|
+
## Common Pitfalls
|
|
123
|
+
|
|
124
|
+
| Mistake | Fix |
|
|
125
|
+
|---------|-----|
|
|
126
|
+
| 1:1 API mirror with 80 tools | Pick 10 workflow-oriented tools |
|
|
127
|
+
| `description: "wrapper for /users endpoint"` | `description: "Find users by email, role, or team. Use when the user mentions employees, staff, or access"` |
|
|
128
|
+
| Dumping raw JSON responses | Filter to 3-5 fields the agent actually needs |
|
|
129
|
+
| Logging API keys on error | Redact all env vars in error formatters |
|
|
130
|
+
| `exit 1` on transient errors | Retry with exponential backoff, surface final error |
|
|
131
|
+
| Stdout pollution (MCP stdio) | All logs go to **stderr**, stdout is JSON-RPC only |
|
|
132
|
+
|
|
133
|
+
## Related
|
|
134
|
+
|
|
135
|
+
- `mcp-patterns` — protocol reference (auto-loaded knowledge skill)
|
|
136
|
+
- `mcp-specialist` agent — for deep MCP design questions
|
|
137
|
+
- `mcp-testing-engineer` agent — for protocol conformance testing
|
|
138
|
+
- https://modelcontextprotocol.io/
|
|
139
|
+
- https://github.com/anthropics/skills/tree/main/skills/mcp-builder
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: model-routing-patterns
|
|
3
|
+
description: "Loaded when user builds multi-model pipelines (Haiku/Sonnet/Opus). Covers cost-optimized routing, escalation, sub-agent delegation, and fallback chains."
|
|
4
|
+
effort: medium
|
|
5
|
+
user-invocable: false
|
|
6
|
+
allowed-tools: Read
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Model Routing Patterns
|
|
10
|
+
|
|
11
|
+
Three Claude tiers. Using Opus for everything is 10-40x more expensive than it needs to be. Using Haiku for everything loses accuracy on hard tasks. The craft is routing.
|
|
12
|
+
|
|
13
|
+
## Model Characteristics (2026)
|
|
14
|
+
|
|
15
|
+
| Model | Latency | Cost (rel.) | Strengths | When |
|
|
16
|
+
|-------|---------|-------------|-----------|------|
|
|
17
|
+
| Haiku 4.5 | Fastest | 1x | Classification, extraction, simple tools, moderation | Bulk processing, triage, labels |
|
|
18
|
+
| Sonnet 4.6 | Medium | 3-5x | General coding, reasoning, most agent tasks | Default workhorse |
|
|
19
|
+
| Opus 4.7 | Slowest | 15-30x | Complex reasoning, orchestration, architecture, large context | Hard, rare, high-stakes |
|
|
20
|
+
|
|
21
|
+
Ratios are approximate and shift between releases. Re-check pricing before committing a production path.
|
|
22
|
+
|
|
23
|
+
## Pattern 1 — Complexity Router (pre-classify)
|
|
24
|
+
|
|
25
|
+
Cheap model classifies the request, then routes to the right tier:
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
def route(user_message: str) -> str:
|
|
29
|
+
complexity = classify_with_haiku(user_message) # returns: simple | medium | hard
|
|
30
|
+
return {"simple": "haiku", "medium": "sonnet", "hard": "opus"}[complexity]
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Good when ~60% of traffic is simple. Overhead: one Haiku call per request (~100 tokens).
|
|
34
|
+
|
|
35
|
+
## Pattern 2 — Confidence-Based Escalation
|
|
36
|
+
|
|
37
|
+
Try the cheap model first, escalate only when it hesitates:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
def solve(problem: str):
|
|
41
|
+
haiku = call_haiku(problem)
|
|
42
|
+
if haiku.confidence > 0.85:
|
|
43
|
+
return haiku.answer
|
|
44
|
+
sonnet = call_sonnet(problem + haiku.reasoning)
|
|
45
|
+
if sonnet.confidence > 0.8:
|
|
46
|
+
return sonnet.answer
|
|
47
|
+
return call_opus(problem)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Haiku must be prompted to output confidence (e.g. via tool-use structured output — see `json-mode-patterns`). Pure self-reported confidence is noisy; combine with a heuristic (output length, tool calls, hedging words).
|
|
51
|
+
|
|
52
|
+
## Pattern 3 — Sub-agent Delegation (Opus orchestrates, Haiku workers)
|
|
53
|
+
|
|
54
|
+
Orchestrator reasons about the plan, workers execute atomic steps:
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
Opus (planner)
|
|
58
|
+
├── Haiku (extract_dates_from_doc_1)
|
|
59
|
+
├── Haiku (extract_dates_from_doc_2)
|
|
60
|
+
├── Haiku (extract_dates_from_doc_3)
|
|
61
|
+
└── Opus (synthesize all extractions into timeline)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Real example: `/orchestrate` in ai-toolkit runs Opus as planner, subagents (model per agent's frontmatter) as workers. See `app/agents/*.md` — each agent sets `model:` explicitly.
|
|
65
|
+
|
|
66
|
+
## Pattern 4 — Fallback Chain (resilience, not cost)
|
|
67
|
+
|
|
68
|
+
When primary is rate-limited or errors, degrade gracefully:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
def call_with_fallback(messages):
|
|
72
|
+
for model in ["claude-opus-4-7", "claude-sonnet-4-6", "claude-haiku-4-5"]:
|
|
73
|
+
try:
|
|
74
|
+
return client.messages.create(model=model, messages=messages, ...)
|
|
75
|
+
except (RateLimitError, OverloadedError):
|
|
76
|
+
continue
|
|
77
|
+
raise AllModelsExhausted()
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Useful in production, not for cost optimization — you lose quality on fallback.
|
|
81
|
+
|
|
82
|
+
## Pattern 5 — Task-Specific Routing
|
|
83
|
+
|
|
84
|
+
Skip generic complexity scoring when you know the task type:
|
|
85
|
+
|
|
86
|
+
| Task | Route |
|
|
87
|
+
|------|-------|
|
|
88
|
+
| Commit message from diff | Haiku |
|
|
89
|
+
| Summarize 5-10 lines | Haiku |
|
|
90
|
+
| Classify intent | Haiku |
|
|
91
|
+
| Fix a failing test | Sonnet |
|
|
92
|
+
| Write new feature | Sonnet |
|
|
93
|
+
| Code review, architecture decision | Opus |
|
|
94
|
+
| Multi-agent orchestration | Opus |
|
|
95
|
+
| Complex debugging across systems | Opus |
|
|
96
|
+
|
|
97
|
+
Encode this as a map in code, not a prompt.
|
|
98
|
+
|
|
99
|
+
## Anti-patterns
|
|
100
|
+
|
|
101
|
+
| Anti-pattern | Consequence | Fix |
|
|
102
|
+
|--------------|-------------|-----|
|
|
103
|
+
| Opus for everything | 10-40x bill | Start with Sonnet, measure, demote |
|
|
104
|
+
| Haiku for code review | Misses subtle bugs | Sonnet minimum for code quality |
|
|
105
|
+
| Router overhead > savings | Haiku classifier eats the margin | Skip router if >80% of traffic is one tier |
|
|
106
|
+
| Different prompts per tier | Maintenance nightmare | Same prompt, just swap model |
|
|
107
|
+
| No telemetry | Can't optimize | Log model + tokens + cost per request |
|
|
108
|
+
|
|
109
|
+
## Measuring
|
|
110
|
+
|
|
111
|
+
Track per-route:
|
|
112
|
+
- Cost per request
|
|
113
|
+
- Latency p50/p95
|
|
114
|
+
- Quality score (human-labeled or auto-evaluated)
|
|
115
|
+
- Escalation rate (how often you fell back to a bigger model)
|
|
116
|
+
|
|
117
|
+
Target: move the Pareto curve — cheaper at equal quality OR better at equal cost.
|
|
118
|
+
|
|
119
|
+
## Related
|
|
120
|
+
|
|
121
|
+
- `llm-ops-engineer` agent — production routing strategy
|
|
122
|
+
- `prompt-caching-patterns` — stack caching on top of routing
|
|
123
|
+
- `json-mode-patterns` — structured confidence from Haiku
|
|
124
|
+
- Anthropic cookbook: https://github.com/anthropics/claude-cookbooks — see "sub-agents" notebook
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: prompt-caching-patterns
|
|
3
|
+
description: "Loaded when user builds with Anthropic API and needs to cut cost or latency via prompt caching. Covers TTL, cache breakpoints, stacking, invalidation, and measuring hit rate."
|
|
4
|
+
effort: medium
|
|
5
|
+
user-invocable: false
|
|
6
|
+
allowed-tools: Read
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Prompt Caching Patterns
|
|
10
|
+
|
|
11
|
+
Anthropic's prompt caching cuts input-token cost by ~90% on cached prefixes and reduces latency. Worth learning because one mistake (putting a dynamic value before a stable prefix) disables the whole cache.
|
|
12
|
+
|
|
13
|
+
## Cache Mechanics
|
|
14
|
+
|
|
15
|
+
- **TTL**: default 5 minutes; `ttl: "1h"` for 1-hour cache (higher base cost but longer-lived).
|
|
16
|
+
- **Minimum size**: 1024 tokens per cache block for Sonnet/Opus, 2048 for Haiku.
|
|
17
|
+
- **Max breakpoints**: 4 per request.
|
|
18
|
+
- **Order matters**: everything BEFORE a `cache_control` block is part of that cache key. Dynamic content AFTER the cached block doesn't break the cache.
|
|
19
|
+
|
|
20
|
+
## Anatomy of a Cached Request
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from anthropic import Anthropic
|
|
24
|
+
|
|
25
|
+
client = Anthropic()
|
|
26
|
+
response = client.messages.create(
|
|
27
|
+
model="claude-opus-4-7",
|
|
28
|
+
max_tokens=1024,
|
|
29
|
+
system=[
|
|
30
|
+
{
|
|
31
|
+
"type": "text",
|
|
32
|
+
"text": LONG_SYSTEM_PROMPT, # stable across requests
|
|
33
|
+
"cache_control": {"type": "ephemeral"}
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
messages=[
|
|
37
|
+
{
|
|
38
|
+
"role": "user",
|
|
39
|
+
"content": [
|
|
40
|
+
{"type": "text", "text": LARGE_DOCUMENT_CONTEXT,
|
|
41
|
+
"cache_control": {"type": "ephemeral"}},
|
|
42
|
+
{"type": "text", "text": user_question} # dynamic
|
|
43
|
+
]
|
|
44
|
+
}
|
|
45
|
+
]
|
|
46
|
+
)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Layering Pattern (4 breakpoints)
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
[ system prompt ] ← breakpoint 1 (most stable)
|
|
53
|
+
[ tool definitions ] ← breakpoint 2
|
|
54
|
+
[ long reference docs ] ← breakpoint 3
|
|
55
|
+
[ conversation history up to turn N ] ← breakpoint 4
|
|
56
|
+
[ current user message ] ← not cached (dynamic)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Put the MOST stable content earliest. A change to breakpoint 2 invalidates 3 and 4.
|
|
60
|
+
|
|
61
|
+
## Anti-patterns
|
|
62
|
+
|
|
63
|
+
| Pattern | Problem | Fix |
|
|
64
|
+
|---------|---------|-----|
|
|
65
|
+
| Timestamp in system prompt | Every request is unique | Remove timestamp, or put it AFTER the cache block |
|
|
66
|
+
| User name inserted into cached text | Cache misses per user | Inject user name AFTER the cache block |
|
|
67
|
+
| Reordering tool definitions across requests | Cache invalidated | Sort tools deterministically |
|
|
68
|
+
| Retrying with exponential jitter that changes prompt | Cache miss on retry | Keep the exact same prefix on retries |
|
|
69
|
+
| Caching <1024 tokens | Silently uncached | Merge with adjacent content or drop the breakpoint |
|
|
70
|
+
|
|
71
|
+
## Measuring Hit Rate
|
|
72
|
+
|
|
73
|
+
Response includes:
|
|
74
|
+
```python
|
|
75
|
+
response.usage.cache_creation_input_tokens # written this request
|
|
76
|
+
response.usage.cache_read_input_tokens # read from cache (billed ~10%)
|
|
77
|
+
response.usage.input_tokens # not cached
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Target ratio for a well-tuned loop: `cache_read / (cache_read + input) > 0.7`. Below that, you're leaving money on the table.
|
|
81
|
+
|
|
82
|
+
## When NOT to Cache
|
|
83
|
+
|
|
84
|
+
- One-shot calls (cost of writing cache > savings)
|
|
85
|
+
- Prompts under ~1500 tokens
|
|
86
|
+
- Content that changes every request (user input, current weather, live data)
|
|
87
|
+
- Hot path with <1 request per 5 min (cache expires unused)
|
|
88
|
+
|
|
89
|
+
## TypeScript SDK
|
|
90
|
+
|
|
91
|
+
```typescript
|
|
92
|
+
const response = await anthropic.messages.create({
|
|
93
|
+
model: "claude-opus-4-7",
|
|
94
|
+
max_tokens: 1024,
|
|
95
|
+
system: [
|
|
96
|
+
{ type: "text", text: LONG_SYSTEM, cache_control: { type: "ephemeral" } }
|
|
97
|
+
],
|
|
98
|
+
messages: [
|
|
99
|
+
{
|
|
100
|
+
role: "user",
|
|
101
|
+
content: [
|
|
102
|
+
{ type: "text", text: LARGE_CONTEXT, cache_control: { type: "ephemeral" } },
|
|
103
|
+
{ type: "text", text: userQuestion }
|
|
104
|
+
]
|
|
105
|
+
}
|
|
106
|
+
]
|
|
107
|
+
});
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Related
|
|
111
|
+
|
|
112
|
+
- `claude-api` skill — full Anthropic SDK patterns
|
|
113
|
+
- `llm-ops-engineer` agent — production caching strategy
|
|
114
|
+
- Anthropic docs: https://docs.claude.com/en/docs/build-with-claude/prompt-caching
|
|
@@ -21,20 +21,47 @@ import sys
|
|
|
21
21
|
from typing import Any
|
|
22
22
|
|
|
23
23
|
|
|
24
|
+
# Order matters: earlier patterns win. `docs` and `test` come first so a
|
|
25
|
+
# README under docs/ or a test file named e.g. `role_test.py` is not miscategorised
|
|
26
|
+
# as `security`. Patterns use word boundaries or path anchors to avoid matching
|
|
27
|
+
# substrings in unrelated filenames.
|
|
24
28
|
CATEGORY_PATTERNS: dict[str, str] = {
|
|
25
|
-
"
|
|
26
|
-
"test": r"(test_|_test
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
29
|
+
"docs": r"(^|/)(readme|changelog|license)\b|(^|/)docs/|\.md$",
|
|
30
|
+
"test": r"(^|/)tests?/|__tests__|(^|/)test_|_test\.|\.test\.|\.spec\.",
|
|
31
|
+
"migration": r"(^|/)(migrations?|alembic)(/|$)|\bschema_migrate\b|\b(migrate|migration)\.[a-z]+$",
|
|
32
|
+
"infra": r"(^|/)(docker|kubernetes|k8s|terraform|ansible|\.github)(/|$)|\.(dockerfile|tf)$|(^|/)ci(/|$)|(^|/)deploy(/|_|-|$)",
|
|
33
|
+
"security": r"\b\w*(auth|login|password|passwd|token|secret|crypto|session|permission|role|access|oauth|jwt|mfa|totp)\w*\b",
|
|
34
|
+
"config": r"(^|/)(config|settings)(/|\.[a-z]+$)|\.(yml|yaml|json|toml|env|ini|cfg)$|\.lock$",
|
|
31
35
|
}
|
|
32
36
|
|
|
37
|
+
# Secret detection patterns. Combines quoted and unquoted env-style assignments,
|
|
38
|
+
# common cloud provider prefixes, JWTs, and PEM private key headers.
|
|
33
39
|
SECRET_PATTERNS: list[str] = [
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
r
|
|
37
|
-
|
|
40
|
+
# Quoted secret assignments: API_KEY = "..." / secret: '...'
|
|
41
|
+
# Allow prefix/suffix word chars so `some_password`, `MY_API_KEY`, `accessToken` match.
|
|
42
|
+
r'(?i)\b\w*(?:api[_-]?key|secret[_-]?key|password|passwd|token|bearer|access[_-]?key)\w*\s*[=:]\s*["\'][^"\']{8,}["\']',
|
|
43
|
+
# Unquoted env-style: API_KEY=abcdef12345678 or API_KEY=SECRET_VALUE_XYZ.
|
|
44
|
+
# The RHS must look like a secret (no snake_case identifiers) — either a
|
|
45
|
+
# dotted/hyphenated/alphanumeric blob without underscores, or an ALL_CAPS
|
|
46
|
+
# constant. This excludes ``request_token = generate_token_v2_legacy`` style
|
|
47
|
+
# function refs.
|
|
48
|
+
r'(?i)\b\w*(?:api[_-]?key|secret[_-]?key|password|passwd|token|bearer|access[_-]?key)\w*\s*=\s*(?:[A-Za-z0-9\-\.]{12,}|(?-i:[A-Z][A-Z0-9_]{11,}))\b',
|
|
49
|
+
# AWS access key ID
|
|
50
|
+
r"\bAKIA[0-9A-Z]{16}\b",
|
|
51
|
+
# OpenAI / Anthropic-style keys
|
|
52
|
+
r"\bsk-[a-zA-Z0-9]{20,}\b",
|
|
53
|
+
# GitHub PAT
|
|
54
|
+
r"\bghp_[a-zA-Z0-9]{36}\b",
|
|
55
|
+
# GitHub fine-grained PAT
|
|
56
|
+
r"\bgithub_pat_[A-Za-z0-9_]{82}\b",
|
|
57
|
+
# Google API key
|
|
58
|
+
r"\bAIza[0-9A-Za-z_\-]{35}\b",
|
|
59
|
+
# Slack tokens (bot, app, user, refresh, config)
|
|
60
|
+
r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b",
|
|
61
|
+
# JWT
|
|
62
|
+
r"\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b",
|
|
63
|
+
# PEM private key header
|
|
64
|
+
r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----",
|
|
38
65
|
]
|
|
39
66
|
|
|
40
67
|
|
|
@@ -54,33 +81,154 @@ def categorize(path: str) -> str:
|
|
|
54
81
|
return "logic"
|
|
55
82
|
|
|
56
83
|
|
|
84
|
+
def _base_ref_exists(base: str) -> bool:
|
|
85
|
+
"""Return True if ``base`` resolves to a valid git ref."""
|
|
86
|
+
result = subprocess.run(
|
|
87
|
+
["git", "rev-parse", "--verify", "--quiet", f"{base}^{{commit}}"],
|
|
88
|
+
stdout=subprocess.DEVNULL,
|
|
89
|
+
stderr=subprocess.DEVNULL,
|
|
90
|
+
)
|
|
91
|
+
return result.returncode == 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run_diff(args: list[str]) -> tuple[str, int]:
|
|
95
|
+
"""Run a git diff command and return (stdout, returncode)."""
|
|
96
|
+
result = subprocess.run(
|
|
97
|
+
["git", "diff", *args],
|
|
98
|
+
capture_output=True,
|
|
99
|
+
text=True,
|
|
100
|
+
)
|
|
101
|
+
return result.stdout, result.returncode
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _parse_numstat_z(raw: str) -> list[tuple[int, int, str]]:
|
|
105
|
+
"""Parse ``git diff --numstat -z`` output into ``(adds, dels, path)`` triples.
|
|
106
|
+
|
|
107
|
+
With ``-z`` each record is NUL-terminated. Rename entries are emitted
|
|
108
|
+
as three NUL-separated tokens: ``additions\\tdeletions\\t`` then
|
|
109
|
+
``old_path`` NUL ``new_path`` NUL. The new path is reported.
|
|
110
|
+
"""
|
|
111
|
+
records: list[tuple[int, int, str]] = []
|
|
112
|
+
tokens = raw.split("\0")
|
|
113
|
+
i = 0
|
|
114
|
+
while i < len(tokens):
|
|
115
|
+
token = tokens[i]
|
|
116
|
+
if not token:
|
|
117
|
+
i += 1
|
|
118
|
+
continue
|
|
119
|
+
parts = token.split("\t")
|
|
120
|
+
if len(parts) < 2:
|
|
121
|
+
i += 1
|
|
122
|
+
continue
|
|
123
|
+
add_str, del_str = parts[0], parts[1]
|
|
124
|
+
try:
|
|
125
|
+
add = int(add_str) if add_str != "-" else 0
|
|
126
|
+
delete = int(del_str) if del_str != "-" else 0
|
|
127
|
+
except ValueError:
|
|
128
|
+
i += 1
|
|
129
|
+
continue
|
|
130
|
+
if len(parts) >= 3 and parts[2]:
|
|
131
|
+
# Normal entry: "adds\tdels\tpath"
|
|
132
|
+
records.append((add, delete, parts[2]))
|
|
133
|
+
i += 1
|
|
134
|
+
else:
|
|
135
|
+
# Rename entry: "adds\tdels\t" then old\0new
|
|
136
|
+
if i + 2 < len(tokens):
|
|
137
|
+
new_path = tokens[i + 2]
|
|
138
|
+
if new_path:
|
|
139
|
+
records.append((add, delete, new_path))
|
|
140
|
+
i += 3
|
|
141
|
+
else:
|
|
142
|
+
i += 1
|
|
143
|
+
return records
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _scan_secrets(diff_content: str) -> list[dict[str, Any]]:
|
|
147
|
+
"""Walk a unified diff, tracking current file path and line number for each
|
|
148
|
+
added line, and report any lines matching a secret pattern.
|
|
149
|
+
|
|
150
|
+
Returns a list of ``{"file": path, "line": file_line, "preview": snippet}``.
|
|
151
|
+
"""
|
|
152
|
+
findings: list[dict[str, Any]] = []
|
|
153
|
+
current_file: str | None = None
|
|
154
|
+
new_line_no: int | None = None
|
|
155
|
+
hunk_re = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
|
156
|
+
for raw_line in diff_content.split("\n"):
|
|
157
|
+
if raw_line.startswith("+++ "):
|
|
158
|
+
# Format: "+++ b/path/to/file" or "+++ /dev/null"
|
|
159
|
+
target = raw_line[4:].strip()
|
|
160
|
+
if target == "/dev/null":
|
|
161
|
+
current_file = None
|
|
162
|
+
elif target.startswith("b/"):
|
|
163
|
+
current_file = target[2:]
|
|
164
|
+
else:
|
|
165
|
+
current_file = target
|
|
166
|
+
new_line_no = None
|
|
167
|
+
continue
|
|
168
|
+
if raw_line.startswith("--- "):
|
|
169
|
+
continue
|
|
170
|
+
if raw_line.startswith("@@"):
|
|
171
|
+
match = hunk_re.match(raw_line)
|
|
172
|
+
if match:
|
|
173
|
+
new_line_no = int(match.group(1))
|
|
174
|
+
continue
|
|
175
|
+
if new_line_no is None:
|
|
176
|
+
continue
|
|
177
|
+
if raw_line.startswith("+") and not raw_line.startswith("+++"):
|
|
178
|
+
for pattern in SECRET_PATTERNS:
|
|
179
|
+
if re.search(pattern, raw_line):
|
|
180
|
+
findings.append(
|
|
181
|
+
{
|
|
182
|
+
"file": current_file,
|
|
183
|
+
"line": new_line_no,
|
|
184
|
+
"preview": raw_line[:80],
|
|
185
|
+
}
|
|
186
|
+
)
|
|
187
|
+
break
|
|
188
|
+
new_line_no += 1
|
|
189
|
+
elif raw_line.startswith("-"):
|
|
190
|
+
# Deletion does not advance the new-file line counter
|
|
191
|
+
continue
|
|
192
|
+
else:
|
|
193
|
+
# Context line advances the new-file line counter
|
|
194
|
+
new_line_no += 1
|
|
195
|
+
return findings
|
|
196
|
+
|
|
197
|
+
|
|
57
198
|
def main() -> None:
|
|
58
199
|
"""Entry point: analyze diff and print JSON risk report to stdout."""
|
|
59
200
|
base = sys.argv[1] if len(sys.argv) > 1 else "main"
|
|
60
201
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
202
|
+
warnings: list[str] = []
|
|
203
|
+
|
|
204
|
+
# Determine scope: branch diff if base exists, otherwise staged-only.
|
|
205
|
+
if _base_ref_exists(base):
|
|
206
|
+
scope = f"{base}...HEAD"
|
|
207
|
+
numstat_raw, rc1 = _run_diff(["--numstat", "-z", f"{base}...HEAD"])
|
|
208
|
+
diff_raw, rc2 = _run_diff([f"{base}...HEAD"])
|
|
209
|
+
if rc1 != 0 or rc2 != 0:
|
|
210
|
+
scope = "staged"
|
|
211
|
+
warnings.append(
|
|
212
|
+
f"branch diff against {base!r} failed; fell back to staged changes"
|
|
213
|
+
)
|
|
214
|
+
numstat_raw, _ = _run_diff(["--numstat", "-z", "--cached"])
|
|
215
|
+
diff_raw, _ = _run_diff(["--cached"])
|
|
216
|
+
else:
|
|
217
|
+
scope = "staged"
|
|
218
|
+
warnings.append(
|
|
219
|
+
f"base ref {base!r} not found; reporting staged changes only"
|
|
220
|
+
)
|
|
221
|
+
print(
|
|
222
|
+
f"[diff-analyzer] base ref {base!r} not found; using --cached",
|
|
223
|
+
file=sys.stderr,
|
|
70
224
|
)
|
|
71
|
-
|
|
225
|
+
numstat_raw, _ = _run_diff(["--numstat", "-z", "--cached"])
|
|
226
|
+
diff_raw, _ = _run_diff(["--cached"])
|
|
227
|
+
|
|
72
228
|
files: list[dict[str, Any]] = []
|
|
73
229
|
total_add, total_del = 0, 0
|
|
74
230
|
|
|
75
|
-
for
|
|
76
|
-
if not line.strip():
|
|
77
|
-
continue
|
|
78
|
-
parts = line.split("\t")
|
|
79
|
-
if len(parts) < 3:
|
|
80
|
-
continue
|
|
81
|
-
add = int(parts[0]) if parts[0] != "-" else 0
|
|
82
|
-
delete = int(parts[1]) if parts[1] != "-" else 0
|
|
83
|
-
path = parts[2]
|
|
231
|
+
for add, delete, path in _parse_numstat_z(numstat_raw):
|
|
84
232
|
cat = categorize(path)
|
|
85
233
|
if cat in ("security", "migration") or add > 100:
|
|
86
234
|
risk = "high"
|
|
@@ -112,23 +260,8 @@ def main() -> None:
|
|
|
112
260
|
if f["additions"] > 20
|
|
113
261
|
]
|
|
114
262
|
|
|
115
|
-
# Secrets scan on added lines
|
|
116
|
-
|
|
117
|
-
["git", "diff", f"{base}...HEAD"],
|
|
118
|
-
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
|
|
119
|
-
)
|
|
120
|
-
if r.returncode != 0:
|
|
121
|
-
r = subprocess.run(
|
|
122
|
-
["git", "diff", "--cached"],
|
|
123
|
-
capture_output=True, text=True,
|
|
124
|
-
)
|
|
125
|
-
diff_content = r.stdout.strip()
|
|
126
|
-
secrets: list[dict[str, Any]] = []
|
|
127
|
-
for i, line in enumerate(diff_content.split("\n")):
|
|
128
|
-
if line.startswith("+") and not line.startswith("+++"):
|
|
129
|
-
for p in SECRET_PATTERNS:
|
|
130
|
-
if re.search(p, line):
|
|
131
|
-
secrets.append({"line": i, "preview": line[:80]})
|
|
263
|
+
# Secrets scan on added lines with accurate file + file-line tracking
|
|
264
|
+
secrets = _scan_secrets(diff_raw)
|
|
132
265
|
|
|
133
266
|
# Risk score
|
|
134
267
|
has_security = any(f["category"] == "security" for f in files)
|
|
@@ -150,8 +283,12 @@ def main() -> None:
|
|
|
150
283
|
else:
|
|
151
284
|
coverage = "none"
|
|
152
285
|
|
|
286
|
+
if not files:
|
|
287
|
+
warnings.append("no changes detected")
|
|
288
|
+
|
|
153
289
|
result: dict[str, Any] = {
|
|
154
290
|
"base": base,
|
|
291
|
+
"scope": scope,
|
|
155
292
|
"files_changed": len(files),
|
|
156
293
|
"additions": total_add,
|
|
157
294
|
"deletions": total_del,
|
|
@@ -162,6 +299,7 @@ def main() -> None:
|
|
|
162
299
|
"test_coverage_estimate": coverage,
|
|
163
300
|
"secrets_scan": secrets,
|
|
164
301
|
"parallel_review_recommended": risk_score == "high" or len(files) > 10,
|
|
302
|
+
"warnings": warnings,
|
|
165
303
|
}
|
|
166
304
|
print(json.dumps(result, indent=2))
|
|
167
305
|
|
|
@@ -185,9 +185,9 @@ Three tiers determine how to approach a task:
|
|
|
185
185
|
|
|
186
186
|
| Type | Field | Invocation | Count |
|
|
187
187
|
|------|-------|-----------|-------|
|
|
188
|
-
| Task | `disable-model-invocation: true` | User via `/skill` only |
|
|
188
|
+
| Task | `disable-model-invocation: true` | User via `/skill` only | 32 |
|
|
189
189
|
| Hybrid | (neither) | User via `/skill` + agent knowledge | 31 |
|
|
190
|
-
| Knowledge | `user-invocable: false` | Claude auto-loads |
|
|
190
|
+
| Knowledge | `user-invocable: false` | Claude auto-loads | 36 |
|
|
191
191
|
|
|
192
192
|
## Multi-Agent Execution
|
|
193
193
|
|