@anvia/cli 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/chunk-ERCP4EIZ.js +607 -0
- package/dist/chunk-ERCP4EIZ.js.map +1 -0
- package/dist/cli.js +278 -74
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.js +21 -3
- package/dist/skills/anvia-agent/SKILL.md +59 -0
- package/dist/skills/anvia-agent/references/agent-options.md +83 -0
- package/dist/skills/anvia-agent/references/providers.md +22 -0
- package/dist/skills/anvia-agent/references/teams.md +84 -0
- package/dist/skills/anvia-agent/references/tools.md +71 -0
- package/dist/skills/anvia-agent/scripts/check-agent.sh +99 -0
- package/dist/skills/anvia-channels/SKILL.md +59 -0
- package/dist/skills/anvia-channels/references/adapters.md +53 -0
- package/dist/skills/anvia-channels/references/channel-agent.md +52 -0
- package/dist/skills/anvia-channels/references/delivery.md +53 -0
- package/dist/skills/anvia-channels/scripts/check-channels.sh +79 -0
- package/dist/skills/anvia-chat/SKILL.md +61 -0
- package/dist/skills/anvia-chat/references/react-ui.md +104 -0
- package/dist/skills/anvia-chat/references/server-protocol.md +53 -0
- package/dist/skills/anvia-chat/references/transports-state.md +71 -0
- package/dist/skills/anvia-chat/scripts/check-chat-boundary.sh +70 -0
- package/dist/skills/anvia-evals/SKILL.md +51 -0
- package/dist/skills/anvia-evals/references/judges.md +61 -0
- package/dist/skills/anvia-evals/references/metrics.md +45 -0
- package/dist/skills/anvia-evals/references/running.md +62 -0
- package/dist/skills/anvia-evals/scripts/check-evals.sh +73 -0
- package/dist/skills/anvia-mcp/SKILL.md +49 -0
- package/dist/skills/anvia-mcp/references/clients.md +73 -0
- package/dist/skills/anvia-mcp/references/safety.md +25 -0
- package/dist/skills/anvia-mcp/scripts/check-mcp.sh +73 -0
- package/dist/skills/anvia-pipeline/SKILL.md +52 -0
- package/dist/skills/anvia-pipeline/references/agents-extract.md +62 -0
- package/dist/skills/anvia-pipeline/references/steps-compose.md +63 -0
- package/dist/skills/anvia-pipeline/scripts/check-pipeline.sh +69 -0
- package/dist/skills/anvia-rag/SKILL.md +50 -0
- package/dist/skills/anvia-rag/references/graph-rag.md +115 -0
- package/dist/skills/anvia-rag/references/pipeline.md +81 -0
- package/dist/skills/anvia-rag/references/rag-tool.md +43 -0
- package/dist/skills/anvia-rag/references/stores.md +68 -0
- package/dist/skills/anvia-rag/scripts/check-rag.sh +75 -0
- package/dist/skills/anvia-studio/SKILL.md +45 -0
- package/dist/skills/anvia-studio/references/inspect.md +33 -0
- package/dist/skills/anvia-studio/references/observe.md +34 -0
- package/dist/skills/anvia-studio/references/serve.md +65 -0
- package/dist/skills/anvia-studio/scripts/check-studio.sh +59 -0
- package/dist/skills/release-notes/SKILL.md +18 -0
- package/dist/skills/release-notes/references/style.md +6 -0
- package/dist/skills/release-notes/scripts/draft.sh +22 -0
- package/package.json +2 -2
- package/dist/chunk-IH4ZNWHB.js +0 -285
- package/dist/chunk-IH4ZNWHB.js.map +0 -1
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Running Evals
|
|
2
|
+
|
|
3
|
+
## Suite vs CLI
|
|
4
|
+
|
|
5
|
+
```ts
|
|
6
|
+
import { agentEvalTarget, runEvalCli, runEvalSuite } from "@anvia/core/evals";
|
|
7
|
+
|
|
8
|
+
// Library use: inspect results programmatically.
|
|
9
|
+
const result = await runEvalSuite({ name: "support-agent-target", cases, target, metrics });
|
|
10
|
+
console.log({
|
|
11
|
+
passed: result.metrics.passed,
|
|
12
|
+
failed: result.metrics.failed,
|
|
13
|
+
invalid: result.metrics.invalid,
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
// CLI/CI use: pretty table, asserted expectations, process exit code.
|
|
17
|
+
await runEvalCli({
|
|
18
|
+
name: "support-basic-metrics",
|
|
19
|
+
cases,
|
|
20
|
+
target,
|
|
21
|
+
metrics,
|
|
22
|
+
expectations,
|
|
23
|
+
exitCode: true,
|
|
24
|
+
});
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
- `runEvalSuite` returns full per-case, per-metric results — use it in scripts
|
|
28
|
+
and notebooks. It also takes execution controls for larger suites:
|
|
29
|
+
`concurrency`, `caseTimeoutMs`, `failFast`, `caseIds`/`caseFilter`, `shard`,
|
|
30
|
+
`signal`, `onProgress`, plus reporters and `cost`/`targetUsage` capture.
|
|
31
|
+
- `runEvalCli` prints tables (`printEvalResult` / `formatEvalResult` under the
|
|
32
|
+
hood), asserts `expectations`, and with `exitCode: true` fails CI on
|
|
33
|
+
regressions. Always set `exitCode: true` in CI. Output is tunable via
|
|
34
|
+
`format: "pretty" | "json" | "quiet"`, `redact`, output writers, and
|
|
35
|
+
`maxValueLength`.
|
|
36
|
+
- `expectations.outcomes` pins known outcomes per case — including intentional
|
|
37
|
+
failures (negative controls), so a "fixed" negative control fails loudly
|
|
38
|
+
instead of silently flipping green. `expectations.totals` pins aggregate
|
|
39
|
+
counts per metric or for the whole run; `defineEvalExpectations`,
|
|
40
|
+
`assertEvalTotals`, `assertEvalOutcomes`, and `evalExitCode` are the
|
|
41
|
+
programmatic helpers behind the CLI.
|
|
42
|
+
|
|
43
|
+
## Targets
|
|
44
|
+
|
|
45
|
+
- Plain functions: `target: async (input) => answer(input)`.
|
|
46
|
+
- Agents: wrap with `agentEvalTarget({ agent, request: ({ input }) => ({ prompt: input }) })`
|
|
47
|
+
and project `output.output` in metrics (see `references/metrics.md`).
|
|
48
|
+
- `defineEvalCases` / `defineEvalSuite` / `createEvalTypes` add type safety to
|
|
49
|
+
large suites — adopt them when cases grow past a handful.
|
|
50
|
+
|
|
51
|
+
## Case design
|
|
52
|
+
|
|
53
|
+
- Every case needs `id`, `input`, `expected`. Keep ids stable — `expectations`
|
|
54
|
+
and history key off them.
|
|
55
|
+
- Include negative controls: inputs the target must get wrong (or must refuse),
|
|
56
|
+
pinned as expected failures. Without them you cannot tell a strict metric
|
|
57
|
+
from a broken target.
|
|
58
|
+
- For RAG cases add `context` (what the generator saw) and `retrievalContext`
|
|
59
|
+
(what retrieval returned) so grounding metrics have something to check.
|
|
60
|
+
- Report evals to observability (Langfuse eval reporting, trace refs via
|
|
61
|
+
`resolveEvalTraceRef`) when runs must be auditable — see the cookbook's
|
|
62
|
+
langfuse eval-reporting example.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Heuristic checks for Anvia eval code in an app directory.
|
|
3
|
+
# Usage: sh scripts/check-evals.sh [--dir <app-root>]
|
|
4
|
+
# Fails with a list of violations; passes silently with "evals OK".
|
|
5
|
+
|
|
6
|
+
DIR="."
|
|
7
|
+
if [ "$1" = "--dir" ] && [ -n "$2" ]; then
|
|
8
|
+
DIR="$2"
|
|
9
|
+
fi
|
|
10
|
+
|
|
11
|
+
ROOTS_FOUND=0
|
|
12
|
+
for root in "$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server" "$DIR/evals"; do
|
|
13
|
+
if [ -d "$root" ]; then
|
|
14
|
+
ROOTS_FOUND=1
|
|
15
|
+
break
|
|
16
|
+
fi
|
|
17
|
+
done
|
|
18
|
+
if [ "$ROOTS_FOUND" -eq 0 ]; then
|
|
19
|
+
echo "ERROR: no src/app/lib/server/evals directory under '$DIR' — nothing was checked."
|
|
20
|
+
echo "Run from the app root or pass --dir <app-root>."
|
|
21
|
+
exit 1
|
|
22
|
+
fi
|
|
23
|
+
|
|
24
|
+
fail=0
|
|
25
|
+
violation() {
|
|
26
|
+
echo "VIOLATION: $1"
|
|
27
|
+
fail=1
|
|
28
|
+
}
|
|
29
|
+
warning() {
|
|
30
|
+
echo "WARNING: $1"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
FILES=$(grep -rln --include='*.ts' \
|
|
34
|
+
-e 'runEvalSuite' -e 'runEvalCli' -e 'llmJudge' -e 'llmScore' -e 'gEval(' \
|
|
35
|
+
"$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server" "$DIR/evals" 2>/dev/null | sort -u)
|
|
36
|
+
|
|
37
|
+
if [ -z "$FILES" ]; then
|
|
38
|
+
echo "evals OK (no eval code found)"
|
|
39
|
+
exit 0
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
# 1. Eval runs need a stable name (history and expectations key off it).
|
|
43
|
+
if echo "$FILES" | xargs grep -l -e 'runEvalSuite' -e 'runEvalCli' 2>/dev/null | grep -q .; then
|
|
44
|
+
echo "$FILES" | xargs grep -h -A6 -e 'runEvalSuite(' -e 'runEvalCli(' 2>/dev/null | grep -q 'name:' || {
|
|
45
|
+
violation "runEvalSuite/runEvalCli without name (see references/running.md)."
|
|
46
|
+
}
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
# 2. llmScore needs a threshold; llmJudge needs a passes predicate.
|
|
50
|
+
if echo "$FILES" | xargs grep -l 'llmScore(' 2>/dev/null | grep -q .; then
|
|
51
|
+
echo "$FILES" | xargs grep -h -A8 'llmScore(' 2>/dev/null | grep -q 'threshold' || {
|
|
52
|
+
violation "llmScore without threshold (see references/judges.md)."
|
|
53
|
+
}
|
|
54
|
+
fi
|
|
55
|
+
if echo "$FILES" | xargs grep -l 'llmJudge(' 2>/dev/null | grep -q .; then
|
|
56
|
+
echo "$FILES" | xargs grep -h -A8 'llmJudge(' 2>/dev/null | grep -q 'passes' || {
|
|
57
|
+
violation "llmJudge without passes predicate (see references/judges.md)."
|
|
58
|
+
}
|
|
59
|
+
fi
|
|
60
|
+
|
|
61
|
+
# 3. CI runs should fail the build on regression (warning only).
|
|
62
|
+
if echo "$FILES" | xargs grep -l 'runEvalCli' 2>/dev/null | grep -q .; then
|
|
63
|
+
echo "$FILES" | xargs grep -h -A20 'runEvalCli(' 2>/dev/null | grep -q 'exitCode' || {
|
|
64
|
+
warning "runEvalCli without exitCode will not fail CI (see references/running.md)."
|
|
65
|
+
}
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
if [ "$fail" -eq 0 ]; then
|
|
69
|
+
echo "evals OK"
|
|
70
|
+
exit 0
|
|
71
|
+
fi
|
|
72
|
+
echo "See skills/anvia-evals/references/ for fixes."
|
|
73
|
+
exit 1
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: anvia-mcp
|
|
3
|
+
description: Connect MCP servers to Anvia agents — clients, transports, tool discovery, URL safety, and lifecycle.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Anvia MCP Skill
|
|
7
|
+
|
|
8
|
+
Use this skill when the user wants tools from an MCP server: configuring an
|
|
9
|
+
`McpClient`, choosing stdio vs Streamable HTTP, handling version negotiation
|
|
10
|
+
and URL safety, grouping servers, or wiring them into an `Agent`.
|
|
11
|
+
|
|
12
|
+
## Process
|
|
13
|
+
|
|
14
|
+
1. Configure clients (`references/clients.md`) — one `McpClient` per server.
|
|
15
|
+
2. Apply the safety rules (`references/safety.md`) — especially for HTTP servers.
|
|
16
|
+
3. Run `scripts/check-mcp.sh` from the app root before claiming done.
|
|
17
|
+
|
|
18
|
+
## Minimal slice
|
|
19
|
+
|
|
20
|
+
```ts
|
|
21
|
+
import { Agent } from "@anvia/core/agent";
|
|
22
|
+
import { McpClient } from "@anvia/mcp";
|
|
23
|
+
|
|
24
|
+
const counterMcp = new McpClient({
|
|
25
|
+
name: "counter",
|
|
26
|
+
transport: { type: "stdio", command: "tsx", args: ["./mcp-counter-server.ts"] },
|
|
27
|
+
});
|
|
28
|
+
const counterServer = await counterMcp.connect();
|
|
29
|
+
|
|
30
|
+
try {
|
|
31
|
+
const agent = new Agent({
|
|
32
|
+
id: "agent",
|
|
33
|
+
model: agentModel,
|
|
34
|
+
instructions: "Use MCP tools for arithmetic and counter updates.",
|
|
35
|
+
mcpServers: [counterServer], // the only path for MCP tools — never via skills
|
|
36
|
+
maxTurns: 3,
|
|
37
|
+
});
|
|
38
|
+
for await (const event of agent.stream({ prompt: "Add 8 and 13." })) {
|
|
39
|
+
if (event.type === "response") console.log("final:", event.text);
|
|
40
|
+
}
|
|
41
|
+
} finally {
|
|
42
|
+
await counterMcp.close();
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Output
|
|
47
|
+
|
|
48
|
+
Prefer stdio for local servers, Streamable HTTP for remote ones. Point to the
|
|
49
|
+
relevant reference file instead of pasting its contents into chat.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# MCP Clients
|
|
2
|
+
|
|
3
|
+
`@anvia/mcp` owns connections, transports, tool discovery, and cleanup (official
|
|
4
|
+
MCP TypeScript SDK v2 client under the hood). `@anvia/core` keeps only the
|
|
5
|
+
registration contracts the `Agent` consumes. Construction performs no I/O.
|
|
6
|
+
|
|
7
|
+
## Transports
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { McpClient, McpClientGroup } from "@anvia/mcp";
|
|
11
|
+
|
|
12
|
+
// Local subprocess.
|
|
13
|
+
const filesystem = new McpClient({
|
|
14
|
+
name: "filesystem",
|
|
15
|
+
versionNegotiation: { mode: "auto" },
|
|
16
|
+
transport: {
|
|
17
|
+
type: "stdio",
|
|
18
|
+
command: "npx",
|
|
19
|
+
args: ["-y", "@modelcontextprotocol/server-filesystem", "./workspace"],
|
|
20
|
+
},
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
// Remote server.
|
|
24
|
+
const github = new McpClient({
|
|
25
|
+
name: "github",
|
|
26
|
+
transport: {
|
|
27
|
+
type: "streamableHttp",
|
|
28
|
+
url: "https://mcp.example.com/mcp",
|
|
29
|
+
headers: { authorization: `Bearer ${process.env.MCP_TOKEN}` },
|
|
30
|
+
},
|
|
31
|
+
tools: { prefix: "github_" },
|
|
32
|
+
});
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
- Every client needs a unique `name` — it identifies the server in logs,
|
|
36
|
+
errors, and the Studio MCP inspector.
|
|
37
|
+
- `tools.prefix` namespaces remote tools (`github_...`) so two servers never
|
|
38
|
+
collide. Set it whenever more than one server is connected.
|
|
39
|
+
- Read tokens from the environment at the boundary; never commit them.
|
|
40
|
+
- Remote servers that need real OAuth take `authProvider` (an
|
|
41
|
+
`OAuthClientProvider`) instead of — not alongside — a `headers.authorization`
|
|
42
|
+
value; the client rejects the combination. `reconnectionOptions` tunes
|
|
43
|
+
reconnect behavior and `sessionId` resumes a known streamable-HTTP session.
|
|
44
|
+
- Adapters not covered above: pass `transport: { type: "custom", ... }` with
|
|
45
|
+
your own MCP `Transport` implementation; URL safety and discovery behavior
|
|
46
|
+
are unchanged.
|
|
47
|
+
|
|
48
|
+
## Connect and wire
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
const mcp = await McpClientGroup.connect({ clients: [filesystem, github] });
|
|
52
|
+
const agent = new Agent({ id: "assistant", model, mcpServers: mcp.servers });
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
await agent.generate({ prompt: "Find the issue and update it." });
|
|
56
|
+
} finally {
|
|
57
|
+
await mcp.close();
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
- `connect()` discovers every tool page once and returns a frozen registration
|
|
62
|
+
snapshot. A single client also exposes `connect()` returning one server.
|
|
63
|
+
- The Agent only sees `mcpServers`. MCP tools must never be passed via
|
|
64
|
+
`Agent.skills` — the Agent rejects them there.
|
|
65
|
+
- Always `close()` clients (try/finally). To adopt changed remote tools,
|
|
66
|
+
reconnect and rebuild the Agent — snapshots do not refresh in place.
|
|
67
|
+
|
|
68
|
+
## Version negotiation
|
|
69
|
+
|
|
70
|
+
By default `@anvia/mcp` requires the modern MCP `2026-07-28` protocol and fails
|
|
71
|
+
clearly otherwise. For older servers set `versionNegotiation: { mode: "auto" }`
|
|
72
|
+
(probe modern, fall back to the legacy handshake) or `{ mode: "legacy" }` for a
|
|
73
|
+
known 2025-era server. Prefer `auto` over `legacy` unless the server is pinned.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# MCP Safety
|
|
2
|
+
|
|
3
|
+
Streamable HTTP connections enforce Anvia URL safety by default:
|
|
4
|
+
|
|
5
|
+
- No custom `fetch`. The transport owns its HTTP method, body, abort signal,
|
|
6
|
+
session, and protocol headers — arbitrary `RequestInit` fields are not exposed.
|
|
7
|
+
- Static headers are explicit transport config. They are sent only to the exact
|
|
8
|
+
MCP endpoint, never attached to OAuth requests, and endpoint redirects fail
|
|
9
|
+
instead of forwarding credentials.
|
|
10
|
+
- A static `authorization` header cannot be combined with `authProvider`.
|
|
11
|
+
|
|
12
|
+
## Private networks
|
|
13
|
+
|
|
14
|
+
For an intentionally local or private-network server, set
|
|
15
|
+
`ssrfProtection: "disabled"` on that transport. This lifts hostname and DNS
|
|
16
|
+
restrictions for the whole transport — redirects and OAuth discovery included —
|
|
17
|
+
while still requiring HTTP(S). Use it only when the application owns and trusts
|
|
18
|
+
that network boundary, and expect `check-mcp.sh` to flag it for review.
|
|
19
|
+
|
|
20
|
+
## Instructions are metadata
|
|
21
|
+
|
|
22
|
+
MCP server instructions remain inspectable metadata; they are not added to Agent
|
|
23
|
+
instructions. If a server's behavior matters, write it into your own
|
|
24
|
+
`instructions` or tool descriptions — never assume the agent has seen the
|
|
25
|
+
server's text.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Heuristic checks for Anvia MCP code in an app directory.
|
|
3
|
+
# Usage: sh scripts/check-mcp.sh [--dir <app-root>]
|
|
4
|
+
# Fails with a list of violations; passes silently with "mcp OK".
|
|
5
|
+
|
|
6
|
+
DIR="."
|
|
7
|
+
if [ "$1" = "--dir" ] && [ -n "$2" ]; then
|
|
8
|
+
DIR="$2"
|
|
9
|
+
fi
|
|
10
|
+
|
|
11
|
+
ROOTS_FOUND=0
|
|
12
|
+
for root in "$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server"; do
|
|
13
|
+
if [ -d "$root" ]; then
|
|
14
|
+
ROOTS_FOUND=1
|
|
15
|
+
break
|
|
16
|
+
fi
|
|
17
|
+
done
|
|
18
|
+
if [ "$ROOTS_FOUND" -eq 0 ]; then
|
|
19
|
+
echo "ERROR: no src/app/lib/server directory under '$DIR' — nothing was checked."
|
|
20
|
+
echo "Run from the app root or pass --dir <app-root>."
|
|
21
|
+
exit 1
|
|
22
|
+
fi
|
|
23
|
+
|
|
24
|
+
fail=0
|
|
25
|
+
violation() {
|
|
26
|
+
echo "VIOLATION: $1"
|
|
27
|
+
fail=1
|
|
28
|
+
}
|
|
29
|
+
warning() {
|
|
30
|
+
echo "WARNING: $1"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
FILES=$(grep -rln --include='*.ts' \
|
|
34
|
+
-e 'new McpClient(' -e 'McpClientGroup' -e 'mcpServers' \
|
|
35
|
+
"$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server" 2>/dev/null | sort -u)
|
|
36
|
+
|
|
37
|
+
if [ -z "$FILES" ]; then
|
|
38
|
+
echo "mcp OK (no MCP code found)"
|
|
39
|
+
exit 0
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
# 1. Every client needs a unique name.
|
|
43
|
+
if echo "$FILES" | xargs grep -l 'new McpClient(' 2>/dev/null | grep -q .; then
|
|
44
|
+
echo "$FILES" | xargs grep -h -A5 'new McpClient(' 2>/dev/null | grep -q 'name:' || {
|
|
45
|
+
violation "new McpClient without name (see references/clients.md)."
|
|
46
|
+
}
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
# 2. Remote transports need an explicit URL.
|
|
50
|
+
if echo "$FILES" | xargs grep -l 'streamableHttp' 2>/dev/null | grep -q .; then
|
|
51
|
+
echo "$FILES" | xargs grep -h -A6 'streamableHttp' 2>/dev/null | grep -q 'url:' || {
|
|
52
|
+
violation "streamableHttp transport without url (see references/clients.md)."
|
|
53
|
+
}
|
|
54
|
+
fi
|
|
55
|
+
|
|
56
|
+
# 3. Disabled SSRF protection must be intentional (warning only).
|
|
57
|
+
if echo "$FILES" | xargs grep -l 'ssrfProtection.*disabled' 2>/dev/null | grep -q .; then
|
|
58
|
+
warning "ssrfProtection disabled — confirm the app owns that network boundary (see references/safety.md)."
|
|
59
|
+
fi
|
|
60
|
+
|
|
61
|
+
# 4. Connected clients should be closed (warning only).
|
|
62
|
+
if echo "$FILES" | xargs grep -l -e '\.connect(' 2>/dev/null | grep -q .; then
|
|
63
|
+
echo "$FILES" | xargs grep -h -e '\.close(' 2>/dev/null | grep -q '.close(' || {
|
|
64
|
+
warning "MCP connect without close — leak risk, use try/finally (see references/clients.md)."
|
|
65
|
+
}
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
if [ "$fail" -eq 0 ]; then
|
|
69
|
+
echo "mcp OK"
|
|
70
|
+
exit 0
|
|
71
|
+
fi
|
|
72
|
+
echo "See skills/anvia-mcp/references/ for fixes."
|
|
73
|
+
exit 1
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: anvia-pipeline
|
|
3
|
+
description: Build deterministic multi-step work with Anvia Pipelines — steps, composition, parallel branches, batch runs, agent and extractor stages.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Anvia Pipeline Skill
|
|
7
|
+
|
|
8
|
+
Use this skill when the work is a fixed sequence of steps, not a model-driven
|
|
9
|
+
loop: transforms, parsing, scoring, fan-out/fan-in, batch runs, or pipelines
|
|
10
|
+
with one bounded agent or extractor stage inside.
|
|
11
|
+
|
|
12
|
+
## Pipeline vs agent
|
|
13
|
+
|
|
14
|
+
- Reach for `Pipeline` when you can name the steps upfront. Reach for `Agent`
|
|
15
|
+
(see the `anvia-agent` skill) when the model must decide what to do next.
|
|
16
|
+
- The common failure is an agent with rising `maxTurns` doing a fixed sequence
|
|
17
|
+
— rewrite that as a pipeline with one `.agent()` stage instead.
|
|
18
|
+
|
|
19
|
+
## Process
|
|
20
|
+
|
|
21
|
+
1. Model the flow as stages (`references/steps-compose.md`) — step, compose,
|
|
22
|
+
parallel, batch.
|
|
23
|
+
2. Add model stages only where judgment is needed (`references/agents-extract.md`).
|
|
24
|
+
3. Run `scripts/check-pipeline.sh` from the app root before claiming done.
|
|
25
|
+
|
|
26
|
+
## Minimal slice
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
import { Pipeline } from "@anvia/core/pipeline";
|
|
30
|
+
import { z } from "zod";
|
|
31
|
+
|
|
32
|
+
const normalizeIncident = new Pipeline({ id: "normalize-incident", inputSchema: z.string() })
|
|
33
|
+
.step({ id: "trim", run: ({ input }) => input.trim() })
|
|
34
|
+
.step({ id: "collapse-whitespace", run: ({ input }) => input.replace(/\s+/g, " ") })
|
|
35
|
+
.step({
|
|
36
|
+
id: "summarize",
|
|
37
|
+
run: ({ input }) => ({
|
|
38
|
+
normalized: input,
|
|
39
|
+
wordCount: input.split(" ").length,
|
|
40
|
+
priority: input.toLowerCase().includes("outage") ? "high" : "normal",
|
|
41
|
+
}),
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
const result = await normalizeIncident.run({ input: " Checkout outage reported. " });
|
|
45
|
+
console.log(result.output);
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Output
|
|
49
|
+
|
|
50
|
+
Keep stages small, named, and typed — each stage's output is the next stage's
|
|
51
|
+
input. Point to the relevant reference file instead of pasting its contents
|
|
52
|
+
into chat.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Agent and Extractor Stages
|
|
2
|
+
|
|
3
|
+
## Agent stages
|
|
4
|
+
|
|
5
|
+
Embed one bounded agent where a step needs judgment. The stage maps pipeline
|
|
6
|
+
input to an agent request explicitly, and `suspension` is required:
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
const executiveUpdate = new Pipeline({ id: "executive-update", inputSchema: z.array(z.string()) })
|
|
10
|
+
.step({
|
|
11
|
+
id: "format-notes",
|
|
12
|
+
run: ({ input }) => input.map((note) => `- ${note}`).join("\n"),
|
|
13
|
+
})
|
|
14
|
+
.agent({
|
|
15
|
+
id: "analyze",
|
|
16
|
+
agent: analyst,
|
|
17
|
+
suspension: "reject",
|
|
18
|
+
request: ({ input }) => ({
|
|
19
|
+
prompt: `Prepare an executive update from these notes:\n\n${input}`,
|
|
20
|
+
}),
|
|
21
|
+
});
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
- Keep the agent's instructions narrow (one task, facts-only, visible final
|
|
25
|
+
text) — the pipeline already handles orchestration, so the agent must not
|
|
26
|
+
coordinate.
|
|
27
|
+
- Prefer approval-free agents inside pipelines. A tool approval inside an agent
|
|
28
|
+
stage suspends the run (`PipelineAgentSuspensionError`); if you need
|
|
29
|
+
approvals, surface them at the pipeline boundary instead of burying them
|
|
30
|
+
mid-flow.
|
|
31
|
+
|
|
32
|
+
## Extractor stages
|
|
33
|
+
|
|
34
|
+
Extractor stages turn unstructured text into schema-validated objects. Reach
|
|
35
|
+
for them when a step's job is "pull fields out of this text" rather than
|
|
36
|
+
open-ended judgment; the schema is the contract downstream steps program
|
|
37
|
+
against:
|
|
38
|
+
|
|
39
|
+
```ts
|
|
40
|
+
const ticketed = pipeline.extract({
|
|
41
|
+
id: "extract-ticket",
|
|
42
|
+
text: ({ input }) => input.body,
|
|
43
|
+
model,
|
|
44
|
+
outputSchema: z.object({ title: z.string(), priority: z.enum(["low", "high"]) }),
|
|
45
|
+
instructions: "Extract the support ticket fields.",
|
|
46
|
+
});
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`text` maps the previous stage's output to the source text; `model`,
|
|
50
|
+
`instructions`, `retries`, `temperature`, `maxTokens`, and `providerOptions`
|
|
51
|
+
behave like the single extraction call. The cookbook's extractor-pipeline
|
|
52
|
+
example shows a full flow.
|
|
53
|
+
|
|
54
|
+
## Observing runs
|
|
55
|
+
|
|
56
|
+
`run` / `runBatch` accept an `observer` (`stage_started` / `stage_completed` /
|
|
57
|
+
`stage_failed`), plus `abortSignal`, `metadata`, `runId`, and
|
|
58
|
+
`failOnObserverError`; `graph()` prints the stage graph. Pipelines also carry
|
|
59
|
+
`observability` options from construction, and Studio replays run history stage
|
|
60
|
+
by stage (see the `anvia-studio` skill). When a pipeline misbehaves, replay the
|
|
61
|
+
failing run in Studio before changing stages — most pipeline bugs are wrong
|
|
62
|
+
intermediate outputs, visible at the stage boundary.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Steps and Composition
|
|
2
|
+
|
|
3
|
+
Pipelines come from `@anvia/core/pipeline`. The constructor takes an `id` and a
|
|
4
|
+
zod `inputSchema` — inputs are parsed at the boundary, and a non-zod schema
|
|
5
|
+
throws. Every stage takes an `id`:
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
const ticketSummary = new Pipeline({ id: "ticket-summary", inputSchema: z.string() })
|
|
9
|
+
.compose({ id: "parse", pipeline: parseTicket })
|
|
10
|
+
.compose({ id: "score", pipeline: scoreTicket });
|
|
11
|
+
|
|
12
|
+
const { output: summary } = await ticketSummary.run({ input: rawTicket });
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Stage kinds
|
|
16
|
+
|
|
17
|
+
- `.step({ id, run })` — a pure function of `{ input }`. `run` may be async.
|
|
18
|
+
Prefer many small named steps over one big closure.
|
|
19
|
+
- `.compose({ id, pipeline })` — nest a whole pipeline as a stage. Build
|
|
20
|
+
parse/score/format sub-pipelines independently, then compose them.
|
|
21
|
+
- `.parallel({ id, branches })` — fan out over named sub-pipelines, then merge:
|
|
22
|
+
|
|
23
|
+
```ts
|
|
24
|
+
const triage = new Pipeline({ id: "triage", inputSchema: z.string() })
|
|
25
|
+
.parallel({
|
|
26
|
+
id: "signals",
|
|
27
|
+
branches: { classification: classifyText, signals: extractSignals, priority: estimatePriority },
|
|
28
|
+
})
|
|
29
|
+
.step({
|
|
30
|
+
id: "merge",
|
|
31
|
+
run: ({ input: { classification, signals, priority } }) => ({
|
|
32
|
+
...classification,
|
|
33
|
+
...signals,
|
|
34
|
+
...priority,
|
|
35
|
+
}),
|
|
36
|
+
});
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Branch outputs arrive keyed by branch name — destructure them in the merge step.
|
|
40
|
+
Keep branches independent; a branch that needs another branch's output belongs
|
|
41
|
+
downstream, not in the same fan-out.
|
|
42
|
+
|
|
43
|
+
## Batch runs
|
|
44
|
+
|
|
45
|
+
```ts
|
|
46
|
+
const batch = await normalizeIncident.runBatch({
|
|
47
|
+
inputs: ["Payment latency for EU customers.", "Search outage for the admin dashboard."],
|
|
48
|
+
concurrency: 2,
|
|
49
|
+
});
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`runBatch` maps the same pipeline over inputs with bounded concurrency. Use it
|
|
53
|
+
instead of hand-rolled `Promise.all` loops — backpressure and error
|
|
54
|
+
aggregation come free.
|
|
55
|
+
|
|
56
|
+
## Rules
|
|
57
|
+
|
|
58
|
+
- Every pipeline and stage needs an `id`. Ids show up in traces, logs, and
|
|
59
|
+
Studio replay — `step-1`, `step-2` waste that surface.
|
|
60
|
+
- Type the seams: `inputSchema` on the pipeline, object outputs from steps. An
|
|
61
|
+
untyped middle step turns every downstream stage into guesswork.
|
|
62
|
+
- Deterministic stages first, model stages last and few (see
|
|
63
|
+
`references/agents-extract.md`).
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Heuristic checks for Anvia Pipeline code in an app directory.
|
|
3
|
+
# Usage: sh scripts/check-pipeline.sh [--dir <app-root>]
|
|
4
|
+
# Fails with a list of violations; passes silently with "pipeline OK".
|
|
5
|
+
|
|
6
|
+
DIR="."
|
|
7
|
+
if [ "$1" = "--dir" ] && [ -n "$2" ]; then
|
|
8
|
+
DIR="$2"
|
|
9
|
+
fi
|
|
10
|
+
|
|
11
|
+
ROOTS_FOUND=0
|
|
12
|
+
for root in "$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server"; do
|
|
13
|
+
if [ -d "$root" ]; then
|
|
14
|
+
ROOTS_FOUND=1
|
|
15
|
+
break
|
|
16
|
+
fi
|
|
17
|
+
done
|
|
18
|
+
if [ "$ROOTS_FOUND" -eq 0 ]; then
|
|
19
|
+
echo "ERROR: no src/app/lib/server directory under '$DIR' — nothing was checked."
|
|
20
|
+
echo "Run from the app root or pass --dir <app-root>."
|
|
21
|
+
exit 1
|
|
22
|
+
fi
|
|
23
|
+
|
|
24
|
+
fail=0
|
|
25
|
+
violation() {
|
|
26
|
+
echo "VIOLATION: $1"
|
|
27
|
+
fail=1
|
|
28
|
+
}
|
|
29
|
+
warning() {
|
|
30
|
+
echo "WARNING: $1"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
FILES=$(grep -rln --include='*.ts' \
|
|
34
|
+
-e 'new Pipeline(' \
|
|
35
|
+
"$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server" 2>/dev/null | sort -u)
|
|
36
|
+
|
|
37
|
+
if [ -z "$FILES" ]; then
|
|
38
|
+
echo "pipeline OK (no Pipeline code found)"
|
|
39
|
+
exit 0
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
# 1. Pipelines need an id and a zod inputSchema (constructor throws without a schema).
|
|
43
|
+
if echo "$FILES" | xargs grep -h -A4 'new Pipeline(' 2>/dev/null | grep -q 'new Pipeline('; then
|
|
44
|
+
echo "$FILES" | xargs grep -h -A4 'new Pipeline(' 2>/dev/null | grep -q 'id:' || {
|
|
45
|
+
violation "new Pipeline without id (see references/steps-compose.md)."
|
|
46
|
+
}
|
|
47
|
+
echo "$FILES" | xargs grep -h -A4 'new Pipeline(' 2>/dev/null | grep -q 'inputSchema' || {
|
|
48
|
+
violation "new Pipeline without inputSchema (see references/steps-compose.md)."
|
|
49
|
+
}
|
|
50
|
+
fi
|
|
51
|
+
|
|
52
|
+
# 2. Agent stages require an explicit suspension policy.
|
|
53
|
+
if echo "$FILES" | xargs grep -l '\.agent(' 2>/dev/null | grep -q .; then
|
|
54
|
+
echo "$FILES" | xargs grep -h -A6 '\.agent(' 2>/dev/null | grep -q 'suspension' || {
|
|
55
|
+
violation "pipeline .agent stage without suspension (see references/agents-extract.md)."
|
|
56
|
+
}
|
|
57
|
+
fi
|
|
58
|
+
|
|
59
|
+
# 3. A pipeline with no stages runs nothing (warning only).
|
|
60
|
+
if echo "$FILES" | xargs grep -L -e '\.step(' -e '\.compose(' -e '\.agent(' -e '\.parallel(' -e '\.extract(' 2>/dev/null | grep -q .; then
|
|
61
|
+
warning "Pipeline file defines no stages — check for an unfinished pipeline (see references/steps-compose.md)."
|
|
62
|
+
fi
|
|
63
|
+
|
|
64
|
+
if [ "$fail" -eq 0 ]; then
|
|
65
|
+
echo "pipeline OK"
|
|
66
|
+
exit 0
|
|
67
|
+
fi
|
|
68
|
+
echo "See skills/anvia-pipeline/references/ for fixes."
|
|
69
|
+
exit 1
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: anvia-rag
|
|
3
|
+
description: Build retrieval and RAG with Anvia — chunking, embeddings, vector stores, knowledge graphs, filters, and search tools.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Anvia RAG Skill
|
|
7
|
+
|
|
8
|
+
Use this skill when the user wants retrieval over their own data: chunking
|
|
9
|
+
documents, embedding them, picking a vector store, filtering results, or
|
|
10
|
+
exposing retrieval to an agent as a search tool.
|
|
11
|
+
|
|
12
|
+
## Process
|
|
13
|
+
|
|
14
|
+
1. Chunk and embed the corpus (`references/pipeline.md`).
|
|
15
|
+
2. Pick the store (`references/stores.md`) — `InMemoryVectorStore` first, a
|
|
16
|
+
client store when data must persist. For connection questions, add a graph
|
|
17
|
+
(`references/graph-rag.md`).
|
|
18
|
+
3. Expose retrieval to the agent (`references/rag-tool.md`, `references/graph-rag.md`) — search tool or
|
|
19
|
+
`VectorContext`, not raw vectors in the prompt.
|
|
20
|
+
4. Run `scripts/check-rag.sh` from the app root before claiming done.
|
|
21
|
+
|
|
22
|
+
## Minimal slice
|
|
23
|
+
|
|
24
|
+
```ts
|
|
25
|
+
import { embedDocuments } from "@anvia/core/embeddings";
|
|
26
|
+
import { InMemoryVectorStore, retrieveDocuments } from "@anvia/core/vector-store";
|
|
27
|
+
import { loadTransformersEmbeddingModel } from "@anvia/transformers";
|
|
28
|
+
|
|
29
|
+
const embeddingModel = await loadTransformersEmbeddingModel({ modelId: "Xenova/all-MiniLM-L6-v2" });
|
|
30
|
+
const { documents: embedded } = await embedDocuments({
|
|
31
|
+
model: embeddingModel,
|
|
32
|
+
documents: notes,
|
|
33
|
+
id: (note) => note.id,
|
|
34
|
+
content: (note) => `${note.title}\n${note.body}`,
|
|
35
|
+
metadata: (note) => ({ topic: note.topic }),
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const store = InMemoryVectorStore.fromDocuments({ documents: embedded });
|
|
39
|
+
const results = await retrieveDocuments({
|
|
40
|
+
store,
|
|
41
|
+
model: embeddingModel,
|
|
42
|
+
query: "market risk",
|
|
43
|
+
topK: 1,
|
|
44
|
+
});
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Output
|
|
48
|
+
|
|
49
|
+
Keep the pipeline explicit: load → chunk → embed → upsert → retrieve. Point to
|
|
50
|
+
the relevant reference file instead of pasting its contents into chat.
|