@llmtune/cli 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tools/permissions.js +3 -3
- package/package.json +8 -2
- package/docs/SKILL_AUTHORING.md +0 -175
- package/llmtune-session-1780260929719.json +0 -6
- package/scripts/qa-full.js +0 -440
- package/scripts/smoke-test.js +0 -142
|
@@ -4,7 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.PermissionManager = void 0;
|
|
7
|
-
const prompts_1 =
|
|
7
|
+
const prompts_1 = require("@inquirer/prompts");
|
|
8
8
|
const chalk_1 = __importDefault(require("chalk"));
|
|
9
9
|
class PermissionManager {
|
|
10
10
|
config;
|
|
@@ -35,14 +35,14 @@ class PermissionManager {
|
|
|
35
35
|
: JSON.stringify(input).slice(0, 100);
|
|
36
36
|
console.log(chalk_1.default.yellow(`\n⚠ ${toolName} wants to execute:`));
|
|
37
37
|
console.log(chalk_1.default.dim(preview));
|
|
38
|
-
const confirmed = await prompts_1.
|
|
38
|
+
const confirmed = await (0, prompts_1.confirm)({
|
|
39
39
|
message: `Allow ${toolName}? (y/N)`,
|
|
40
40
|
default: false,
|
|
41
41
|
});
|
|
42
42
|
if (!confirmed) {
|
|
43
43
|
return { behavior: "deny", message: "User denied" };
|
|
44
44
|
}
|
|
45
|
-
const alwaysTrust = await prompts_1.
|
|
45
|
+
const alwaysTrust = await (0, prompts_1.confirm)({
|
|
46
46
|
message: `Trust ${toolName} for this session? (y/N)`,
|
|
47
47
|
default: false,
|
|
48
48
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@llmtune/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "LLMTune CLI -AI CLI Agent powered by llmtune.io",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -13,7 +13,13 @@
|
|
|
13
13
|
"lint": "tsc --noEmit",
|
|
14
14
|
"test": "npm run build && node scripts/smoke-test.js"
|
|
15
15
|
},
|
|
16
|
-
"keywords": [
|
|
16
|
+
"keywords": [
|
|
17
|
+
"llmtune",
|
|
18
|
+
"cli",
|
|
19
|
+
"ai",
|
|
20
|
+
"agent",
|
|
21
|
+
"coding"
|
|
22
|
+
],
|
|
17
23
|
"license": "MIT",
|
|
18
24
|
"engines": {
|
|
19
25
|
"node": ">=18.0.0"
|
package/docs/SKILL_AUTHORING.md
DELETED
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
# Skill Authoring Guide
|
|
2
|
-
|
|
3
|
-
Skills are reusable AI workflows that extend the LLMTune CLI. Each skill is defined by a `SKILL.md` file inside a named directory.
|
|
4
|
-
|
|
5
|
-
## Quick Start
|
|
6
|
-
|
|
7
|
-
Create a skill in `~/.llmtune/skills/` or `.llmtune/skills/` in your project:
|
|
8
|
-
|
|
9
|
-
```
|
|
10
|
-
~/.llmtune/skills/
|
|
11
|
-
my-skill/
|
|
12
|
-
SKILL.md
|
|
13
|
-
```
|
|
14
|
-
|
|
15
|
-
## SKILL.md Format
|
|
16
|
-
|
|
17
|
-
A skill file has two parts: **YAML frontmatter** (optional) and **Markdown body** (the prompt template).
|
|
18
|
-
|
|
19
|
-
```markdown
|
|
20
|
-
---
|
|
21
|
-
description: "Fix ESLint errors in a file"
|
|
22
|
-
user-invocable: true
|
|
23
|
-
allowed-tools: [read, edit, glob, grep]
|
|
24
|
-
arguments: file_path
|
|
25
|
-
trust: local
|
|
26
|
-
---
|
|
27
|
-
|
|
28
|
-
You are a linting expert. Fix all ESLint errors in the file at `{{file_path}}`.
|
|
29
|
-
|
|
30
|
-
1. Read the file
|
|
31
|
-
2. Identify lint errors
|
|
32
|
-
3. Fix each error while preserving the original intent
|
|
33
|
-
4. Report what you changed
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
## Frontmatter Fields
|
|
37
|
-
|
|
38
|
-
| Field | Type | Default | Description |
|
|
39
|
-
|-------|------|---------|-------------|
|
|
40
|
-
| `description` | string | (required) | Short description shown in `/skills` list |
|
|
41
|
-
| `user-invocable` | boolean | `true` | Whether users can invoke with `/<skill-name>` |
|
|
42
|
-
| `allowed-tools` | string[] | all tools | Restrict which tools the skill can use |
|
|
43
|
-
| `arguments` | string or string[] | none | Argument names for substitution |
|
|
44
|
-
| `trust` | string | `local` | Trust level: `local`, `community`, `verified`, `signed` |
|
|
45
|
-
| `when-to-use` | string | none | Hint for when the agent should auto-invoke this skill |
|
|
46
|
-
|
|
47
|
-
## Argument Substitution
|
|
48
|
-
|
|
49
|
-
Arguments are referenced in the body using `{{arg_name}}` syntax:
|
|
50
|
-
|
|
51
|
-
```markdown
|
|
52
|
-
---
|
|
53
|
-
arguments: file_path, style
|
|
54
|
-
---
|
|
55
|
-
|
|
56
|
-
Refactor the file at `{{file_path}}` to use `{{style}}` coding style.
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
When invoked: `/my-skill src/auth.ts functional`
|
|
60
|
-
|
|
61
|
-
- `{{file_path}}` becomes `src/auth.ts`
|
|
62
|
-
- `{{style}}` becomes `functional`
|
|
63
|
-
|
|
64
|
-
## Trust Levels
|
|
65
|
-
|
|
66
|
-
Skills have four trust levels that determine what tools they can access:
|
|
67
|
-
|
|
68
|
-
| Level | Tools Allowed | Use Case |
|
|
69
|
-
|-------|--------------|----------|
|
|
70
|
-
| **local** | All | Skills you create yourself in `~/.llmtune/skills/` |
|
|
71
|
-
| **community** | read, glob, grep only | Skills installed from marketplace |
|
|
72
|
-
| **verified** | All | Skills reviewed by the LLMTune team |
|
|
73
|
-
| **signed** | All | Cryptographically signed skills with verified authors |
|
|
74
|
-
|
|
75
|
-
## Invoking Skills
|
|
76
|
-
|
|
77
|
-
### From the CLI REPL
|
|
78
|
-
|
|
79
|
-
```
|
|
80
|
-
> /explain-code src/auth.ts
|
|
81
|
-
> /fix-lint src/utils/helpers.ts
|
|
82
|
-
> /generate-test src/services/user.ts
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
### From Commander (non-interactive)
|
|
86
|
-
|
|
87
|
-
```bash
|
|
88
|
-
llmtune skills run explain-code --args "src/auth.ts"
|
|
89
|
-
```
|
|
90
|
-
|
|
91
|
-
## Built-in Skills
|
|
92
|
-
|
|
93
|
-
The LLMTune CLI includes these built-in skills:
|
|
94
|
-
|
|
95
|
-
| Skill | Description | Arguments |
|
|
96
|
-
|-------|-------------|-----------|
|
|
97
|
-
| `explain-code` | Explain code with a clear breakdown | `file_path` |
|
|
98
|
-
| `fix-lint` | Auto-fix linting errors | `file_path` |
|
|
99
|
-
| `generate-test` | Generate unit tests for a file | `file_path` |
|
|
100
|
-
| `security-review` | Review code for security issues | `file_path` |
|
|
101
|
-
|
|
102
|
-
## Publishing to the Marketplace
|
|
103
|
-
|
|
104
|
-
```bash
|
|
105
|
-
# Sign your skill
|
|
106
|
-
llmtune skills sign my-skill
|
|
107
|
-
|
|
108
|
-
# Publish to marketplace
|
|
109
|
-
llmtune skills publish my-skill
|
|
110
|
-
|
|
111
|
-
# Install from marketplace
|
|
112
|
-
llmtune skills install security-review
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
## Examples
|
|
116
|
-
|
|
117
|
-
### Code Review Skill
|
|
118
|
-
|
|
119
|
-
```markdown
|
|
120
|
-
---
|
|
121
|
-
description: "Review code for best practices and potential issues"
|
|
122
|
-
allowed-tools: [read, glob, grep]
|
|
123
|
-
arguments: file_path
|
|
124
|
-
---
|
|
125
|
-
|
|
126
|
-
Review the code at `{{file_path}}` for:
|
|
127
|
-
|
|
128
|
-
1. **Security vulnerabilities** - XSS, injection, auth issues
|
|
129
|
-
2. **Performance problems** - N+1 queries, unnecessary re-renders, memory leaks
|
|
130
|
-
3. **Code quality** - Naming, structure, DRY violations
|
|
131
|
-
4. **Error handling** - Missing error cases, swallowed errors
|
|
132
|
-
5. **Type safety** - Any `any` types, missing null checks
|
|
133
|
-
|
|
134
|
-
Provide specific, actionable feedback with line numbers.
|
|
135
|
-
```
|
|
136
|
-
|
|
137
|
-
### Database Migration Skill
|
|
138
|
-
|
|
139
|
-
```markdown
|
|
140
|
-
---
|
|
141
|
-
description: "Generate a Prisma migration for schema changes"
|
|
142
|
-
allowed-tools: [read, write, edit, bash]
|
|
143
|
-
arguments: description
|
|
144
|
-
---
|
|
145
|
-
|
|
146
|
-
The user wants to make this database change: {{description}}
|
|
147
|
-
|
|
148
|
-
1. Read the current prisma/schema.prisma
|
|
149
|
-
2. Make the necessary schema changes
|
|
150
|
-
3. Generate the migration using `npx prisma migrate dev --name <descriptive-name>`
|
|
151
|
-
4. Report what changed
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
### API Endpoint Skill
|
|
155
|
-
|
|
156
|
-
```markdown
|
|
157
|
-
---
|
|
158
|
-
description: "Scaffold a new REST API endpoint"
|
|
159
|
-
allowed-tools: [read, write, edit, glob, grep]
|
|
160
|
-
arguments: method, path
|
|
161
|
-
---
|
|
162
|
-
|
|
163
|
-
Create a new API endpoint:
|
|
164
|
-
|
|
165
|
-
- Method: `{{method}}`
|
|
166
|
-
- Path: `{{path}}`
|
|
167
|
-
|
|
168
|
-
1. Find existing route files to understand the pattern
|
|
169
|
-
2. Create the route handler
|
|
170
|
-
3. Add input validation
|
|
171
|
-
4. Add error handling
|
|
172
|
-
5. Register the route
|
|
173
|
-
|
|
174
|
-
Follow the existing code patterns in the project.
|
|
175
|
-
```
|
package/scripts/qa-full.js
DELETED
|
@@ -1,440 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* Full QA checklist runner for @llmtune/cli
|
|
4
|
-
* Usage: node scripts/qa-full.js
|
|
5
|
-
*/
|
|
6
|
-
const { execSync, spawnSync, spawn } = require("child_process")
|
|
7
|
-
const fs = require("fs")
|
|
8
|
-
const path = require("path")
|
|
9
|
-
const os = require("os")
|
|
10
|
-
const readline = require("readline")
|
|
11
|
-
|
|
12
|
-
const CLI_DIR = path.join(__dirname, "..")
|
|
13
|
-
const QA_DIR = path.join(CLI_DIR, ".qa-tmp")
|
|
14
|
-
const QA_FILE = path.join(QA_DIR, "qa-test.txt")
|
|
15
|
-
|
|
16
|
-
const results = []
|
|
17
|
-
|
|
18
|
-
function record(id, name, pass, detail = "") {
|
|
19
|
-
results.push({ id, name, pass, detail })
|
|
20
|
-
const icon = pass ? "PASS" : "FAIL"
|
|
21
|
-
console.log(`[${icon}] ${id}: ${name}${detail ? " — " + detail : ""}`)
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function run(cmd, opts = {}) {
|
|
25
|
-
return execSync(cmd, {
|
|
26
|
-
encoding: "utf-8",
|
|
27
|
-
cwd: opts.cwd || CLI_DIR,
|
|
28
|
-
timeout: opts.timeout || 60000,
|
|
29
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
30
|
-
...opts,
|
|
31
|
-
})
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function loadApiKey() {
|
|
35
|
-
const configPath = path.join(os.homedir(), ".llmtune", "config.json")
|
|
36
|
-
if (!fs.existsSync(configPath)) return null
|
|
37
|
-
const cfg = JSON.parse(fs.readFileSync(configPath, "utf-8"))
|
|
38
|
-
return cfg.apiKey || null
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
async function replSession(inputs, opts = {}) {
|
|
42
|
-
const cwd = opts.cwd || CLI_DIR
|
|
43
|
-
const input = inputs.join("\n") + "\n"
|
|
44
|
-
return new Promise((resolve, reject) => {
|
|
45
|
-
const child = spawn("llmtune", ["chat", ...(opts.model ? ["-m", opts.model] : [])], {
|
|
46
|
-
cwd,
|
|
47
|
-
shell: true,
|
|
48
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
49
|
-
env: { ...process.env, ...(opts.noAuth ? { LLMTUNE_QA_NO_AUTH: "1" } : {}) },
|
|
50
|
-
})
|
|
51
|
-
let stdout = ""
|
|
52
|
-
let stderr = ""
|
|
53
|
-
child.stdout.on("data", (d) => { stdout += d.toString() })
|
|
54
|
-
child.stderr.on("data", (d) => { stderr += d.toString() })
|
|
55
|
-
child.on("close", (code) => resolve({ code, stdout, stderr }))
|
|
56
|
-
child.on("error", reject)
|
|
57
|
-
child.stdin.write(input)
|
|
58
|
-
child.stdin.end()
|
|
59
|
-
})
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
async function agentPrompt(prompt, opts = {}) {
|
|
63
|
-
const { createClient } = require("../dist/auth/client")
|
|
64
|
-
const { Conversation } = require("../dist/agent/conversation")
|
|
65
|
-
const { ToolRegistry } = require("../dist/tools/registry")
|
|
66
|
-
const { runAgentLoop } = require("../dist/agent/loop")
|
|
67
|
-
const { readTool } = require("../dist/tools/tools/read")
|
|
68
|
-
const { writeTool } = require("../dist/tools/tools/write")
|
|
69
|
-
const { editTool } = require("../dist/tools/tools/edit")
|
|
70
|
-
const { bashTool } = require("../dist/tools/tools/bash")
|
|
71
|
-
const { globTool } = require("../dist/tools/tools/glob")
|
|
72
|
-
const { grepTool } = require("../dist/tools/tools/grep")
|
|
73
|
-
const { webFetchTool } = require("../dist/tools/tools/web-fetch")
|
|
74
|
-
|
|
75
|
-
const allTools = [readTool, writeTool, editTool, bashTool, globTool, grepTool, webFetchTool]
|
|
76
|
-
const registry = new ToolRegistry()
|
|
77
|
-
for (const t of opts.tools || allTools) registry.register(t)
|
|
78
|
-
|
|
79
|
-
const client = createClient()
|
|
80
|
-
const conversation = new Conversation(opts.model || "z-ai/GLM-5.1")
|
|
81
|
-
const cwd = opts.cwd || CLI_DIR
|
|
82
|
-
|
|
83
|
-
const result = await runAgentLoop(client, conversation, registry, prompt, {
|
|
84
|
-
model: opts.model || "z-ai/GLM-5.1",
|
|
85
|
-
maxTurns: opts.maxTurns || 5,
|
|
86
|
-
stream: false,
|
|
87
|
-
cwd,
|
|
88
|
-
workspaceRoot: cwd,
|
|
89
|
-
permissions: opts.permissions,
|
|
90
|
-
})
|
|
91
|
-
|
|
92
|
-
return { result, conversation, stdout: result.finalText }
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
async function main() {
|
|
96
|
-
console.log("=" .repeat(60))
|
|
97
|
-
console.log("LLMTune CLI — Full QA Checklist")
|
|
98
|
-
console.log("Package: @llmtune/cli from npm")
|
|
99
|
-
console.log("=" .repeat(60))
|
|
100
|
-
console.log("")
|
|
101
|
-
|
|
102
|
-
fs.mkdirSync(QA_DIR, { recursive: true })
|
|
103
|
-
|
|
104
|
-
// ─── A1: Entry commands ───
|
|
105
|
-
console.log("\n--- A1: CLI Entry Commands ---")
|
|
106
|
-
try {
|
|
107
|
-
const ver = run("llmtune --version").trim()
|
|
108
|
-
record("qa-1a", "Version", ver === "0.1.5", `got ${ver}`)
|
|
109
|
-
} catch (e) {
|
|
110
|
-
record("qa-1a", "Version", false, e.message)
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
try {
|
|
114
|
-
const cfg = run("llmtune config")
|
|
115
|
-
const ok = cfg.includes("api.llmtune.io") && cfg.includes("GLM")
|
|
116
|
-
record("qa-1b", "Config", ok)
|
|
117
|
-
} catch (e) {
|
|
118
|
-
record("qa-1b", "Config", false, e.message)
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
try {
|
|
122
|
-
const models = run("llmtune models", { timeout: 120000 })
|
|
123
|
-
record("qa-1c", "Models list", models.length > 50, `${models.split("\n").length} lines`)
|
|
124
|
-
} catch (e) {
|
|
125
|
-
record("qa-1c", "Models list", false, e.message)
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
// ─── Build dist for programmatic tests ───
|
|
129
|
-
try {
|
|
130
|
-
run("npm run build", { cwd: CLI_DIR })
|
|
131
|
-
} catch (e) {
|
|
132
|
-
console.warn("Build warning:", e.message)
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// ─── A6: Context / git (no path error) ───
|
|
136
|
-
console.log("\n--- A6: Context Builder ---")
|
|
137
|
-
try {
|
|
138
|
-
const { collectGitContext } = require("../dist/context/git-context")
|
|
139
|
-
collectGitContext(CLI_DIR)
|
|
140
|
-
record("qa-6a", "Git context (no path error)", true)
|
|
141
|
-
} catch (e) {
|
|
142
|
-
record("qa-6a", "Git context", false, e.message)
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
try {
|
|
146
|
-
const { buildContextPrompt } = require("../dist/context/builder")
|
|
147
|
-
const ctx = await buildContextPrompt(CLI_DIR, CLI_DIR, { model: "z-ai/GLM-5.1", useCache: false })
|
|
148
|
-
const ok = ctx.prompt.includes("LLMTune Agent") && ctx.sections.length > 0
|
|
149
|
-
record("qa-6b", "Context prompt has identity + sections", ok, `${ctx.sections.length} sections`)
|
|
150
|
-
} catch (e) {
|
|
151
|
-
record("qa-6b", "Context prompt", false, e.message)
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
// ─── A3: Tools ───
|
|
155
|
-
console.log("\n--- A3: Tool Execution ---")
|
|
156
|
-
|
|
157
|
-
await (async () => {
|
|
158
|
-
try {
|
|
159
|
-
const { result } = await agentPrompt(
|
|
160
|
-
"Use read tool ONLY. Read package.json in current directory and reply with ONLY the version field value, nothing else.",
|
|
161
|
-
{ tools: [require("../dist/tools/tools/read").readTool], maxTurns: 3 },
|
|
162
|
-
)
|
|
163
|
-
const ok = result.totalToolCalls >= 1 && (result.finalText.includes("0.1.5") || result.finalText.includes("1."))
|
|
164
|
-
record("qa-3-read", "read tool", ok, `${result.totalToolCalls} tool calls`)
|
|
165
|
-
} catch (e) {
|
|
166
|
-
record("qa-3-read", "read tool", false, e.message)
|
|
167
|
-
}
|
|
168
|
-
})()
|
|
169
|
-
|
|
170
|
-
await (async () => {
|
|
171
|
-
try {
|
|
172
|
-
const writePath = QA_FILE.replace(/\\/g, "/")
|
|
173
|
-
const { result } = await agentPrompt(
|
|
174
|
-
`Use write tool ONLY. Write file at path "${writePath}" with exact content: QA_WRITE_OK`,
|
|
175
|
-
{ tools: [require("../dist/tools/tools/write").writeTool], maxTurns: 3 },
|
|
176
|
-
)
|
|
177
|
-
const content = fs.existsSync(QA_FILE) ? fs.readFileSync(QA_FILE, "utf-8") : ""
|
|
178
|
-
const ok = result.totalToolCalls >= 1 && content.includes("QA_WRITE_OK")
|
|
179
|
-
record("qa-3-write", "write tool", ok, content.slice(0, 40))
|
|
180
|
-
} catch (e) {
|
|
181
|
-
record("qa-3-write", "write tool", false, e.message)
|
|
182
|
-
}
|
|
183
|
-
})()
|
|
184
|
-
|
|
185
|
-
await (async () => {
|
|
186
|
-
try {
|
|
187
|
-
fs.writeFileSync(QA_FILE, "QA_WRITE_OK", "utf-8")
|
|
188
|
-
const { result } = await agentPrompt(
|
|
189
|
-
`Use edit tool ONLY. In file "${QA_FILE.replace(/\\/g, "/")}" replace QA_WRITE_OK with QA_EDIT_OK`,
|
|
190
|
-
{ tools: [require("../dist/tools/tools/edit").editTool], maxTurns: 3 },
|
|
191
|
-
)
|
|
192
|
-
const content = fs.readFileSync(QA_FILE, "utf-8")
|
|
193
|
-
const ok = result.totalToolCalls >= 1 && content.includes("QA_EDIT_OK")
|
|
194
|
-
record("qa-3-edit", "edit tool", ok, content)
|
|
195
|
-
} catch (e) {
|
|
196
|
-
record("qa-3-edit", "edit tool", false, e.message)
|
|
197
|
-
}
|
|
198
|
-
})()
|
|
199
|
-
|
|
200
|
-
await (async () => {
|
|
201
|
-
try {
|
|
202
|
-
const { result } = await agentPrompt(
|
|
203
|
-
"Use bash tool ONLY. Run command: echo QA_BASH_OK — reply with the output only.",
|
|
204
|
-
{ tools: [require("../dist/tools/tools/bash").bashTool], maxTurns: 3 },
|
|
205
|
-
)
|
|
206
|
-
const ok = result.totalToolCalls >= 1 && (result.finalText.includes("QA_BASH_OK") || result.totalToolCalls > 0)
|
|
207
|
-
record("qa-3-bash", "bash tool", ok, `${result.totalToolCalls} calls`)
|
|
208
|
-
} catch (e) {
|
|
209
|
-
record("qa-3-bash", "bash tool", false, e.message)
|
|
210
|
-
}
|
|
211
|
-
})()
|
|
212
|
-
|
|
213
|
-
await (async () => {
|
|
214
|
-
try {
|
|
215
|
-
const { result } = await agentPrompt(
|
|
216
|
-
"Use bash tool ONLY. Run: echo %USERNAME% (Windows cmd). Show output.",
|
|
217
|
-
{ tools: [require("../dist/tools/tools/bash").bashTool], maxTurns: 3 },
|
|
218
|
-
)
|
|
219
|
-
record("qa-9-win", "Windows bash (echo %USERNAME%)", result.totalToolCalls >= 1, result.finalText.slice(0, 60))
|
|
220
|
-
} catch (e) {
|
|
221
|
-
record("qa-9-win", "Windows bash", false, e.message)
|
|
222
|
-
}
|
|
223
|
-
})()
|
|
224
|
-
|
|
225
|
-
await (async () => {
|
|
226
|
-
try {
|
|
227
|
-
const { result } = await agentPrompt(
|
|
228
|
-
"Use glob tool ONLY. Find *.ts files in src/ directory. List count in one sentence.",
|
|
229
|
-
{ tools: [require("../dist/tools/tools/glob").globTool], maxTurns: 3 },
|
|
230
|
-
)
|
|
231
|
-
record("qa-3-glob", "glob tool", result.totalToolCalls >= 1, `${result.totalToolCalls} calls`)
|
|
232
|
-
} catch (e) {
|
|
233
|
-
record("qa-3-glob", "glob tool", false, e.message)
|
|
234
|
-
}
|
|
235
|
-
})()
|
|
236
|
-
|
|
237
|
-
await (async () => {
|
|
238
|
-
try {
|
|
239
|
-
const { result } = await agentPrompt(
|
|
240
|
-
"Use grep tool ONLY. Search for 'dispatchAsync' in src/ folder. Say if found.",
|
|
241
|
-
{ tools: [require("../dist/tools/tools/grep").grepTool], maxTurns: 3 },
|
|
242
|
-
)
|
|
243
|
-
record("qa-3-grep", "grep tool", result.totalToolCalls >= 1, `${result.totalToolCalls} calls`)
|
|
244
|
-
} catch (e) {
|
|
245
|
-
record("qa-3-grep", "grep tool", false, e.message)
|
|
246
|
-
}
|
|
247
|
-
})()
|
|
248
|
-
|
|
249
|
-
await (async () => {
|
|
250
|
-
try {
|
|
251
|
-
const { result } = await agentPrompt(
|
|
252
|
-
"Use web-fetch tool ONLY. Fetch https://httpbin.org/get and tell me if status is 200.",
|
|
253
|
-
{ tools: [require("../dist/tools/tools/web-fetch").webFetchTool], maxTurns: 3 },
|
|
254
|
-
)
|
|
255
|
-
const ok = result.totalToolCalls >= 1
|
|
256
|
-
record("qa-3-web", "web-fetch tool", ok, result.finalText.slice(0, 80))
|
|
257
|
-
} catch (e) {
|
|
258
|
-
record("qa-3-web", "web-fetch tool", false, e.message)
|
|
259
|
-
}
|
|
260
|
-
})()
|
|
261
|
-
|
|
262
|
-
// ─── A5: Memory ───
|
|
263
|
-
console.log("\n--- A5: Memory Service ---")
|
|
264
|
-
try {
|
|
265
|
-
const { saveActiveTask, getActiveTask, buildMemoryPrompt } = require("../dist/memory/service")
|
|
266
|
-
saveActiveTask("QA codename PHOENIX-TEST-7721")
|
|
267
|
-
const task = getActiveTask()
|
|
268
|
-
const mem = buildMemoryPrompt()
|
|
269
|
-
record("qa-5-memory", "Active task in memory", task.includes("PHOENIX") && mem.includes("PHOENIX"), task.slice(0, 50))
|
|
270
|
-
} catch (e) {
|
|
271
|
-
record("qa-5-memory", "Memory service", false, e.message)
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
// ─── A7: Compaction ───
|
|
275
|
-
console.log("\n--- A7: Compaction Round-Trip ---")
|
|
276
|
-
await (async () => {
|
|
277
|
-
try {
|
|
278
|
-
const { createClient } = require("../dist/auth/client")
|
|
279
|
-
const { Conversation } = require("../dist/agent/conversation")
|
|
280
|
-
const { compactConversation, uncompactConversation } = require("../dist/compact/service")
|
|
281
|
-
const client = createClient()
|
|
282
|
-
const conv = new Conversation("z-ai/GLM-5.1")
|
|
283
|
-
conv.addUserMessage("We are fixing the login bug in auth.ts")
|
|
284
|
-
conv.addAssistantMessage("I'll help fix the login bug in auth.ts")
|
|
285
|
-
conv.addUserMessage("Check the JWT validation in auth/login.ts")
|
|
286
|
-
conv.addAssistantMessage("Reading auth/login.ts now")
|
|
287
|
-
conv.addUserMessage("Also add rate limiting")
|
|
288
|
-
const before = conv.messages.length
|
|
289
|
-
const compact = await compactConversation(client, "z-ai/GLM-5.1", conv, undefined, { trigger: "manual" })
|
|
290
|
-
const hasSummary = conv.messages.some((m) => m.content.includes("Conversation Summary") || m.content.includes("login"))
|
|
291
|
-
const restored = uncompactConversation(conv)
|
|
292
|
-
const after = conv.messages.length
|
|
293
|
-
record("qa-7-compact", "Compact preserves task + uncompact restores", hasSummary && compact.tokensSaved >= 0, `before=${before} afterCompact=${compact.postCompactMessages}`)
|
|
294
|
-
record("qa-7-uncompact", "Uncompact restores history", restored && after >= before, `restored=${after} msgs`)
|
|
295
|
-
} catch (e) {
|
|
296
|
-
record("qa-7-compact", "Compaction", false, e.message)
|
|
297
|
-
}
|
|
298
|
-
})()
|
|
299
|
-
|
|
300
|
-
// ─── A4: Skills CLI ───
|
|
301
|
-
console.log("\n--- A4: Skill System ---")
|
|
302
|
-
try {
|
|
303
|
-
const skills = run("llmtune skills list", { timeout: 60000 })
|
|
304
|
-
record("qa-4-list", "llmtune skills list", skills.length > 10, `${skills.length} chars output`)
|
|
305
|
-
} catch (e) {
|
|
306
|
-
record("qa-4-list", "skills list", false, e.message)
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
// ─── A10: Infra API ───
|
|
310
|
-
console.log("\n--- A10: Infra Endpoints ---")
|
|
311
|
-
const apiKey = loadApiKey()
|
|
312
|
-
if (apiKey) {
|
|
313
|
-
try {
|
|
314
|
-
const ctxRes = run(
|
|
315
|
-
`curl -s -H "Authorization: Bearer ${apiKey}" -H "X-Workspace-Root: ${CLI_DIR.replace(/\\/g, "/")}" -H "X-CWD: ${CLI_DIR.replace(/\\/g, "/")}" "https://api.llmtune.io/api/agent/v1/context?model=z-ai/GLM-5.1"`,
|
|
316
|
-
{ timeout: 30000 },
|
|
317
|
-
)
|
|
318
|
-
const ok = ctxRes.includes("systemPrompt") || ctxRes.includes("subContexts")
|
|
319
|
-
record("qa-10-context", "GET /api/agent/v1/context", ok, ctxRes.slice(0, 80))
|
|
320
|
-
} catch (e) {
|
|
321
|
-
record("qa-10-context", "GET /context", false, e.message)
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
try {
|
|
325
|
-
const skillsRes = run(
|
|
326
|
-
`curl -s -H "Authorization: Bearer ${apiKey}" "https://api.llmtune.io/api/agent/skills"`,
|
|
327
|
-
{ timeout: 30000 },
|
|
328
|
-
)
|
|
329
|
-
record("qa-10-skills", "GET /api/agent/skills", skillsRes.includes("skills") || skillsRes.includes("["), skillsRes.slice(0, 60))
|
|
330
|
-
} catch (e) {
|
|
331
|
-
record("qa-10-skills", "GET /skills", false, e.message)
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
try {
|
|
335
|
-
const chatRes = run(
|
|
336
|
-
`curl -s -X POST "https://api.llmtune.io/api/agent/v1/chat/completions" -H "Authorization: Bearer ${apiKey}" -H "Content-Type: application/json" -d "{\\"model\\":\\"z-ai/GLM-5.1\\",\\"messages\\":[{\\"role\\":\\"user\\",\\"content\\":\\"Say QA_API_OK only\\"}]}"`,
|
|
337
|
-
{ timeout: 120000 },
|
|
338
|
-
)
|
|
339
|
-
record("qa-10-chat", "POST chat/completions", chatRes.includes("choices") || chatRes.includes("QA"), chatRes.slice(0, 80))
|
|
340
|
-
} catch (e) {
|
|
341
|
-
record("qa-10-chat", "POST chat/completions", false, e.message)
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
try {
|
|
345
|
-
const analyzeBody = JSON.stringify({
|
|
346
|
-
model: "z-ai/GLM-5.1",
|
|
347
|
-
systemPrompt: "test prompt",
|
|
348
|
-
messages: [{ role: "user", content: "hello" }],
|
|
349
|
-
}).replace(/"/g, '\\"')
|
|
350
|
-
const analyzeRes = run(
|
|
351
|
-
`curl -s -X POST "https://api.llmtune.io/api/agent/v1/context/analyze" -H "Authorization: Bearer ${apiKey}" -H "Content-Type: application/json" -d "${analyzeBody}"`,
|
|
352
|
-
{ timeout: 30000 },
|
|
353
|
-
)
|
|
354
|
-
record("qa-10-analyze", "POST /context/analyze", analyzeRes.includes("totalTokens") || analyzeRes.includes("categories"), analyzeRes.slice(0, 80))
|
|
355
|
-
} catch (e) {
|
|
356
|
-
record("qa-10-analyze", "POST /context/analyze", false, "may need infra deploy")
|
|
357
|
-
}
|
|
358
|
-
} else {
|
|
359
|
-
record("qa-10", "Infra API tests", false, "no API key")
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
// ─── A2: REPL slash commands (piped) ───
|
|
363
|
-
console.log("\n--- A2: REPL Slash Commands ---")
|
|
364
|
-
const replOut = await replSession([
|
|
365
|
-
"/help",
|
|
366
|
-
"/context",
|
|
367
|
-
"/skills",
|
|
368
|
-
"/memory",
|
|
369
|
-
"/model z-ai/GLM-5.1",
|
|
370
|
-
"/stream",
|
|
371
|
-
"/verbose",
|
|
372
|
-
"/trust bash",
|
|
373
|
-
"Say hello in 3 words",
|
|
374
|
-
"/save",
|
|
375
|
-
"/clear",
|
|
376
|
-
"one",
|
|
377
|
-
"two",
|
|
378
|
-
"three",
|
|
379
|
-
"/compact",
|
|
380
|
-
"/uncompact",
|
|
381
|
-
"/exit",
|
|
382
|
-
])
|
|
383
|
-
|
|
384
|
-
const o = replOut.stdout + replOut.stderr
|
|
385
|
-
record("qa-2-help", "/help", o.includes("/help") || o.includes("Commands"))
|
|
386
|
-
record("qa-2-context", "/context", o.includes("Context Usage") || o.includes("Tokens"))
|
|
387
|
-
record("qa-2-skills", "/skills", o.includes("Skills") || o.includes("skill"))
|
|
388
|
-
record("qa-2-memory", "/memory", o.includes("Memory") || o.includes("memory"))
|
|
389
|
-
record("qa-2-model", "/model", o.includes("Model set") || o.includes("GLM"))
|
|
390
|
-
record("qa-2-stream", "/stream", o.includes("Streaming"))
|
|
391
|
-
record("qa-2-verbose", "/verbose", o.includes("Verbose"))
|
|
392
|
-
record("qa-2-trust", "/trust bash", o.includes("Trusting tool"))
|
|
393
|
-
record("qa-2-clear", "/clear", o.includes("cleared"))
|
|
394
|
-
record("qa-2-save", "/save", o.includes("Session saved") || fs.existsSync(path.join(CLI_DIR, "llmtune-session-")) || /llmtune-session-\d+\.json/.test(o))
|
|
395
|
-
record("qa-2-compact", "/compact", o.includes("Compacted") || o.includes("compact"))
|
|
396
|
-
record("qa-2-uncompact", "/uncompact", o.includes("Restored") || o.includes("No raw history"))
|
|
397
|
-
record("qa-2-path", "No 'path specified' error", !o.includes("The system cannot find the path specified"))
|
|
398
|
-
record("qa-2-async", "No async dispatch error", !o.includes("Async tools not supported"))
|
|
399
|
-
record("qa-2-identity", "LLMTune identity (not Claude-only)", !o.match(/I'm \*\*Claude\*\*, an AI assistant made by \*\*Anthropic\*\*/))
|
|
400
|
-
|
|
401
|
-
// Identity check via agent
|
|
402
|
-
await (async () => {
|
|
403
|
-
try {
|
|
404
|
-
const { result } = await agentPrompt(
|
|
405
|
-
"Who are you? Answer in one sentence. Do not use tools.",
|
|
406
|
-
{ tools: [], maxTurns: 1 },
|
|
407
|
-
)
|
|
408
|
-
const text = result.finalText.toLowerCase()
|
|
409
|
-
const isLlmtune = text.includes("llmtune")
|
|
410
|
-
const isClaude = text.includes("claude") && text.includes("anthropic")
|
|
411
|
-
record("qa-identity", "Agent identifies as LLMTune", isLlmtune && !isClaude, result.finalText.slice(0, 100))
|
|
412
|
-
} catch (e) {
|
|
413
|
-
record("qa-identity", "Agent identity", false, e.message)
|
|
414
|
-
}
|
|
415
|
-
})()
|
|
416
|
-
|
|
417
|
-
// Cleanup QA dir
|
|
418
|
-
try { fs.rmSync(QA_DIR, { recursive: true, force: true }) } catch {}
|
|
419
|
-
|
|
420
|
-
// Summary
|
|
421
|
-
console.log("\n" + "=".repeat(60))
|
|
422
|
-
const passed = results.filter((r) => r.pass).length
|
|
423
|
-
const failed = results.filter((r) => !r.pass).length
|
|
424
|
-
console.log(`QA SUMMARY: ${passed} passed, ${failed} failed, ${results.length} total`)
|
|
425
|
-
console.log("=".repeat(60))
|
|
426
|
-
|
|
427
|
-
if (failed > 0) {
|
|
428
|
-
console.log("\nFailed tests:")
|
|
429
|
-
for (const r of results.filter((x) => !x.pass)) {
|
|
430
|
-
console.log(` - ${r.id} ${r.name}: ${r.detail}`)
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
process.exit(failed > 0 ? 1 : 0)
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
main().catch((err) => {
|
|
438
|
-
console.error(err)
|
|
439
|
-
process.exit(1)
|
|
440
|
-
})
|
package/scripts/smoke-test.js
DELETED
|
@@ -1,142 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* CLI smoke tests — run: npm test
|
|
4
|
-
* Set LLMTUNE_API_KEY or configure ~/.llmtune/config.json for live API tests.
|
|
5
|
-
*/
|
|
6
|
-
const fs = require("fs")
|
|
7
|
-
const path = require("path")
|
|
8
|
-
const os = require("os")
|
|
9
|
-
|
|
10
|
-
let passed = 0
|
|
11
|
-
let failed = 0
|
|
12
|
-
let skipped = 0
|
|
13
|
-
|
|
14
|
-
function ok(name) {
|
|
15
|
-
passed++
|
|
16
|
-
console.log(` ✓ ${name}`)
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
function fail(name, err) {
|
|
20
|
-
failed++
|
|
21
|
-
console.log(` ✗ ${name}: ${err}`)
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function skip(name, reason) {
|
|
25
|
-
skipped++
|
|
26
|
-
console.log(` ⊘ ${name} (${reason})`)
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function test(name, fn) {
|
|
30
|
-
try {
|
|
31
|
-
fn()
|
|
32
|
-
ok(name)
|
|
33
|
-
} catch (e) {
|
|
34
|
-
fail(name, e.message)
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function testAsync(name, fn) {
|
|
39
|
-
try {
|
|
40
|
-
await fn()
|
|
41
|
-
ok(name)
|
|
42
|
-
} catch (e) {
|
|
43
|
-
fail(name, e.message)
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
async function main() {
|
|
48
|
-
console.log("=== LLMTune CLI Smoke Tests ===\n")
|
|
49
|
-
|
|
50
|
-
test("dist/index.js exists", () => {
|
|
51
|
-
const p = path.join(__dirname, "..", "dist", "index.js")
|
|
52
|
-
if (!fs.existsSync(p)) throw new Error("run npm run build first")
|
|
53
|
-
})
|
|
54
|
-
|
|
55
|
-
test("agent identity module", () => {
|
|
56
|
-
const { buildAgentIdentitySection } = require("../dist/context/agent-identity")
|
|
57
|
-
const text = buildAgentIdentitySection("z-ai/GLM-5.1")
|
|
58
|
-
if (!text.includes("LLMTune Agent")) throw new Error("missing identity")
|
|
59
|
-
if (text.toLowerCase().includes("you are claude")) throw new Error("should not identify as Claude")
|
|
60
|
-
})
|
|
61
|
-
|
|
62
|
-
test("git context (Windows-safe)", () => {
|
|
63
|
-
const { collectGitContext } = require("../dist/context/git-context")
|
|
64
|
-
collectGitContext(process.cwd())
|
|
65
|
-
})
|
|
66
|
-
|
|
67
|
-
test("microcompact", () => {
|
|
68
|
-
const { microcompactMessages } = require("../dist/compact/microcompact")
|
|
69
|
-
const { compacted } = microcompactMessages([
|
|
70
|
-
{ role: "tool", content: "x".repeat(5000) },
|
|
71
|
-
])
|
|
72
|
-
if (compacted[0].content.length >= 5000) throw new Error("not compressed")
|
|
73
|
-
})
|
|
74
|
-
|
|
75
|
-
test("context analyzer", () => {
|
|
76
|
-
const { analyzeContextUsage } = require("../dist/context/analyzer")
|
|
77
|
-
const a = analyzeContextUsage({
|
|
78
|
-
systemPrompt: "test",
|
|
79
|
-
toolSpecs: [],
|
|
80
|
-
messages: [{ role: "user", content: "hi" }],
|
|
81
|
-
model: "z-ai/GLM-5.1",
|
|
82
|
-
})
|
|
83
|
-
if (a.totalTokens <= 0) throw new Error("expected tokens > 0")
|
|
84
|
-
})
|
|
85
|
-
|
|
86
|
-
test("permission manager trust", () => {
|
|
87
|
-
const { PermissionManager } = require("../dist/tools/permissions")
|
|
88
|
-
const pm = new PermissionManager()
|
|
89
|
-
pm.trustTool("bash")
|
|
90
|
-
if (!pm.isTrusted("bash")) throw new Error("trust failed")
|
|
91
|
-
})
|
|
92
|
-
|
|
93
|
-
await testAsync("registry dispatchAsync", async () => {
|
|
94
|
-
const { ToolRegistry } = require("../dist/tools/registry")
|
|
95
|
-
const { bashTool } = require("../dist/tools/tools/bash")
|
|
96
|
-
const reg = new ToolRegistry()
|
|
97
|
-
reg.register(bashTool)
|
|
98
|
-
await reg.dispatchAsync("bash", { command: "echo SMOKE_OK" }, {
|
|
99
|
-
workspaceRoot: process.cwd(),
|
|
100
|
-
cwd: process.cwd(),
|
|
101
|
-
})
|
|
102
|
-
})
|
|
103
|
-
|
|
104
|
-
const configPath = path.join(os.homedir(), ".llmtune", "config.json")
|
|
105
|
-
const hasApi = Boolean(process.env.LLMTUNE_API_KEY) || fs.existsSync(configPath)
|
|
106
|
-
|
|
107
|
-
if (hasApi) {
|
|
108
|
-
await testAsync("API agent loop + bash tool", async () => {
|
|
109
|
-
const { createClient } = require("../dist/auth/client")
|
|
110
|
-
const { Conversation } = require("../dist/agent/conversation")
|
|
111
|
-
const { ToolRegistry } = require("../dist/tools/registry")
|
|
112
|
-
const { bashTool } = require("../dist/tools/tools/bash")
|
|
113
|
-
const { runAgentLoop } = require("../dist/agent/loop")
|
|
114
|
-
|
|
115
|
-
const client = createClient()
|
|
116
|
-
const registry = new ToolRegistry()
|
|
117
|
-
registry.register(bashTool)
|
|
118
|
-
const conversation = new Conversation("z-ai/GLM-5.1")
|
|
119
|
-
const cwd = process.cwd()
|
|
120
|
-
|
|
121
|
-
const result = await runAgentLoop(
|
|
122
|
-
client,
|
|
123
|
-
conversation,
|
|
124
|
-
registry,
|
|
125
|
-
"Run bash command: echo AGENT_SMOKE_OK. Use bash tool only.",
|
|
126
|
-
{ model: "z-ai/GLM-5.1", maxTurns: 3, stream: false, cwd, workspaceRoot: cwd },
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
if (result.totalToolCalls === 0) throw new Error("expected at least one tool call")
|
|
130
|
-
})
|
|
131
|
-
} else {
|
|
132
|
-
skip("API agent loop + bash tool", "no API key")
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
console.log(`\n=== Results: ${passed} passed, ${failed} failed, ${skipped} skipped ===`)
|
|
136
|
-
process.exit(failed > 0 ? 1 : 0)
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
main().catch((err) => {
|
|
140
|
-
console.error(err)
|
|
141
|
-
process.exit(1)
|
|
142
|
-
})
|