lastlight 0.5.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +14 -0
- package/README.md +39 -0
- package/agent-context/rules.md +0 -38
- package/agent-context/soul.md +0 -11
- package/dist/cli.js +24 -0
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +3 -3
- package/dist/config.js.map +1 -1
- package/dist/engine/agent-executor.d.ts +2 -139
- package/dist/engine/agent-executor.js +76 -996
- package/dist/engine/agent-executor.js.map +1 -1
- package/dist/engine/executors/backends.d.ts +41 -0
- package/dist/engine/executors/backends.js +541 -0
- package/dist/engine/executors/backends.js.map +1 -0
- package/dist/engine/executors/shared.d.ts +189 -0
- package/dist/engine/executors/shared.js +612 -0
- package/dist/engine/executors/shared.js.map +1 -0
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +1 -1
- package/dist/sandbox/index.js.map +1 -1
- package/dist/sandbox/smol.d.ts +162 -0
- package/dist/sandbox/smol.integration.test.d.ts +1 -0
- package/dist/sandbox/smol.integration.test.js +130 -0
- package/dist/sandbox/smol.integration.test.js.map +1 -0
- package/dist/sandbox/smol.js +485 -0
- package/dist/sandbox/smol.js.map +1 -0
- package/dist/skills-install.d.ts +12 -0
- package/dist/skills-install.js +179 -0
- package/dist/skills-install.js.map +1 -0
- package/package.json +4 -2
- package/plugins/lastlight/.claude-plugin/plugin.json +12 -0
- package/plugins/lastlight/README.md +44 -0
- package/plugins/lastlight/skills/lastlight-client/SKILL.md +82 -0
- package/plugins/lastlight/skills/lastlight-evals/SKILL.md +130 -0
- package/plugins/lastlight/skills/lastlight-evals/references/instance-schema.md +84 -0
- package/plugins/lastlight/skills/lastlight-evals/references/models-json.md +42 -0
- package/plugins/lastlight/skills/lastlight-overlay/SKILL.md +72 -0
- package/plugins/lastlight/skills/lastlight-overlay/references/forking.md +55 -0
- package/plugins/lastlight/skills/lastlight-overlay/references/overlay-layout.md +52 -0
- package/plugins/lastlight/skills/lastlight-server/SKILL.md +124 -0
- package/plugins/lastlight/skills/lastlight-server/references/env-schema.md +103 -0
- package/plugins/lastlight/skills/lastlight-server/references/operations.md +60 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-local `lastlight skills …` — install the Last Light Claude Code skills
|
|
3
|
+
* into a local Claude Code instance.
|
|
4
|
+
*
|
|
5
|
+
* The skills live in this package's `plugins/lastlight/` tree (shipped via the
|
|
6
|
+
* package `files` allowlist) and are also exposed as a Claude Code marketplace
|
|
7
|
+
* via `.claude-plugin/marketplace.json` at the package root. Two install paths:
|
|
8
|
+
*
|
|
9
|
+
* 1. **Marketplace** (preferred when the `claude` CLI is present): register the
|
|
10
|
+
* bundled marketplace by local path and install the plugin. Using the
|
|
11
|
+
* bundled path keeps the installed skills version-matched to this CLI and
|
|
12
|
+
* works offline — no dependency on the core repo being public.
|
|
13
|
+
* 2. **Copy fallback** (no `claude` CLI): copy each `skills/<name>/` directory
|
|
14
|
+
* into the scope's skills dir (`~/.claude/skills` for user,
|
|
15
|
+
* `<cwd>/.claude/skills` for project), which Claude Code auto-discovers with
|
|
16
|
+
* no marketplace at all.
|
|
17
|
+
*
|
|
18
|
+
* Like `lastlight fork` / `lastlight server`, this operates on local files — not
|
|
19
|
+
* over HTTP.
|
|
20
|
+
*/
|
|
21
|
+
import fs from "node:fs";
|
|
22
|
+
import os from "node:os";
|
|
23
|
+
import path from "node:path";
|
|
24
|
+
import { fileURLToPath } from "node:url";
|
|
25
|
+
import { spawnSync } from "node:child_process";
|
|
26
|
+
import chalk from "chalk";
|
|
27
|
+
/** Marketplace + plugin identifiers — must match `.claude-plugin/marketplace.json`. */
|
|
28
|
+
const MARKETPLACE_NAME = "lastlight-skills";
|
|
29
|
+
const PLUGIN_NAME = "lastlight";
|
|
30
|
+
// ── bundle resolution ────────────────────────────────────────────────────────
|
|
31
|
+
/**
|
|
32
|
+
* Package root that holds `.claude-plugin/marketplace.json` and
|
|
33
|
+
* `plugins/<plugin>/`. Resolves for both dev (`src/skills-install.ts` → repo
|
|
34
|
+
* root) and compiled/installed (`dist/skills-install.js` → package root).
|
|
35
|
+
*/
|
|
36
|
+
function bundleRoot() {
|
|
37
|
+
return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
38
|
+
}
|
|
39
|
+
function pluginDir() {
|
|
40
|
+
return path.join(bundleRoot(), "plugins", PLUGIN_NAME);
|
|
41
|
+
}
|
|
42
|
+
function skillsSrcDir() {
|
|
43
|
+
return path.join(pluginDir(), "skills");
|
|
44
|
+
}
|
|
45
|
+
/** Names of the bundled skill directories (each holding a SKILL.md). */
|
|
46
|
+
function bundledSkillNames() {
|
|
47
|
+
const dir = skillsSrcDir();
|
|
48
|
+
try {
|
|
49
|
+
return fs
|
|
50
|
+
.readdirSync(dir, { withFileTypes: true })
|
|
51
|
+
.filter((e) => e.isDirectory() && fs.existsSync(path.join(dir, e.name, "SKILL.md")))
|
|
52
|
+
.map((e) => e.name)
|
|
53
|
+
.sort();
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function assertBundlePresent() {
|
|
60
|
+
if (bundledSkillNames().length === 0) {
|
|
61
|
+
throw new Error(`No bundled skills found under ${skillsSrcDir()} — the package may be built without the plugins/ assets.`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// ── scope paths ──────────────────────────────────────────────────────────────
|
|
65
|
+
function scopeSkillsDir(scope) {
|
|
66
|
+
const base = scope === "project" ? process.cwd() : os.homedir();
|
|
67
|
+
return path.join(base, ".claude", "skills");
|
|
68
|
+
}
|
|
69
|
+
// ── claude CLI detection ─────────────────────────────────────────────────────
|
|
70
|
+
function hasClaudeCli() {
|
|
71
|
+
const probe = spawnSync("claude", ["--version"], { stdio: "ignore" });
|
|
72
|
+
return probe.status === 0;
|
|
73
|
+
}
|
|
74
|
+
/** Run a `claude` subcommand, inheriting stdio so the user sees progress. */
|
|
75
|
+
function claude(args) {
|
|
76
|
+
const res = spawnSync("claude", args, { stdio: "inherit" });
|
|
77
|
+
return res.status === 0;
|
|
78
|
+
}
|
|
79
|
+
// ── copy fallback ────────────────────────────────────────────────────────────
|
|
80
|
+
function copySkills(scope) {
|
|
81
|
+
const destRoot = scopeSkillsDir(scope);
|
|
82
|
+
fs.mkdirSync(destRoot, { recursive: true });
|
|
83
|
+
const installed = [];
|
|
84
|
+
for (const name of bundledSkillNames()) {
|
|
85
|
+
const src = path.join(skillsSrcDir(), name);
|
|
86
|
+
const dest = path.join(destRoot, name);
|
|
87
|
+
// Bundle skills are managed artifacts — replace any existing copy so it
|
|
88
|
+
// tracks this CLI version.
|
|
89
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
90
|
+
fs.cpSync(src, dest, { recursive: true });
|
|
91
|
+
installed.push(name);
|
|
92
|
+
}
|
|
93
|
+
return installed;
|
|
94
|
+
}
|
|
95
|
+
// ── commands ─────────────────────────────────────────────────────────────────
|
|
96
|
+
export async function skillsInstall(opts) {
|
|
97
|
+
assertBundlePresent();
|
|
98
|
+
const scope = opts.scope ?? "user";
|
|
99
|
+
const names = bundledSkillNames();
|
|
100
|
+
// Preferred: register the bundled marketplace + install via the claude CLI.
|
|
101
|
+
if (!opts.noMarketplace && hasClaudeCli()) {
|
|
102
|
+
console.log(chalk.dim(`Using the claude CLI (marketplace: ${bundleRoot()})`));
|
|
103
|
+
// `marketplace add` may report "already added" — non-fatal; proceed to install.
|
|
104
|
+
claude(["plugin", "marketplace", "add", bundleRoot()]);
|
|
105
|
+
const ok = claude(["plugin", "install", `${PLUGIN_NAME}@${MARKETPLACE_NAME}`, "--scope", scope]);
|
|
106
|
+
if (ok) {
|
|
107
|
+
console.log(chalk.green(`✓ Installed ${PLUGIN_NAME}@${MARKETPLACE_NAME}`) +
|
|
108
|
+
chalk.dim(` (${scope} scope) — ${names.length} skills`));
|
|
109
|
+
console.log(chalk.dim(" Start a new Claude Code session to use them."));
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
console.log(chalk.yellow("claude plugin install failed — falling back to copying skills."));
|
|
113
|
+
}
|
|
114
|
+
// Fallback: copy skill dirs into the scope's skills directory.
|
|
115
|
+
const installed = copySkills(scope);
|
|
116
|
+
const destRoot = scopeSkillsDir(scope);
|
|
117
|
+
console.log(chalk.green(`✓ Copied ${installed.length} skills`) +
|
|
118
|
+
chalk.dim(` → ${destRoot}`));
|
|
119
|
+
for (const n of installed)
|
|
120
|
+
console.log(chalk.dim(` • ${n}`));
|
|
121
|
+
console.log(chalk.dim(" Start a new Claude Code session to use them."));
|
|
122
|
+
}
|
|
123
|
+
export async function skillsList(opts) {
|
|
124
|
+
assertBundlePresent();
|
|
125
|
+
const names = bundledSkillNames();
|
|
126
|
+
const userDir = scopeSkillsDir("user");
|
|
127
|
+
const projDir = scopeSkillsDir("project");
|
|
128
|
+
console.log(chalk.bold(`Bundled Last Light skills (${names.length})`));
|
|
129
|
+
for (const n of names) {
|
|
130
|
+
const marks = [];
|
|
131
|
+
if (fs.existsSync(path.join(userDir, n)))
|
|
132
|
+
marks.push("user");
|
|
133
|
+
if (fs.existsSync(path.join(projDir, n)))
|
|
134
|
+
marks.push("project");
|
|
135
|
+
const where = marks.length ? chalk.green(` [installed: ${marks.join(", ")}]`) : chalk.dim(" [not installed]");
|
|
136
|
+
console.log(` • ${n}${where}`);
|
|
137
|
+
}
|
|
138
|
+
console.log(chalk.dim(`\nInstall: lastlight skills install [--scope user|project]\n` +
|
|
139
|
+
`Marketplace: ${PLUGIN_NAME}@${MARKETPLACE_NAME} (${bundleRoot()})`));
|
|
140
|
+
}
|
|
141
|
+
export async function skillsUninstall(opts) {
|
|
142
|
+
const scope = opts.scope ?? "user";
|
|
143
|
+
// Best-effort marketplace uninstall if the claude CLI is present.
|
|
144
|
+
if (!opts.noMarketplace && hasClaudeCli()) {
|
|
145
|
+
claude(["plugin", "uninstall", `${PLUGIN_NAME}@${MARKETPLACE_NAME}`, "--scope", scope]);
|
|
146
|
+
}
|
|
147
|
+
// Remove any copied skill dirs in this scope.
|
|
148
|
+
const destRoot = scopeSkillsDir(scope);
|
|
149
|
+
let removed = 0;
|
|
150
|
+
for (const n of bundledSkillNames()) {
|
|
151
|
+
const dest = path.join(destRoot, n);
|
|
152
|
+
if (fs.existsSync(dest)) {
|
|
153
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
154
|
+
removed++;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
console.log(chalk.green(`✓ Uninstalled Last Light skills`) +
|
|
158
|
+
chalk.dim(` (${scope} scope${removed ? `, removed ${removed} copied dirs` : ""})`));
|
|
159
|
+
}
|
|
160
|
+
/** Entry point dispatched from `src/cli.ts` for `lastlight skills …`. */
|
|
161
|
+
export async function skills(args, opts) {
|
|
162
|
+
const sub = args[0] ?? "install";
|
|
163
|
+
switch (sub) {
|
|
164
|
+
case "install":
|
|
165
|
+
return skillsInstall(opts);
|
|
166
|
+
case "list":
|
|
167
|
+
return skillsList(opts);
|
|
168
|
+
case "uninstall":
|
|
169
|
+
case "remove":
|
|
170
|
+
return skillsUninstall(opts);
|
|
171
|
+
default:
|
|
172
|
+
console.error("Usage:\n" +
|
|
173
|
+
" lastlight skills install [--scope user|project] [--no-marketplace]\n" +
|
|
174
|
+
" lastlight skills list\n" +
|
|
175
|
+
" lastlight skills uninstall [--scope user|project]");
|
|
176
|
+
process.exit(1);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
//# sourceMappingURL=skills-install.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skills-install.js","sourceRoot":"","sources":["../src/skills-install.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,uFAAuF;AACvF,MAAM,gBAAgB,GAAG,kBAAkB,CAAC;AAC5C,MAAM,WAAW,GAAG,WAAW,CAAC;AAWhC,gFAAgF;AAEhF;;;;GAIG;AACH,SAAS,UAAU;IACjB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,YAAY;IACnB,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED,wEAAwE;AACxE,SAAS,iBAAiB;IACxB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,IAAI,CAAC;QACH,OAAO,EAAE;aACN,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aACzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;aACnF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,IAAI,iBAAiB,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,EAAE,0DAA0D,CAC1G,CAAC;IACJ,CAAC;AACH,CAAC;AAED,gFAAgF;AAEhF,SAAS,cAAc,CAAC,KAAiB;IACvC,MAAM,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;IAChE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAED,gFAAgF;AAEhF,SAAS,YAAY;IACnB,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtE,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,6EAA6E;AAC7E,SAAS,MAAM,CAAC,IAAc;IAC5B,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAC5D,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,gFAAgF;AAEhF,SAAS,UAAU,CAAC,KAAiB;IACnC,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACvC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,iBAAiB,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvC,wEAAwE;QACxE,2BAA2B;QAC3B,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,gFAAgF;AAEhF,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAgB;IAClD,mBAAmB,EAAE,CAAC;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACnC,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;IAElC,4EAA4E;IAC5E,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,YAAY,EAAE,EAAE,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sCAAsC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9E,gFAAgF;QAChF,MAAM,CAAC,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,WAAW,IAAI,gBAAgB,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;QACjG,IAAI,EAAE,EAAE,CAAC;YACP,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,eAAe,WAAW,IAAI,gBAAgB,EAAE,CAAC;gBAC3D,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,aAAa,KAAK,CAAC,MAAM,SAAS,CAAC,CAC1D,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;YACzE,OAAO;QACT,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,gEAAgE,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED,+DAA+D;IAC/D,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,YAAY,SAAS,CAAC,MAAM,SAAS,CAAC;QAChD,KAAK,CAAC,GAAG,CAAC,MAAM,QAAQ,EAAE,CAAC,CAC9B,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAgB;IAC/C,mBAAmB,EAAE,CAAC;IACtB,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAC9G,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,8DAA8D;QAC5D,gBAAgB,WAAW,IAAI,gBAAgB,KAAK,UAAU,EAAE,GAAG,CACtE,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAgB;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACnC,kEAAkE;IAClE,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,YAAY,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,WAAW,IAAI,gBAAgB,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,8CAA8C;IAC9C,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,iBAAiB,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,iCAAiC,CAAC;QAC5C,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,OAAO,CAAC,CAAC,CAAC,aAAa,OAAO,cAAc,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CACrF,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAc,EAAE,IAAgB;IAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;IACjC,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,SAAS;YACZ,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1B,KAAK,WAAW,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;QAC/B;YACE,OAAO,CAAC,KAAK,CACX,UAAU;gBACR,wEAAwE;gBACxE,2BAA2B;gBAC3B,qDAAqD,CACxD,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "lastlight",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "GitHub repository maintenance agent — Agent SDK harness",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
"agent-context",
|
|
25
25
|
"deploy",
|
|
26
26
|
"sandbox.Dockerfile",
|
|
27
|
-
"docker-compose.yml"
|
|
27
|
+
"docker-compose.yml",
|
|
28
|
+
".claude-plugin",
|
|
29
|
+
"plugins"
|
|
28
30
|
],
|
|
29
31
|
"repository": {
|
|
30
32
|
"type": "git",
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json.schemastore.org/claude-code-plugin.json",
|
|
3
|
+
"name": "lastlight",
|
|
4
|
+
"displayName": "Last Light",
|
|
5
|
+
"version": "0.6.2",
|
|
6
|
+
"description": "Install, configure and operate Last Light (GitHub maintenance agent) — its server, CLI client, deployment overlay, and the Last Light Evals harness.",
|
|
7
|
+
"author": { "name": "Clifton Cunningham" },
|
|
8
|
+
"homepage": "https://github.com/cliftonc/lastlight",
|
|
9
|
+
"repository": "https://github.com/cliftonc/lastlight",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"keywords": ["lastlight", "github-agent", "evals", "devops", "setup", "docker"]
|
|
12
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Last Light — Claude Code plugin
|
|
2
|
+
|
|
3
|
+
A bundle of [Claude Code](https://docs.claude.com/en/docs/claude-code) skills
|
|
4
|
+
that help you install, configure and operate **Last Light** (a GitHub
|
|
5
|
+
repository-maintenance agent) and **Last Light Evals** (its eval harness).
|
|
6
|
+
|
|
7
|
+
These are *Claude Code* skills — they run on your machine and drive the
|
|
8
|
+
`lastlight` / `lastlight-evals` CLIs for you. They are distinct from Last
|
|
9
|
+
Light's *internal* sandbox skills under the repo's top-level `skills/` dir
|
|
10
|
+
(which are staged into the agent's own workflows).
|
|
11
|
+
|
|
12
|
+
## Skills
|
|
13
|
+
|
|
14
|
+
| Skill | Use it when you want to… |
|
|
15
|
+
|-------|--------------------------|
|
|
16
|
+
| `lastlight-server` | Install & configure a Last Light **server** (the agent + docker stack) on a host. |
|
|
17
|
+
| `lastlight-client` | Point the `lastlight` **CLI client** at an existing server and log in. |
|
|
18
|
+
| `lastlight-overlay` | Create a deployment **overlay** instance and fork/customize workflows, prompts, skills, or the agent persona. |
|
|
19
|
+
| `lastlight-evals` | Scaffold and run a **Last Light Evals** workspace (datasets, models, model comparisons). |
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
**Fastest — via the lastlight CLI** (installs the version-matched skills bundled
|
|
24
|
+
with the CLI, no marketplace registration needed):
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npm i -g lastlight
|
|
28
|
+
lastlight skills install # → ~/.claude/skills (user scope)
|
|
29
|
+
lastlight skills install --scope project # → ./.claude/skills (this repo only)
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
**Via the Claude Code plugin marketplace** (from a checkout of this repo):
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
claude plugin marketplace add ./ # or: cliftonc/lastlight if public
|
|
36
|
+
claude plugin install lastlight@lastlight-skills
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
**Manual** — copy any `skills/<name>/` directory here into `~/.claude/skills/`
|
|
40
|
+
(personal, all projects) or a project's `.claude/skills/`. Claude Code
|
|
41
|
+
auto-discovers them on the next session.
|
|
42
|
+
|
|
43
|
+
After installing, start a new Claude Code session and say e.g. *"set up a Last
|
|
44
|
+
Light server"* or *"scaffold a Last Light evals workspace"*.
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: lastlight-client
|
|
3
|
+
description: Install the `lastlight` CLI and connect it as a CLIENT to an existing Last Light server — log in, save the token, and verify the connection. Use when the user wants to "connect / point my lastlight CLI at a server", "log in to Last Light", "set up the lastlight client", or run lastlight commands against a remote instance. For standing up the server itself use lastlight-server; for editing a deployment's config use lastlight-overlay.
|
|
4
|
+
version: 1.0.0
|
|
5
|
+
tags: [lastlight, client, cli, login, auth]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# Install & configure the Last Light CLI client
|
|
9
|
+
|
|
10
|
+
The `lastlight` CLI is a thin HTTP client: it POSTs triggers and reads a running
|
|
11
|
+
instance's admin API. As a *client* it needs only the instance URL and an auth
|
|
12
|
+
token, saved to `~/.lastlight/config.json` (mode 0600).
|
|
13
|
+
|
|
14
|
+
## 1. Install the CLI
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
command -v lastlight >/dev/null && echo "installed" || npm i -g lastlight
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## 2. Get the server URL
|
|
21
|
+
|
|
22
|
+
Ask the user for the instance URL (e.g. `https://lastlight.example.com`). If they
|
|
23
|
+
don't know it, they need it from whoever runs the server.
|
|
24
|
+
|
|
25
|
+
## 3. Log in
|
|
26
|
+
|
|
27
|
+
Two paths — pick based on environment:
|
|
28
|
+
|
|
29
|
+
- **Browser handoff (default, interactive desktop):**
|
|
30
|
+
```bash
|
|
31
|
+
lastlight login https://lastlight.example.com
|
|
32
|
+
```
|
|
33
|
+
Opens the dashboard; the user authenticates with whatever method the server
|
|
34
|
+
has (password / Slack / GitHub OAuth) and the token is captured automatically.
|
|
35
|
+
|
|
36
|
+
- **Headless / no browser (SSH, CI, server box):**
|
|
37
|
+
```bash
|
|
38
|
+
lastlight login https://lastlight.example.com --password
|
|
39
|
+
```
|
|
40
|
+
Prompts for the admin password and POSTs it to `/admin/api/login`. Requires the
|
|
41
|
+
server to have `ADMIN_PASSWORD` set.
|
|
42
|
+
|
|
43
|
+
The token (≈7-day TTL) is saved to `~/.lastlight/config.json`.
|
|
44
|
+
|
|
45
|
+
## 4. Verify
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
lastlight status
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Confirm: **Server healthy**, **Token valid**. If the token shows
|
|
52
|
+
`expired/invalid`, re-run `lastlight login`.
|
|
53
|
+
|
|
54
|
+
## Overrides & alternatives
|
|
55
|
+
|
|
56
|
+
- Skip the saved file with env vars or flags (precedence: flags → env → saved
|
|
57
|
+
file → default `http://localhost:8644`):
|
|
58
|
+
```bash
|
|
59
|
+
LASTLIGHT_URL=https://ll.example.com LASTLIGHT_TOKEN=... lastlight status
|
|
60
|
+
lastlight status --url https://ll.example.com --token ...
|
|
61
|
+
```
|
|
62
|
+
- `lastlight logout` clears the saved config.
|
|
63
|
+
- One-shot wizard: `lastlight setup --client` is equivalent to `lastlight login`.
|
|
64
|
+
|
|
65
|
+
## What you can do once connected
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
lastlight chat "hello" # chat with the bot (REPL if no message)
|
|
69
|
+
lastlight owner/repo#123 # triage an issue (default, cheap)
|
|
70
|
+
lastlight build owner/repo#123 # full build cycle
|
|
71
|
+
lastlight triage|review owner/repo[#N] # repo scan or single issue/PR
|
|
72
|
+
lastlight health|security owner/repo # repo-level report
|
|
73
|
+
lastlight workflow list|log <id> # inspect runs
|
|
74
|
+
lastlight session list|log <id> -f # tail a sandbox session
|
|
75
|
+
lastlight logs search "<text>" # search errors / transcripts
|
|
76
|
+
lastlight approvals list|approve|reject
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Done when
|
|
80
|
+
|
|
81
|
+
`lastlight status` reports the server healthy and the token valid. Report the
|
|
82
|
+
connected instance URL and a couple of commands the user can run next.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: lastlight-evals
|
|
3
|
+
description: Scaffold, configure and run a Last Light EVALS workspace — the harness that runs Last Light's real workflows against a mocked GitHub and grades them deterministically. Use when the user wants to "set up / scaffold Last Light Evals", "create an evals workspace or instance", "run evals", "compare models", or author new eval cases (triage / code-fix instances). GitHub is mocked, so no real GitHub token is needed — only a model provider API key.
|
|
4
|
+
version: 1.1.0
|
|
5
|
+
tags: [lastlight, evals, benchmark, models, swe-bench]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# Set up & run Last Light Evals
|
|
9
|
+
|
|
10
|
+
`lastlight-evals` runs Last Light's **real** production workflows (issue-triage,
|
|
11
|
+
build, …) end-to-end against a mocked GitHub, grades the results deterministically
|
|
12
|
+
(no LLM-as-judge), and compares models on pass rate, cost, and latency. It's a
|
|
13
|
+
thin CLI on top of the `lastlight` core package (via the `lastlight/evals`
|
|
14
|
+
barrel), so it exercises the same workflows/skills production does. SWE-bench
|
|
15
|
+
compatible. **Node 24+.**
|
|
16
|
+
|
|
17
|
+
## 1. Check prerequisites
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
node --version # need >= 24
|
|
21
|
+
command -v lastlight-evals >/dev/null && echo "installed" || npm i -g lastlight-evals
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## 2. Scaffold a workspace
|
|
25
|
+
|
|
26
|
+
`init` is **non-interactive** when there's no TTY (piped/agent/CI) — it never
|
|
27
|
+
blocks on a prompt. Two layouts:
|
|
28
|
+
|
|
29
|
+
### Recommended — Separate (point evals at your existing deployment overlay)
|
|
30
|
+
|
|
31
|
+
If you already have a deployment overlay repo (e.g. `your-org/lastlight-instance`),
|
|
32
|
+
this is the default. It runs your **real** overlay (its config, workflows, skills,
|
|
33
|
+
persona) and keeps the eval datasets out of your deployment repo:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
lastlight-evals init my-evals --clone your-org/lastlight-instance
|
|
37
|
+
cd my-evals && lastlight-evals run # bare run — overlay + datasets auto-detected
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
This produces:
|
|
41
|
+
- `instance/` — your overlay, cloned as its **own git checkout** (git-ignored
|
|
42
|
+
here; `cd instance && git pull` to update it)
|
|
43
|
+
- `evals/datasets/` — seeded from the built-in `triage` + `code-fix` samples (this
|
|
44
|
+
is what you edit; **always created from defaults at init**)
|
|
45
|
+
- `evals/models.json` — a copy of the built-in model registry
|
|
46
|
+
- `.gitignore` (ignores `instance/` + `eval-results/`), `README.md`
|
|
47
|
+
|
|
48
|
+
The runner **auto-detects** `./instance` as the overlay and `./evals/datasets` as
|
|
49
|
+
the dataset root, so a bare `lastlight-evals run` "just works" — no `--overlay` flag.
|
|
50
|
+
Re-running `init --clone` is idempotent: it won't re-clone an existing `instance/`.
|
|
51
|
+
|
|
52
|
+
### Plain — self-contained overlay + evals (no existing overlay repo)
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
lastlight-evals init my-evals # or: lastlight-evals init (→ ./lastlight-evals-workspace)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The workspace **is** its own overlay: it gets `workflows/`, `skills/`,
|
|
59
|
+
`agent-context/` placeholders + its own `config.yaml` alongside `evals/`. Run it
|
|
60
|
+
with `--overlay .`. In a TTY it offers `git init` + a private `gh repo create`
|
|
61
|
+
(`--no-git` to skip, `--yes` to take the non-interactive path).
|
|
62
|
+
|
|
63
|
+
## 3. Configure providers (`.env`)
|
|
64
|
+
|
|
65
|
+
Create `.env` in the workspace with **at least one** provider key. GitHub is
|
|
66
|
+
mocked end-to-end, so **no real GitHub token is needed** (the harness sets a
|
|
67
|
+
dummy one internally).
|
|
68
|
+
|
|
69
|
+
```dotenv
|
|
70
|
+
# any one (or more) of:
|
|
71
|
+
OPENAI_API_KEY=sk-...
|
|
72
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
73
|
+
FIREWORKS_API_KEY=fw-...
|
|
74
|
+
OPENROUTER_API_KEY=sk-or-...
|
|
75
|
+
DEEPSEEK_API_KEY=...
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Useful optional vars: `EVAL_MODELS` (comma-list override), `LASTLIGHT_EVALS_OUT`
|
|
79
|
+
(scorecard dir, default `./eval-results/`), `LASTLIGHT_CORE_DIR` (point at a local
|
|
80
|
+
lastlight checkout to eval un-published asset edits), `CI=1` (don't open a
|
|
81
|
+
browser). See **`references/models-json.md`** for the model registry format and
|
|
82
|
+
how `--compare` is key-gated.
|
|
83
|
+
|
|
84
|
+
## 4. Run
|
|
85
|
+
|
|
86
|
+
From inside the workspace. For the **Separate** layout the overlay (`./instance`)
|
|
87
|
+
and datasets (`./evals/datasets`) are auto-detected — no `--overlay` needed. For
|
|
88
|
+
the **Plain** layout pass `--overlay .`.
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
cd my-evals
|
|
92
|
+
lastlight-evals run # default model, triage tier (auto overlay+datasets)
|
|
93
|
+
lastlight-evals run triage # one tier
|
|
94
|
+
lastlight-evals run triage code-fix # multiple tiers
|
|
95
|
+
lastlight-evals run triage --model haiku # fuzzy match in models.json
|
|
96
|
+
lastlight-evals run triage --model openai/gpt-5.5,anthropic/claude-opus-4-8
|
|
97
|
+
lastlight-evals run --compare # cross-vendor set (only models whose envKey is present)
|
|
98
|
+
lastlight-evals run triage --runs 3 # repeat each case 3× (worst-case verdict, mean metrics)
|
|
99
|
+
lastlight-evals run triage --no-open # don't open the report
|
|
100
|
+
# Plain layout: add --overlay . (e.g. lastlight-evals run triage --overlay .)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Output lands in `./eval-results/<tiers>/`: `index.html` (scorecard),
|
|
104
|
+
`scorecard.json`, `predictions.jsonl` (SWE-bench format). Re-render a report
|
|
105
|
+
without re-running: `lastlight-evals report ./eval-results/triage`.
|
|
106
|
+
|
|
107
|
+
## 5. Author eval cases (optional)
|
|
108
|
+
|
|
109
|
+
Two tiers ship: **triage** (cheap, issue-triage) and **code-fix** (heavy, build
|
|
110
|
+
workflow with held-out tests). To add cases or a custom tier, read
|
|
111
|
+
**`references/instance-schema.md`** — it has the `SweBenchInstance` schema, the
|
|
112
|
+
exact files to create for each tier, and worked examples.
|
|
113
|
+
|
|
114
|
+
Quick shape (paths are relative to the workspace's `evals/` dir):
|
|
115
|
+
- **Triage case:** append a `SweBenchInstance` to `evals/datasets/triage/instances.json`
|
|
116
|
+
with `issue`, `triage_gold`, and `expect_github`.
|
|
117
|
+
- **Code-fix case:** add the instance to `evals/datasets/code-fix/instances.json` **and**
|
|
118
|
+
create `evals/datasets/code-fix/repos/<instance_id>/` (fixture repo at base) +
|
|
119
|
+
`evals/datasets/code-fix/tests/<instance_id>/` (held-out tests applied at grade time).
|
|
120
|
+
- **Custom tier:** a new `evals/datasets/<tier>/` with `tier.json` +
|
|
121
|
+
`instances.json` (+ `repos/` & `tests/` for code-fix-style tiers). Discovery is
|
|
122
|
+
automatic — no code change.
|
|
123
|
+
|
|
124
|
+
## Done when
|
|
125
|
+
|
|
126
|
+
The workspace is scaffolded (Separate: overlay in `instance/`, evals at root;
|
|
127
|
+
Plain: self-overlay), `.env` has a working provider key, and a bare
|
|
128
|
+
`lastlight-evals run` produces a scorecard under `eval-results/`. Report the
|
|
129
|
+
workspace path, the layout used, the provider(s) configured, and the run/compare
|
|
130
|
+
command.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Eval instance schema & authoring cases
|
|
2
|
+
|
|
3
|
+
An eval case is a **`SweBenchInstance`** — SWE-bench-compatible core fields plus
|
|
4
|
+
Last Light extensions (the GitHub fixtures + behavioral expectations that let the
|
|
5
|
+
harness drive and grade the real workflow against a mocked GitHub).
|
|
6
|
+
|
|
7
|
+
Datasets are discovered from (overlay > user > built-in): `<overlay>/evals/datasets/`,
|
|
8
|
+
`--datasets <dir>` / `LASTLIGHT_EVALS_DATASETS`, and the package's built-in
|
|
9
|
+
`datasets/`. A tier is a directory with a `tier.json` and an `instances.json`.
|
|
10
|
+
|
|
11
|
+
## `tier.json`
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{ "name": "triage", "defaultWorkflow": "issue-triage", "description": "..." }
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## `SweBenchInstance` fields
|
|
18
|
+
|
|
19
|
+
```jsonc
|
|
20
|
+
{
|
|
21
|
+
// ── SWE-bench core ──
|
|
22
|
+
"instance_id": "triage__my-case", // unique id
|
|
23
|
+
"repo": "owner/repo", // logical; fixture origin is a local bare repo
|
|
24
|
+
"base_commit": "0000000...", // code-fix only
|
|
25
|
+
"problem_statement": "short issue text",
|
|
26
|
+
"patch": "...", // gold patch — reference only, NOT graded
|
|
27
|
+
"test_patch": "...", // held-out tests (code-fix)
|
|
28
|
+
"FAIL_TO_PASS": ["test id 1"], // must go red→green (code-fix)
|
|
29
|
+
"PASS_TO_PASS": ["test id 2"], // must stay green (code-fix)
|
|
30
|
+
|
|
31
|
+
// ── Last Light extensions ──
|
|
32
|
+
"workflow": "issue-triage", // optional; defaults to the tier's defaultWorkflow
|
|
33
|
+
"issue": { // seed state for the fake GitHub
|
|
34
|
+
"number": 110, "title": "...", "body": "...",
|
|
35
|
+
"labels": [], "user": "alice",
|
|
36
|
+
"comments": [{ "user": "bob", "body": "..." }],
|
|
37
|
+
"state": "open"
|
|
38
|
+
},
|
|
39
|
+
"triage_gold": { "category": "bug", "state": "ready-for-agent" }, // triage grading
|
|
40
|
+
"expect_github": { // behavioral assertions on recorded GitHub calls
|
|
41
|
+
"labels_added": ["bug"],
|
|
42
|
+
"labels_absent": ["wontfix"],
|
|
43
|
+
"issue_closed": false,
|
|
44
|
+
"comment_matches": "(?i)thanks",
|
|
45
|
+
"pr_opened": { "base": "main", "head_is_branch": true, "title_matches": "(?i)fix" }
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Every `expect_github` field is optional — only the present ones are checked.
|
|
51
|
+
|
|
52
|
+
## Add a triage case
|
|
53
|
+
|
|
54
|
+
Append a `SweBenchInstance` to `datasets/triage/instances.json` with `issue`,
|
|
55
|
+
`triage_gold`, and the `expect_github` assertions (e.g. `labels_added`). That's
|
|
56
|
+
it — triage is graded on the triage decision + GitHub mutations.
|
|
57
|
+
|
|
58
|
+
## Add a code-fix case (three things, keyed by `instance_id`)
|
|
59
|
+
|
|
60
|
+
1. **`datasets/code-fix/instances.json`** — append the instance with
|
|
61
|
+
`base_commit`, `FAIL_TO_PASS`, `PASS_TO_PASS`, `issue`, and `expect_github`
|
|
62
|
+
(e.g. `pr_opened`).
|
|
63
|
+
2. **`datasets/code-fix/repos/<instance_id>/`** — the fixture repo at the base
|
|
64
|
+
commit (the buggy code *before* the fix; no held-out tests here).
|
|
65
|
+
3. **`datasets/code-fix/tests/<instance_id>/`** — the held-out test files, copied
|
|
66
|
+
into the repo at grade time and run to compute `FAIL_TO_PASS` / `PASS_TO_PASS`.
|
|
67
|
+
|
|
68
|
+
## Add a custom tier
|
|
69
|
+
|
|
70
|
+
Create `datasets/<tier-name>/` with `tier.json` (`name`, `defaultWorkflow`,
|
|
71
|
+
`description`) + `instances.json`. For code-fix-style tiers also add `repos/<id>/`
|
|
72
|
+
and `tests/<id>/`. Discovery auto-finds it — no code change. Run it with
|
|
73
|
+
`lastlight-evals run <tier-name> --overlay .`.
|
|
74
|
+
|
|
75
|
+
## Grading (how a case passes)
|
|
76
|
+
|
|
77
|
+
- **Behavioral:** did the workflow take the expected GitHub actions
|
|
78
|
+
(`expect_github`)?
|
|
79
|
+
- **Triage:** did the decision match `triage_gold`?
|
|
80
|
+
- **Execution (code-fix):** all `FAIL_TO_PASS` green AND all `PASS_TO_PASS` still
|
|
81
|
+
green after applying held-out tests.
|
|
82
|
+
- With `--runs N` (N>1) the binary verdict is **worst-case** (passes only if every
|
|
83
|
+
trial passed); the scorecard also shows per-verdict pass counts to expose
|
|
84
|
+
variance.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Model registry — `models.json`
|
|
2
|
+
|
|
3
|
+
The eval harness picks models from a `models.json`. Resolution: the scaffolded
|
|
4
|
+
`evals/models.json` is picked up automatically (or pass `--models-file <path>`).
|
|
5
|
+
|
|
6
|
+
```json
|
|
7
|
+
{
|
|
8
|
+
"default": "openai/gpt-5.4-mini",
|
|
9
|
+
"compare": [
|
|
10
|
+
{ "id": "openai/gpt-5.5", "label": "GPT-5.5", "provider": "OpenAI", "envKey": "OPENAI_API_KEY" },
|
|
11
|
+
{ "id": "anthropic/claude-opus-4-8", "label": "Claude Opus 4.8","provider": "Anthropic", "envKey": "ANTHROPIC_API_KEY" },
|
|
12
|
+
{ "id": "fireworks/accounts/fireworks/models/glm-5p2", "label": "GLM-5.2", "provider": "Fireworks (open)", "envKey": "FIREWORKS_API_KEY" }
|
|
13
|
+
]
|
|
14
|
+
}
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Fields:
|
|
18
|
+
- **`default`** — the single model `run` uses when not `--compare`.
|
|
19
|
+
- **`compare`** — the cross-vendor set `--compare` runs. **Each entry runs only if
|
|
20
|
+
its `envKey` is present** in the environment, so you can list many and only the
|
|
21
|
+
ones you have keys for execute. Add/remove freely.
|
|
22
|
+
- Per entry: `id` is the agentic-pi/pi-ai `provider/model` spec; `label` is the
|
|
23
|
+
scorecard display name; `provider` is a grouping label; `envKey` is the env var
|
|
24
|
+
that gates it.
|
|
25
|
+
|
|
26
|
+
## Selecting models on the CLI
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
lastlight-evals run triage --model haiku # fuzzy substring match against ids/labels
|
|
30
|
+
lastlight-evals run triage --model openai/gpt-5.5 # exact provider/model id
|
|
31
|
+
lastlight-evals run triage --model glm,deepseek # comma-list
|
|
32
|
+
lastlight-evals run --compare # the whole compare set (key-gated)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
You can also override the set with `EVAL_MODELS=<comma-list>` in the environment.
|
|
36
|
+
|
|
37
|
+
## Provider keys
|
|
38
|
+
|
|
39
|
+
Set whichever provider keys you have in the workspace `.env`:
|
|
40
|
+
`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `FIREWORKS_API_KEY` (GLM / DeepSeek /
|
|
41
|
+
GPT-OSS), `OPENROUTER_API_KEY`, `DEEPSEEK_API_KEY`. No GitHub credentials needed —
|
|
42
|
+
GitHub is mocked.
|