@stage-labs/metro 0.1.0-beta.279 → 0.1.0-beta.280
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/node_modules/@metro-labs/core/src/skill-source.ts +96 -0
- package/runtime/node_modules/@metro-labs/daemon/src/boot/boot.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/api.ts +30 -5
- package/runtime/node_modules/@metro-labs/daemon/src/claude/github-api.ts +12 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/github-fetch.ts +160 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/github-skills.ts +257 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/github-validate.ts +53 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/skills.ts +2 -0
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/package.json +1 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/skill-source.ts +96 -0
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/app.ts +4 -1
- package/runtime/sdk-runner/src/runner.ts +26 -3
- package/runtime/sdk-runner/src/skill-work.ts +76 -0
- package/runtime/sdk-runner/src/skills-files.ts +169 -0
- package/runtime/sdk-runner/src/skills.ts +150 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.280",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
"./automation-types": "./src/automation-types.ts",
|
|
14
14
|
"./call": "./src/call.ts",
|
|
15
15
|
"./secure-fs": "./src/secure-fs.ts",
|
|
16
|
+
"./skill-source": "./src/skill-source.ts",
|
|
16
17
|
"./version": "./src/version.ts",
|
|
17
18
|
"./train-error": "./src/train-error.ts",
|
|
18
19
|
"./trains/protocol": "./src/protocol.ts",
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { isRecord } from './is-record.js';
|
|
3
|
+
|
|
4
|
+
export const SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
5
|
+
export const SKILL_HASH = /^[a-f0-9]{64}$/;
|
|
6
|
+
export const SKILL_COMMIT = /^[a-f0-9]{40}$/;
|
|
7
|
+
export const SKILL_FILE_MAX = 256 * 1024;
|
|
8
|
+
export const SKILL_TOTAL_MAX = 8 * 1024 * 1024;
|
|
9
|
+
export const SKILL_FILES_MAX = 256;
|
|
10
|
+
export const SKILL_MANIFEST_MAX = 128 * 1024;
|
|
11
|
+
|
|
12
|
+
export interface SkillSource {
|
|
13
|
+
repository: string;
|
|
14
|
+
ref: string;
|
|
15
|
+
folder: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface SkillFile {
|
|
19
|
+
path: string;
|
|
20
|
+
size: number;
|
|
21
|
+
sha256: string;
|
|
22
|
+
executable?: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface SkillGeneration {
|
|
26
|
+
id: string;
|
|
27
|
+
source: (SkillSource & { repositoryId: number; commit: string }) | null;
|
|
28
|
+
createdAt: string;
|
|
29
|
+
skills: { name: string; description: string }[];
|
|
30
|
+
files: SkillFile[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface SkillActivation {
|
|
34
|
+
generation: string | null;
|
|
35
|
+
loading: string | null;
|
|
36
|
+
rollback?: string | null;
|
|
37
|
+
appliedAt: string | null;
|
|
38
|
+
problem: string | null;
|
|
39
|
+
shadowed: string[];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export const skillSourceRoot = (claudeDir: string): string => join(claudeDir, 'metro-github');
|
|
43
|
+
export const skillRelease = (root: string, id: string): string => {
|
|
44
|
+
if (!SKILL_HASH.test(id)) throw new Error('Invalid skills generation.');
|
|
45
|
+
return join(root, 'releases', id);
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export function skillPath(path: string): boolean {
|
|
49
|
+
const parts = path.split('/');
|
|
50
|
+
return path.length <= 240 && parts.length >= 2 && parts.length <= 8 && SKILL_NAME.test(parts[0] ?? '') &&
|
|
51
|
+
parts.every((part) => /^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$/.test(part)) &&
|
|
52
|
+
!parts.slice(1, -1).includes('SKILL.md') && !(parts.length > 2 && parts.at(-1) === 'SKILL.md');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function manifestSkill(value: unknown): SkillGeneration['skills'][number] {
|
|
56
|
+
if (!isRecord(value) || typeof value.name !== 'string' || !SKILL_NAME.test(value.name) || typeof value.description !== 'string' || value.description.length > 1024) throw new Error('Invalid skill name or description.');
|
|
57
|
+
return { name: value.name, description: value.description };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function fileSize(value: unknown): number {
|
|
61
|
+
if (typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > SKILL_FILE_MAX) throw new Error('Invalid skills file size.');
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function manifestFile(value: unknown): SkillFile {
|
|
66
|
+
if (!isRecord(value)) throw new Error('Invalid skills file.');
|
|
67
|
+
if (typeof value.path !== 'string' || !skillPath(value.path) || typeof value.sha256 !== 'string' || !SKILL_HASH.test(value.sha256)) throw new Error('Invalid skills file path or hash.');
|
|
68
|
+
return { path: value.path, size: fileSize(value.size), sha256: value.sha256, ...(value.executable === true ? { executable: true } : {}) };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function manifestSource(s: unknown): SkillGeneration['source'] {
|
|
72
|
+
if (s === null) return null;
|
|
73
|
+
if (!isRecord(s) || typeof s.repository !== 'string' || typeof s.ref !== 'string' || typeof s.folder !== 'string' || typeof s.repositoryId !== 'number' || typeof s.commit !== 'string' || !SKILL_COMMIT.test(s.commit)) throw new Error('Invalid skills provenance.');
|
|
74
|
+
return { repository: s.repository, ref: s.ref, folder: s.folder, repositoryId: s.repositoryId, commit: s.commit };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function checkFiles(skills: SkillGeneration['skills'], files: SkillFile[]): void {
|
|
78
|
+
if (files.reduce((sum, file) => sum + file.size, 0) > SKILL_TOTAL_MAX || new Set(files.map((f) => f.path.toLowerCase())).size !== files.length || new Set(skills.map((s) => s.name)).size !== skills.length) throw new Error('Duplicate or oversized skills files.');
|
|
79
|
+
if (files.some((f) => !skills.some((s) => f.path.startsWith(`${s.name}/`))) || skills.some((s) => !files.some((f) => f.path === `${s.name}/SKILL.md`))) throw new Error('Every skill needs its own SKILL.md.');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function boundedGeneration(generation: SkillGeneration): SkillGeneration {
|
|
83
|
+
if (Buffer.byteLength(JSON.stringify(generation)) > SKILL_MANIFEST_MAX) throw new Error('Skills manifest is too large.');
|
|
84
|
+
return generation;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function skillGeneration(raw: unknown): SkillGeneration {
|
|
88
|
+
if (!isRecord(raw) || typeof raw.id !== 'string' || !SKILL_HASH.test(raw.id) || typeof raw.createdAt !== 'string' || !Array.isArray(raw.skills) || !Array.isArray(raw.files)) throw new Error('Invalid skills manifest.');
|
|
89
|
+
if (raw.skills.length > 64 || raw.files.length > SKILL_FILES_MAX) throw new Error('Skills manifest is too large.');
|
|
90
|
+
const skills = raw.skills.map(manifestSkill);
|
|
91
|
+
const files = raw.files.map(manifestFile);
|
|
92
|
+
const source = manifestSource(raw.source);
|
|
93
|
+
checkFiles(skills, files);
|
|
94
|
+
if (source === null) checkFiles([], files);
|
|
95
|
+
return boundedGeneration({ id: raw.id, source, createdAt: raw.createdAt, skills, files });
|
|
96
|
+
}
|
|
@@ -32,6 +32,7 @@ import { followRelease } from '../metro-user/root-follow.js';
|
|
|
32
32
|
import { unwatchSession, watchSession } from '../claude/session.js';
|
|
33
33
|
import { liveEvents, memoryRoutine, tryClaudeSetup } from '../claude/setup.js';
|
|
34
34
|
import { tryMemoryJob } from '../claude/memory-routine.js';
|
|
35
|
+
import { githubSkills } from '../claude/github-skills.js';
|
|
35
36
|
import { applyLocalOwner } from './local-owner.js';
|
|
36
37
|
import { installBearerSessions } from '../routes/bearer.js';
|
|
37
38
|
import { ensureLocalAgent, localOwner } from '../agents/file-admin.js';
|
|
@@ -195,6 +196,7 @@ async function startClaude(): Promise<void> {
|
|
|
195
196
|
log.warn({ err: errMsg(err) }, 'plugin: could not ensure the Claude Code plugin');
|
|
196
197
|
});
|
|
197
198
|
tryClaudeSetup();
|
|
199
|
+
try { githubSkills().watch(); } catch { log.warn('github-skills: saved source could not be loaded'); }
|
|
198
200
|
tryMemoryJob(memoryRoutine());
|
|
199
201
|
watchSession();
|
|
200
202
|
}
|
|
@@ -3,6 +3,8 @@ import { errMsg, log } from '@metro-labs/core/log';
|
|
|
3
3
|
import { apiFailure, apiSession, requireAdmin, cors, readJsonBody, sendJson, type ApiSession } from '@metro-labs/http/api-http';
|
|
4
4
|
import { ApiError } from '@metro-labs/http/api-error';
|
|
5
5
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
6
|
+
import { githubSkillsAnswer } from './github-api.js';
|
|
7
|
+
import { githubSkills, type GitHubSkills } from './github-skills.js';
|
|
6
8
|
import { listClaudeSettings, SETTINGS_MAX, writeClaudeSettings } from './settings.js';
|
|
7
9
|
import {
|
|
8
10
|
createClaudeSkill,
|
|
@@ -60,6 +62,7 @@ export interface ClaudeApiDeps extends SetupApiDeps {
|
|
|
60
62
|
login?: LoginDeps;
|
|
61
63
|
claudeLogins?: ClaudeLoginDeps;
|
|
62
64
|
version?: VersionDeps;
|
|
65
|
+
skillSource?: GitHubSkills;
|
|
63
66
|
}
|
|
64
67
|
|
|
65
68
|
function projectOf(query: URLSearchParams): string {
|
|
@@ -132,7 +135,7 @@ async function created(req: IncomingMessage, path: string, dir: string): Promise
|
|
|
132
135
|
return createClaudeSkill(body.name, typeof body.text === 'string' ? body.text : undefined, dir);
|
|
133
136
|
}
|
|
134
137
|
|
|
135
|
-
const ADMIN_ONLY = /^\/api\/claude\/(login|session|version|setup)(\/|$)/;
|
|
138
|
+
const ADMIN_ONLY = /^\/api\/claude\/(login|session|version|setup|skill-source)(\/|$)/;
|
|
136
139
|
const LOGIN = 'login';
|
|
137
140
|
const SESSION = 'session';
|
|
138
141
|
const SETUP = 'setup';
|
|
@@ -220,15 +223,37 @@ const SINGLETONS: Record<string, (req: IncomingMessage, deps: ClaudeApiDeps, ses
|
|
|
220
223
|
[VERSION]: versionAnswer,
|
|
221
224
|
};
|
|
222
225
|
|
|
226
|
+
function skillsAnswer(req: IncomingMessage, path: string, search: string, dir: string, source: GitHubSkills): unknown {
|
|
227
|
+
const segments = parts(path);
|
|
228
|
+
if (req.method === 'GET' && segments.length === 1) {
|
|
229
|
+
const listing = source.listing();
|
|
230
|
+
return { skills: [...listClaudeSkills(dir), ...listing.rows], skillSource: listing.skillSource };
|
|
231
|
+
}
|
|
232
|
+
const item = decodeURIComponent(segments[1] ?? '');
|
|
233
|
+
if (segments.length === 2 && item.startsWith('github:')) {
|
|
234
|
+
if (req.method !== 'GET') throw new ApiError('GitHub skills are read-only here. Edit the repository or remove its source.', 409);
|
|
235
|
+
return source.read(item);
|
|
236
|
+
}
|
|
237
|
+
return fileAnswer(req, path, search, dir);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const sourceOf = (deps: ClaudeApiDeps): GitHubSkills => deps.skillSource ?? githubSkills();
|
|
241
|
+
|
|
242
|
+
function fileAnswer(req: IncomingMessage, path: string, search: string, dir: string): unknown {
|
|
243
|
+
if (req.method === 'PUT') return writeAnswer(req, path, search, dir);
|
|
244
|
+
if (req.method === 'POST') return created(req, path, dir);
|
|
245
|
+
return answer(req.method ?? 'GET', path, new URLSearchParams(search), dir);
|
|
246
|
+
}
|
|
247
|
+
|
|
223
248
|
function routed(req: IncomingMessage, path: string, search: string, deps: ClaudeApiDeps, session: ApiSession): unknown {
|
|
224
249
|
const dir = (deps.dir ?? claudeDir)();
|
|
225
250
|
const [head = '', item = ''] = parts(path);
|
|
226
251
|
if (head === LOGIN) return loginAnswer(req, item, deps);
|
|
252
|
+
if (head === 'skill-source' && parts(path).length === 1) return githubSkillsAnswer(req, deps.skillSource);
|
|
253
|
+
if (head === 'skills') return skillsAnswer(req, path, search, dir, sourceOf(deps));
|
|
227
254
|
const single = item === '' ? SINGLETONS[head] : undefined;
|
|
228
255
|
if (single !== undefined) return single(req, deps, session);
|
|
229
|
-
|
|
230
|
-
if (req.method === 'POST') return created(req, path, dir);
|
|
231
|
-
return answer(req.method ?? 'GET', path, new URLSearchParams(search), dir);
|
|
256
|
+
return fileAnswer(req, path, search, dir);
|
|
232
257
|
}
|
|
233
258
|
|
|
234
259
|
const TEXT = 'text/plain; charset=utf-8';
|
|
@@ -276,7 +301,7 @@ export function handleClaudeRequest(
|
|
|
276
301
|
sendJson(req, res, 200, sessionSnapshot(deps.session));
|
|
277
302
|
return;
|
|
278
303
|
}
|
|
279
|
-
if (req.method !== 'GET' && ADMIN_ONLY.test(path)) requireAdmin(session);
|
|
304
|
+
if (req.method !== 'GET' && ADMIN_ONLY.test(path.replace(/\/{2,}/g, '/'))) requireAdmin(session);
|
|
280
305
|
if (await streamed(req, res, path, search, (deps.dir ?? claudeDir)())) return;
|
|
281
306
|
sendJson(req, res, 200, await routed(req, path, search, deps, session));
|
|
282
307
|
})
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { IncomingMessage } from 'node:http';
|
|
2
|
+
import { readJsonBody } from '@metro-labs/http/api-http';
|
|
3
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
4
|
+
import { githubSkills, type GitHubSkills } from './github-skills.js';
|
|
5
|
+
|
|
6
|
+
export async function githubSkillsAnswer(req: IncomingMessage, source: GitHubSkills = githubSkills()): Promise<object> {
|
|
7
|
+
if (req.method === 'GET') return source.view();
|
|
8
|
+
if (req.method === 'PUT') return source.configure(await readJsonBody(req, 4096));
|
|
9
|
+
if (req.method === 'DELETE') return source.remove();
|
|
10
|
+
if (req.method === 'POST') { await source.sync(true); return source.view(); }
|
|
11
|
+
throw new ApiError('method not allowed', 405);
|
|
12
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
|
+
import { SKILL_COMMIT, SKILL_FILE_MAX, SKILL_FILES_MAX, SKILL_TOTAL_MAX, skillPath, type SkillSource } from '@metro-labs/core/skill-source';
|
|
4
|
+
|
|
5
|
+
export class GitHubFailure extends Error {
|
|
6
|
+
constructor(message: string, readonly retryAt = Date.now() + 60_000, readonly rateLimited = false) { super(message); }
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface GitHubSnapshot {
|
|
10
|
+
repositoryId: number;
|
|
11
|
+
commit: string;
|
|
12
|
+
etag: string | null;
|
|
13
|
+
files: Map<string, Buffer>;
|
|
14
|
+
executables?: string[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
interface Entry { path: string; mode: string; type: string; sha: string; size: number }
|
|
18
|
+
type Request = (url: string, init: RequestInit) => Promise<Response>;
|
|
19
|
+
const API = 'https://api.github.com/repos/';
|
|
20
|
+
const RESPONSE_MAX = 2 * 1024 * 1024;
|
|
21
|
+
|
|
22
|
+
async function boundedJson(response: Response, text: boolean): Promise<unknown> {
|
|
23
|
+
if (response.body === null) throw new GitHubFailure('GitHub returned an empty response.');
|
|
24
|
+
const reader = response.body.getReader();
|
|
25
|
+
const chunks: Uint8Array[] = [];
|
|
26
|
+
let bytes = 0;
|
|
27
|
+
try {
|
|
28
|
+
for (;;) {
|
|
29
|
+
const { done, value } = await reader.read();
|
|
30
|
+
if (done) break;
|
|
31
|
+
bytes += value.byteLength;
|
|
32
|
+
if (bytes > RESPONSE_MAX) throw new GitHubFailure('GitHub returned too much data. Choose a smaller skills folder.');
|
|
33
|
+
chunks.push(value);
|
|
34
|
+
}
|
|
35
|
+
const body = Buffer.concat(chunks).toString('utf8');
|
|
36
|
+
return text ? body.trim() : JSON.parse(body) as unknown;
|
|
37
|
+
} finally { await reader.cancel().catch(() => undefined); }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function refused(response: Response): GitHubFailure {
|
|
41
|
+
const retry = Number(response.headers.get('retry-after'));
|
|
42
|
+
const reset = Number(response.headers.get('x-ratelimit-reset')) * 1000;
|
|
43
|
+
if (response.status === 429 || (response.status === 403 && (response.headers.get('x-ratelimit-remaining') === '0' || retry > 0))) {
|
|
44
|
+
return new GitHubFailure('GitHub rate limit reached. Sync will retry later.', Math.max(Date.now() + 60_000, Date.now() + (Number.isFinite(retry) ? retry * 1000 : 0), Number.isFinite(reset) ? reset : 0), true);
|
|
45
|
+
}
|
|
46
|
+
if ([401, 403, 404].includes(response.status)) return new GitHubFailure('GitHub access was refused. Check the repository, ref, token expiry and organization approval. The loaded skills are kept.', Date.now() + 15 * 60_000);
|
|
47
|
+
return new GitHubFailure('GitHub could not provide the skills. The loaded skills are kept.');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function headers(token: string, etag: string | null | undefined, shaOnly: boolean): Record<string, string> {
|
|
51
|
+
return { Authorization: `Bearer ${token}`, Accept: shaOnly ? 'application/vnd.github.sha' : 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28', ...(etag ? { 'If-None-Match': etag } : {}) };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export class GitHubReader {
|
|
55
|
+
private readonly base: string;
|
|
56
|
+
private count = 0;
|
|
57
|
+
private readonly signal = AbortSignal.timeout(120_000);
|
|
58
|
+
constructor(private readonly source: SkillSource, private readonly token: string, private readonly request: Request = fetch) {
|
|
59
|
+
this.base = `${API}${source.repository}`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
private async get(path: string, etag?: string | null, shaOnly = false): Promise<{ data: unknown; etag: string | null; unchanged: boolean }> {
|
|
63
|
+
if (++this.count > SKILL_FILES_MAX + 12) throw new GitHubFailure('The skills folder needs too many GitHub requests.');
|
|
64
|
+
let response: Response;
|
|
65
|
+
try {
|
|
66
|
+
response = await this.request(`${this.base}${path}`, {
|
|
67
|
+
headers: headers(this.token, etag, shaOnly),
|
|
68
|
+
redirect: 'manual', signal: AbortSignal.any([this.signal, AbortSignal.timeout(20_000)]),
|
|
69
|
+
});
|
|
70
|
+
} catch { throw new GitHubFailure('GitHub is unreachable. The loaded skills are kept.'); }
|
|
71
|
+
if (response.status === 304) return { data: null, etag: etag ?? null, unchanged: true };
|
|
72
|
+
if (!response.ok) { await response.body?.cancel(); throw refused(response); }
|
|
73
|
+
try { return { data: await boundedJson(response, shaOnly), etag: response.headers.get('etag'), unchanged: false }; }
|
|
74
|
+
catch (err) { if (err instanceof GitHubFailure) throw err; throw new GitHubFailure('GitHub returned invalid skills data.'); }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
private async tree(sha: string, recursive: boolean): Promise<Entry[]> {
|
|
78
|
+
const { data } = await this.get(`/git/trees/${sha}${recursive ? '?recursive=1' : ''}`);
|
|
79
|
+
if (!isRecord(data) || data.truncated !== false || !Array.isArray(data.tree) || data.tree.length > 4096) throw new GitHubFailure('The GitHub tree is incomplete or too large. Choose a smaller skills folder.');
|
|
80
|
+
return data.tree.map((raw: unknown) => {
|
|
81
|
+
if (!isRecord(raw) || typeof raw.path !== 'string' || typeof raw.mode !== 'string' || typeof raw.type !== 'string' || typeof raw.sha !== 'string' || !SKILL_COMMIT.test(raw.sha)) throw new GitHubFailure('GitHub returned an invalid tree.');
|
|
82
|
+
return { path: raw.path, mode: raw.mode, type: raw.type, sha: raw.sha, size: typeof raw.size === 'number' ? raw.size : 0 };
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
private async folder(tree: string): Promise<string> {
|
|
87
|
+
let sha = tree;
|
|
88
|
+
for (const segment of this.source.folder.split('/').filter(Boolean)) {
|
|
89
|
+
const entry = (await this.tree(sha, false)).find((row) => row.path === segment);
|
|
90
|
+
if (entry?.mode !== '040000' || entry.type !== 'tree') throw new GitHubFailure('The selected skills folder is missing or is not a regular directory.');
|
|
91
|
+
sha = entry.sha;
|
|
92
|
+
}
|
|
93
|
+
return sha;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private async blob(entry: Entry): Promise<Buffer> {
|
|
97
|
+
const { data } = await this.get(`/git/blobs/${entry.sha}`);
|
|
98
|
+
if (!isRecord(data) || data.encoding !== 'base64' || typeof data.content !== 'string' || data.size !== entry.size) throw new GitHubFailure('GitHub returned an invalid skill file.');
|
|
99
|
+
const encoded = data.content.replace(/\n/g, '');
|
|
100
|
+
if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded)) throw new GitHubFailure('GitHub returned an invalid skill file.');
|
|
101
|
+
const bytes = Buffer.from(encoded, 'base64');
|
|
102
|
+
const hash = createHash('sha1').update(`blob ${bytes.byteLength}\0`).update(bytes).digest('hex');
|
|
103
|
+
if (bytes.byteLength !== entry.size || hash !== entry.sha) throw new GitHubFailure('A skill file has an unexpected size or Git object hash.');
|
|
104
|
+
return bytes;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private async repository(expected: number | null): Promise<number> {
|
|
108
|
+
const repo = (await this.get('')).data;
|
|
109
|
+
if (!isRecord(repo) || typeof repo.id !== 'number' || typeof repo.full_name !== 'string' || repo.full_name.toLowerCase() !== this.source.repository.toLowerCase() || (expected !== null && repo.id !== expected)) throw new GitHubFailure('The GitHub repository identity changed. Reconnect it explicitly.');
|
|
110
|
+
return repo.id;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async snapshot(repositoryId: number | null, commit: string | null, etag: string | null): Promise<GitHubSnapshot | null> {
|
|
114
|
+
const id = await this.repository(repositoryId);
|
|
115
|
+
const resolved = await this.get(`/commits/${encodeURIComponent(this.source.ref)}`, etag, true);
|
|
116
|
+
if (resolved.unchanged) return null;
|
|
117
|
+
if (typeof resolved.data !== 'string' || !SKILL_COMMIT.test(resolved.data)) throw new GitHubFailure('GitHub did not resolve an immutable commit.');
|
|
118
|
+
if (resolved.data === commit) return null;
|
|
119
|
+
const data = immutableCommit((await this.get(`/git/commits/${resolved.data}`)).data);
|
|
120
|
+
if (data.sha !== resolved.data) throw new GitHubFailure('GitHub returned a different commit.');
|
|
121
|
+
const entries = checkedEntries(await this.tree(await this.folder(data.tree), true));
|
|
122
|
+
const files = await this.blobs(entries);
|
|
123
|
+
return { repositoryId: id, commit: data.sha, etag: resolved.etag, files, executables: entries.filter((entry) => entry.mode === '100755').map((entry) => entry.path) };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private async blobs(entries: Entry[]): Promise<Map<string, Buffer>> {
|
|
127
|
+
const queue = [...entries];
|
|
128
|
+
const files = new Map<string, Buffer>();
|
|
129
|
+
const worker = async (): Promise<void> => {
|
|
130
|
+
for (let entry = queue.shift(); entry !== undefined; entry = queue.shift()) files.set(entry.path, await this.blob(entry));
|
|
131
|
+
};
|
|
132
|
+
try { await Promise.all(Array.from({ length: Math.min(4, queue.length) }, worker)); }
|
|
133
|
+
catch (err) { queue.length = 0; throw err; }
|
|
134
|
+
return files;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function immutableCommit(data: unknown): { sha: string; tree: string } {
|
|
139
|
+
if (!isRecord(data) || typeof data.sha !== 'string' || !SKILL_COMMIT.test(data.sha) || !isRecord(data.tree) || typeof data.tree.sha !== 'string' || !SKILL_COMMIT.test(data.tree.sha)) throw new GitHubFailure('GitHub did not resolve an immutable commit.');
|
|
140
|
+
return { sha: data.sha, tree: data.tree.sha };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function regularFile(entry: Entry): void {
|
|
144
|
+
if (entry.type !== 'blob' || !['100644', '100755'].includes(entry.mode)) throw new GitHubFailure('Symlinks, submodules and special files are not supported in skills.');
|
|
145
|
+
if (!skillPath(entry.path) || entry.size < 0 || entry.size > SKILL_FILE_MAX || !Number.isInteger(entry.size)) throw new GitHubFailure('A skills path or file size is not supported. Use skill-name/SKILL.md and regular resources, up to 256 KiB each.');
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function checkedEntries(entries: Entry[]): Entry[] {
|
|
149
|
+
const files: Entry[] = [];
|
|
150
|
+
let total = 0;
|
|
151
|
+
for (const entry of entries) {
|
|
152
|
+
if (entry.mode === '040000' && entry.type === 'tree') continue;
|
|
153
|
+
regularFile(entry);
|
|
154
|
+
total += entry.size;
|
|
155
|
+
files.push(entry);
|
|
156
|
+
}
|
|
157
|
+
if (files.length > SKILL_FILES_MAX || total > SKILL_TOTAL_MAX) throw new GitHubFailure('Skills exceed 256 files or 8 MiB.');
|
|
158
|
+
if (new Set(files.map((entry) => entry.path.toLowerCase())).size !== files.length) throw new GitHubFailure('Skills contain duplicate file paths.');
|
|
159
|
+
return files;
|
|
160
|
+
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { existsSync, readFileSync as readLocal, statSync as statLocal } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { Readable } from 'node:stream';
|
|
4
|
+
import { log } from '@metro-labs/core/log';
|
|
5
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
6
|
+
import { writeSecure } from '@metro-labs/core/secure-fs';
|
|
7
|
+
import { SKILL_HASH, SKILL_MANIFEST_MAX, skillGeneration, skillRelease, skillSourceRoot, type SkillGeneration } from '@metro-labs/core/skill-source';
|
|
8
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
9
|
+
import { readFileSync, statSync } from '../agent-user/agent-fs.js';
|
|
10
|
+
import { receiveHomeFile, writeHomeText } from '../agent-user/home-fs.js';
|
|
11
|
+
import { agentsDir } from '../agents/files.js';
|
|
12
|
+
import { claudeDir } from './files.js';
|
|
13
|
+
import { readSetupState } from './setup-state.js';
|
|
14
|
+
import { GitHubFailure, GitHubReader } from './github-fetch.js';
|
|
15
|
+
import { githubSource, validatedGeneration } from './github-validate.js';
|
|
16
|
+
import type { ClaudeSkill } from './skills.js';
|
|
17
|
+
|
|
18
|
+
type Configuration = ReturnType<typeof githubSource>;
|
|
19
|
+
interface Stored {
|
|
20
|
+
source: Configuration | null;
|
|
21
|
+
prepared: SkillGeneration | null;
|
|
22
|
+
etag: string | null;
|
|
23
|
+
checkedAt: string | null;
|
|
24
|
+
retryAt: number;
|
|
25
|
+
problem: string | null;
|
|
26
|
+
rateLimit: number;
|
|
27
|
+
repositoryId: number | null;
|
|
28
|
+
}
|
|
29
|
+
export interface GitHubSkillsDeps {
|
|
30
|
+
agents: string;
|
|
31
|
+
claude: string;
|
|
32
|
+
sdk(): boolean;
|
|
33
|
+
request?: (url: string, init: RequestInit) => Promise<Response>;
|
|
34
|
+
}
|
|
35
|
+
const EMPTY: Stored = { source: null, prepared: null, etag: null, checkedAt: null, retryAt: 0, problem: null, rateLimit: 0, repositoryId: null };
|
|
36
|
+
const savedString = (value: unknown): string | null => typeof value === 'string' ? value : null;
|
|
37
|
+
const savedNumber = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
|
38
|
+
|
|
39
|
+
function readStored(path: string): Stored {
|
|
40
|
+
if (!existsSync(path)) return { ...EMPTY };
|
|
41
|
+
try {
|
|
42
|
+
if (statLocal(path).size > SKILL_MANIFEST_MAX * 2) throw new Error('size');
|
|
43
|
+
const raw: unknown = JSON.parse(readLocal(path, 'utf8'));
|
|
44
|
+
if (!isRecord(raw)) throw new Error('shape');
|
|
45
|
+
return {
|
|
46
|
+
source: raw.source === null ? null : githubSource(raw.source),
|
|
47
|
+
prepared: raw.prepared === null ? null : skillGeneration(raw.prepared),
|
|
48
|
+
etag: savedString(raw.etag), checkedAt: savedString(raw.checkedAt),
|
|
49
|
+
retryAt: savedNumber(raw.retryAt), rateLimit: savedNumber(raw.rateLimit), problem: savedString(raw.problem),
|
|
50
|
+
repositoryId: typeof raw.repositoryId === 'number' ? raw.repositoryId : null,
|
|
51
|
+
};
|
|
52
|
+
} catch { throw new Error('The saved GitHub skills source is unreadable. Restore it before changing the source.'); }
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function agentJson(path: string): unknown {
|
|
56
|
+
try {
|
|
57
|
+
if (statSync(path).size > SKILL_MANIFEST_MAX) return null;
|
|
58
|
+
return JSON.parse(readFileSync(path, 'utf8')) as unknown;
|
|
59
|
+
} catch { return null; }
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export class GitHubSkills {
|
|
63
|
+
private readonly path: string;
|
|
64
|
+
readonly root: string;
|
|
65
|
+
private stored: Stored = { ...EMPTY };
|
|
66
|
+
private unreadable = false;
|
|
67
|
+
private busy = false;
|
|
68
|
+
private epoch = 0;
|
|
69
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
70
|
+
|
|
71
|
+
constructor(private readonly deps: GitHubSkillsDeps) {
|
|
72
|
+
this.path = join(deps.agents, 'github-skills.json');
|
|
73
|
+
this.root = skillSourceRoot(deps.claude);
|
|
74
|
+
try { this.stored = readStored(this.path); }
|
|
75
|
+
catch { this.unreadable = true; }
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private readable(): void {
|
|
79
|
+
if (!this.unreadable) return;
|
|
80
|
+
this.stored = readStored(this.path);
|
|
81
|
+
this.unreadable = false;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
private save(): void {
|
|
85
|
+
this.readable();
|
|
86
|
+
writeSecure(this.path, JSON.stringify(this.stored));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private activation(): { generation: string | null; appliedAt: string | null; problem: string | null; shadowed: string[] } {
|
|
90
|
+
const raw = agentJson(join(this.root, 'status.json'));
|
|
91
|
+
if (!isRecord(raw)) return { generation: null, appliedAt: null, problem: null, shadowed: [] };
|
|
92
|
+
return {
|
|
93
|
+
generation: typeof raw.generation === 'string' && SKILL_HASH.test(raw.generation) ? raw.generation : null,
|
|
94
|
+
appliedAt: typeof raw.appliedAt === 'string' ? raw.appliedAt : null,
|
|
95
|
+
problem: typeof raw.problem === 'string' ? raw.problem.slice(0, 300) : null,
|
|
96
|
+
shadowed: Array.isArray(raw.shadowed) ? raw.shadowed.filter((name): name is string => typeof name === 'string').slice(0, 64) : [],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private loaded(activation = this.activation()): SkillGeneration | null {
|
|
101
|
+
const id = activation.generation;
|
|
102
|
+
if (id === null) return null;
|
|
103
|
+
try { return skillGeneration(agentJson(join(skillRelease(this.root, id), 'manifest.json'))); }
|
|
104
|
+
catch { return null; }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
view(activation = this.activation(), loaded = this.loaded(activation)): object {
|
|
108
|
+
const { source, prepared, checkedAt, problem } = this.stored;
|
|
109
|
+
return {
|
|
110
|
+
supported: this.deps.sdk(),
|
|
111
|
+
source: source === null ? null : { repository: source.repository, ref: source.ref, folder: source.folder, hasToken: true },
|
|
112
|
+
checking: this.busy, checkedAt, problem: this.unreadable ? 'The saved GitHub source is unreadable. Restore its configuration before changing it. Local skills are still available.' : problem,
|
|
113
|
+
prepared: prepared === null ? null : { generation: prepared.id, source: prepared.source, skillCount: prepared.skills.length },
|
|
114
|
+
loaded: loaded?.source ?? null, activation,
|
|
115
|
+
pending: prepared !== null && prepared.id !== activation.generation,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
listing(): { rows: ClaudeSkill[]; skillSource: object } {
|
|
120
|
+
const activation = this.activation();
|
|
121
|
+
const loaded = this.loaded(activation);
|
|
122
|
+
return { rows: this.rows(activation, loaded), skillSource: this.view(activation, loaded) };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
rows(activation = this.activation(), loaded = this.loaded(activation)): ClaudeSkill[] {
|
|
126
|
+
if (loaded === null) return [];
|
|
127
|
+
const shadowed = activation.shadowed;
|
|
128
|
+
return loaded.skills.map((skill) => ({
|
|
129
|
+
id: `github:${skill.name}`, name: skill.name, title: skill.name, description: skill.description,
|
|
130
|
+
path: join(skillRelease(this.root, loaded.id), 'skills', skill.name, 'SKILL.md'), editable: false,
|
|
131
|
+
updatedAt: loaded.createdAt, managed: true, shadowed: shadowed.includes(skill.name),
|
|
132
|
+
}));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
read(id: string): ClaudeSkill & { text: string } {
|
|
136
|
+
const row = this.rows().find((skill) => skill.id === id);
|
|
137
|
+
if (row === undefined) throw new ApiError('no such skill', 404);
|
|
138
|
+
if (statSync(row.path).size > 256 * 1024) throw new ApiError('The managed skill changed on disk. Sync it again.', 409);
|
|
139
|
+
return { ...row, text: readFileSync(row.path, 'utf8') };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async configure(raw: unknown): Promise<object> {
|
|
143
|
+
if (!this.deps.sdk()) throw new ApiError('GitHub skills require the Agent SDK runner. Local skills are unchanged.', 409);
|
|
144
|
+
if (this.busy) throw new ApiError('A skills sync is already running. Try again when it finishes.', 409);
|
|
145
|
+
const source = githubSource(raw);
|
|
146
|
+
this.readable();
|
|
147
|
+
const loaded = this.loaded();
|
|
148
|
+
this.epoch += 1;
|
|
149
|
+
this.stored = { ...this.stored, source, prepared: loaded, etag: null, retryAt: 0, problem: null, repositoryId: null };
|
|
150
|
+
if (loaded === null) await this.publish(validatedGeneration(null, null), new Map(), this.epoch);
|
|
151
|
+
else { this.save(); this.offer(); }
|
|
152
|
+
await this.sync(true);
|
|
153
|
+
return this.view();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async remove(): Promise<object> {
|
|
157
|
+
this.readable();
|
|
158
|
+
this.epoch += 1;
|
|
159
|
+
this.stored = { ...EMPTY };
|
|
160
|
+
this.save();
|
|
161
|
+
await this.publish(validatedGeneration(null, null), new Map(), this.epoch);
|
|
162
|
+
return this.view();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private waiting(force: boolean): boolean {
|
|
166
|
+
if (Date.now() < this.stored.rateLimit) return true;
|
|
167
|
+
if (force) return false;
|
|
168
|
+
return Date.now() < this.stored.retryAt || (this.stored.prepared !== null && this.stored.prepared.id !== this.activation().generation);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private reader(source: Configuration, known: SkillGeneration['source'], force: boolean): ReturnType<GitHubReader['snapshot']> {
|
|
172
|
+
return new GitHubReader(source, source.token, this.deps.request).snapshot(this.stored.repositoryId, force ? null : known?.commit ?? null, force ? null : this.stored.etag);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
private async fetch(source: Configuration, epoch: number, force: boolean): Promise<void> {
|
|
176
|
+
const previous = this.stored.prepared?.source;
|
|
177
|
+
const same = previous?.repository === source.repository && previous.ref === source.ref && previous.folder === source.folder;
|
|
178
|
+
const known = same ? previous : null;
|
|
179
|
+
const snapshot = await this.reader(source, known, force);
|
|
180
|
+
if (epoch !== this.epoch) return;
|
|
181
|
+
if (snapshot !== null) {
|
|
182
|
+
const generation = validatedGeneration(source, snapshot);
|
|
183
|
+
this.stored.etag = snapshot.etag;
|
|
184
|
+
this.stored.repositoryId = snapshot.repositoryId;
|
|
185
|
+
await this.publish(generation, snapshot.files, epoch);
|
|
186
|
+
if (epoch !== this.epoch) return;
|
|
187
|
+
}
|
|
188
|
+
this.stored.checkedAt = new Date().toISOString();
|
|
189
|
+
this.stored.problem = null;
|
|
190
|
+
this.stored.retryAt = Date.now() + 60_000;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async sync(force = false): Promise<void> {
|
|
194
|
+
if (force && !this.deps.sdk()) throw new ApiError('GitHub skills require the Agent SDK runner. Local skills are unchanged.', 409);
|
|
195
|
+
this.readable();
|
|
196
|
+
this.offer();
|
|
197
|
+
const source = this.stored.source;
|
|
198
|
+
if (source === null || this.busy || !this.deps.sdk() || this.waiting(force)) return;
|
|
199
|
+
const epoch = this.epoch;
|
|
200
|
+
this.busy = true;
|
|
201
|
+
try { await this.fetch(source, epoch, force); }
|
|
202
|
+
catch (err) {
|
|
203
|
+
if (epoch !== this.epoch) return;
|
|
204
|
+
this.failure(err);
|
|
205
|
+
} finally {
|
|
206
|
+
this.busy = false;
|
|
207
|
+
this.save();
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
private failure(err: unknown): void {
|
|
212
|
+
this.stored.problem = err instanceof GitHubFailure ? err.message : 'The repository failed skill validation or staging. Keep one folder per skill, safe metadata and regular files. The loaded skills are kept.';
|
|
213
|
+
this.stored.retryAt = err instanceof GitHubFailure ? err.retryAt : Date.now() + 5 * 60_000;
|
|
214
|
+
if (err instanceof GitHubFailure && err.rateLimited) this.stored.rateLimit = err.retryAt;
|
|
215
|
+
this.stored.checkedAt = new Date().toISOString();
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
private offer(): void {
|
|
219
|
+
const generation = this.stored.prepared;
|
|
220
|
+
if (generation === null) return;
|
|
221
|
+
const pending = agentJson(join(this.root, 'pending.json'));
|
|
222
|
+
if (isRecord(pending) && pending.id === generation.id) return;
|
|
223
|
+
writeHomeText(join(this.root, 'pending.json'), JSON.stringify(generation), 0o600);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
private async publish(generation: SkillGeneration, files: Map<string, Buffer>, epoch: number): Promise<void> {
|
|
227
|
+
const release = skillRelease(this.root, generation.id);
|
|
228
|
+
const deadline = Date.now() + 120_000;
|
|
229
|
+
for (const [path, bytes] of files) {
|
|
230
|
+
if (epoch !== this.epoch) return;
|
|
231
|
+
if (Date.now() > deadline) throw new Error('Staging skills timed out.');
|
|
232
|
+
const mode = generation.files.find((file) => file.path === path)?.executable === true ? 0o700 : 0o600;
|
|
233
|
+
await receiveHomeFile(join(release, 'skills', path), Readable.from(bytes), mode);
|
|
234
|
+
}
|
|
235
|
+
if (epoch !== this.epoch) return;
|
|
236
|
+
writeHomeText(join(release, 'manifest.json'), JSON.stringify(generation), 0o600);
|
|
237
|
+
this.stored.prepared = generation;
|
|
238
|
+
this.save();
|
|
239
|
+
this.offer();
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
watch(): void {
|
|
243
|
+
if (this.timer !== null) return;
|
|
244
|
+
const tick = (): void => {
|
|
245
|
+
this.sync().catch(() => { log.warn('github-skills: could not save sync state'); });
|
|
246
|
+
this.timer = setTimeout(tick, 60_000 + Math.floor(Math.random() * 5_000));
|
|
247
|
+
this.timer.unref();
|
|
248
|
+
};
|
|
249
|
+
tick();
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
let current: GitHubSkills | null = null;
|
|
254
|
+
export function githubSkills(): GitHubSkills {
|
|
255
|
+
current ??= new GitHubSkills({ agents: agentsDir(), claude: claudeDir(), sdk: () => readSetupState(agentsDir()).runner === 'sdk' });
|
|
256
|
+
return current;
|
|
257
|
+
}
|