@skilljit/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.d.ts +18 -0
- package/dist/catalog.js +234 -0
- package/dist/catalog.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/github.d.ts +16 -0
- package/dist/ingest/github.js +56 -0
- package/dist/ingest/github.js.map +1 -0
- package/dist/ingest/parse.d.ts +17 -0
- package/dist/ingest/parse.js +49 -0
- package/dist/ingest/parse.js.map +1 -0
- package/dist/ingest/sources.d.ts +11 -0
- package/dist/ingest/sources.js +14 -0
- package/dist/ingest/sources.js.map +1 -0
- package/dist/paths.d.ts +2 -0
- package/dist/paths.js +8 -0
- package/dist/paths.js.map +1 -0
- package/dist/tokens.d.ts +55 -0
- package/dist/tokens.js +73 -0
- package/dist/tokens.js.map +1 -0
- package/dist/types.d.ts +37 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +30 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { SkillRecord, SkillSearchHit, ToolRecord, ToolSearchHit } from "./types.js";
|
|
2
|
+
export declare class Catalog {
|
|
3
|
+
private db;
|
|
4
|
+
constructor(dbPath: string);
|
|
5
|
+
close(): void;
|
|
6
|
+
upsertSkills(skills: SkillRecord[]): void;
|
|
7
|
+
getSkill(id: string): SkillRecord | undefined;
|
|
8
|
+
searchSkills(query: string, limit?: number): SkillSearchHit[];
|
|
9
|
+
count(): number;
|
|
10
|
+
/** Name + description for every cataloged skill — used to compute the
|
|
11
|
+
* "what this would have cost every turn" baseline without paying to
|
|
12
|
+
* load every skill's full body. */
|
|
13
|
+
listSkillMeta(): Pick<SkillRecord, "name" | "description">[];
|
|
14
|
+
upsertTools(tools: ToolRecord[]): void;
|
|
15
|
+
searchTools(query: string, limit?: number): ToolSearchHit[];
|
|
16
|
+
removeToolsForServer(server: string): void;
|
|
17
|
+
toolCount(): number;
|
|
18
|
+
}
|
package/dist/catalog.js
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
const SCHEMA = `
|
|
5
|
+
CREATE TABLE IF NOT EXISTS skills (
|
|
6
|
+
id TEXT PRIMARY KEY,
|
|
7
|
+
name TEXT NOT NULL,
|
|
8
|
+
source TEXT NOT NULL,
|
|
9
|
+
description TEXT NOT NULL,
|
|
10
|
+
body TEXT NOT NULL,
|
|
11
|
+
install_count INTEGER,
|
|
12
|
+
audit_status TEXT,
|
|
13
|
+
updated_at TEXT NOT NULL
|
|
14
|
+
);
|
|
15
|
+
|
|
16
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
|
|
17
|
+
id UNINDEXED,
|
|
18
|
+
name,
|
|
19
|
+
description,
|
|
20
|
+
tokenize = 'porter unicode61'
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS tools (
|
|
24
|
+
id TEXT PRIMARY KEY,
|
|
25
|
+
server TEXT NOT NULL,
|
|
26
|
+
name TEXT NOT NULL,
|
|
27
|
+
description TEXT NOT NULL,
|
|
28
|
+
input_schema TEXT NOT NULL,
|
|
29
|
+
updated_at TEXT NOT NULL
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5(
|
|
33
|
+
id UNINDEXED,
|
|
34
|
+
name,
|
|
35
|
+
description,
|
|
36
|
+
tokenize = 'porter unicode61'
|
|
37
|
+
);
|
|
38
|
+
`;
|
|
39
|
+
/**
|
|
40
|
+
* Turn a free-text query into a permissive FTS5 MATCH expression:
|
|
41
|
+
* each alphanumeric token, quoted (so hyphens/punctuation in the raw
|
|
42
|
+
* query can't break MATCH syntax) and OR'd together. OR-recall is
|
|
43
|
+
* intentional: skill_find/tool_find return cheap candidates, and the
|
|
44
|
+
* caller (Claude) can re-query in different words if the first list
|
|
45
|
+
* doesn't have what it needs — that beats forcing one-shot precision
|
|
46
|
+
* out of a token search.
|
|
47
|
+
*/
|
|
48
|
+
function toFtsQuery(query) {
|
|
49
|
+
const tokens = query
|
|
50
|
+
.toLowerCase()
|
|
51
|
+
.match(/[a-z0-9]+/g)
|
|
52
|
+
?.filter((t) => t.length > 0);
|
|
53
|
+
if (!tokens || tokens.length === 0)
|
|
54
|
+
return null;
|
|
55
|
+
return tokens.map((t) => `"${t}"`).join(" OR ");
|
|
56
|
+
}
|
|
57
|
+
export class Catalog {
|
|
58
|
+
db;
|
|
59
|
+
constructor(dbPath) {
|
|
60
|
+
if (dbPath !== ":memory:") {
|
|
61
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
62
|
+
}
|
|
63
|
+
this.db = new Database(dbPath);
|
|
64
|
+
this.db.pragma("journal_mode = WAL");
|
|
65
|
+
this.db.exec(SCHEMA);
|
|
66
|
+
}
|
|
67
|
+
close() {
|
|
68
|
+
this.db.close();
|
|
69
|
+
}
|
|
70
|
+
// ---- Skills ----------------------------------------------------------
|
|
71
|
+
upsertSkills(skills) {
|
|
72
|
+
const upsert = this.db.prepare(`
|
|
73
|
+
INSERT INTO skills (id, name, source, description, body, install_count, audit_status, updated_at)
|
|
74
|
+
VALUES (@id, @name, @source, @description, @body, @installCount, @auditStatus, @updatedAt)
|
|
75
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
76
|
+
name = excluded.name,
|
|
77
|
+
source = excluded.source,
|
|
78
|
+
description = excluded.description,
|
|
79
|
+
body = excluded.body,
|
|
80
|
+
install_count = excluded.install_count,
|
|
81
|
+
audit_status = excluded.audit_status,
|
|
82
|
+
updated_at = excluded.updated_at
|
|
83
|
+
`);
|
|
84
|
+
const deleteFts = this.db.prepare(`DELETE FROM skills_fts WHERE id = ?`);
|
|
85
|
+
const insertFts = this.db.prepare(`INSERT INTO skills_fts (id, name, description) VALUES (?, ?, ?)`);
|
|
86
|
+
const tx = this.db.transaction((rows) => {
|
|
87
|
+
for (const s of rows) {
|
|
88
|
+
upsert.run({
|
|
89
|
+
id: s.id,
|
|
90
|
+
name: s.name,
|
|
91
|
+
source: s.source,
|
|
92
|
+
description: s.description,
|
|
93
|
+
body: s.body,
|
|
94
|
+
installCount: s.installCount ?? null,
|
|
95
|
+
auditStatus: s.auditStatus ?? "unaudited",
|
|
96
|
+
updatedAt: s.updatedAt,
|
|
97
|
+
});
|
|
98
|
+
deleteFts.run(s.id);
|
|
99
|
+
insertFts.run(s.id, s.name, s.description);
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
tx(skills);
|
|
103
|
+
}
|
|
104
|
+
getSkill(id) {
|
|
105
|
+
const row = this.db.prepare(`SELECT * FROM skills WHERE id = ?`).get(id);
|
|
106
|
+
if (!row)
|
|
107
|
+
return undefined;
|
|
108
|
+
return rowToSkill(row);
|
|
109
|
+
}
|
|
110
|
+
searchSkills(query, limit = 8) {
|
|
111
|
+
const ftsQuery = toFtsQuery(query);
|
|
112
|
+
if (!ftsQuery)
|
|
113
|
+
return [];
|
|
114
|
+
const rows = this.db
|
|
115
|
+
.prepare(`
|
|
116
|
+
SELECT s.id, s.name, s.source, s.description, s.install_count, s.audit_status, s.updated_at,
|
|
117
|
+
bm25(skills_fts) AS rank
|
|
118
|
+
FROM skills_fts
|
|
119
|
+
JOIN skills s ON s.id = skills_fts.id
|
|
120
|
+
WHERE skills_fts MATCH ?
|
|
121
|
+
ORDER BY rank
|
|
122
|
+
LIMIT ?
|
|
123
|
+
`)
|
|
124
|
+
.all(ftsQuery, limit);
|
|
125
|
+
return rows.map((row) => ({
|
|
126
|
+
skill: {
|
|
127
|
+
id: row.id,
|
|
128
|
+
name: row.name,
|
|
129
|
+
source: row.source,
|
|
130
|
+
description: row.description,
|
|
131
|
+
installCount: row.install_count ?? undefined,
|
|
132
|
+
auditStatus: row.audit_status ?? "unaudited",
|
|
133
|
+
updatedAt: row.updated_at,
|
|
134
|
+
},
|
|
135
|
+
rank: row.rank,
|
|
136
|
+
}));
|
|
137
|
+
}
|
|
138
|
+
count() {
|
|
139
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM skills`).get();
|
|
140
|
+
return row.c;
|
|
141
|
+
}
|
|
142
|
+
/** Name + description for every cataloged skill — used to compute the
|
|
143
|
+
* "what this would have cost every turn" baseline without paying to
|
|
144
|
+
* load every skill's full body. */
|
|
145
|
+
listSkillMeta() {
|
|
146
|
+
return this.db.prepare(`SELECT name, description FROM skills`).all();
|
|
147
|
+
}
|
|
148
|
+
// ---- Tools -------------------------------------------------------------
|
|
149
|
+
upsertTools(tools) {
|
|
150
|
+
const upsert = this.db.prepare(`
|
|
151
|
+
INSERT INTO tools (id, server, name, description, input_schema, updated_at)
|
|
152
|
+
VALUES (@id, @server, @name, @description, @inputSchema, @updatedAt)
|
|
153
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
154
|
+
server = excluded.server,
|
|
155
|
+
name = excluded.name,
|
|
156
|
+
description = excluded.description,
|
|
157
|
+
input_schema = excluded.input_schema,
|
|
158
|
+
updated_at = excluded.updated_at
|
|
159
|
+
`);
|
|
160
|
+
const deleteFts = this.db.prepare(`DELETE FROM tools_fts WHERE id = ?`);
|
|
161
|
+
const insertFts = this.db.prepare(`INSERT INTO tools_fts (id, name, description) VALUES (?, ?, ?)`);
|
|
162
|
+
const tx = this.db.transaction((rows) => {
|
|
163
|
+
for (const t of rows) {
|
|
164
|
+
upsert.run({
|
|
165
|
+
id: t.id,
|
|
166
|
+
server: t.server,
|
|
167
|
+
name: t.name,
|
|
168
|
+
description: t.description,
|
|
169
|
+
inputSchema: JSON.stringify(t.inputSchema),
|
|
170
|
+
updatedAt: t.updatedAt,
|
|
171
|
+
});
|
|
172
|
+
deleteFts.run(t.id);
|
|
173
|
+
insertFts.run(t.id, t.name, t.description);
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
tx(tools);
|
|
177
|
+
}
|
|
178
|
+
searchTools(query, limit = 8) {
|
|
179
|
+
const ftsQuery = toFtsQuery(query);
|
|
180
|
+
if (!ftsQuery)
|
|
181
|
+
return [];
|
|
182
|
+
const rows = this.db
|
|
183
|
+
.prepare(`
|
|
184
|
+
SELECT t.id, t.server, t.name, t.description, t.input_schema, t.updated_at,
|
|
185
|
+
bm25(tools_fts) AS rank
|
|
186
|
+
FROM tools_fts
|
|
187
|
+
JOIN tools t ON t.id = tools_fts.id
|
|
188
|
+
WHERE tools_fts MATCH ?
|
|
189
|
+
ORDER BY rank
|
|
190
|
+
LIMIT ?
|
|
191
|
+
`)
|
|
192
|
+
.all(ftsQuery, limit);
|
|
193
|
+
return rows.map((row) => ({
|
|
194
|
+
tool: {
|
|
195
|
+
id: row.id,
|
|
196
|
+
server: row.server,
|
|
197
|
+
name: row.name,
|
|
198
|
+
description: row.description,
|
|
199
|
+
inputSchema: JSON.parse(row.input_schema),
|
|
200
|
+
updatedAt: row.updated_at,
|
|
201
|
+
},
|
|
202
|
+
rank: row.rank,
|
|
203
|
+
}));
|
|
204
|
+
}
|
|
205
|
+
removeToolsForServer(server) {
|
|
206
|
+
const ids = this.db.prepare(`SELECT id FROM tools WHERE server = ?`).all(server);
|
|
207
|
+
const deleteTool = this.db.prepare(`DELETE FROM tools WHERE id = ?`);
|
|
208
|
+
const deleteFts = this.db.prepare(`DELETE FROM tools_fts WHERE id = ?`);
|
|
209
|
+
const tx = this.db.transaction(() => {
|
|
210
|
+
for (const { id } of ids) {
|
|
211
|
+
deleteTool.run(id);
|
|
212
|
+
deleteFts.run(id);
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
tx();
|
|
216
|
+
}
|
|
217
|
+
toolCount() {
|
|
218
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM tools`).get();
|
|
219
|
+
return row.c;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
function rowToSkill(row) {
|
|
223
|
+
return {
|
|
224
|
+
id: row.id,
|
|
225
|
+
name: row.name,
|
|
226
|
+
source: row.source,
|
|
227
|
+
description: row.description,
|
|
228
|
+
body: row.body,
|
|
229
|
+
installCount: row.install_count ?? undefined,
|
|
230
|
+
auditStatus: row.audit_status ?? "unaudited",
|
|
231
|
+
updatedAt: row.updated_at,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=catalog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../src/catalog.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCd,CAAC;AAEF;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,MAAM,GAAG,KAAK;SACjB,WAAW,EAAE;SACb,KAAK,CAAC,YAAY,CAAC;QACpB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,OAAO,OAAO;IACV,EAAE,CAAoB;IAE9B,YAAY,MAAc;QACxB,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YAC1B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,yEAAyE;IAEzE,YAAY,CAAC,MAAqB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;KAW9B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iEAAiE,CAAC,CAAC;QAErG,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAmB,EAAE,EAAE;YACrD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,CAAC;oBACT,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,WAAW,EAAE,CAAC,CAAC,WAAW;oBAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI;oBACpC,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,WAAW;oBACzC,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;gBACH,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,MAAM,CAAC,CAAC;IACb,CAAC;IAED,QAAQ,CAAC,EAAU;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAQ,CAAC;QAChF,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,KAAK,GAAG,CAAC;QACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;;;;;OAQD,CACA;aACA,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE;gBACL,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;gBAC5C,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,WAAW;gBAC5C,SAAS,EAAE,GAAG,CAAC,UAAU;aAC1B;YACD,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;QACvF,OAAO,GAAG,CAAC,CAAC,CAAC;IACf,CAAC;IAED;;uCAEmC;IACnC,aAAa;QACX,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAG/D,CAAC;IACN,CAAC;IAED,2EAA2E;IAE3E,WAAW,CAAC,KAAmB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;KAS9B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC;QAEpG,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAkB,EAAE,EAAE;YACpD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,CAAC;oBACT,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;oBAC1B,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;oBAC1C,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;gBACH,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,CAAC;IACZ,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,KAAK,GAAG,CAAC;QAClC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;;;;;OAQD,CACA;aACA,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,IAAI,EAAE;gBACJ,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC;gBACzC,SAAS,EAAE,GAAG,CAAC,UAAU;aAC1B;YACD,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,oBAAoB,CAAC,MAAc;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAqB,CAAC;QACrG,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;QACrE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;QACxE,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClC,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACnB,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,EAAE,CAAC;IACP,CAAC;IAED,SAAS;QACP,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAAmB,CAAC;QACtF,OAAO,GAAG,CAAC,CAAC,CAAC;IACf,CAAC;CACF;AAED,SAAS,UAAU,CAAC,GAAQ;IAC1B,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;QAC5C,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,WAAW;QAC5C,SAAS,EAAE,GAAG,CAAC,UAAU;KAC1B,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { Catalog } from "./catalog.js";
|
|
2
|
+
export { countTokens, estimateSkillMetadataTokens, TokenLedger } from "./tokens.js";
|
|
3
|
+
export type { TokenStats } from "./tokens.js";
|
|
4
|
+
export type { SkillRecord, SkillSearchHit, ToolRecord, ToolSearchHit } from "./types.js";
|
|
5
|
+
export { defaultCatalogPath } from "./paths.js";
|
|
6
|
+
export { parseSkillMd } from "./ingest/parse.js";
|
|
7
|
+
export { ingestGithubRepo } from "./ingest/github.js";
|
|
8
|
+
export type { GithubIngestOptions } from "./ingest/github.js";
|
|
9
|
+
export { DEFAULT_GITHUB_SOURCES } from "./ingest/sources.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { Catalog } from "./catalog.js";
|
|
2
|
+
export { countTokens, estimateSkillMetadataTokens, TokenLedger } from "./tokens.js";
|
|
3
|
+
export { defaultCatalogPath } from "./paths.js";
|
|
4
|
+
export { parseSkillMd } from "./ingest/parse.js";
|
|
5
|
+
export { ingestGithubRepo } from "./ingest/github.js";
|
|
6
|
+
export { DEFAULT_GITHUB_SOURCES } from "./ingest/sources.js";
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGpF,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { SkillRecord } from "../types.js";
|
|
2
|
+
export interface GithubIngestOptions {
|
|
3
|
+
/** Injectable for testing; defaults to the global fetch. */
|
|
4
|
+
fetchImpl?: typeof fetch;
|
|
5
|
+
/** Personal access token (or SKILLJIT_GITHUB_TOKEN env var) for the 5000/hr rate limit. */
|
|
6
|
+
token?: string;
|
|
7
|
+
/** Override branch/ref instead of the repo's default branch. */
|
|
8
|
+
ref?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Ingest every SKILL.md found in a GitHub repo, via the git trees API
|
|
12
|
+
* (one recursive call, no per-directory listing) and raw.githubusercontent.com
|
|
13
|
+
* for content — both public, unauthenticated-friendly endpoints, so this
|
|
14
|
+
* scales to skilljit's default repo list without needing a GitHub App.
|
|
15
|
+
*/
|
|
16
|
+
export declare function ingestGithubRepo(owner: string, repo: string, opts?: GithubIngestOptions): Promise<SkillRecord[]>;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { parseSkillMd } from "./parse.js";
|
|
2
|
+
async function githubApiFetch(url, opts) {
|
|
3
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
4
|
+
const token = opts.token ?? process.env.SKILLJIT_GITHUB_TOKEN;
|
|
5
|
+
const res = await fetchImpl(url, {
|
|
6
|
+
headers: {
|
|
7
|
+
Accept: "application/vnd.github+json",
|
|
8
|
+
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
|
9
|
+
},
|
|
10
|
+
});
|
|
11
|
+
if (res.status === 403 || res.status === 429) {
|
|
12
|
+
const body = await res.text().catch(() => "");
|
|
13
|
+
throw new Error(`GitHub API rate limit hit while ingesting (status ${res.status}). ` +
|
|
14
|
+
`Set SKILLJIT_GITHUB_TOKEN to raise the limit from 60/hr to 5000/hr. ${body}`);
|
|
15
|
+
}
|
|
16
|
+
if (!res.ok) {
|
|
17
|
+
throw new Error(`GitHub API request failed: ${res.status} ${url}`);
|
|
18
|
+
}
|
|
19
|
+
return res;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Ingest every SKILL.md found in a GitHub repo, via the git trees API
|
|
23
|
+
* (one recursive call, no per-directory listing) and raw.githubusercontent.com
|
|
24
|
+
* for content — both public, unauthenticated-friendly endpoints, so this
|
|
25
|
+
* scales to skilljit's default repo list without needing a GitHub App.
|
|
26
|
+
*/
|
|
27
|
+
export async function ingestGithubRepo(owner, repo, opts = {}) {
|
|
28
|
+
const source = `github:${owner}/${repo}`;
|
|
29
|
+
let ref = opts.ref;
|
|
30
|
+
if (!ref) {
|
|
31
|
+
const repoRes = await githubApiFetch(`https://api.github.com/repos/${owner}/${repo}`, opts);
|
|
32
|
+
const repoJson = (await repoRes.json());
|
|
33
|
+
ref = repoJson.default_branch;
|
|
34
|
+
}
|
|
35
|
+
const treeRes = await githubApiFetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`, opts);
|
|
36
|
+
const treeJson = (await treeRes.json());
|
|
37
|
+
if (treeJson.truncated) {
|
|
38
|
+
// eslint-disable-next-line no-console
|
|
39
|
+
console.warn(`skilljit: ${owner}/${repo}'s file tree was truncated by GitHub's API — some SKILL.md files may be missed.`);
|
|
40
|
+
}
|
|
41
|
+
const skillPaths = treeJson.tree.filter((e) => e.type === "blob" && /(^|\/)SKILL\.md$/.test(e.path));
|
|
42
|
+
const results = [];
|
|
43
|
+
for (const entry of skillPaths) {
|
|
44
|
+
const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${entry.path}`;
|
|
45
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
46
|
+
const res = await fetchImpl(rawUrl);
|
|
47
|
+
if (!res.ok)
|
|
48
|
+
continue;
|
|
49
|
+
const content = await res.text();
|
|
50
|
+
const parsed = parseSkillMd(content, { source, path: entry.path });
|
|
51
|
+
if (parsed)
|
|
52
|
+
results.push(parsed);
|
|
53
|
+
}
|
|
54
|
+
return results;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=github.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/ingest/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAY1C,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAyB;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IAC9D,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAC/B,OAAO,EAAE;YACP,MAAM,EAAE,6BAA6B;YACrC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD;KACa,CAAC,CAAC;IAClB,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,qDAAqD,GAAG,CAAC,MAAM,KAAK;YAClE,uEAAuE,IAAI,EAAE,CAChF,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAa,EACb,IAAY,EACZ,OAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,UAAU,KAAK,IAAI,IAAI,EAAE,CAAC;IACzC,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IACnB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,gCAAgC,KAAK,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QAC5F,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAA+B,CAAC;QACtE,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,cAAc,CAClC,gCAAgC,KAAK,IAAI,IAAI,cAAc,GAAG,cAAc,EAC5E,IAAI,CACL,CAAC;IACF,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAGrC,CAAC;IACF,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,sCAAsC;QACtC,OAAO,CAAC,IAAI,CACV,aAAa,KAAK,IAAI,IAAI,iFAAiF,CAC5G,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAErG,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,qCAAqC,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACzF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,SAAS;QACtB,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { SkillRecord } from "../types.js";
|
|
2
|
+
export interface ParseContext {
|
|
3
|
+
/** e.g. "github:anthropics/skills" */
|
|
4
|
+
source: string;
|
|
5
|
+
/** repo-relative path to the SKILL.md file, used to build a stable id */
|
|
6
|
+
path: string;
|
|
7
|
+
now?: () => string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Parse a SKILL.md file's content into a SkillRecord, enforcing the same
|
|
11
|
+
* frontmatter constraints Anthropic's Agent Skills spec requires (max
|
|
12
|
+
* lengths, allowed name charset, no reserved words, no XML tags). A file
|
|
13
|
+
* that doesn't satisfy the spec is skipped (returns null) rather than
|
|
14
|
+
* ingested malformed — a bad upstream skill should disappear quietly, not
|
|
15
|
+
* corrupt the local catalog.
|
|
16
|
+
*/
|
|
17
|
+
export declare function parseSkillMd(content: string, ctx: ParseContext): SkillRecord | null;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import YAML from "yaml";
|
|
2
|
+
const NAME_RE = /^[a-z0-9-]{1,64}$/;
|
|
3
|
+
const RESERVED_WORDS = ["anthropic", "claude"];
|
|
4
|
+
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
|
|
5
|
+
/**
|
|
6
|
+
* Parse a SKILL.md file's content into a SkillRecord, enforcing the same
|
|
7
|
+
* frontmatter constraints Anthropic's Agent Skills spec requires (max
|
|
8
|
+
* lengths, allowed name charset, no reserved words, no XML tags). A file
|
|
9
|
+
* that doesn't satisfy the spec is skipped (returns null) rather than
|
|
10
|
+
* ingested malformed — a bad upstream skill should disappear quietly, not
|
|
11
|
+
* corrupt the local catalog.
|
|
12
|
+
*/
|
|
13
|
+
export function parseSkillMd(content, ctx) {
|
|
14
|
+
const match = FRONTMATTER_RE.exec(content);
|
|
15
|
+
if (!match)
|
|
16
|
+
return null;
|
|
17
|
+
let frontmatter;
|
|
18
|
+
try {
|
|
19
|
+
frontmatter = YAML.parse(match[1]);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
if (!frontmatter || typeof frontmatter !== "object")
|
|
25
|
+
return null;
|
|
26
|
+
const { name, description } = frontmatter;
|
|
27
|
+
if (typeof name !== "string" || typeof description !== "string")
|
|
28
|
+
return null;
|
|
29
|
+
if (!NAME_RE.test(name))
|
|
30
|
+
return null;
|
|
31
|
+
if (RESERVED_WORDS.some((w) => name.includes(w)))
|
|
32
|
+
return null;
|
|
33
|
+
if (description.length === 0 || description.length > 1024)
|
|
34
|
+
return null;
|
|
35
|
+
if (/<[^>]+>/.test(name) || /<[^>]+>/.test(description))
|
|
36
|
+
return null;
|
|
37
|
+
const body = match[2] ?? "";
|
|
38
|
+
const dir = ctx.path.replace(/\/SKILL\.md$/i, "").replace(/^\.\//, "");
|
|
39
|
+
const id = dir && dir !== ctx.path ? `${ctx.source}/${dir}` : `${ctx.source}/${name}`;
|
|
40
|
+
return {
|
|
41
|
+
id,
|
|
42
|
+
name,
|
|
43
|
+
source: ctx.source,
|
|
44
|
+
description,
|
|
45
|
+
body,
|
|
46
|
+
updatedAt: (ctx.now ?? (() => new Date().toISOString()))(),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=parse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse.js","sourceRoot":"","sources":["../../src/ingest/parse.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,MAAM,OAAO,GAAG,mBAAmB,CAAC;AACpC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AAC/C,MAAM,cAAc,GAAG,6CAA6C,CAAC;AAUrE;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,GAAiB;IAC7D,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,IAAI,WAAoB,CAAC;IACzB,IAAI,CAAC;QACH,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC,WAAW,IAAI,OAAO,WAAW,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,WAAsC,CAAC;IACrE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,WAAW,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC;IACvE,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAErE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACvE,MAAM,EAAE,GAAG,GAAG,IAAI,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IAEtF,OAAO;QACL,EAAE;QACF,IAAI;QACJ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,IAAI;QACJ,SAAS,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE;KAC3D,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default first-party / high-quality repos to scan for SKILL.md files.
|
|
3
|
+
* This is the "direct repos" leg of catalog sourcing described in the
|
|
4
|
+
* design doc — deliberately small and curated rather than an attempt to
|
|
5
|
+
* mirror all 2,800+ repos skills.sh indexes. `skilljit sync --repo owner/name`
|
|
6
|
+
* lets a user add their own.
|
|
7
|
+
*/
|
|
8
|
+
export declare const DEFAULT_GITHUB_SOURCES: {
|
|
9
|
+
owner: string;
|
|
10
|
+
repo: string;
|
|
11
|
+
}[];
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default first-party / high-quality repos to scan for SKILL.md files.
|
|
3
|
+
* This is the "direct repos" leg of catalog sourcing described in the
|
|
4
|
+
* design doc — deliberately small and curated rather than an attempt to
|
|
5
|
+
* mirror all 2,800+ repos skills.sh indexes. `skilljit sync --repo owner/name`
|
|
6
|
+
* lets a user add their own.
|
|
7
|
+
*/
|
|
8
|
+
export const DEFAULT_GITHUB_SOURCES = [
|
|
9
|
+
{ owner: "anthropics", repo: "skills" },
|
|
10
|
+
{ owner: "vercel-labs", repo: "agent-skills" },
|
|
11
|
+
{ owner: "obra", repo: "superpowers" },
|
|
12
|
+
{ owner: "wshobson", repo: "agents" },
|
|
13
|
+
];
|
|
14
|
+
//# sourceMappingURL=sources.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sources.js","sourceRoot":"","sources":["../../src/ingest/sources.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAsC;IACvE,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvC,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE;IAC9C,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE;IACtC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE;CACtC,CAAC"}
|
package/dist/paths.d.ts
ADDED
package/dist/paths.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import os from "node:os";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/** Where the local skilljit catalog db lives: ~/.skilljit/catalog.db. */
|
|
4
|
+
export function defaultCatalogPath() {
|
|
5
|
+
const home = process.env.SKILLJIT_HOME ?? os.homedir();
|
|
6
|
+
return path.join(home, ".skilljit", "catalog.db");
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=paths.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paths.js","sourceRoot":"","sources":["../src/paths.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,yEAAyE;AACzE,MAAM,UAAU,kBAAkB;IAChC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IACvD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;AACpD,CAAC"}
|
package/dist/tokens.d.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { SkillRecord } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Token count for a string, via a real BPE tokenizer (gpt-tokenizer's
|
|
4
|
+
* cl100k encoding). This is an approximation of Claude's own tokenizer —
|
|
5
|
+
* Anthropic doesn't publish one — but it is a real, auditable count of a
|
|
6
|
+
* real BPE encoding, not a heuristic like "chars / 4". We say so in every
|
|
7
|
+
* place this number is surfaced (see TokenLedger.stats()).
|
|
8
|
+
*/
|
|
9
|
+
export declare function countTokens(text: string): number;
|
|
10
|
+
/**
|
|
11
|
+
* Approximate the per-turn cost of one skill's always-loaded metadata:
|
|
12
|
+
* its `name` + `description`, formatted the way Claude Code renders it
|
|
13
|
+
* in the system prompt (see Anthropic's Agent Skills docs example:
|
|
14
|
+
* "pdf-processing - Extract text... Use when...").
|
|
15
|
+
*/
|
|
16
|
+
export declare function estimateSkillMetadataTokens(skill: Pick<SkillRecord, "name" | "description">): number;
|
|
17
|
+
export interface TokenStats {
|
|
18
|
+
/** What every turn would have cost with all catalogued skills/tools loaded as static metadata. */
|
|
19
|
+
baselineTokens: number;
|
|
20
|
+
/** What skilljit's fixed tool surface + returned candidates actually cost. */
|
|
21
|
+
actualTokens: number;
|
|
22
|
+
/** baselineTokens - actualTokens, floored at 0. */
|
|
23
|
+
savedTokens: number;
|
|
24
|
+
/** How many baseline skill/tool metadata entries were counted. */
|
|
25
|
+
baselineEntries: number;
|
|
26
|
+
/** Human-readable caveat about the tokenizer used. */
|
|
27
|
+
method: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Tracks two running totals for one skilljit session:
|
|
31
|
+
* - baseline: the metadata cost every catalogued skill/tool WOULD have
|
|
32
|
+
* added to every turn, had it been installed the traditional way.
|
|
33
|
+
* - actual: the real size of what skilljit actually returned over MCP
|
|
34
|
+
* (tool schemas + tool call results), nothing hypothetical.
|
|
35
|
+
*
|
|
36
|
+
* `stats()` reports the delta. We only count things that were genuinely
|
|
37
|
+
* computed here — no invented multipliers — because this is the number
|
|
38
|
+
* users will screenshot, and the whole pitch is that it's honest.
|
|
39
|
+
*/
|
|
40
|
+
export declare class TokenLedger {
|
|
41
|
+
private baseline;
|
|
42
|
+
private actual;
|
|
43
|
+
private entries;
|
|
44
|
+
recordBaselineSkill(skill: Pick<SkillRecord, "name" | "description">): void;
|
|
45
|
+
recordBaselineTool(tool: {
|
|
46
|
+
name: string;
|
|
47
|
+
description: string;
|
|
48
|
+
inputSchema: unknown;
|
|
49
|
+
}): void;
|
|
50
|
+
recordActual(label: string, payload: string): void;
|
|
51
|
+
baselineTokens(): number;
|
|
52
|
+
actualTokens(): number;
|
|
53
|
+
stats(): TokenStats;
|
|
54
|
+
reset(): void;
|
|
55
|
+
}
|
package/dist/tokens.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { countTokens as gptCountTokens } from "gpt-tokenizer";
|
|
2
|
+
/**
|
|
3
|
+
* Token count for a string, via a real BPE tokenizer (gpt-tokenizer's
|
|
4
|
+
* cl100k encoding). This is an approximation of Claude's own tokenizer —
|
|
5
|
+
* Anthropic doesn't publish one — but it is a real, auditable count of a
|
|
6
|
+
* real BPE encoding, not a heuristic like "chars / 4". We say so in every
|
|
7
|
+
* place this number is surfaced (see TokenLedger.stats()).
|
|
8
|
+
*/
|
|
9
|
+
export function countTokens(text) {
|
|
10
|
+
if (!text)
|
|
11
|
+
return 0;
|
|
12
|
+
return gptCountTokens(text);
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Approximate the per-turn cost of one skill's always-loaded metadata:
|
|
16
|
+
* its `name` + `description`, formatted the way Claude Code renders it
|
|
17
|
+
* in the system prompt (see Anthropic's Agent Skills docs example:
|
|
18
|
+
* "pdf-processing - Extract text... Use when...").
|
|
19
|
+
*/
|
|
20
|
+
export function estimateSkillMetadataTokens(skill) {
|
|
21
|
+
return countTokens(`${skill.name} - ${skill.description}`);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Tracks two running totals for one skilljit session:
|
|
25
|
+
* - baseline: the metadata cost every catalogued skill/tool WOULD have
|
|
26
|
+
* added to every turn, had it been installed the traditional way.
|
|
27
|
+
* - actual: the real size of what skilljit actually returned over MCP
|
|
28
|
+
* (tool schemas + tool call results), nothing hypothetical.
|
|
29
|
+
*
|
|
30
|
+
* `stats()` reports the delta. We only count things that were genuinely
|
|
31
|
+
* computed here — no invented multipliers — because this is the number
|
|
32
|
+
* users will screenshot, and the whole pitch is that it's honest.
|
|
33
|
+
*/
|
|
34
|
+
export class TokenLedger {
|
|
35
|
+
baseline = 0;
|
|
36
|
+
actual = 0;
|
|
37
|
+
entries = 0;
|
|
38
|
+
recordBaselineSkill(skill) {
|
|
39
|
+
this.baseline += estimateSkillMetadataTokens(skill);
|
|
40
|
+
this.entries += 1;
|
|
41
|
+
}
|
|
42
|
+
recordBaselineTool(tool) {
|
|
43
|
+
this.baseline += countTokens(`${tool.name} ${tool.description} ${JSON.stringify(tool.inputSchema)}`);
|
|
44
|
+
this.entries += 1;
|
|
45
|
+
}
|
|
46
|
+
recordActual(label, payload) {
|
|
47
|
+
this.actual += countTokens(payload);
|
|
48
|
+
}
|
|
49
|
+
baselineTokens() {
|
|
50
|
+
return this.baseline;
|
|
51
|
+
}
|
|
52
|
+
actualTokens() {
|
|
53
|
+
return this.actual;
|
|
54
|
+
}
|
|
55
|
+
stats() {
|
|
56
|
+
const saved = Math.max(0, this.baseline - this.actual);
|
|
57
|
+
return {
|
|
58
|
+
baselineTokens: this.baseline,
|
|
59
|
+
actualTokens: this.actual,
|
|
60
|
+
savedTokens: saved,
|
|
61
|
+
baselineEntries: this.entries,
|
|
62
|
+
method: "Counted with gpt-tokenizer (cl100k BPE) as an approximation of Claude's tokenizer; " +
|
|
63
|
+
"baseline = metadata for every catalogued skill/tool touched this session, actual = " +
|
|
64
|
+
"what skilljit's fixed tool surface really returned over MCP.",
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
reset() {
|
|
68
|
+
this.baseline = 0;
|
|
69
|
+
this.actual = 0;
|
|
70
|
+
this.entries = 0;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=tokens.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,eAAe,CAAC;AAG9D;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAAgD;IAC1F,OAAO,WAAW,CAAC,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;AAC7D,CAAC;AAeD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,WAAW;IACd,QAAQ,GAAG,CAAC,CAAC;IACb,MAAM,GAAG,CAAC,CAAC;IACX,OAAO,GAAG,CAAC,CAAC;IAEpB,mBAAmB,CAAC,KAAgD;QAClE,IAAI,CAAC,QAAQ,IAAI,2BAA2B,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,kBAAkB,CAAC,IAAiE;QAClF,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,OAAe;QACzC,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,KAAK;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QACvD,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,QAAQ;YAC7B,YAAY,EAAE,IAAI,CAAC,MAAM;YACzB,WAAW,EAAE,KAAK;YAClB,eAAe,EAAE,IAAI,CAAC,OAAO;YAC7B,MAAM,EACJ,qFAAqF;gBACrF,qFAAqF;gBACrF,8DAA8D;SACjE,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;IACnB,CAAC;CACF"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/** A single skill record as stored in the local catalog. */
|
|
2
|
+
export interface SkillRecord {
|
|
3
|
+
/** Stable unique id, e.g. "vercel-labs/agent-skills/pdf-processing". */
|
|
4
|
+
id: string;
|
|
5
|
+
/** Short name from SKILL.md frontmatter, e.g. "pdf-processing". */
|
|
6
|
+
name: string;
|
|
7
|
+
/** Where this skill came from, e.g. "github:vercel-labs/agent-skills". */
|
|
8
|
+
source: string;
|
|
9
|
+
/** The `description` field from SKILL.md frontmatter. */
|
|
10
|
+
description: string;
|
|
11
|
+
/** Full SKILL.md body (Level 2 content), loaded lazily by consumers. */
|
|
12
|
+
body: string;
|
|
13
|
+
/** Optional install-count / popularity signal from the upstream registry. */
|
|
14
|
+
installCount?: number;
|
|
15
|
+
/** Audit status surfaced by the upstream registry, if any. */
|
|
16
|
+
auditStatus?: "pass" | "warn" | "fail" | "unaudited";
|
|
17
|
+
/** ISO timestamp of when this record was last refreshed. */
|
|
18
|
+
updatedAt: string;
|
|
19
|
+
}
|
|
20
|
+
/** A search hit: a skill plus its relevance rank (lower = more relevant). */
|
|
21
|
+
export interface SkillSearchHit {
|
|
22
|
+
skill: Omit<SkillRecord, "body">;
|
|
23
|
+
rank: number;
|
|
24
|
+
}
|
|
25
|
+
/** An MCP upstream tool schema as cataloged for tool_find/tool_call routing. */
|
|
26
|
+
export interface ToolRecord {
|
|
27
|
+
id: string;
|
|
28
|
+
server: string;
|
|
29
|
+
name: string;
|
|
30
|
+
description: string;
|
|
31
|
+
inputSchema: unknown;
|
|
32
|
+
updatedAt: string;
|
|
33
|
+
}
|
|
34
|
+
export interface ToolSearchHit {
|
|
35
|
+
tool: ToolRecord;
|
|
36
|
+
rank: number;
|
|
37
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/package.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@skilljit/core",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Local catalog, full-text search, and token accounting for skilljit",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"license": "MIT",
|
|
9
|
+
"publishConfig": {
|
|
10
|
+
"access": "public"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"dist"
|
|
14
|
+
],
|
|
15
|
+
"scripts": {
|
|
16
|
+
"build": "tsc -p tsconfig.json",
|
|
17
|
+
"test": "vitest run",
|
|
18
|
+
"test:watch": "vitest"
|
|
19
|
+
},
|
|
20
|
+
"dependencies": {
|
|
21
|
+
"better-sqlite3": "^13.0.3",
|
|
22
|
+
"gpt-tokenizer": "^4.0.0",
|
|
23
|
+
"yaml": "^2.9.0"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"@types/better-sqlite3": "^7.6.11",
|
|
27
|
+
"typescript": "^5.7.0",
|
|
28
|
+
"vitest": "^4.1.11"
|
|
29
|
+
}
|
|
30
|
+
}
|