@skilljit/core 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/catalog.d.ts +15 -0
- package/dist/catalog.js +56 -2
- package/dist/catalog.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/files.d.ts +4 -0
- package/dist/ingest/files.js +15 -0
- package/dist/ingest/files.js.map +1 -0
- package/dist/ingest/git-local.d.ts +30 -0
- package/dist/ingest/git-local.js +104 -0
- package/dist/ingest/git-local.js.map +1 -0
- package/dist/ingest/github.d.ts +6 -0
- package/dist/ingest/github.js +49 -12
- package/dist/ingest/github.js.map +1 -1
- package/dist/types.d.ts +11 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -16,7 +16,12 @@ const catalog = new Catalog("~/.skilljit/catalog.db");
|
|
|
16
16
|
const hits = catalog.searchSkills("postgres migration", 3);
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
+
Also exports `ingestGithubRepo` (GitHub API ingestion) and `ingestLocalGitRepo`
|
|
20
|
+
(clone + `git worktree` ingestion for any git remote — self-hosted, GitLab, an
|
|
21
|
+
SSH-authenticated private repo), plus a persistent cross-session token ledger on
|
|
22
|
+
`Catalog` so `skilljit_stats()` can report totals across every tab sharing a catalog.
|
|
23
|
+
|
|
19
24
|
See the [main repo](https://github.com/aqibsidd/skilljit) for the full design,
|
|
20
|
-
the fixed
|
|
25
|
+
the fixed tool surface, and benchmark numbers.
|
|
21
26
|
|
|
22
27
|
MIT licensed.
|
package/dist/catalog.d.ts
CHANGED
|
@@ -2,6 +2,11 @@ import type { SkillRecord, SkillSearchHit, ToolRecord, ToolSearchHit } from "./t
|
|
|
2
2
|
export declare class Catalog {
|
|
3
3
|
private db;
|
|
4
4
|
constructor(dbPath: string);
|
|
5
|
+
/** Additive, idempotent schema migrations for catalogs created by older
|
|
6
|
+
* skilljit versions — CREATE TABLE IF NOT EXISTS only runs for a table
|
|
7
|
+
* that doesn't exist yet, so a pre-existing catalog.db needs this to
|
|
8
|
+
* pick up new columns. */
|
|
9
|
+
private migrate;
|
|
5
10
|
close(): void;
|
|
6
11
|
upsertSkills(skills: SkillRecord[]): void;
|
|
7
12
|
getSkill(id: string): SkillRecord | undefined;
|
|
@@ -15,4 +20,14 @@ export declare class Catalog {
|
|
|
15
20
|
searchTools(query: string, limit?: number): ToolSearchHit[];
|
|
16
21
|
removeToolsForServer(server: string): void;
|
|
17
22
|
toolCount(): number;
|
|
23
|
+
/** Adds to the all-time baseline total and counts one more session. */
|
|
24
|
+
addGlobalLedgerBaseline(deltaTokens: number): void;
|
|
25
|
+
addGlobalLedgerActual(deltaTokens: number): void;
|
|
26
|
+
/** Call once per server startup — this is what makes sessionCount count tabs. */
|
|
27
|
+
recordGlobalSessionStart(): void;
|
|
28
|
+
getGlobalLedgerTotals(): {
|
|
29
|
+
baselineTokens: number;
|
|
30
|
+
actualTokens: number;
|
|
31
|
+
sessionCount: number;
|
|
32
|
+
};
|
|
18
33
|
}
|
package/dist/catalog.js
CHANGED
|
@@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS skills (
|
|
|
8
8
|
source TEXT NOT NULL,
|
|
9
9
|
description TEXT NOT NULL,
|
|
10
10
|
body TEXT NOT NULL,
|
|
11
|
+
files_json TEXT,
|
|
11
12
|
install_count INTEGER,
|
|
12
13
|
audit_status TEXT,
|
|
13
14
|
updated_at TEXT NOT NULL
|
|
@@ -20,6 +21,13 @@ CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
|
|
|
20
21
|
tokenize = 'porter unicode61'
|
|
21
22
|
);
|
|
22
23
|
|
|
24
|
+
CREATE TABLE IF NOT EXISTS ledger_totals (
|
|
25
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
26
|
+
baseline_tokens INTEGER NOT NULL DEFAULT 0,
|
|
27
|
+
actual_tokens INTEGER NOT NULL DEFAULT 0,
|
|
28
|
+
session_count INTEGER NOT NULL DEFAULT 0
|
|
29
|
+
);
|
|
30
|
+
|
|
23
31
|
CREATE TABLE IF NOT EXISTS tools (
|
|
24
32
|
id TEXT PRIMARY KEY,
|
|
25
33
|
server TEXT NOT NULL,
|
|
@@ -63,6 +71,18 @@ export class Catalog {
|
|
|
63
71
|
this.db = new Database(dbPath);
|
|
64
72
|
this.db.pragma("journal_mode = WAL");
|
|
65
73
|
this.db.exec(SCHEMA);
|
|
74
|
+
this.migrate();
|
|
75
|
+
}
|
|
76
|
+
/** Additive, idempotent schema migrations for catalogs created by older
|
|
77
|
+
* skilljit versions — CREATE TABLE IF NOT EXISTS only runs for a table
|
|
78
|
+
* that doesn't exist yet, so a pre-existing catalog.db needs this to
|
|
79
|
+
* pick up new columns. */
|
|
80
|
+
migrate() {
|
|
81
|
+
const columns = this.db.prepare(`PRAGMA table_info(skills)`).all();
|
|
82
|
+
if (!columns.some((c) => c.name === "files_json")) {
|
|
83
|
+
this.db.exec(`ALTER TABLE skills ADD COLUMN files_json TEXT`);
|
|
84
|
+
}
|
|
85
|
+
this.db.exec(`INSERT OR IGNORE INTO ledger_totals (id, baseline_tokens, actual_tokens, session_count) VALUES (1, 0, 0, 0)`);
|
|
66
86
|
}
|
|
67
87
|
close() {
|
|
68
88
|
this.db.close();
|
|
@@ -70,13 +90,14 @@ export class Catalog {
|
|
|
70
90
|
// ---- Skills ----------------------------------------------------------
|
|
71
91
|
upsertSkills(skills) {
|
|
72
92
|
const upsert = this.db.prepare(`
|
|
73
|
-
INSERT INTO skills (id, name, source, description, body, install_count, audit_status, updated_at)
|
|
74
|
-
VALUES (@id, @name, @source, @description, @body, @installCount, @auditStatus, @updatedAt)
|
|
93
|
+
INSERT INTO skills (id, name, source, description, body, files_json, install_count, audit_status, updated_at)
|
|
94
|
+
VALUES (@id, @name, @source, @description, @body, @filesJson, @installCount, @auditStatus, @updatedAt)
|
|
75
95
|
ON CONFLICT(id) DO UPDATE SET
|
|
76
96
|
name = excluded.name,
|
|
77
97
|
source = excluded.source,
|
|
78
98
|
description = excluded.description,
|
|
79
99
|
body = excluded.body,
|
|
100
|
+
files_json = excluded.files_json,
|
|
80
101
|
install_count = excluded.install_count,
|
|
81
102
|
audit_status = excluded.audit_status,
|
|
82
103
|
updated_at = excluded.updated_at
|
|
@@ -91,6 +112,7 @@ export class Catalog {
|
|
|
91
112
|
source: s.source,
|
|
92
113
|
description: s.description,
|
|
93
114
|
body: s.body,
|
|
115
|
+
filesJson: s.files && s.files.length > 0 ? JSON.stringify(s.files) : null,
|
|
94
116
|
installCount: s.installCount ?? null,
|
|
95
117
|
auditStatus: s.auditStatus ?? "unaudited",
|
|
96
118
|
updatedAt: s.updatedAt,
|
|
@@ -218,6 +240,37 @@ export class Catalog {
|
|
|
218
240
|
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM tools`).get();
|
|
219
241
|
return row.c;
|
|
220
242
|
}
|
|
243
|
+
// ---- Cross-session token ledger ----------------------------------------
|
|
244
|
+
// Every skilljit session/tab shares this catalog.db, so these totals
|
|
245
|
+
// accumulate across all of them — the fix for "N tabs open means N tabs
|
|
246
|
+
// each independently paying the traditional baseline cost."
|
|
247
|
+
/** Adds to the all-time baseline total and counts one more session. */
|
|
248
|
+
addGlobalLedgerBaseline(deltaTokens) {
|
|
249
|
+
if (deltaTokens === 0)
|
|
250
|
+
return;
|
|
251
|
+
this.db
|
|
252
|
+
.prepare(`UPDATE ledger_totals SET baseline_tokens = baseline_tokens + ? WHERE id = 1`)
|
|
253
|
+
.run(deltaTokens);
|
|
254
|
+
}
|
|
255
|
+
addGlobalLedgerActual(deltaTokens) {
|
|
256
|
+
if (deltaTokens === 0)
|
|
257
|
+
return;
|
|
258
|
+
this.db.prepare(`UPDATE ledger_totals SET actual_tokens = actual_tokens + ? WHERE id = 1`).run(deltaTokens);
|
|
259
|
+
}
|
|
260
|
+
/** Call once per server startup — this is what makes sessionCount count tabs. */
|
|
261
|
+
recordGlobalSessionStart() {
|
|
262
|
+
this.db.exec(`UPDATE ledger_totals SET session_count = session_count + 1 WHERE id = 1`);
|
|
263
|
+
}
|
|
264
|
+
getGlobalLedgerTotals() {
|
|
265
|
+
const row = this.db
|
|
266
|
+
.prepare(`SELECT baseline_tokens, actual_tokens, session_count FROM ledger_totals WHERE id = 1`)
|
|
267
|
+
.get();
|
|
268
|
+
return {
|
|
269
|
+
baselineTokens: row.baseline_tokens,
|
|
270
|
+
actualTokens: row.actual_tokens,
|
|
271
|
+
sessionCount: row.session_count,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
221
274
|
}
|
|
222
275
|
function rowToSkill(row) {
|
|
223
276
|
return {
|
|
@@ -226,6 +279,7 @@ function rowToSkill(row) {
|
|
|
226
279
|
source: row.source,
|
|
227
280
|
description: row.description,
|
|
228
281
|
body: row.body,
|
|
282
|
+
files: row.files_json ? JSON.parse(row.files_json) : undefined,
|
|
229
283
|
installCount: row.install_count ?? undefined,
|
|
230
284
|
auditStatus: row.audit_status ?? "unaudited",
|
|
231
285
|
updatedAt: row.updated_at,
|
package/dist/catalog.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../src/catalog.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,MAAM,GAAG
|
|
1
|
+
{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../src/catalog.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Cd,CAAC;AAEF;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,MAAM,GAAG,KAAK;SACjB,WAAW,EAAE;SACb,KAAK,CAAC,YAAY,CAAC;QACpB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,OAAO,OAAO;IACV,EAAE,CAAoB;IAE9B,YAAY,MAAc;QACxB,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YAC1B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED;;;8BAG0B;IAClB,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,GAAG,EAAwB,CAAC;QACzF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,IAAI,CACV,6GAA6G,CAC9G,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,yEAAyE;IAEzE,YAAY,CAAC,MAAqB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;KAY9B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iEAAiE,CAAC,CAAC;QAErG,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAmB,EAAE,EAAE;YACrD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,CAAC;oBACT,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,WAAW,EAAE,CAAC,CAAC,WAAW;oBAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;oBACzE,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI;oBACpC,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,WAAW;oBACzC,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;gBACH,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,MAAM,CAAC,CAAC;IACb,CAAC;IAED,QAAQ,CAAC,EAAU;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAQ,CAAC;QAChF,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,KAAK,GAAG,CAAC;QACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;;;;;OAQD,CACA;aACA,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE;gBACL,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;gBAC5C,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,WAAW;gBAC5C,SAAS,EAAE,GAAG,CAAC,UAAU;aAC1B;YACD,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;QACvF,OAAO,GAAG,CAAC,CAAC,CAAC;IACf,CAAC;IAED;;uCAEmC;IACnC,aAAa;QACX,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAG/D,CAAC;IACN,CAAC;IAED,2EAA2E;IAE3E,WAAW,CAAC,KAAmB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;KAS9B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC;QAEpG,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAkB,EAAE,EAAE;YACpD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,CAAC;oBACT,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;oBAC1B,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;oBAC1C,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;gBACH,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,CAAC;IACZ,CAAC;IAED,WAAW,CAAC,KAAa,EAAE,KAAK,GAAG,CAAC;QAClC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;;;;;OAQD,CACA;aACA,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,IAAI,EAAE;gBACJ,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC;gBACzC,SAAS,EAAE,GAAG,CAAC,UAAU;aAC1B;YACD,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,oBAAoB,CAAC,MAAc;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAqB,CAAC;QACrG,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;QACrE,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;QACxE,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClC,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACnB,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,EAAE,CAAC;IACP,CAAC;IAED,SAAS;QACP,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAAmB,CAAC;QACtF,OAAO,GAAG,CAAC,CAAC,CAAC;IACf,CAAC;IAED,2EAA2E;IAC3E,qEAAqE;IACrE,wEAAwE;IACxE,4DAA4D;IAE5D,uEAAuE;IACvE,uBAAuB,CAAC,WAAmB;QACzC,IAAI,WAAW,KAAK,CAAC;YAAE,OAAO;QAC9B,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,6EAA6E,CAAC;aACtF,GAAG,CAAC,WAAW,CAAC,CAAC;IACtB,CAAC;IAED,qBAAqB,CAAC,WAAmB;QACvC,IAAI,WAAW,KAAK,CAAC;YAAE,OAAO;QAC9B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yEAAyE,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC9G,CAAC;IAED,iFAAiF;IACjF,wBAAwB;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;IAC1F,CAAC;IAED,qBAAqB;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,sFAAsF,CAAC;aAC/F,GAAG,EAA+E,CAAC;QACtF,OAAO;YACL,cAAc,EAAE,GAAG,CAAC,eAAe;YACnC,YAAY,EAAE,GAAG,CAAC,aAAa;YAC/B,YAAY,EAAE,GAAG,CAAC,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AAED,SAAS,UAAU,CAAC,GAAQ;IAC1B,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,KAAK,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;QAC9D,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;QAC5C,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,WAAW;QAC5C,SAAS,EAAE,GAAG,CAAC,UAAU;KAC1B,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
export { Catalog } from "./catalog.js";
|
|
2
2
|
export { countTokens, estimateSkillMetadataTokens, TokenLedger } from "./tokens.js";
|
|
3
3
|
export type { TokenStats } from "./tokens.js";
|
|
4
|
-
export type { SkillRecord, SkillSearchHit, ToolRecord, ToolSearchHit } from "./types.js";
|
|
4
|
+
export type { SkillRecord, SkillFile, SkillSearchHit, ToolRecord, ToolSearchHit } from "./types.js";
|
|
5
5
|
export { defaultCatalogPath } from "./paths.js";
|
|
6
6
|
export { parseSkillMd } from "./ingest/parse.js";
|
|
7
7
|
export { ingestGithubRepo } from "./ingest/github.js";
|
|
8
8
|
export type { GithubIngestOptions } from "./ingest/github.js";
|
|
9
|
+
export { ingestLocalGitRepo } from "./ingest/git-local.js";
|
|
10
|
+
export type { LocalGitIngestOptions, ExecFileFn } from "./ingest/git-local.js";
|
|
9
11
|
export { DEFAULT_GITHUB_SOURCES } from "./ingest/sources.js";
|
package/dist/index.js
CHANGED
|
@@ -3,5 +3,6 @@ export { countTokens, estimateSkillMetadataTokens, TokenLedger } from "./tokens.
|
|
|
3
3
|
export { defaultCatalogPath } from "./paths.js";
|
|
4
4
|
export { parseSkillMd } from "./ingest/parse.js";
|
|
5
5
|
export { ingestGithubRepo } from "./ingest/github.js";
|
|
6
|
+
export { ingestLocalGitRepo } from "./ingest/git-local.js";
|
|
6
7
|
export { DEFAULT_GITHUB_SOURCES } from "./ingest/sources.js";
|
|
7
8
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGpF,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGpF,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Best-effort check so bundled-file ingestion doesn't fetch/read binary
|
|
2
|
+
* assets as text (and corrupt them) — skipped files just aren't offered
|
|
3
|
+
* via skill_read_file, rather than being stored garbled. */
|
|
4
|
+
export declare function isProbablyBinaryPath(filePath: string): boolean;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const BINARY_EXTENSIONS = new Set([
|
|
2
|
+
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".bmp",
|
|
3
|
+
".pdf", ".zip", ".tar", ".gz", ".woff", ".woff2", ".ttf", ".otf",
|
|
4
|
+
".mp3", ".mp4", ".mov", ".wav", ".exe", ".dll", ".so", ".dylib",
|
|
5
|
+
]);
|
|
6
|
+
/** Best-effort check so bundled-file ingestion doesn't fetch/read binary
|
|
7
|
+
* assets as text (and corrupt them) — skipped files just aren't offered
|
|
8
|
+
* via skill_read_file, rather than being stored garbled. */
|
|
9
|
+
export function isProbablyBinaryPath(filePath) {
|
|
10
|
+
const dot = filePath.lastIndexOf(".");
|
|
11
|
+
if (dot === -1)
|
|
12
|
+
return false;
|
|
13
|
+
return BINARY_EXTENSIONS.has(filePath.slice(dot).toLowerCase());
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/ingest/files.ts"],"names":[],"mappings":"AAAA,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IACxD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM;IAChE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ;CAChE,CAAC,CAAC;AAEH;;4DAE4D;AAC5D,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7B,OAAO,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { SkillRecord } from "../types.js";
|
|
2
|
+
export interface ExecFileFn {
|
|
3
|
+
(command: string, args: string[]): Promise<{
|
|
4
|
+
stdout: string;
|
|
5
|
+
stderr: string;
|
|
6
|
+
}>;
|
|
7
|
+
}
|
|
8
|
+
export interface LocalGitIngestOptions {
|
|
9
|
+
/** Where bare mirror clones are cached between syncs. Defaults to
|
|
10
|
+
* ~/.skilljit/git-cache. */
|
|
11
|
+
cacheDir?: string;
|
|
12
|
+
/** Branch/tag/commit to check out. Defaults to the remote's HEAD. */
|
|
13
|
+
ref?: string;
|
|
14
|
+
/** Injectable for tests; defaults to shelling out to the real `git`. */
|
|
15
|
+
execFileImpl?: ExecFileFn;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Ingest every SKILL.md in an arbitrary git repository — self-hosted,
|
|
19
|
+
* GitLab, Bitbucket, an internal server, or a private GitHub repo reached
|
|
20
|
+
* over SSH — anywhere the `git` binary on this machine can already
|
|
21
|
+
* authenticate (SSH key, git-credential helper), no GitHub API token
|
|
22
|
+
* needed at all.
|
|
23
|
+
*
|
|
24
|
+
* Uses a persistent bare mirror clone under `cacheDir` plus a throwaway
|
|
25
|
+
* `git worktree add` per sync, instead of a fresh clone every time: the
|
|
26
|
+
* first sync pays for a full clone, every sync after that is just a
|
|
27
|
+
* `git fetch` (cheap, incremental) followed by a worktree checkout. The
|
|
28
|
+
* worktree is removed again once ingestion finishes.
|
|
29
|
+
*/
|
|
30
|
+
export declare function ingestLocalGitRepo(url: string, opts?: LocalGitIngestOptions): Promise<SkillRecord[]>;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { promisify } from "node:util";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import crypto from "node:crypto";
|
|
7
|
+
import { parseSkillMd } from "./parse.js";
|
|
8
|
+
import { isProbablyBinaryPath } from "./files.js";
|
|
9
|
+
const execFileAsync = promisify(execFile);
|
|
10
|
+
const IGNORED_DIR_NAMES = new Set([".git", "node_modules", ".venv", "__pycache__", "dist"]);
|
|
11
|
+
function cacheKeyForUrl(url) {
|
|
12
|
+
return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
13
|
+
}
|
|
14
|
+
function walkSkillDirs(root, dir, out) {
|
|
15
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
16
|
+
if (entry.isDirectory()) {
|
|
17
|
+
if (IGNORED_DIR_NAMES.has(entry.name))
|
|
18
|
+
continue;
|
|
19
|
+
walkSkillDirs(root, path.join(dir, entry.name), out);
|
|
20
|
+
}
|
|
21
|
+
else if (entry.isFile() && entry.name === "SKILL.md") {
|
|
22
|
+
out.push(path.relative(root, dir));
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Ingest every SKILL.md in an arbitrary git repository — self-hosted,
|
|
28
|
+
* GitLab, Bitbucket, an internal server, or a private GitHub repo reached
|
|
29
|
+
* over SSH — anywhere the `git` binary on this machine can already
|
|
30
|
+
* authenticate (SSH key, git-credential helper), no GitHub API token
|
|
31
|
+
* needed at all.
|
|
32
|
+
*
|
|
33
|
+
* Uses a persistent bare mirror clone under `cacheDir` plus a throwaway
|
|
34
|
+
* `git worktree add` per sync, instead of a fresh clone every time: the
|
|
35
|
+
* first sync pays for a full clone, every sync after that is just a
|
|
36
|
+
* `git fetch` (cheap, incremental) followed by a worktree checkout. The
|
|
37
|
+
* worktree is removed again once ingestion finishes.
|
|
38
|
+
*/
|
|
39
|
+
export async function ingestLocalGitRepo(url, opts = {}) {
|
|
40
|
+
const run = opts.execFileImpl ?? ((cmd, args) => execFileAsync(cmd, args));
|
|
41
|
+
const cacheDir = opts.cacheDir ?? path.join(os.homedir(), ".skilljit", "git-cache");
|
|
42
|
+
fs.mkdirSync(cacheDir, { recursive: true });
|
|
43
|
+
const bareDir = path.join(cacheDir, `${cacheKeyForUrl(url)}.git`);
|
|
44
|
+
const source = `git:${url}`;
|
|
45
|
+
if (!fs.existsSync(bareDir)) {
|
|
46
|
+
// --mirror, not --bare: a plain --bare clone doesn't set up a fetch
|
|
47
|
+
// refspec that updates local branch refs (and HEAD's target) on a
|
|
48
|
+
// later `git fetch` — only --mirror does, which is what makes the
|
|
49
|
+
// "clone once, fetch cheaply after" reuse actually pick up new commits.
|
|
50
|
+
await run("git", ["clone", "--mirror", "--quiet", url, bareDir]);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
await run("git", ["--git-dir", bareDir, "fetch", "--quiet", "--prune", "origin"]);
|
|
54
|
+
}
|
|
55
|
+
const checkoutRef = opts.ref ?? "HEAD";
|
|
56
|
+
const worktreeDir = fs.mkdtempSync(path.join(os.tmpdir(), "skilljit-wt-"));
|
|
57
|
+
// A worktree can't be added at a path that already exists as an empty
|
|
58
|
+
// dir in some git versions; remove it and let `worktree add` create it.
|
|
59
|
+
fs.rmdirSync(worktreeDir);
|
|
60
|
+
try {
|
|
61
|
+
await run("git", ["--git-dir", bareDir, "worktree", "add", "--detach", "--quiet", worktreeDir, checkoutRef]);
|
|
62
|
+
const skillDirs = [];
|
|
63
|
+
walkSkillDirs(worktreeDir, worktreeDir, skillDirs);
|
|
64
|
+
const results = [];
|
|
65
|
+
for (const relDir of skillDirs) {
|
|
66
|
+
const absDir = path.join(worktreeDir, relDir);
|
|
67
|
+
const skillMdPath = path.join(absDir, "SKILL.md");
|
|
68
|
+
const content = fs.readFileSync(skillMdPath, "utf8");
|
|
69
|
+
const skillPath = relDir === "" ? "SKILL.md" : `${relDir}/SKILL.md`;
|
|
70
|
+
const parsed = parseSkillMd(content, { source, path: skillPath });
|
|
71
|
+
if (!parsed)
|
|
72
|
+
continue;
|
|
73
|
+
if (relDir !== "") {
|
|
74
|
+
const files = [];
|
|
75
|
+
collectSiblingFiles(absDir, absDir, files);
|
|
76
|
+
if (files.length > 0)
|
|
77
|
+
parsed.files = files;
|
|
78
|
+
}
|
|
79
|
+
results.push(parsed);
|
|
80
|
+
}
|
|
81
|
+
return results;
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
await run("git", ["--git-dir", bareDir, "worktree", "remove", "--force", worktreeDir]).catch(() => { });
|
|
85
|
+
fs.rmSync(worktreeDir, { recursive: true, force: true });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function collectSiblingFiles(root, dir, out) {
|
|
89
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
90
|
+
if (entry.isDirectory()) {
|
|
91
|
+
if (IGNORED_DIR_NAMES.has(entry.name))
|
|
92
|
+
continue;
|
|
93
|
+
collectSiblingFiles(root, path.join(dir, entry.name), out);
|
|
94
|
+
}
|
|
95
|
+
else if (entry.isFile() && entry.name !== "SKILL.md" && entry.name !== ".git") {
|
|
96
|
+
const abs = path.join(dir, entry.name);
|
|
97
|
+
const rel = path.relative(root, abs);
|
|
98
|
+
if (isProbablyBinaryPath(rel))
|
|
99
|
+
continue;
|
|
100
|
+
out.push({ path: rel, content: fs.readFileSync(abs, "utf8") });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=git-local.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"git-local.js","sourceRoot":"","sources":["../../src/ingest/git-local.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAGlD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE1C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;AAgB5F,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,GAAW,EAAE,GAAa;IAC7D,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACjE,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YAChD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,OAA8B,EAAE;IAEhC,MAAM,GAAG,GAAe,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IACpF,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,GAAG,EAAE,CAAC;IAE5B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,oEAAoE;QACpE,kEAAkE;QAClE,kEAAkE;QAClE,wEAAwE;QACxE,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC;IACvC,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;IAC3E,sEAAsE;IACtE,wEAAwE;IACxE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1B,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;QAE7G,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,aAAa,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAEnD,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACrD,MAAM,SAAS,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,WAAW,CAAC;YACpE,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM;gBAAE,SAAS;YAEtB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;gBAClB,MAAM,KAAK,GAAyB,EAAE,CAAC;gBACvC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;gBAC3C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YAC7C,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACvG,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,GAAW,EAAE,GAAsC;IAC5F,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACjE,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YAChD,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAChF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACrC,IAAI,oBAAoB,CAAC,GAAG,CAAC;gBAAE,SAAS;YACxC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/dist/ingest/github.d.ts
CHANGED
|
@@ -12,5 +12,11 @@ export interface GithubIngestOptions {
|
|
|
12
12
|
* (one recursive call, no per-directory listing) and raw.githubusercontent.com
|
|
13
13
|
* for content — both public, unauthenticated-friendly endpoints, so this
|
|
14
14
|
* scales to skilljit's default repo list without needing a GitHub App.
|
|
15
|
+
*
|
|
16
|
+
* Also bundles each skill's sibling files (references, helper scripts) —
|
|
17
|
+
* anything else living under the same directory as its SKILL.md — so
|
|
18
|
+
* skill_read_file has something to serve. A skill whose SKILL.md sits at
|
|
19
|
+
* the repo root is skipped for bundling (its "siblings" would be the
|
|
20
|
+
* entire rest of the repo, not files that belong to it).
|
|
15
21
|
*/
|
|
16
22
|
export declare function ingestGithubRepo(owner: string, repo: string, opts?: GithubIngestOptions): Promise<SkillRecord[]>;
|
package/dist/ingest/github.js
CHANGED
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import { parseSkillMd } from "./parse.js";
|
|
2
|
+
import { isProbablyBinaryPath } from "./files.js";
|
|
3
|
+
function resolveToken(opts) {
|
|
4
|
+
return opts.token ?? process.env.SKILLJIT_GITHUB_TOKEN;
|
|
5
|
+
}
|
|
6
|
+
function authHeaders(opts) {
|
|
7
|
+
const token = resolveToken(opts);
|
|
8
|
+
return token ? { Authorization: `Bearer ${token}` } : {};
|
|
9
|
+
}
|
|
2
10
|
async function githubApiFetch(url, opts) {
|
|
3
11
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
4
|
-
const token = opts.token ?? process.env.SKILLJIT_GITHUB_TOKEN;
|
|
5
12
|
const res = await fetchImpl(url, {
|
|
6
13
|
headers: {
|
|
7
14
|
Accept: "application/vnd.github+json",
|
|
8
|
-
...(
|
|
15
|
+
...authHeaders(opts),
|
|
9
16
|
},
|
|
10
17
|
});
|
|
11
18
|
if (res.status === 403 || res.status === 429) {
|
|
@@ -18,11 +25,29 @@ async function githubApiFetch(url, opts) {
|
|
|
18
25
|
}
|
|
19
26
|
return res;
|
|
20
27
|
}
|
|
28
|
+
/** Fetch a raw file's content, authenticated — this is what makes private
|
|
29
|
+
* repos actually work. Without the Authorization header here, GitHub
|
|
30
|
+
* returns 404 for any private repo's raw content even though the caller
|
|
31
|
+
* has a valid token, and the ingest loop below silently skips the file. */
|
|
32
|
+
async function fetchRawFile(owner, repo, ref, filePath, opts) {
|
|
33
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
34
|
+
const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`;
|
|
35
|
+
const res = await fetchImpl(rawUrl, { headers: authHeaders(opts) });
|
|
36
|
+
if (!res.ok)
|
|
37
|
+
return null;
|
|
38
|
+
return res.text();
|
|
39
|
+
}
|
|
21
40
|
/**
|
|
22
41
|
* Ingest every SKILL.md found in a GitHub repo, via the git trees API
|
|
23
42
|
* (one recursive call, no per-directory listing) and raw.githubusercontent.com
|
|
24
43
|
* for content — both public, unauthenticated-friendly endpoints, so this
|
|
25
44
|
* scales to skilljit's default repo list without needing a GitHub App.
|
|
45
|
+
*
|
|
46
|
+
* Also bundles each skill's sibling files (references, helper scripts) —
|
|
47
|
+
* anything else living under the same directory as its SKILL.md — so
|
|
48
|
+
* skill_read_file has something to serve. A skill whose SKILL.md sits at
|
|
49
|
+
* the repo root is skipped for bundling (its "siblings" would be the
|
|
50
|
+
* entire rest of the repo, not files that belong to it).
|
|
26
51
|
*/
|
|
27
52
|
export async function ingestGithubRepo(owner, repo, opts = {}) {
|
|
28
53
|
const source = `github:${owner}/${repo}`;
|
|
@@ -38,18 +63,30 @@ export async function ingestGithubRepo(owner, repo, opts = {}) {
|
|
|
38
63
|
// eslint-disable-next-line no-console
|
|
39
64
|
console.warn(`skilljit: ${owner}/${repo}'s file tree was truncated by GitHub's API — some SKILL.md files may be missed.`);
|
|
40
65
|
}
|
|
41
|
-
const
|
|
66
|
+
const blobPaths = treeJson.tree.filter((e) => e.type === "blob").map((e) => e.path);
|
|
67
|
+
const skillPaths = blobPaths.filter((p) => /(^|\/)SKILL\.md$/.test(p));
|
|
42
68
|
const results = [];
|
|
43
|
-
for (const
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
69
|
+
for (const skillPath of skillPaths) {
|
|
70
|
+
const content = await fetchRawFile(owner, repo, ref, skillPath, opts);
|
|
71
|
+
if (content === null)
|
|
72
|
+
continue;
|
|
73
|
+
const parsed = parseSkillMd(content, { source, path: skillPath });
|
|
74
|
+
if (!parsed)
|
|
48
75
|
continue;
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
76
|
+
const dir = skillPath.replace(/\/SKILL\.md$/i, "");
|
|
77
|
+
if (dir !== skillPath) {
|
|
78
|
+
const siblingPaths = blobPaths.filter((p) => p !== skillPath && p.startsWith(`${dir}/`) && !isProbablyBinaryPath(p));
|
|
79
|
+
const files = [];
|
|
80
|
+
for (const siblingPath of siblingPaths) {
|
|
81
|
+
const siblingContent = await fetchRawFile(owner, repo, ref, siblingPath, opts);
|
|
82
|
+
if (siblingContent === null)
|
|
83
|
+
continue;
|
|
84
|
+
files.push({ path: siblingPath.slice(dir.length + 1), content: siblingContent });
|
|
85
|
+
}
|
|
86
|
+
if (files.length > 0)
|
|
87
|
+
parsed.files = files;
|
|
88
|
+
}
|
|
89
|
+
results.push(parsed);
|
|
53
90
|
}
|
|
54
91
|
return results;
|
|
55
92
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/ingest/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/ingest/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAYlD,SAAS,YAAY,CAAC,IAAyB;IAC7C,OAAO,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;AACzD,CAAC;AAED,SAAS,WAAW,CAAC,IAAyB;IAC5C,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAyB;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAC/B,OAAO,EAAE;YACP,MAAM,EAAE,6BAA6B;YACrC,GAAG,WAAW,CAAC,IAAI,CAAC;SACrB;KACa,CAAC,CAAC;IAClB,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,qDAAqD,GAAG,CAAC,MAAM,KAAK;YAClE,uEAAuE,IAAI,EAAE,CAChF,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;2EAG2E;AAC3E,KAAK,UAAU,YAAY,CACzB,KAAa,EACb,IAAY,EACZ,GAAW,EACX,QAAgB,EAChB,IAAyB;IAEzB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,MAAM,GAAG,qCAAqC,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;IACvF,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,EAAiB,CAAC,CAAC;IACnF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IACzB,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAa,EACb,IAAY,EACZ,OAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,UAAU,KAAK,IAAI,IAAI,EAAE,CAAC;IACzC,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IACnB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,gCAAgC,KAAK,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QAC5F,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAA+B,CAAC;QACtE,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,cAAc,CAClC,gCAAgC,KAAK,IAAI,IAAI,cAAc,GAAG,cAAc,EAC5E,IAAI,CACL,CAAC;IACF,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAGrC,CAAC;IACF,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,sCAAsC;QACtC,OAAO,CAAC,IAAI,CACV,aAAa,KAAK,IAAI,IAAI,iFAAiF,CAC5G,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACpF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvE,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACtE,IAAI,OAAO,KAAK,IAAI;YAAE,SAAS;QAC/B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;QACnD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAC9E,CAAC;YACF,MAAM,KAAK,GAAyB,EAAE,CAAC;YACvC,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;gBACvC,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;gBAC/E,IAAI,cAAc,KAAK,IAAI;oBAAE,SAAS;gBACtC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/** A bundled non-SKILL.md file (e.g. a reference doc or helper script)
|
|
2
|
+
* living alongside a skill in its source directory. Path is relative to
|
|
3
|
+
* the skill's own directory, e.g. "references/foo.md". */
|
|
4
|
+
export interface SkillFile {
|
|
5
|
+
path: string;
|
|
6
|
+
content: string;
|
|
7
|
+
}
|
|
1
8
|
/** A single skill record as stored in the local catalog. */
|
|
2
9
|
export interface SkillRecord {
|
|
3
10
|
/** Stable unique id, e.g. "vercel-labs/agent-skills/pdf-processing". */
|
|
@@ -10,6 +17,9 @@ export interface SkillRecord {
|
|
|
10
17
|
description: string;
|
|
11
18
|
/** Full SKILL.md body (Level 2 content), loaded lazily by consumers. */
|
|
12
19
|
body: string;
|
|
20
|
+
/** Bundled reference docs / scripts from the skill's directory, loaded
|
|
21
|
+
* lazily via skill_read_file — never eagerly returned by skill_load. */
|
|
22
|
+
files?: SkillFile[];
|
|
13
23
|
/** Optional install-count / popularity signal from the upstream registry. */
|
|
14
24
|
installCount?: number;
|
|
15
25
|
/** Audit status surfaced by the upstream registry, if any. */
|
|
@@ -19,7 +29,7 @@ export interface SkillRecord {
|
|
|
19
29
|
}
|
|
20
30
|
/** A search hit: a skill plus its relevance rank (lower = more relevant). */
|
|
21
31
|
export interface SkillSearchHit {
|
|
22
|
-
skill: Omit<SkillRecord, "body">;
|
|
32
|
+
skill: Omit<SkillRecord, "body" | "files">;
|
|
23
33
|
rank: number;
|
|
24
34
|
}
|
|
25
35
|
/** An MCP upstream tool schema as cataloged for tool_find/tool_call routing. */
|