@nanmicoder/dsh-skills-hub 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +22 -0
- package/README.md +32 -82
- package/README_EN.md +32 -82
- package/README_ZH.md +2 -0
- package/assets/readme/marketplace.png +0 -0
- package/compatibility.json +1 -1
- package/docs/CATALOG.md +39 -0
- package/docs/COMPATIBILITY.md +46 -0
- package/docs/CONFIGURATION.md +38 -0
- package/docs/CONTRACT.md +35 -12
- package/docs/RELEASE.md +5 -5
- package/lib/catalog/catalog.js +89 -0
- package/lib/catalog/skills.json +8673 -0
- package/lib/client.js +3349 -1728
- package/lib/client.js.map +1 -1
- package/lib/config.js +2 -0
- package/lib/index.js +16 -6
- package/lib/market/cache.js +104 -30
- package/lib/market/clawhub-provider.js +139 -22
- package/lib/market/disk-cache.js +113 -0
- package/lib/market/install-service.js +24 -11
- package/lib/market/market-service.js +161 -32
- package/lib/market/provider-fetch.js +9 -2
- package/lib/market/skillhub-provider.js +6 -2
- package/lib/market/types.js +2 -1
- package/lib/skills/installed.js +5 -0
- package/lib/skills/management.js +182 -4
- package/lib/skills/root.js +1 -1
- package/lib/types/catalog/catalog.d.ts +73 -0
- package/lib/types/client/api.d.ts +45 -8
- package/lib/types/client/components/CategoryBar.d.ts +13 -0
- package/lib/types/client/components/FilterBar.d.ts +4 -0
- package/lib/types/client/components/InstallConfirmDialog.d.ts +20 -11
- package/lib/types/client/components/InstalledSkillDetail.d.ts +24 -0
- package/lib/types/client/components/InstalledSkills.d.ts +18 -1
- package/lib/types/client/components/MarkdownView.d.ts +7 -1
- package/lib/types/client/components/MarketSkillDetail.d.ts +34 -0
- package/lib/types/client/components/SkillAvatar.d.ts +1 -1
- package/lib/types/client/components/SkillDetailShell.d.ts +86 -0
- package/lib/types/client/components/SkillFiles.d.ts +32 -0
- package/lib/types/client/components/SourceStatusBar.d.ts +4 -0
- package/lib/types/client/icons.d.ts +4 -0
- package/lib/types/client/locales.d.ts +94 -0
- package/lib/types/client/relative-time.d.ts +13 -0
- package/lib/types/client/skill-insights.d.ts +30 -0
- package/lib/types/client/skill-markdown.d.ts +39 -0
- package/lib/types/client/state.d.ts +36 -7
- package/lib/types/config.d.ts +2 -0
- package/lib/types/market/cache.d.ts +62 -14
- package/lib/types/market/clawhub-provider.d.ts +15 -0
- package/lib/types/market/disk-cache.d.ts +16 -0
- package/lib/types/market/install-service.d.ts +1 -0
- package/lib/types/market/market-service.d.ts +40 -6
- package/lib/types/market/provider-fetch.d.ts +2 -0
- package/lib/types/market/types.d.ts +29 -0
- package/lib/types/skills/installed.d.ts +4 -0
- package/lib/types/skills/management.d.ts +63 -2
- package/lib/types/skills/root.d.ts +8 -0
- package/lib/types/web-routes.d.ts +3 -1
- package/lib/web-routes.js +85 -8
- package/package.json +1 -1
- package/release-notes/v0.0.1.md +1 -1
- package/release-notes/v0.0.2.md +12 -0
- package/release-notes/v0.0.3.md +12 -0
- package/lib/types/client/components/InstallStateBadge.d.ts +0 -11
- package/lib/types/client/components/SecurityBadge.d.ts +0 -16
- package/lib/types/client/components/SkillDetailView.d.ts +0 -21
package/lib/config.js
CHANGED
|
@@ -27,6 +27,7 @@ export const SKILLS_HUB_DEFAULTS = {
|
|
|
27
27
|
requestRetries: 1,
|
|
28
28
|
allowUninstall: true,
|
|
29
29
|
pageSize: 24,
|
|
30
|
+
cacheTtlMinutes: 60,
|
|
30
31
|
};
|
|
31
32
|
/**
|
|
32
33
|
* `null` must stay a legal value for `skillsRoot` (it is the "use the DSH
|
|
@@ -48,4 +49,5 @@ export const Config = z.object({
|
|
|
48
49
|
requestRetries: z.natural().default(SKILLS_HUB_DEFAULTS.requestRetries),
|
|
49
50
|
allowUninstall: z.boolean().default(SKILLS_HUB_DEFAULTS.allowUninstall),
|
|
50
51
|
pageSize: z.natural().min(1).max(100).default(SKILLS_HUB_DEFAULTS.pageSize),
|
|
52
|
+
cacheTtlMinutes: z.natural().min(1).max(1440).default(SKILLS_HUB_DEFAULTS.cacheTtlMinutes),
|
|
51
53
|
});
|
package/lib/index.js
CHANGED
|
@@ -15,11 +15,13 @@
|
|
|
15
15
|
*
|
|
16
16
|
* @module dsh-skills-hub
|
|
17
17
|
*/
|
|
18
|
+
import * as path from 'node:path';
|
|
18
19
|
import { Config, SKILLS_HUB_DEFAULTS } from "./config.js";
|
|
20
|
+
import { configureMarketCache } from "./market/cache.js";
|
|
19
21
|
import { configureProviderFetch } from "./market/provider-fetch.js";
|
|
20
22
|
import { setInstalledLookup, resetInstalledLookup } from "./market/market-service.js";
|
|
21
23
|
import { installedLookupFrom, scanInstalledSkills } from "./skills/installed.js";
|
|
22
|
-
import { resolveSkillsRoot, resolveSkillsScanRoots } from "./skills/root.js";
|
|
24
|
+
import { resolveDshHome, resolveSkillsRoot, resolveSkillsScanRoots } from "./skills/root.js";
|
|
23
25
|
import { registerSkillsHubRoutes, setSkillsHubPageSize, } from "./web-routes.js";
|
|
24
26
|
export const name = 'skills-hub';
|
|
25
27
|
/**
|
|
@@ -52,6 +54,7 @@ function resolveConfig(config) {
|
|
|
52
54
|
requestRetries: config?.requestRetries ?? SKILLS_HUB_DEFAULTS.requestRetries,
|
|
53
55
|
allowUninstall: config?.allowUninstall ?? SKILLS_HUB_DEFAULTS.allowUninstall,
|
|
54
56
|
pageSize: config?.pageSize ?? SKILLS_HUB_DEFAULTS.pageSize,
|
|
57
|
+
cacheTtlMinutes: config?.cacheTtlMinutes ?? SKILLS_HUB_DEFAULTS.cacheTtlMinutes,
|
|
55
58
|
};
|
|
56
59
|
}
|
|
57
60
|
/**
|
|
@@ -101,6 +104,11 @@ export function apply(ctx, config) {
|
|
|
101
104
|
retries: resolved.requestRetries,
|
|
102
105
|
});
|
|
103
106
|
setSkillsHubPageSize(resolved.pageSize);
|
|
107
|
+
configureMarketCache({
|
|
108
|
+
directory: path.join(resolveDshHome(), 'cache', 'skills-hub', 'v1'),
|
|
109
|
+
ttlMs: resolved.cacheTtlMinutes * 60_000,
|
|
110
|
+
onError: (error) => ctx.logger.warn(`skills-hub: persistent cache unavailable: ${String(error)}`),
|
|
111
|
+
});
|
|
104
112
|
let disposed = false;
|
|
105
113
|
/** Rebuild the installed index and hand it to the market layer. */
|
|
106
114
|
const refreshInstalledIndex = async () => {
|
|
@@ -135,11 +143,12 @@ export function apply(ctx, config) {
|
|
|
135
143
|
// republishing it).
|
|
136
144
|
resetInstalledLookup();
|
|
137
145
|
}, 'skills-hub: activation state');
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
146
|
+
// Apply is synchronous, but the first HTTP read must wait for install
|
|
147
|
+
// annotations even when a persisted snapshot arrives before the scan does.
|
|
148
|
+
const initialScan = refreshInstalledIndex().catch((error) => {
|
|
149
|
+
ctx.logger.warn(`skills-hub: installed-skill index refresh failed: ${String(error)}`);
|
|
150
|
+
});
|
|
151
|
+
void initialScan.then(() => refreshSkillCatalog(ctx));
|
|
143
152
|
// The Web server may bind before or after this plugin (the Loader activates
|
|
144
153
|
// rows concurrently), and older compositions name the service `httpServer`.
|
|
145
154
|
// Bind lazily on the first service event that makes a server reachable.
|
|
@@ -159,6 +168,7 @@ export function apply(ctx, config) {
|
|
|
159
168
|
skillsRoot: () => resolveSkillsRoot(resolved.skillsRoot),
|
|
160
169
|
allowUninstall: () => resolved.allowUninstall,
|
|
161
170
|
rescan,
|
|
171
|
+
ready: () => initialScan,
|
|
162
172
|
};
|
|
163
173
|
// `ctx.effect` runs the factory immediately; `dsh-host-webserver` throws on a
|
|
164
174
|
// duplicate route path, so the flag may only be set once registration has
|
package/lib/market/cache.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Skills Market —
|
|
2
|
+
* Skills Market — memory + persistent TTL cache with stale-while-error support, plus
|
|
3
3
|
* per-source health tracking.
|
|
4
4
|
*
|
|
5
5
|
* Only *upstream* data is cached here (list pages, details, file bodies).
|
|
@@ -7,56 +7,128 @@
|
|
|
7
7
|
* and is deliberately never cached: a skill installed a second ago must show up
|
|
8
8
|
* as installed on the very next list call.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* The host configures disk storage at activation. Without a directory the
|
|
11
|
+
* cache stays memory-only, so provider tests never touch a user's Harness home.
|
|
12
12
|
*/
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
/** Per-payload TTLs: cheap-to-refetch list pages expire sooner than file bodies. */
|
|
13
|
+
import { DiskMarketCache, MAX_CACHE_ENTRIES, MAX_STALE_AGE_MS } from "./disk-cache.js";
|
|
14
|
+
/** Default snapshot lifetime; activation can override it with cacheTtlMinutes. */
|
|
16
15
|
export const MARKET_TTL = {
|
|
17
|
-
list:
|
|
18
|
-
search:
|
|
19
|
-
detail:
|
|
20
|
-
files:
|
|
21
|
-
fileContent:
|
|
16
|
+
list: 60 * 60_000,
|
|
17
|
+
search: 60 * 60_000,
|
|
18
|
+
detail: 60 * 60_000,
|
|
19
|
+
files: 60 * 60_000,
|
|
20
|
+
fileContent: 60 * 60_000,
|
|
22
21
|
};
|
|
23
|
-
|
|
22
|
+
const stats = {
|
|
23
|
+
hits: 0,
|
|
24
|
+
misses: 0,
|
|
25
|
+
staleServed: 0,
|
|
26
|
+
upstreamRequests: 0,
|
|
27
|
+
forcedRefreshes: 0,
|
|
28
|
+
};
|
|
29
|
+
/** Record one cache event. Kept tiny so it can sit on the request path. */
|
|
30
|
+
export function noteMarketStat(entry) {
|
|
31
|
+
stats[entry] += 1;
|
|
32
|
+
}
|
|
33
|
+
/** Snapshot of the counters; the caller cannot mutate the live object. */
|
|
34
|
+
export function getMarketStats() {
|
|
35
|
+
return { ...stats };
|
|
36
|
+
}
|
|
37
|
+
export class MarketCache {
|
|
38
|
+
options;
|
|
24
39
|
entries = new Map();
|
|
25
|
-
|
|
26
|
-
|
|
40
|
+
disk;
|
|
41
|
+
warned = false;
|
|
42
|
+
ttlMs;
|
|
43
|
+
constructor(options = {}) {
|
|
44
|
+
this.options = options;
|
|
45
|
+
this.ttlMs = options.ttlMs ?? MARKET_TTL.list;
|
|
46
|
+
if (options.directory)
|
|
47
|
+
this.disk = new DiskMarketCache(options.directory);
|
|
48
|
+
}
|
|
49
|
+
warn(error) {
|
|
50
|
+
if (this.warned)
|
|
51
|
+
return;
|
|
52
|
+
this.warned = true;
|
|
53
|
+
this.options.onError?.(error);
|
|
54
|
+
}
|
|
55
|
+
remember(key, entry) {
|
|
56
|
+
this.entries.delete(key);
|
|
57
|
+
this.entries.set(key, entry);
|
|
58
|
+
if (this.entries.size > MAX_CACHE_ENTRIES) {
|
|
59
|
+
const oldest = this.entries.keys().next().value;
|
|
60
|
+
if (oldest !== undefined)
|
|
61
|
+
this.entries.delete(oldest);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async read(key) {
|
|
65
|
+
const existing = this.entries.get(key);
|
|
66
|
+
if (existing)
|
|
67
|
+
return existing;
|
|
68
|
+
try {
|
|
69
|
+
const restored = await this.disk?.read(key);
|
|
70
|
+
// A concurrent upstream response may have replaced the snapshot while
|
|
71
|
+
// the disk read was pending. Never overwrite that newer memory entry.
|
|
72
|
+
const current = this.entries.get(key);
|
|
73
|
+
if (current)
|
|
74
|
+
return current;
|
|
75
|
+
if (restored)
|
|
76
|
+
this.remember(key, restored);
|
|
77
|
+
return restored;
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
// A corrupt or unwritable cache must never break the market request.
|
|
81
|
+
this.warn(error);
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Read a fresh entry.
|
|
87
|
+
*
|
|
88
|
+
* The record carries `storedAt` on purpose: a hit is a *snapshot*, and the
|
|
89
|
+
* caller must be able to say how old it is. Returning only the payload is how
|
|
90
|
+
* the panel ended up reporting "fetched just now" for data it had not touched
|
|
91
|
+
* in ten minutes.
|
|
92
|
+
*/
|
|
93
|
+
async getRecord(key) {
|
|
94
|
+
const entry = await this.read(key);
|
|
27
95
|
if (!entry)
|
|
28
96
|
return undefined;
|
|
29
|
-
if (Date.now()
|
|
97
|
+
if (Date.now() >= Math.min(entry.expiresAt, entry.storedAt + this.ttlMs))
|
|
30
98
|
return undefined;
|
|
31
99
|
// LRU touch: re-insert so the oldest key is the least recently used one.
|
|
32
100
|
this.entries.delete(key);
|
|
33
101
|
this.entries.set(key, entry);
|
|
34
|
-
return entry.value;
|
|
102
|
+
return { value: entry.value, storedAt: entry.storedAt };
|
|
35
103
|
}
|
|
36
104
|
/** Returns the entry even when expired — used for the stale-while-error fallback. */
|
|
37
|
-
getStale(key) {
|
|
38
|
-
const entry = this.
|
|
39
|
-
if (!entry)
|
|
105
|
+
async getStale(key) {
|
|
106
|
+
const entry = await this.read(key);
|
|
107
|
+
if (!entry || Date.now() - entry.storedAt > MAX_STALE_AGE_MS)
|
|
40
108
|
return undefined;
|
|
41
109
|
return { value: entry.value, storedAt: entry.storedAt };
|
|
42
110
|
}
|
|
43
|
-
set(key, value, ttlMs) {
|
|
44
|
-
if (this.entries.has(key))
|
|
45
|
-
this.entries.delete(key);
|
|
111
|
+
async set(key, value, ttlMs = this.ttlMs) {
|
|
46
112
|
const now = Date.now();
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (oldest !== undefined)
|
|
52
|
-
this.entries.delete(oldest);
|
|
113
|
+
const entry = { value, expiresAt: now + ttlMs, storedAt: now };
|
|
114
|
+
this.remember(key, entry);
|
|
115
|
+
try {
|
|
116
|
+
await this.disk?.write(key, entry);
|
|
53
117
|
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
this.warn(error);
|
|
120
|
+
}
|
|
121
|
+
return now;
|
|
54
122
|
}
|
|
55
123
|
clear() {
|
|
56
124
|
this.entries.clear();
|
|
57
125
|
}
|
|
58
126
|
}
|
|
59
|
-
export
|
|
127
|
+
export let marketCache = new MarketCache();
|
|
128
|
+
/** Replace memory state when activating a profile; disk entries are read lazily. */
|
|
129
|
+
export function configureMarketCache(options) {
|
|
130
|
+
marketCache = new MarketCache(options);
|
|
131
|
+
}
|
|
60
132
|
function freshHealth() {
|
|
61
133
|
return {
|
|
62
134
|
clawhub: { status: 'ok' },
|
|
@@ -106,8 +178,10 @@ export function resetSourceHealth() {
|
|
|
106
178
|
sourceHealth[source] = fresh[source];
|
|
107
179
|
}
|
|
108
180
|
}
|
|
109
|
-
/** Test hook: drop
|
|
181
|
+
/** Test hook: drop memory payloads, reset source health and the counters. */
|
|
110
182
|
export function resetMarketCache() {
|
|
111
183
|
marketCache.clear();
|
|
112
184
|
resetSourceHealth();
|
|
185
|
+
for (const entry of Object.keys(stats))
|
|
186
|
+
stats[entry] = 0;
|
|
113
187
|
}
|
|
@@ -7,7 +7,11 @@
|
|
|
7
7
|
* - GET /api/v1/skills/{slug} → {skill, latestVersion, owner, metadata, moderation}
|
|
8
8
|
* - GET /api/v1/skills/{slug}/versions/{v} → {version:{license, files[], security}}
|
|
9
9
|
* - GET /api/v1/skills/{slug}/file?path= → raw file text
|
|
10
|
+
*
|
|
11
|
+
* Slugs are not unique: every endpoint accepts `?owner=`, and without it a
|
|
12
|
+
* shared slug answers 409 AMBIGUOUS_SKILL_SLUG. See `clawhubOwnerFor`.
|
|
10
13
|
*/
|
|
14
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
11
15
|
import { getProviderBase, providerFetch, providerFetchJson, readResponseTextWithLimit, MarketHttpError, } from "./provider-fetch.js";
|
|
12
16
|
import { markSourceHealth } from "./cache.js";
|
|
13
17
|
import { detectMarketLanguage, MARKET_ERROR_CODES, MARKET_LIMITS, MarketUpstreamError, skillId, } from "./types.js";
|
|
@@ -344,14 +348,77 @@ function usableFileEntries(files) {
|
|
|
344
348
|
}
|
|
345
349
|
return out;
|
|
346
350
|
}
|
|
347
|
-
// ClawHub slugs are not unique across owners.
|
|
348
|
-
//
|
|
349
|
-
//
|
|
351
|
+
// ClawHub slugs are not unique across owners. Every read is therefore made
|
|
352
|
+
// for one owner, chosen in this order:
|
|
353
|
+
// 1. the owner the reader's request names (the card it came from);
|
|
354
|
+
// 2. the owner the curated catalogue pins for that slug;
|
|
355
|
+
// 3. an owner resolved earlier for the slug (remembered below);
|
|
356
|
+
// 4. otherwise, on 409 AMBIGUOUS_SKILL_SLUG, the most-downloaded candidate.
|
|
357
|
+
// The registry lists candidates oldest-first, and for popular slugs that is
|
|
358
|
+
// routinely a later copy by another author — "first match" opened (and
|
|
359
|
+
// installed) somebody else's skill.
|
|
350
360
|
const ownerCache = new Map();
|
|
361
|
+
/**
|
|
362
|
+
* Owners the curated catalogue already knows. A catalogue entry names the exact
|
|
363
|
+
* skill it recommends, so its owner must win over any guessing.
|
|
364
|
+
*/
|
|
365
|
+
const ownerHints = new Map();
|
|
366
|
+
/** The owner one detail/file/install operation was asked for. */
|
|
367
|
+
const requestedOwner = new AsyncLocalStorage();
|
|
368
|
+
/** Shape of a ClawHub owner handle (also the route's validation rule). */
|
|
369
|
+
export const CLAWHUB_OWNER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/;
|
|
370
|
+
/** Most candidates probed when resolving an ambiguous slug without an owner. */
|
|
371
|
+
const MAX_AMBIGUOUS_PROBES = 8;
|
|
372
|
+
export function setClawhubOwnerHints(hints) {
|
|
373
|
+
ownerHints.clear();
|
|
374
|
+
for (const [slug, owner] of hints)
|
|
375
|
+
ownerHints.set(slug, owner);
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Run one operation pinned to `owner` for `slug`. Without an owner the
|
|
379
|
+
* operation runs unpinned and falls back to the hint/cache/probe order.
|
|
380
|
+
*/
|
|
381
|
+
export function withClawhubOwner(slug, owner, operation) {
|
|
382
|
+
if (!owner)
|
|
383
|
+
return operation();
|
|
384
|
+
return requestedOwner.run({ slug, owner }, operation);
|
|
385
|
+
}
|
|
386
|
+
/** The owner a read of `slug` will use right now, if one is known before asking upstream. */
|
|
387
|
+
export function clawhubOwnerFor(slug) {
|
|
388
|
+
const requested = requestedOwner.getStore();
|
|
389
|
+
if (requested?.slug === slug)
|
|
390
|
+
return requested.owner;
|
|
391
|
+
return ownerHints.get(slug) ?? ownerCache.get(slug);
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Pick the candidate readers most plausibly mean: the most-downloaded one.
|
|
395
|
+
* Probes are owner-qualified detail reads; a candidate that fails to answer
|
|
396
|
+
* simply does not compete.
|
|
397
|
+
*/
|
|
398
|
+
async function resolveAmbiguousOwner(slug, candidates) {
|
|
399
|
+
const base = getProviderBase('clawhub');
|
|
400
|
+
const probed = await Promise.all(candidates.slice(0, MAX_AMBIGUOUS_PROBES).map(async (owner) => {
|
|
401
|
+
const url = new URL(`/api/v1/skills/${encodeURIComponent(slug)}`, base);
|
|
402
|
+
url.searchParams.set('owner', owner);
|
|
403
|
+
try {
|
|
404
|
+
const res = await providerFetch('clawhub', url.toString());
|
|
405
|
+
if (!res.ok)
|
|
406
|
+
return { owner, downloads: -1 };
|
|
407
|
+
const { content } = await readResponseTextWithLimit('clawhub', res, MARKET_LIMITS.maxTotalSize, 'ambiguous-slug probe');
|
|
408
|
+
const parsed = JSON.parse(content);
|
|
409
|
+
return { owner, downloads: asNumber(parsed.skill?.stats?.downloads) ?? 0 };
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
return { owner, downloads: -1 };
|
|
413
|
+
}
|
|
414
|
+
}));
|
|
415
|
+
const best = probed.filter((entry) => entry.downloads >= 0).sort((a, b) => b.downloads - a.downloads)[0];
|
|
416
|
+
return best?.owner;
|
|
417
|
+
}
|
|
351
418
|
async function clawhubFetch(url, slug) {
|
|
352
|
-
const
|
|
353
|
-
if (
|
|
354
|
-
url.searchParams.set('owner',
|
|
419
|
+
const knownOwner = clawhubOwnerFor(slug);
|
|
420
|
+
if (knownOwner && !url.searchParams.has('owner')) {
|
|
421
|
+
url.searchParams.set('owner', knownOwner);
|
|
355
422
|
}
|
|
356
423
|
const res = await providerFetch('clawhub', url.toString());
|
|
357
424
|
if (res.status !== 409)
|
|
@@ -371,7 +438,13 @@ async function clawhubFetch(url, slug) {
|
|
|
371
438
|
throw error;
|
|
372
439
|
body = null;
|
|
373
440
|
}
|
|
374
|
-
const
|
|
441
|
+
const candidates = body?.code === 'AMBIGUOUS_SKILL_SLUG'
|
|
442
|
+
? (body.matches ?? []).map((match) => asString(match?.ownerHandle)).filter((owner) => owner !== undefined)
|
|
443
|
+
: [];
|
|
444
|
+
// A pinned owner that still answers 409 is not a guess we may widen.
|
|
445
|
+
const resolvedOwner = knownOwner === undefined && candidates.length > 0
|
|
446
|
+
? await resolveAmbiguousOwner(slug, candidates)
|
|
447
|
+
: undefined;
|
|
375
448
|
if (!resolvedOwner) {
|
|
376
449
|
throw new MarketHttpError('clawhub', 409, MARKET_ERROR_CODES.upstreamError, `clawhub responded 409 for ${url.pathname}`);
|
|
377
450
|
}
|
|
@@ -392,24 +465,61 @@ async function clawhubFetchJson(url, slug) {
|
|
|
392
465
|
throw new MarketUpstreamError('clawhub', MARKET_ERROR_CODES.upstreamBadResponse, 'clawhub returned invalid JSON');
|
|
393
466
|
}
|
|
394
467
|
}
|
|
468
|
+
/**
|
|
469
|
+
* ClawHub's scan of one version: an overall status plus per-scanner verdicts
|
|
470
|
+
* (VirusTotal, skillspector, an LLM review). The overall status decides the
|
|
471
|
+
* badge; each scanner becomes its own report so the reader sees which one
|
|
472
|
+
* objected and why.
|
|
473
|
+
*/
|
|
395
474
|
function mapSecurity(security) {
|
|
396
475
|
const vendorStatus = asString(security?.status);
|
|
397
476
|
if (!vendorStatus)
|
|
398
477
|
return { status: 'unknown', reports: [] };
|
|
399
478
|
const clean = vendorStatus === 'clean';
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
],
|
|
412
|
-
|
|
479
|
+
const reports = [
|
|
480
|
+
{
|
|
481
|
+
vendor: 'clawhub-scan',
|
|
482
|
+
status: vendorStatus,
|
|
483
|
+
statusText: clean
|
|
484
|
+
? security?.hasWarnings === true ? 'Clean (with warnings)' : 'Clean'
|
|
485
|
+
: `Scan status: ${vendorStatus}`,
|
|
486
|
+
reportUrl: asString(security?.virustotalUrl),
|
|
487
|
+
},
|
|
488
|
+
];
|
|
489
|
+
const scanners = [
|
|
490
|
+
['VirusTotal', security?.scanners?.vt],
|
|
491
|
+
['skillspector', security?.scanners?.skillspector],
|
|
492
|
+
['LLM review', security?.scanners?.llm],
|
|
493
|
+
];
|
|
494
|
+
for (const [vendor, scanner] of scanners) {
|
|
495
|
+
const status = asString(scanner?.normalizedStatus) ?? asString(scanner?.status);
|
|
496
|
+
if (!status)
|
|
497
|
+
continue;
|
|
498
|
+
const recommendation = asString(scanner?.recommendation);
|
|
499
|
+
const summary = asString(scanner?.summary);
|
|
500
|
+
const reportUrl = vendor === 'VirusTotal' ? asString(security?.virustotalUrl) : undefined;
|
|
501
|
+
reports.push({
|
|
502
|
+
vendor,
|
|
503
|
+
status,
|
|
504
|
+
statusText: recommendation ? `${status} · ${recommendation}` : status,
|
|
505
|
+
...(summary ? { summary } : {}),
|
|
506
|
+
...(reportUrl ? { reportUrl } : {}),
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
return { status: clean ? 'benign' : 'flagged', reports };
|
|
510
|
+
}
|
|
511
|
+
function changelogOf(version, latest) {
|
|
512
|
+
const text = meaningfulChangelog(latest?.changelog);
|
|
513
|
+
return text ? { changelog: { version, text, publishedAt: asNumber(latest?.createdAt) } } : {};
|
|
514
|
+
}
|
|
515
|
+
/** A release note worth showing: registries stamp placeholders on synced versions. */
|
|
516
|
+
export function meaningfulChangelog(text) {
|
|
517
|
+
const value = asString(text)?.trim();
|
|
518
|
+
if (!value)
|
|
519
|
+
return undefined;
|
|
520
|
+
if (/^synced by .*pipeline$/i.test(value))
|
|
521
|
+
return undefined;
|
|
522
|
+
return value;
|
|
413
523
|
}
|
|
414
524
|
function normalizeListItem(item) {
|
|
415
525
|
return {
|
|
@@ -418,7 +528,7 @@ function normalizeListItem(item) {
|
|
|
418
528
|
slug: item.slug,
|
|
419
529
|
name: asString(item.displayName) || item.slug,
|
|
420
530
|
summary: asString(item.summary) ?? '',
|
|
421
|
-
author: { handle: '' },
|
|
531
|
+
author: { handle: asString(item.ownerHandle) || '' },
|
|
422
532
|
stats: {
|
|
423
533
|
downloads: asNumber(item.stats?.downloads) ?? 0,
|
|
424
534
|
installs: asNumber(item.stats?.installs),
|
|
@@ -505,7 +615,11 @@ export const clawhubProvider = {
|
|
|
505
615
|
}
|
|
506
616
|
const nativeResults = usable.filter((result) => (result.source === undefined || result.source === 'clawhub')
|
|
507
617
|
&& (result.install?.kind === undefined || result.install.kind === 'clawhub'));
|
|
508
|
-
|
|
618
|
+
// A card id is `clawhub:<slug>`, so a page can hold one owner per slug: the
|
|
619
|
+
// most relevant (first) result keeps the slot and carries its owner.
|
|
620
|
+
const seen = new Set();
|
|
621
|
+
const distinct = nativeResults.filter((result) => !seen.has(result.slug) && Boolean(seen.add(result.slug)));
|
|
622
|
+
return { items: distinct.slice(0, limit).map(normalizeSearchResult) };
|
|
509
623
|
},
|
|
510
624
|
async detail(slug) {
|
|
511
625
|
const base = getProviderBase('clawhub');
|
|
@@ -549,6 +663,7 @@ export const clawhubProvider = {
|
|
|
549
663
|
frontmatter = parsed.frontmatter;
|
|
550
664
|
}
|
|
551
665
|
const item = normalizeListItem(data.skill);
|
|
666
|
+
const owner = asString(data.owner?.handle);
|
|
552
667
|
return {
|
|
553
668
|
...item,
|
|
554
669
|
version,
|
|
@@ -559,6 +674,8 @@ export const clawhubProvider = {
|
|
|
559
674
|
},
|
|
560
675
|
securityStatus: security.status,
|
|
561
676
|
securityReports: security.reports.length ? security.reports : undefined,
|
|
677
|
+
...changelogOf(version, data.latestVersion),
|
|
678
|
+
...(owner ? { pageUrl: `https://clawhub.ai/${encodeURIComponent(owner)}/${encodeURIComponent(slug)}` } : {}),
|
|
562
679
|
description: body,
|
|
563
680
|
descriptionFrontmatter: frontmatter,
|
|
564
681
|
license,
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/** Atomic, bounded storage for upstream snapshots; never stores install state. */
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import * as fs from 'node:fs/promises';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
export const MAX_CACHE_ENTRIES = 500;
|
|
6
|
+
export const MAX_STALE_AGE_MS = 7 * 24 * 60 * 60_000;
|
|
7
|
+
const MAX_ENTRY_BYTES = 5 * 1024 * 1024;
|
|
8
|
+
const MAX_DISK_BYTES = 64 * 1024 * 1024;
|
|
9
|
+
const CACHE_FILE = /^[a-f0-9]{64}\.json$/;
|
|
10
|
+
const TEMP_FILE = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}\.tmp$/;
|
|
11
|
+
const MAX_TEMP_AGE_MS = 60 * 60_000;
|
|
12
|
+
function snapshotHash(key, entry) {
|
|
13
|
+
return createHash('sha256').update(JSON.stringify([2, key, entry.storedAt, entry.expiresAt, entry.value])).digest('hex');
|
|
14
|
+
}
|
|
15
|
+
export class DiskMarketCache {
|
|
16
|
+
directory;
|
|
17
|
+
writes = Promise.resolve();
|
|
18
|
+
constructor(directory) {
|
|
19
|
+
this.directory = directory;
|
|
20
|
+
}
|
|
21
|
+
filename(key) {
|
|
22
|
+
return path.join(this.directory, `${createHash('sha256').update(key).digest('hex')}.json`);
|
|
23
|
+
}
|
|
24
|
+
async read(key) {
|
|
25
|
+
let handle;
|
|
26
|
+
try {
|
|
27
|
+
handle = await fs.open(this.filename(key), 'r');
|
|
28
|
+
if ((await handle.stat()).size > MAX_ENTRY_BYTES)
|
|
29
|
+
return undefined;
|
|
30
|
+
const record = JSON.parse(await handle.readFile('utf8'));
|
|
31
|
+
if (!record || record.version !== 2 || record.key !== key || !('value' in record))
|
|
32
|
+
return undefined;
|
|
33
|
+
const { storedAt, expiresAt, value } = record;
|
|
34
|
+
if (typeof storedAt !== 'number' || !Number.isFinite(storedAt) || storedAt < 0
|
|
35
|
+
|| typeof expiresAt !== 'number' || !Number.isFinite(expiresAt) || expiresAt < storedAt
|
|
36
|
+
|| storedAt > Date.now() || Date.now() - storedAt > MAX_STALE_AGE_MS)
|
|
37
|
+
return undefined;
|
|
38
|
+
const entry = { storedAt, expiresAt, value };
|
|
39
|
+
if (record.sha256 !== snapshotHash(key, entry))
|
|
40
|
+
return undefined;
|
|
41
|
+
return entry;
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
if (error.code === 'ENOENT')
|
|
45
|
+
return undefined;
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
await handle?.close();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async write(key, entry) {
|
|
53
|
+
const body = JSON.stringify({ version: 2, key, ...entry, sha256: snapshotHash(key, entry) });
|
|
54
|
+
const oversized = Buffer.byteLength(body) > MAX_ENTRY_BYTES;
|
|
55
|
+
// Serialize publication and pruning within this host. Each response awaits
|
|
56
|
+
// its write, so quitting after reading a page does not lose a delayed flush.
|
|
57
|
+
const pending = this.writes.then(async () => {
|
|
58
|
+
// An unpersistable refresh must not resurrect the previous version after
|
|
59
|
+
// restart. Serialize invalidation with ordinary writes of the same key.
|
|
60
|
+
if (oversized) {
|
|
61
|
+
await fs.rm(this.filename(key), { force: true });
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
await fs.mkdir(this.directory, { recursive: true, mode: 0o700 });
|
|
65
|
+
const temporary = path.join(this.directory, `${randomUUID()}.tmp`);
|
|
66
|
+
try {
|
|
67
|
+
await fs.writeFile(temporary, body, { mode: 0o600, flag: 'wx' });
|
|
68
|
+
await fs.rename(temporary, this.filename(key));
|
|
69
|
+
await this.prune();
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
await fs.rm(temporary, { force: true });
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
this.writes = pending.catch(() => { });
|
|
76
|
+
await pending;
|
|
77
|
+
}
|
|
78
|
+
async prune() {
|
|
79
|
+
const names = (await fs.readdir(this.directory)).filter((name) => CACHE_FILE.test(name) || TEMP_FILE.test(name));
|
|
80
|
+
const records = await Promise.all(names.map(async (name) => {
|
|
81
|
+
const filename = path.join(this.directory, name);
|
|
82
|
+
try {
|
|
83
|
+
const stat = await fs.stat(filename);
|
|
84
|
+
if (TEMP_FILE.test(name)) {
|
|
85
|
+
// A killed host can leave a partial write. Leave recent files alone:
|
|
86
|
+
// another host may still be publishing in this shared home.
|
|
87
|
+
if (Date.now() - stat.mtimeMs > MAX_TEMP_AGE_MS)
|
|
88
|
+
await fs.rm(filename, { force: true });
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
return { filename, size: stat.size, modifiedAt: stat.mtimeMs };
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
if (error.code === 'ENOENT')
|
|
95
|
+
return undefined;
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
}));
|
|
99
|
+
const newest = records.filter((record) => record !== undefined).sort((a, b) => b.modifiedAt - a.modifiedAt);
|
|
100
|
+
let count = 0;
|
|
101
|
+
let bytes = 0;
|
|
102
|
+
for (const record of newest) {
|
|
103
|
+
if (Date.now() - record.modifiedAt > MAX_STALE_AGE_MS
|
|
104
|
+
|| record.size > MAX_ENTRY_BYTES || count >= MAX_CACHE_ENTRIES || bytes + record.size > MAX_DISK_BYTES) {
|
|
105
|
+
await fs.rm(record.filename, { force: true });
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
count += 1;
|
|
109
|
+
bytes += record.size;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|