@nkmc/gateway 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-56RA53VS.js +37 -0
- package/dist/chunk-CZJ75YTV.js +969 -0
- package/dist/chunk-QGM4M3NI.js +37 -0
- package/dist/http.cjs +1772 -0
- package/dist/http.d.cts +49 -0
- package/dist/http.d.ts +49 -0
- package/dist/http.js +748 -0
- package/dist/index.cjs +2436 -0
- package/dist/index.d.cts +436 -0
- package/dist/index.d.ts +436 -0
- package/dist/index.js +1434 -0
- package/dist/proxy-ClPcDgsO.d.cts +283 -0
- package/dist/proxy-qpda1ANS.d.ts +283 -0
- package/dist/proxy.cjs +148 -0
- package/dist/proxy.d.cts +6 -0
- package/dist/proxy.d.ts +6 -0
- package/dist/proxy.js +90 -0
- package/dist/testing.cjs +865 -0
- package/dist/testing.d.cts +12 -0
- package/dist/testing.d.ts +12 -0
- package/dist/testing.js +831 -0
- package/dist/tunnels-BviBEaih.d.cts +12 -0
- package/dist/tunnels-DFHNgmN7.d.ts +12 -0
- package/dist/types-C6JC9oTm.d.cts +21 -0
- package/dist/types-C6JC9oTm.d.ts +21 -0
- package/package.json +47 -0
- package/src/__tests__/sqlite-integration.test.ts +384 -0
- package/src/credential/d1-vault.ts +134 -0
- package/src/credential/memory-vault.ts +50 -0
- package/src/credential/types.ts +16 -0
- package/src/d1/__tests__/sqlite-adapter.test.ts +75 -0
- package/src/d1/sqlite-adapter.ts +59 -0
- package/src/d1/types.ts +22 -0
- package/src/federation/__tests__/d1-peer-store.test.ts +218 -0
- package/src/federation/__tests__/peer-client.test.ts +205 -0
- package/src/federation/__tests__/peer-store.test.ts +114 -0
- package/src/federation/d1-peer-store.ts +164 -0
- package/src/federation/peer-backend.ts +60 -0
- package/src/federation/peer-client.ts +122 -0
- package/src/federation/peer-store.ts +45 -0
- package/src/federation/types.ts +39 -0
- package/src/http/app.ts +152 -0
- package/src/http/lib/dns.ts +30 -0
- package/src/http/middleware/admin-auth.ts +18 -0
- package/src/http/middleware/agent-auth.ts +27 -0
- package/src/http/middleware/publish-auth.ts +39 -0
- package/src/http/routes/__tests__/federation.test.ts +364 -0
- package/src/http/routes/__tests__/peers.test.ts +290 -0
- package/src/http/routes/__tests__/proxy.test.ts +159 -0
- package/src/http/routes/auth.ts +39 -0
- package/src/http/routes/byok.ts +62 -0
- package/src/http/routes/credentials.ts +40 -0
- package/src/http/routes/domains.ts +174 -0
- package/src/http/routes/federation.ts +170 -0
- package/src/http/routes/fs.ts +89 -0
- package/src/http/routes/peers.ts +103 -0
- package/src/http/routes/proxy.ts +57 -0
- package/src/http/routes/registry.ts +222 -0
- package/src/http/routes/tunnels.ts +124 -0
- package/src/http.ts +9 -0
- package/src/index.ts +63 -0
- package/src/metering/d1-store.ts +123 -0
- package/src/metering/memory-store.ts +29 -0
- package/src/metering/pricing-guard.ts +68 -0
- package/src/metering/types.ts +25 -0
- package/src/onboard/apis-guru.ts +64 -0
- package/src/onboard/index.ts +4 -0
- package/src/onboard/manifest.ts +362 -0
- package/src/onboard/pipeline.ts +214 -0
- package/src/onboard/types.ts +72 -0
- package/src/proxy/__tests__/tool-registry.test.ts +93 -0
- package/src/proxy/tool-registry.ts +122 -0
- package/src/proxy.ts +12 -0
- package/src/registry/context7-backend.ts +93 -0
- package/src/registry/context7.ts +54 -0
- package/src/registry/d1-store.ts +242 -0
- package/src/registry/memory-store.ts +101 -0
- package/src/registry/openapi-compiler.ts +284 -0
- package/src/registry/resolver.ts +196 -0
- package/src/registry/rpc-compiler.ts +142 -0
- package/src/registry/skill-parser.ts +119 -0
- package/src/registry/skill-to-config.ts +239 -0
- package/src/registry/source-refresher.ts +83 -0
- package/src/registry/types.ts +129 -0
- package/src/registry/virtual-files.ts +76 -0
- package/src/testing/sqlite-d1.ts +64 -0
- package/src/testing.ts +2 -0
- package/src/tunnel/__tests__/cloudflare-provider.test.ts +255 -0
- package/src/tunnel/__tests__/tunnel.test.ts +542 -0
- package/src/tunnel/cloudflare-provider.ts +121 -0
- package/src/tunnel/memory-store.ts +30 -0
- package/src/tunnel/types.ts +28 -0
- package/test/credential/d1-vault.test.ts +127 -0
- package/test/credential/injection.test.ts +67 -0
- package/test/credential/memory-vault.test.ts +63 -0
- package/test/http/app.test.ts +300 -0
- package/test/http/byok-e2e.test.ts +240 -0
- package/test/http/byok.test.ts +115 -0
- package/test/http/credentials.test.ts +57 -0
- package/test/http/e2e.test.ts +260 -0
- package/test/integration/authenticated-apis.test.ts +185 -0
- package/test/integration/free-apis-e2e.test.ts +222 -0
- package/test/metering/d1-store.test.ts +82 -0
- package/test/metering/memory-store.test.ts +76 -0
- package/test/metering/pricing-guard.test.ts +108 -0
- package/test/onboard/apis-guru.test.ts +57 -0
- package/test/onboard/e2e.test.ts +70 -0
- package/test/onboard/pipeline.test.ts +318 -0
- package/test/onboard/real-apis.test.ts +483 -0
- package/test/registry/compilation-correctness.test.ts +132 -0
- package/test/registry/context7-backend.test.ts +88 -0
- package/test/registry/context7-e2e.test.ts +92 -0
- package/test/registry/context7.test.ts +73 -0
- package/test/registry/d1-store.test.ts +184 -0
- package/test/registry/integration.test.ts +129 -0
- package/test/registry/lazy-mount.test.ts +138 -0
- package/test/registry/memory-store.test.ts +171 -0
- package/test/registry/openapi-compiler.test.ts +267 -0
- package/test/registry/openapi-e2e.test.ts +154 -0
- package/test/registry/passthrough-e2e.test.ts +109 -0
- package/test/registry/resolver-peer.test.ts +299 -0
- package/test/registry/resolver.test.ts +228 -0
- package/test/registry/rpc-compiler.test.ts +112 -0
- package/test/registry/skill-parser.test.ts +151 -0
- package/test/registry/skill-to-config.test.ts +151 -0
- package/test/registry/skill-to-rpc-config.test.ts +142 -0
- package/test/registry/source-refresher.test.ts +90 -0
- package/test/registry/virtual-files.test.ts +96 -0
- package/tsconfig.json +4 -0
- package/tsup.config.ts +8 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { ManifestEntry } from "./types.js";
|
|
2
|
+
|
|
3
|
+
const APIS_GURU_LIST = "https://api.apis.guru/v2/list.json";
|
|
4
|
+
|
|
5
|
+
export interface ApisGuruOptions {
|
|
6
|
+
/** Max number of APIs to return */
|
|
7
|
+
limit?: number;
|
|
8
|
+
/** Filter by keyword in API title/description */
|
|
9
|
+
filter?: string;
|
|
10
|
+
/** Custom fetch function */
|
|
11
|
+
fetchFn?: typeof globalThis.fetch;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface ApisGuruEntry {
|
|
15
|
+
preferred: string;
|
|
16
|
+
versions: Record<string, {
|
|
17
|
+
swaggerUrl?: string;
|
|
18
|
+
openapiVer?: string;
|
|
19
|
+
info?: { title?: string; description?: string; "x-logo"?: { url?: string } };
|
|
20
|
+
}>;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Discover APIs from the apis.guru public directory.
|
|
25
|
+
* Returns ManifestEntry[] ready for the onboard pipeline.
|
|
26
|
+
*/
|
|
27
|
+
export async function discoverFromApisGuru(options?: ApisGuruOptions): Promise<ManifestEntry[]> {
|
|
28
|
+
const fetchFn = options?.fetchFn ?? globalThis.fetch.bind(globalThis);
|
|
29
|
+
const limit = options?.limit ?? 100;
|
|
30
|
+
const filter = options?.filter?.toLowerCase();
|
|
31
|
+
|
|
32
|
+
const resp = await fetchFn(APIS_GURU_LIST);
|
|
33
|
+
if (!resp.ok) throw new Error(`apis.guru fetch failed: ${resp.status}`);
|
|
34
|
+
const catalog = (await resp.json()) as Record<string, ApisGuruEntry>;
|
|
35
|
+
|
|
36
|
+
const entries: ManifestEntry[] = [];
|
|
37
|
+
|
|
38
|
+
for (const [key, api] of Object.entries(catalog)) {
|
|
39
|
+
if (entries.length >= limit) break;
|
|
40
|
+
|
|
41
|
+
const version = api.versions[api.preferred];
|
|
42
|
+
if (!version?.swaggerUrl) continue;
|
|
43
|
+
|
|
44
|
+
const title = version.info?.title ?? key;
|
|
45
|
+
const desc = version.info?.description ?? "";
|
|
46
|
+
|
|
47
|
+
// Filter by keyword if provided
|
|
48
|
+
if (filter) {
|
|
49
|
+
const text = `${key} ${title} ${desc}`.toLowerCase();
|
|
50
|
+
if (!text.includes(filter)) continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Extract domain from the key (format: "domain.com:version" or "domain.com")
|
|
54
|
+
const domain = key.split(":")[0];
|
|
55
|
+
|
|
56
|
+
entries.push({
|
|
57
|
+
domain,
|
|
58
|
+
specUrl: version.swaggerUrl,
|
|
59
|
+
tags: ["apis-guru", "public"],
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return entries;
|
|
64
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { OnboardPipeline, type PipelineOptions } from "./pipeline.js";
|
|
2
|
+
export { discoverFromApisGuru, type ApisGuruOptions } from "./apis-guru.js";
|
|
3
|
+
export { ALL_APIS, FREE_APIS, FREEMIUM_APIS, getSpecOnlyApis } from "./manifest.js";
|
|
4
|
+
export type { ManifestEntry, ManifestAuth, OnboardResult, OnboardReport } from "./types.js";
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
import type { ManifestEntry, RpcManifestDef } from "./types.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Curated manifest of major public APIs with verified OpenAPI spec URLs.
|
|
5
|
+
*
|
|
6
|
+
* Categories:
|
|
7
|
+
* - free: No auth needed, all endpoints public
|
|
8
|
+
* - freemium: Some endpoints public, auth optional
|
|
9
|
+
* - auth-required: Auth needed for all API calls
|
|
10
|
+
*
|
|
11
|
+
* All spec URLs verified reachable as of 2026-02.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
// ── Free / No-Auth APIs ──────────────────────────────────────────────
|
|
15
|
+
|
|
16
|
+
export const FREE_APIS: ManifestEntry[] = [
|
|
17
|
+
{
|
|
18
|
+
domain: "petstore3.swagger.io",
|
|
19
|
+
specUrl: "https://petstore3.swagger.io/api/v3/openapi.json",
|
|
20
|
+
tags: ["demo", "free"],
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
domain: "api.weather.gov",
|
|
24
|
+
specUrl: "https://api.weather.gov/openapi.json",
|
|
25
|
+
tags: ["weather", "government", "free"],
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
domain: "en.wikipedia.org",
|
|
29
|
+
specUrl: "https://en.wikipedia.org/api/rest_v1/?spec",
|
|
30
|
+
tags: ["knowledge", "encyclopedia", "free"],
|
|
31
|
+
},
|
|
32
|
+
];
|
|
33
|
+
|
|
34
|
+
// ── Freemium APIs (public read, auth optional) ──────────────────────
|
|
35
|
+
|
|
36
|
+
export const FREEMIUM_APIS: ManifestEntry[] = [
|
|
37
|
+
{
|
|
38
|
+
domain: "api.github.com",
|
|
39
|
+
specUrl:
|
|
40
|
+
"https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.2022-11-28.json",
|
|
41
|
+
auth: { type: "bearer", token: "${GITHUB_TOKEN}" },
|
|
42
|
+
tags: ["developer-tools", "vcs", "freemium"],
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
domain: "huggingface.co",
|
|
46
|
+
specUrl: "https://huggingface.co/.well-known/openapi.json",
|
|
47
|
+
auth: { type: "bearer", token: "${HF_TOKEN}" },
|
|
48
|
+
tags: ["ai", "ml", "models", "freemium"],
|
|
49
|
+
},
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
// ── Auth-Required APIs (Developer Tools) ─────────────────────────────
|
|
53
|
+
|
|
54
|
+
export const DEVELOPER_TOOL_APIS: ManifestEntry[] = [
|
|
55
|
+
{
|
|
56
|
+
domain: "gitlab.com",
|
|
57
|
+
specUrl:
|
|
58
|
+
"https://gitlab.com/gitlab-org/gitlab/-/raw/master/doc/api/openapi/openapi.yaml",
|
|
59
|
+
auth: { type: "bearer", token: "${GITLAB_TOKEN}" },
|
|
60
|
+
tags: ["developer-tools", "vcs"],
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
domain: "api.vercel.com",
|
|
64
|
+
specUrl: "https://openapi.vercel.sh",
|
|
65
|
+
auth: { type: "bearer", token: "${VERCEL_TOKEN}" },
|
|
66
|
+
tags: ["developer-tools", "hosting"],
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
domain: "sentry.io",
|
|
70
|
+
specUrl:
|
|
71
|
+
"https://raw.githubusercontent.com/getsentry/sentry-api-schema/main/openapi-derefed.json",
|
|
72
|
+
auth: { type: "bearer", token: "${SENTRY_AUTH_TOKEN}" },
|
|
73
|
+
tags: ["developer-tools", "monitoring"],
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
domain: "api.pagerduty.com",
|
|
77
|
+
specUrl:
|
|
78
|
+
"https://raw.githubusercontent.com/PagerDuty/api-schema/main/reference/REST/openapiv3.json",
|
|
79
|
+
auth: { type: "bearer", token: "${PAGERDUTY_TOKEN}" },
|
|
80
|
+
tags: ["developer-tools", "incident-management"],
|
|
81
|
+
},
|
|
82
|
+
];
|
|
83
|
+
|
|
84
|
+
// ── AI / ML APIs ─────────────────────────────────────────────────────
|
|
85
|
+
|
|
86
|
+
export const AI_APIS: ManifestEntry[] = [
|
|
87
|
+
{
|
|
88
|
+
domain: "api.mistral.ai",
|
|
89
|
+
specUrl:
|
|
90
|
+
"https://raw.githubusercontent.com/mistralai/platform-docs-public/main/openapi.yaml",
|
|
91
|
+
auth: { type: "bearer", token: "${MISTRAL_API_KEY}" },
|
|
92
|
+
tags: ["ai", "llm"],
|
|
93
|
+
disabled: true, // upstream YAML spec has unescaped quotes in example data
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
domain: "api.openai.com",
|
|
97
|
+
specUrl:
|
|
98
|
+
"https://raw.githubusercontent.com/openai/openai-openapi/manual_spec/openapi.yaml",
|
|
99
|
+
auth: { type: "bearer", token: "${OPENAI_API_KEY}" },
|
|
100
|
+
tags: ["ai", "llm"],
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
domain: "openrouter.ai",
|
|
104
|
+
specUrl: "https://openrouter.ai/openapi.json",
|
|
105
|
+
auth: { type: "bearer", token: "${OPENROUTER_API_KEY}" },
|
|
106
|
+
tags: ["ai", "llm", "gateway"],
|
|
107
|
+
},
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
// ── Cloud / Infrastructure APIs ──────────────────────────────────────
|
|
111
|
+
|
|
112
|
+
export const CLOUD_APIS: ManifestEntry[] = [
|
|
113
|
+
{
|
|
114
|
+
domain: "api.cloudflare.com",
|
|
115
|
+
specUrl:
|
|
116
|
+
"https://raw.githubusercontent.com/cloudflare/api-schemas/main/openapi.yaml",
|
|
117
|
+
auth: { type: "bearer", token: "${CLOUDFLARE_API_TOKEN}" },
|
|
118
|
+
tags: ["cloud", "cdn", "dns"],
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
domain: "api.digitalocean.com",
|
|
122
|
+
specUrl:
|
|
123
|
+
"https://raw.githubusercontent.com/digitalocean/openapi/main/specification/DigitalOcean-public.v2.yaml",
|
|
124
|
+
auth: { type: "bearer", token: "${DIGITALOCEAN_TOKEN}" },
|
|
125
|
+
tags: ["cloud", "infrastructure"],
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
domain: "fly.io",
|
|
129
|
+
specUrl: "https://docs.machines.dev/spec/openapi3.json",
|
|
130
|
+
auth: { type: "bearer", token: "${FLY_API_TOKEN}" },
|
|
131
|
+
tags: ["cloud", "deployment"],
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
domain: "api.render.com",
|
|
135
|
+
specUrl:
|
|
136
|
+
"https://api-docs.render.com/v1.0/openapi/render-public-api-1.json",
|
|
137
|
+
auth: { type: "bearer", token: "${RENDER_API_KEY}" },
|
|
138
|
+
tags: ["cloud", "deployment"],
|
|
139
|
+
},
|
|
140
|
+
];
|
|
141
|
+
|
|
142
|
+
// ── Productivity / Project Management ────────────────────────────────
|
|
143
|
+
|
|
144
|
+
export const PRODUCTIVITY_APIS: ManifestEntry[] = [
|
|
145
|
+
{
|
|
146
|
+
domain: "api.notion.com",
|
|
147
|
+
specUrl:
|
|
148
|
+
"https://raw.githubusercontent.com/makenotion/notion-mcp-server/main/scripts/notion-openapi.json",
|
|
149
|
+
auth: { type: "bearer", token: "${NOTION_API_KEY}" },
|
|
150
|
+
tags: ["productivity", "database"],
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
domain: "app.asana.com",
|
|
154
|
+
specUrl:
|
|
155
|
+
"https://raw.githubusercontent.com/Asana/openapi/master/defs/asana_oas.yaml",
|
|
156
|
+
auth: { type: "bearer", token: "${ASANA_ACCESS_TOKEN}" },
|
|
157
|
+
tags: ["productivity", "project-management"],
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
domain: "jira.atlassian.com",
|
|
161
|
+
specUrl:
|
|
162
|
+
"https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json",
|
|
163
|
+
auth: { type: "bearer", token: "${ATLASSIAN_API_TOKEN}" },
|
|
164
|
+
tags: ["productivity", "project-management"],
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
domain: "api.spotify.com",
|
|
168
|
+
specUrl:
|
|
169
|
+
"https://raw.githubusercontent.com/sonallux/spotify-web-api/main/fixed-spotify-open-api.yml",
|
|
170
|
+
auth: { type: "bearer", token: "${SPOTIFY_ACCESS_TOKEN}" },
|
|
171
|
+
tags: ["media", "music"],
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
domain: "api.getpostman.com",
|
|
175
|
+
specUrl:
|
|
176
|
+
"https://api.apis.guru/v2/specs/getpostman.com/1.20.0/openapi.json",
|
|
177
|
+
auth: { type: "api-key", header: "X-Api-Key", key: "${POSTMAN_API_KEY}" },
|
|
178
|
+
tags: ["developer-tools", "api-testing"],
|
|
179
|
+
},
|
|
180
|
+
];
|
|
181
|
+
|
|
182
|
+
// ── DevOps / CI/CD ──────────────────────────────────────────────────
|
|
183
|
+
|
|
184
|
+
export const DEVOPS_APIS: ManifestEntry[] = [
|
|
185
|
+
{
|
|
186
|
+
domain: "circleci.com",
|
|
187
|
+
specUrl: "https://circleci.com/api/v2/openapi.json",
|
|
188
|
+
auth: { type: "bearer", token: "${CIRCLECI_TOKEN}" },
|
|
189
|
+
tags: ["devops", "ci-cd"],
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
domain: "api.datadoghq.com",
|
|
193
|
+
specUrl:
|
|
194
|
+
"https://raw.githubusercontent.com/DataDog/datadog-api-client-python/master/.generator/schemas/v2/openapi.yaml",
|
|
195
|
+
auth: { type: "api-key", header: "DD-API-KEY", key: "${DATADOG_API_KEY}" },
|
|
196
|
+
tags: ["devops", "monitoring"],
|
|
197
|
+
},
|
|
198
|
+
];
|
|
199
|
+
|
|
200
|
+
// ── Database / BaaS ─────────────────────────────────────────────────
|
|
201
|
+
|
|
202
|
+
export const DATABASE_APIS: ManifestEntry[] = [
|
|
203
|
+
{
|
|
204
|
+
domain: "api.supabase.com",
|
|
205
|
+
specUrl:
|
|
206
|
+
"https://raw.githubusercontent.com/supabase/supabase/master/apps/docs/spec/api_v1_openapi.json",
|
|
207
|
+
auth: { type: "bearer", token: "${SUPABASE_ACCESS_TOKEN}" },
|
|
208
|
+
tags: ["database", "baas"],
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
domain: "api.turso.tech",
|
|
212
|
+
specUrl:
|
|
213
|
+
"https://raw.githubusercontent.com/tursodatabase/turso-docs/main/api-reference/openapi.json",
|
|
214
|
+
auth: { type: "bearer", token: "${TURSO_API_TOKEN}" },
|
|
215
|
+
tags: ["database", "edge"],
|
|
216
|
+
},
|
|
217
|
+
{
|
|
218
|
+
domain: "console.neon.tech",
|
|
219
|
+
specUrl:
|
|
220
|
+
"https://raw.githubusercontent.com/neondatabase/neon-api-python/main/v2.json",
|
|
221
|
+
auth: { type: "bearer", token: "${NEON_API_KEY}" },
|
|
222
|
+
tags: ["database", "serverless-postgres"],
|
|
223
|
+
},
|
|
224
|
+
];
|
|
225
|
+
|
|
226
|
+
// ── Commerce / Payments ──────────────────────────────────────────────
|
|
227
|
+
|
|
228
|
+
export const COMMERCE_APIS: ManifestEntry[] = [
|
|
229
|
+
{
|
|
230
|
+
domain: "api.stripe.com",
|
|
231
|
+
specUrl:
|
|
232
|
+
"https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json",
|
|
233
|
+
auth: { type: "bearer", token: "${STRIPE_SECRET_KEY}" },
|
|
234
|
+
tags: ["commerce", "payments"],
|
|
235
|
+
},
|
|
236
|
+
];
|
|
237
|
+
|
|
238
|
+
// ── Communication APIs ───────────────────────────────────────────────
|
|
239
|
+
|
|
240
|
+
export const COMMUNICATION_APIS: ManifestEntry[] = [
|
|
241
|
+
{
|
|
242
|
+
domain: "slack.com",
|
|
243
|
+
specUrl:
|
|
244
|
+
"https://raw.githubusercontent.com/slackapi/slack-api-specs/master/web-api/slack_web_openapi_v2.json",
|
|
245
|
+
auth: { type: "bearer", token: "${SLACK_BOT_TOKEN}" },
|
|
246
|
+
tags: ["communication", "messaging"],
|
|
247
|
+
},
|
|
248
|
+
{
|
|
249
|
+
domain: "discord.com",
|
|
250
|
+
specUrl:
|
|
251
|
+
"https://raw.githubusercontent.com/discord/discord-api-spec/main/specs/openapi.json",
|
|
252
|
+
auth: { type: "bearer", token: "${DISCORD_BOT_TOKEN}", prefix: "Bot" },
|
|
253
|
+
tags: ["communication", "messaging"],
|
|
254
|
+
},
|
|
255
|
+
{
|
|
256
|
+
domain: "api.twilio.com",
|
|
257
|
+
specUrl:
|
|
258
|
+
"https://raw.githubusercontent.com/twilio/twilio-oai/main/spec/json/twilio_api_v2010.json",
|
|
259
|
+
auth: { type: "basic", token: "${TWILIO_ACCOUNT_SID}:${TWILIO_AUTH_TOKEN}" },
|
|
260
|
+
tags: ["communication", "sms", "voice"],
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
domain: "api.resend.com",
|
|
264
|
+
specUrl:
|
|
265
|
+
"https://raw.githubusercontent.com/resendlabs/resend-openapi/main/resend.yaml",
|
|
266
|
+
auth: { type: "bearer", token: "${RESEND_API_KEY}" },
|
|
267
|
+
tags: ["communication", "email"],
|
|
268
|
+
},
|
|
269
|
+
];
|
|
270
|
+
|
|
271
|
+
// ── JSON-RPC APIs ───────────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
/** Standard EVM methods shared across all Ethereum-compatible RPC providers */
|
|
274
|
+
const EVM_METHODS: RpcManifestDef["methods"] = [
|
|
275
|
+
{ rpcMethod: "eth_blockNumber", description: "Returns the latest block number", resource: "blocks", fsOp: "list" },
|
|
276
|
+
{ rpcMethod: "eth_getBlockByNumber", description: "Returns block by number", resource: "blocks", fsOp: "read" },
|
|
277
|
+
{ rpcMethod: "eth_getBalance", description: "Returns account balance in wei", resource: "balances", fsOp: "read" },
|
|
278
|
+
{ rpcMethod: "eth_getTransactionByHash", description: "Returns transaction by hash", resource: "transactions", fsOp: "read" },
|
|
279
|
+
{ rpcMethod: "eth_getTransactionReceipt", description: "Returns transaction receipt", resource: "receipts", fsOp: "read" },
|
|
280
|
+
{ rpcMethod: "eth_call", description: "Executes a call without creating a transaction", resource: "calls", fsOp: "read" },
|
|
281
|
+
{ rpcMethod: "eth_estimateGas", description: "Estimates gas needed for a transaction", resource: "gas", fsOp: "read" },
|
|
282
|
+
{ rpcMethod: "eth_gasPrice", description: "Returns current gas price in wei" },
|
|
283
|
+
{ rpcMethod: "eth_chainId", description: "Returns the chain ID" },
|
|
284
|
+
{ rpcMethod: "eth_getCode", description: "Returns contract bytecode at address", resource: "code", fsOp: "read" },
|
|
285
|
+
{ rpcMethod: "eth_getLogs", description: "Returns logs matching a filter", resource: "logs", fsOp: "list" },
|
|
286
|
+
{ rpcMethod: "eth_getTransactionCount", description: "Returns the number of transactions sent from an address", resource: "nonces", fsOp: "read" },
|
|
287
|
+
{ rpcMethod: "net_version", description: "Returns the network ID" },
|
|
288
|
+
];
|
|
289
|
+
|
|
290
|
+
export const RPC_APIS: ManifestEntry[] = [
|
|
291
|
+
// ── Free / Public RPC Providers ──────────────────────────────────
|
|
292
|
+
{
|
|
293
|
+
domain: "rpc.ankr.com",
|
|
294
|
+
rpcDef: { url: "https://rpc.ankr.com/eth", convention: "evm", methods: EVM_METHODS },
|
|
295
|
+
tags: ["blockchain", "ethereum", "free"],
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
domain: "cloudflare-eth.com",
|
|
299
|
+
rpcDef: { url: "https://cloudflare-eth.com", convention: "evm", methods: EVM_METHODS },
|
|
300
|
+
tags: ["blockchain", "ethereum", "free"],
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
domain: "ethereum-rpc.publicnode.com",
|
|
304
|
+
rpcDef: { url: "https://ethereum-rpc.publicnode.com", convention: "evm", methods: EVM_METHODS },
|
|
305
|
+
tags: ["blockchain", "ethereum", "free"],
|
|
306
|
+
},
|
|
307
|
+
|
|
308
|
+
// ── Auth-Required RPC Providers ──────────────────────────────────
|
|
309
|
+
{
|
|
310
|
+
domain: "eth-mainnet.g.alchemy.com",
|
|
311
|
+
rpcDef: { url: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}", convention: "evm", methods: EVM_METHODS },
|
|
312
|
+
tags: ["blockchain", "ethereum"],
|
|
313
|
+
},
|
|
314
|
+
{
|
|
315
|
+
domain: "mainnet.infura.io",
|
|
316
|
+
rpcDef: { url: "https://mainnet.infura.io/v3/${INFURA_API_KEY}", convention: "evm", methods: EVM_METHODS },
|
|
317
|
+
tags: ["blockchain", "ethereum"],
|
|
318
|
+
},
|
|
319
|
+
|
|
320
|
+
// ── L2 / Alt Chains (Free) ───────────────────────────────────────
|
|
321
|
+
{
|
|
322
|
+
domain: "arb1.arbitrum.io",
|
|
323
|
+
rpcDef: { url: "https://arb1.arbitrum.io/rpc", convention: "evm", methods: EVM_METHODS },
|
|
324
|
+
tags: ["blockchain", "arbitrum", "l2", "free"],
|
|
325
|
+
},
|
|
326
|
+
{
|
|
327
|
+
domain: "mainnet.optimism.io",
|
|
328
|
+
rpcDef: { url: "https://mainnet.optimism.io", convention: "evm", methods: EVM_METHODS },
|
|
329
|
+
tags: ["blockchain", "optimism", "l2", "free"],
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
domain: "mainnet.base.org",
|
|
333
|
+
rpcDef: { url: "https://mainnet.base.org", convention: "evm", methods: EVM_METHODS },
|
|
334
|
+
tags: ["blockchain", "base", "l2", "free"],
|
|
335
|
+
},
|
|
336
|
+
{
|
|
337
|
+
domain: "polygon-rpc.com",
|
|
338
|
+
rpcDef: { url: "https://polygon-rpc.com", convention: "evm", methods: EVM_METHODS },
|
|
339
|
+
tags: ["blockchain", "polygon", "free"],
|
|
340
|
+
},
|
|
341
|
+
];
|
|
342
|
+
|
|
343
|
+
// ── All APIs combined ────────────────────────────────────────────────
|
|
344
|
+
|
|
345
|
+
export const ALL_APIS: ManifestEntry[] = [
|
|
346
|
+
...FREE_APIS,
|
|
347
|
+
...FREEMIUM_APIS,
|
|
348
|
+
...DEVELOPER_TOOL_APIS,
|
|
349
|
+
...AI_APIS,
|
|
350
|
+
...CLOUD_APIS,
|
|
351
|
+
...PRODUCTIVITY_APIS,
|
|
352
|
+
...DEVOPS_APIS,
|
|
353
|
+
...DATABASE_APIS,
|
|
354
|
+
...COMMERCE_APIS,
|
|
355
|
+
...COMMUNICATION_APIS,
|
|
356
|
+
...RPC_APIS,
|
|
357
|
+
];
|
|
358
|
+
|
|
359
|
+
/** Get APIs that can be onboarded without any credentials (spec is public) */
|
|
360
|
+
export function getSpecOnlyApis(): ManifestEntry[] {
|
|
361
|
+
return ALL_APIS.map((e) => ({ ...e, auth: undefined }));
|
|
362
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import type { HttpAuth } from "@nkmc/agent-fs";
|
|
2
|
+
import { AgentFs } from "@nkmc/agent-fs";
|
|
3
|
+
import type { RegistryStore } from "../registry/types.js";
|
|
4
|
+
import type { CredentialVault } from "../credential/types.js";
|
|
5
|
+
import { compileOpenApiSpec, fetchAndCompile } from "../registry/openapi-compiler.js";
|
|
6
|
+
import { parseSkillMd } from "../registry/skill-parser.js";
|
|
7
|
+
import { compileRpcDef } from "../registry/rpc-compiler.js";
|
|
8
|
+
import { createRegistryResolver } from "../registry/resolver.js";
|
|
9
|
+
import type { ManifestEntry, ManifestAuth, OnboardResult, OnboardReport } from "./types.js";
|
|
10
|
+
|
|
11
|
+
export interface PipelineOptions {
|
|
12
|
+
store: RegistryStore;
|
|
13
|
+
vault?: CredentialVault;
|
|
14
|
+
/** Run smoke tests after registration (default true) */
|
|
15
|
+
smokeTest?: boolean;
|
|
16
|
+
/** Concurrency limit for parallel onboarding (default 5) */
|
|
17
|
+
concurrency?: number;
|
|
18
|
+
/** Custom fetch function */
|
|
19
|
+
fetchFn?: typeof globalThis.fetch;
|
|
20
|
+
/** Progress callback */
|
|
21
|
+
onProgress?: (result: OnboardResult, index: number, total: number) => void;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class OnboardPipeline {
|
|
25
|
+
private store: RegistryStore;
|
|
26
|
+
private vault?: CredentialVault;
|
|
27
|
+
private smokeTest: boolean;
|
|
28
|
+
private concurrency: number;
|
|
29
|
+
private fetchFn: typeof globalThis.fetch;
|
|
30
|
+
private onProgress?: PipelineOptions["onProgress"];
|
|
31
|
+
|
|
32
|
+
constructor(options: PipelineOptions) {
|
|
33
|
+
this.store = options.store;
|
|
34
|
+
this.vault = options.vault;
|
|
35
|
+
this.smokeTest = options.smokeTest !== false;
|
|
36
|
+
this.concurrency = options.concurrency ?? 5;
|
|
37
|
+
this.fetchFn = options.fetchFn ?? globalThis.fetch.bind(globalThis);
|
|
38
|
+
this.onProgress = options.onProgress;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Onboard a single service */
|
|
42
|
+
async onboardOne(entry: ManifestEntry): Promise<OnboardResult> {
|
|
43
|
+
const start = Date.now();
|
|
44
|
+
const base: Omit<OnboardResult, "status" | "durationMs"> = {
|
|
45
|
+
domain: entry.domain,
|
|
46
|
+
source: "none",
|
|
47
|
+
endpoints: 0,
|
|
48
|
+
resources: 0,
|
|
49
|
+
hasCredentials: false,
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
if (entry.disabled) {
|
|
53
|
+
return { ...base, status: "skipped", durationMs: Date.now() - start };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
// Step 1: Compile/parse the service definition
|
|
58
|
+
if (entry.specUrl) {
|
|
59
|
+
const result = await fetchAndCompile(entry.specUrl, { domain: entry.domain }, this.fetchFn);
|
|
60
|
+
await this.store.put(entry.domain, result.record);
|
|
61
|
+
base.source = "openapi";
|
|
62
|
+
base.endpoints = result.record.endpoints.length;
|
|
63
|
+
base.resources = result.resources.length;
|
|
64
|
+
} else if (entry.skillMdUrl) {
|
|
65
|
+
const resp = await this.fetchFn(entry.skillMdUrl);
|
|
66
|
+
if (!resp.ok) throw new Error(`Failed to fetch skill.md: ${resp.status}`);
|
|
67
|
+
const md = await resp.text();
|
|
68
|
+
const record = parseSkillMd(entry.domain, md);
|
|
69
|
+
await this.store.put(entry.domain, record);
|
|
70
|
+
base.source = "wellknown";
|
|
71
|
+
base.endpoints = record.endpoints.length;
|
|
72
|
+
} else if (entry.skillMd) {
|
|
73
|
+
const record = parseSkillMd(entry.domain, entry.skillMd);
|
|
74
|
+
await this.store.put(entry.domain, record);
|
|
75
|
+
base.source = "skillmd";
|
|
76
|
+
base.endpoints = record.endpoints.length;
|
|
77
|
+
} else if (entry.rpcDef) {
|
|
78
|
+
const { record } = compileRpcDef(entry.domain, entry.rpcDef);
|
|
79
|
+
await this.store.put(entry.domain, record);
|
|
80
|
+
base.source = "jsonrpc";
|
|
81
|
+
base.endpoints = record.endpoints.length;
|
|
82
|
+
base.resources = record.source?.rpc?.resources.length ?? 0;
|
|
83
|
+
} else {
|
|
84
|
+
return { ...base, status: "skipped", error: "No spec, skillMdUrl, or skillMd provided", durationMs: Date.now() - start };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Step 2: Store credentials if provided
|
|
88
|
+
if (entry.auth && this.vault) {
|
|
89
|
+
const auth = resolveAuth(entry.auth);
|
|
90
|
+
await this.vault.putPool(entry.domain, auth);
|
|
91
|
+
base.hasCredentials = true;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Step 3: Smoke test
|
|
95
|
+
if (this.smokeTest) {
|
|
96
|
+
base.smokeTest = await this.runSmokeTest(entry.domain, base.hasCredentials);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { ...base, status: "ok", durationMs: Date.now() - start };
|
|
100
|
+
} catch (err) {
|
|
101
|
+
return {
|
|
102
|
+
...base,
|
|
103
|
+
status: "failed",
|
|
104
|
+
error: err instanceof Error ? err.message : String(err),
|
|
105
|
+
durationMs: Date.now() - start,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Onboard many services with controlled concurrency */
|
|
111
|
+
async onboardMany(entries: ManifestEntry[]): Promise<OnboardReport> {
|
|
112
|
+
const start = Date.now();
|
|
113
|
+
const results: OnboardResult[] = [];
|
|
114
|
+
let index = 0;
|
|
115
|
+
|
|
116
|
+
// Process in batches
|
|
117
|
+
for (let i = 0; i < entries.length; i += this.concurrency) {
|
|
118
|
+
const batch = entries.slice(i, i + this.concurrency);
|
|
119
|
+
const batchResults = await Promise.all(
|
|
120
|
+
batch.map(async (entry) => {
|
|
121
|
+
const result = await this.onboardOne(entry);
|
|
122
|
+
const idx = index++;
|
|
123
|
+
this.onProgress?.(result, idx, entries.length);
|
|
124
|
+
return result;
|
|
125
|
+
}),
|
|
126
|
+
);
|
|
127
|
+
results.push(...batchResults);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
total: results.length,
|
|
132
|
+
ok: results.filter((r) => r.status === "ok").length,
|
|
133
|
+
failed: results.filter((r) => r.status === "failed").length,
|
|
134
|
+
skipped: results.filter((r) => r.status === "skipped").length,
|
|
135
|
+
results,
|
|
136
|
+
durationMs: Date.now() - start,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private async runSmokeTest(domain: string, hasCredentials: boolean): Promise<OnboardResult["smokeTest"]> {
|
|
141
|
+
const resolverOpts = this.vault
|
|
142
|
+
? { store: this.store, vault: this.vault, wrapVirtualFiles: false }
|
|
143
|
+
: { store: this.store, wrapVirtualFiles: false };
|
|
144
|
+
|
|
145
|
+
const { onMiss, listDomains } = createRegistryResolver(resolverOpts);
|
|
146
|
+
const fs = new AgentFs({ mounts: [], onMiss, listDomains });
|
|
147
|
+
|
|
148
|
+
const test: NonNullable<OnboardResult["smokeTest"]> = { ls: false, cat: false };
|
|
149
|
+
|
|
150
|
+
// Test ls
|
|
151
|
+
const lsResult = await fs.execute(`ls /${domain}/`);
|
|
152
|
+
test.ls = lsResult.ok === true;
|
|
153
|
+
|
|
154
|
+
// Test cat — only if ls succeeded and we found something to read
|
|
155
|
+
if (test.ls && lsResult.ok) {
|
|
156
|
+
const entries = lsResult.data as string[];
|
|
157
|
+
// Find a readable resource or endpoint
|
|
158
|
+
const resource = entries.find((e) => e.endsWith("/") && !e.startsWith("_"));
|
|
159
|
+
if (resource) {
|
|
160
|
+
// Try to list the resource — this makes a real HTTP call
|
|
161
|
+
const catResult = await fs.execute(`ls /${domain}/${resource}`);
|
|
162
|
+
test.cat = catResult.ok === true;
|
|
163
|
+
test.catEndpoint = `ls /${domain}/${resource}`;
|
|
164
|
+
} else if (entries.includes("_api/")) {
|
|
165
|
+
// Just verify _api listing works
|
|
166
|
+
const apiResult = await fs.execute(`ls /${domain}/_api/`);
|
|
167
|
+
test.cat = apiResult.ok === true;
|
|
168
|
+
test.catEndpoint = `ls /${domain}/_api/`;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return test;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Resolve ${ENV_VAR} references in auth values and return HttpAuth */
|
|
177
|
+
function resolveAuth(auth: ManifestAuth): HttpAuth {
|
|
178
|
+
const resolve = (val?: string): string | undefined => {
|
|
179
|
+
if (!val) return undefined;
|
|
180
|
+
const match = val.match(/^\$\{(\w+)\}$/);
|
|
181
|
+
if (match) {
|
|
182
|
+
const envVal = process.env[match[1]];
|
|
183
|
+
if (!envVal) throw new Error(`Environment variable ${match[1]} is not set`);
|
|
184
|
+
return envVal;
|
|
185
|
+
}
|
|
186
|
+
return val;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
if (auth.type === "bearer") {
|
|
190
|
+
const token = resolve(auth.token);
|
|
191
|
+
if (!token) throw new Error("Bearer auth requires token");
|
|
192
|
+
return { type: "bearer", token, ...(auth.prefix ? { prefix: auth.prefix } : {}) };
|
|
193
|
+
}
|
|
194
|
+
if (auth.type === "api-key") {
|
|
195
|
+
const header = resolve(auth.header);
|
|
196
|
+
const key = resolve(auth.key);
|
|
197
|
+
if (!header || !key) throw new Error("API key auth requires header and key");
|
|
198
|
+
return { type: "api-key", header, key };
|
|
199
|
+
}
|
|
200
|
+
if (auth.type === "basic") {
|
|
201
|
+
const username = resolve(auth.username);
|
|
202
|
+
const password = resolve(auth.password);
|
|
203
|
+
if (!username || !password) throw new Error("Basic auth requires username and password");
|
|
204
|
+
return { type: "basic", username, password };
|
|
205
|
+
}
|
|
206
|
+
if (auth.type === "oauth2") {
|
|
207
|
+
const tokenUrl = resolve(auth.tokenUrl);
|
|
208
|
+
const clientId = resolve(auth.clientId);
|
|
209
|
+
const clientSecret = resolve(auth.clientSecret);
|
|
210
|
+
if (!tokenUrl || !clientId || !clientSecret) throw new Error("OAuth2 auth requires tokenUrl, clientId, and clientSecret");
|
|
211
|
+
return { type: "oauth2", tokenUrl, clientId, clientSecret, ...(auth.scope ? { scope: auth.scope } : {}) };
|
|
212
|
+
}
|
|
213
|
+
throw new Error(`Unknown auth type: ${auth.type}`);
|
|
214
|
+
}
|