@mugwork/mug 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +251 -0
- package/dist/explorer.js +3 -0
- package/dist/packages/email-template/src/email-template.d.ts +18 -0
- package/dist/packages/email-template/src/email-template.js +74 -0
- package/dist/packages/email-template/src/index.d.ts +1 -0
- package/dist/packages/email-template/src/index.js +1 -0
- package/dist/packages/surface-renderer/src/form-renderer.d.ts +117 -0
- package/dist/packages/surface-renderer/src/form-renderer.js +719 -0
- package/dist/packages/surface-renderer/src/index.d.ts +4 -0
- package/dist/packages/surface-renderer/src/index.js +2 -0
- package/dist/packages/surface-renderer/src/portal-renderer.d.ts +177 -0
- package/dist/packages/surface-renderer/src/portal-renderer.js +1089 -0
- package/dist/packages/surface-renderer/src/workspace-home.d.ts +46 -0
- package/dist/packages/surface-renderer/src/workspace-home.js +345 -0
- package/dist/runtime/agent-types.d.ts +48 -0
- package/dist/runtime/agent-types.js +3 -0
- package/dist/runtime/ai-router.d.ts +32 -0
- package/dist/runtime/ai-router.js +112 -0
- package/dist/runtime/app.d.ts +6 -0
- package/dist/runtime/app.js +399 -0
- package/dist/runtime/chunker.d.ts +6 -0
- package/dist/runtime/chunker.js +30 -0
- package/dist/runtime/context.d.ts +115 -0
- package/dist/runtime/context.js +440 -0
- package/dist/runtime/do/workspace-database.d.ts +10 -0
- package/dist/runtime/do/workspace-database.js +199 -0
- package/dist/runtime/form-types.d.ts +143 -0
- package/dist/runtime/form-types.js +1 -0
- package/dist/runtime/runtime.d.ts +9 -0
- package/dist/runtime/runtime.js +7 -0
- package/dist/runtime/source-types.d.ts +15 -0
- package/dist/runtime/source-types.js +1 -0
- package/dist/runtime/source.d.ts +70 -0
- package/dist/runtime/source.js +21 -0
- package/dist/runtime/sync-runtime.d.ts +10 -0
- package/dist/runtime/sync-runtime.js +185 -0
- package/dist/runtime/types.d.ts +21 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/workflow-entrypoint.d.ts +31 -0
- package/dist/runtime/workflow-entrypoint.js +1297 -0
- package/dist/runtime/workflow.d.ts +285 -0
- package/dist/runtime/workflow.js +1008 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +44116 -0
- package/dist/src/commands/ai-gateway-route.d.ts +24 -0
- package/dist/src/commands/ai-gateway-route.js +192 -0
- package/dist/src/commands/auth.d.ts +1 -0
- package/dist/src/commands/auth.js +42 -0
- package/dist/src/commands/billing.d.ts +6 -0
- package/dist/src/commands/billing.js +76 -0
- package/dist/src/commands/brain.d.ts +1 -0
- package/dist/src/commands/brain.js +194 -0
- package/dist/src/commands/demo.d.ts +12 -0
- package/dist/src/commands/demo.js +147 -0
- package/dist/src/commands/deploy.d.ts +1 -0
- package/dist/src/commands/deploy.js +1052 -0
- package/dist/src/commands/dev.d.ts +14 -0
- package/dist/src/commands/dev.js +2818 -0
- package/dist/src/commands/form.d.ts +8 -0
- package/dist/src/commands/form.js +396 -0
- package/dist/src/commands/init.d.ts +1 -0
- package/dist/src/commands/init.js +139 -0
- package/dist/src/commands/issue.d.ts +7 -0
- package/dist/src/commands/issue.js +191 -0
- package/dist/src/commands/login.d.ts +9 -0
- package/dist/src/commands/login.js +163 -0
- package/dist/src/commands/logs.d.ts +8 -0
- package/dist/src/commands/logs.js +113 -0
- package/dist/src/commands/portal.d.ts +2 -0
- package/dist/src/commands/portal.js +111 -0
- package/dist/src/commands/pull.d.ts +3 -0
- package/dist/src/commands/pull.js +184 -0
- package/dist/src/commands/push.d.ts +4 -0
- package/dist/src/commands/push.js +183 -0
- package/dist/src/commands/run.d.ts +6 -0
- package/dist/src/commands/run.js +91 -0
- package/dist/src/commands/secret.d.ts +7 -0
- package/dist/src/commands/secret.js +105 -0
- package/dist/src/commands/shutdown.d.ts +1 -0
- package/dist/src/commands/shutdown.js +46 -0
- package/dist/src/commands/sql.d.ts +8 -0
- package/dist/src/commands/sql.js +142 -0
- package/dist/src/commands/status.d.ts +5 -0
- package/dist/src/commands/status.js +39 -0
- package/dist/src/commands/sync.d.ts +7 -0
- package/dist/src/commands/sync.js +991 -0
- package/dist/src/commands/usage.d.ts +6 -0
- package/dist/src/commands/usage.js +78 -0
- package/dist/src/commands/webhooks.d.ts +1 -0
- package/dist/src/commands/webhooks.js +102 -0
- package/dist/src/commands/workspace.d.ts +23 -0
- package/dist/src/commands/workspace.js +590 -0
- package/dist/src/connector-migration.d.ts +20 -0
- package/dist/src/connector-migration.js +43 -0
- package/dist/src/connector-parser.d.ts +14 -0
- package/dist/src/connector-parser.js +94 -0
- package/dist/src/connector-service/discover.d.ts +37 -0
- package/dist/src/connector-service/discover.js +79 -0
- package/dist/src/connector-service/gather.d.ts +22 -0
- package/dist/src/connector-service/gather.js +89 -0
- package/dist/src/connector-service/init.d.ts +14 -0
- package/dist/src/connector-service/init.js +109 -0
- package/dist/src/connector-service/scaffold.d.ts +17 -0
- package/dist/src/connector-service/scaffold.js +194 -0
- package/dist/src/connector-service/spec-storage.d.ts +8 -0
- package/dist/src/connector-service/spec-storage.js +48 -0
- package/dist/src/connector-service/types.d.ts +57 -0
- package/dist/src/connector-service/types.js +2 -0
- package/dist/src/connector-service/verify.d.ts +24 -0
- package/dist/src/connector-service/verify.js +575 -0
- package/dist/src/email-template.d.ts +2 -0
- package/dist/src/email-template.js +1 -0
- package/dist/src/manifest.d.ts +31 -0
- package/dist/src/manifest.js +25 -0
- package/dist/src/mug-icon.d.ts +1 -0
- package/dist/src/mug-icon.js +12 -0
- package/dist/src/slack-manifest.d.ts +119 -0
- package/dist/src/slack-manifest.js +163 -0
- package/dist/src/source-migration.d.ts +20 -0
- package/dist/src/source-migration.js +43 -0
- package/dist/src/surface-renderer.d.ts +5 -0
- package/dist/src/surface-renderer.js +3 -0
- package/dist/src/templates.d.ts +3 -0
- package/dist/src/templates.js +48 -0
- package/dist/src/version-check.d.ts +1 -0
- package/dist/src/version-check.js +28 -0
- package/dist/src/workflow-parser.d.ts +95 -0
- package/dist/src/workflow-parser.js +526 -0
- package/dist/worker/src/agent-types.d.ts +27 -0
- package/dist/worker/src/agent-types.js +3 -0
- package/dist/worker/src/source-types.d.ts +14 -0
- package/dist/worker/src/source-types.js +1 -0
- package/package.json +90 -0
- package/src/data/model-capabilities.json +171 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import * as acorn from "acorn";
|
|
2
|
+
import * as walk from "acorn-walk";
|
|
3
|
+
import { transformSync } from "esbuild";
|
|
4
|
+
export function parseSourceFile(source, fileName) {
|
|
5
|
+
const tableComments = extractTableComments(source);
|
|
6
|
+
const js = transformSync(source, { loader: "ts", target: "es2022" }).code;
|
|
7
|
+
const ast = acorn.parse(js, { ecmaVersion: 2022, sourceType: "module" });
|
|
8
|
+
let result = null;
|
|
9
|
+
walk.simple(ast, {
|
|
10
|
+
CallExpression(node) {
|
|
11
|
+
const callee = node.callee?.name;
|
|
12
|
+
if (callee !== "connector" && callee !== "source")
|
|
13
|
+
return;
|
|
14
|
+
const args = node.arguments;
|
|
15
|
+
if (args.length < 1 || args[0].type !== "ObjectExpression")
|
|
16
|
+
return;
|
|
17
|
+
const def = args[0];
|
|
18
|
+
const name = getStringProp(def, "name", js);
|
|
19
|
+
const database = getStringProp(def, "database", js);
|
|
20
|
+
const description = getStringProp(def, "description", js);
|
|
21
|
+
const baseUrl = getStringProp(def, "baseUrl", js);
|
|
22
|
+
if (!name || !database)
|
|
23
|
+
return;
|
|
24
|
+
const tables = [];
|
|
25
|
+
const tablesProp = findProp(def, "tables");
|
|
26
|
+
if (tablesProp?.value?.type === "ArrayExpression") {
|
|
27
|
+
let tableIdx = 0;
|
|
28
|
+
for (const el of tablesProp.value.elements ?? []) {
|
|
29
|
+
if (el?.type !== "ObjectExpression")
|
|
30
|
+
continue;
|
|
31
|
+
const tName = getStringProp(el, "name", js);
|
|
32
|
+
const tPK = getStringProp(el, "primaryKey", js);
|
|
33
|
+
const tEndpoint = getStringProp(el, "endpoint", js);
|
|
34
|
+
if (!tName)
|
|
35
|
+
continue;
|
|
36
|
+
const tDesc = tableComments[tableIdx] || undefined;
|
|
37
|
+
tableIdx++;
|
|
38
|
+
tables.push({
|
|
39
|
+
name: tName,
|
|
40
|
+
primaryKey: tPK ?? "id",
|
|
41
|
+
endpoint: tEndpoint,
|
|
42
|
+
description: tDesc,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
result = { name, description, database, tables, baseUrl };
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
return result;
|
|
50
|
+
}
|
|
51
|
+
function extractTableComments(source) {
|
|
52
|
+
const lines = source.split("\n");
|
|
53
|
+
const comments = [];
|
|
54
|
+
for (let i = 0; i < lines.length; i++) {
|
|
55
|
+
const nameMatch = lines[i].match(/^(\s*)name:\s*["']/);
|
|
56
|
+
if (!nameMatch || nameMatch[1].length < 6)
|
|
57
|
+
continue;
|
|
58
|
+
const commentLines = [];
|
|
59
|
+
for (let j = i - 1; j >= 0; j--) {
|
|
60
|
+
const prev = lines[j].trim();
|
|
61
|
+
if (prev === "" || prev === "{")
|
|
62
|
+
continue;
|
|
63
|
+
if (prev.startsWith("//")) {
|
|
64
|
+
commentLines.unshift(prev.replace(/^\/\/\s*/, ""));
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
comments.push(commentLines.join(" "));
|
|
71
|
+
}
|
|
72
|
+
return comments;
|
|
73
|
+
}
|
|
74
|
+
function findProp(obj, name) {
|
|
75
|
+
for (const prop of obj.properties ?? []) {
|
|
76
|
+
if (prop.type === "SpreadElement")
|
|
77
|
+
continue;
|
|
78
|
+
const key = prop.key?.name ?? prop.key?.value;
|
|
79
|
+
if (key === name)
|
|
80
|
+
return prop;
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
function getStringProp(obj, name, source) {
|
|
85
|
+
const prop = findProp(obj, name);
|
|
86
|
+
if (!prop)
|
|
87
|
+
return undefined;
|
|
88
|
+
const val = prop.value;
|
|
89
|
+
if (val?.type === "Literal" && typeof val.value === "string")
|
|
90
|
+
return val.value;
|
|
91
|
+
if (val?.type === "TemplateLiteral")
|
|
92
|
+
return source.slice(val.start + 1, val.end - 1);
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { ApiMetadata, ApiTier, ApiType, AuthConfig } from "./types.js";
|
|
2
|
+
export interface DiscoverInput {
|
|
3
|
+
product: string;
|
|
4
|
+
tier?: ApiTier;
|
|
5
|
+
hasApi?: boolean;
|
|
6
|
+
apiType?: ApiType;
|
|
7
|
+
docsUrl?: string;
|
|
8
|
+
specUrl?: string;
|
|
9
|
+
auth?: AuthConfig;
|
|
10
|
+
alternatives?: string[];
|
|
11
|
+
notes?: string;
|
|
12
|
+
integrations?: {
|
|
13
|
+
zapier?: boolean;
|
|
14
|
+
make?: boolean;
|
|
15
|
+
n8n?: boolean;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
export interface DiscoverResult {
|
|
19
|
+
slug: string;
|
|
20
|
+
meta: ApiMetadata;
|
|
21
|
+
cached: boolean;
|
|
22
|
+
nextStep: NextStep;
|
|
23
|
+
}
|
|
24
|
+
export type NextStep = {
|
|
25
|
+
action: "gather-from-spec";
|
|
26
|
+
specUrl: string;
|
|
27
|
+
} | {
|
|
28
|
+
action: "gather-from-docs";
|
|
29
|
+
docsUrl: string;
|
|
30
|
+
} | {
|
|
31
|
+
action: "gather-from-har";
|
|
32
|
+
reason: string;
|
|
33
|
+
} | {
|
|
34
|
+
action: "no-api";
|
|
35
|
+
reason: string;
|
|
36
|
+
};
|
|
37
|
+
export declare function discover(workspaceRoot: string, input: DiscoverInput): DiscoverResult;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { readMeta, writeMeta, slugify } from "./spec-storage.js";
|
|
2
|
+
export function discover(workspaceRoot, input) {
|
|
3
|
+
const slug = slugify(input.product);
|
|
4
|
+
const existing = readMeta(workspaceRoot, slug);
|
|
5
|
+
if (existing && !inputHasNewData(input)) {
|
|
6
|
+
return {
|
|
7
|
+
slug,
|
|
8
|
+
meta: existing,
|
|
9
|
+
cached: true,
|
|
10
|
+
nextStep: routeNextStep(existing),
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
const hasApi = input.hasApi ?? inferHasApi(input);
|
|
14
|
+
const tier = input.tier ?? inferTier(input);
|
|
15
|
+
const meta = {
|
|
16
|
+
name: input.product,
|
|
17
|
+
slug,
|
|
18
|
+
hasApi,
|
|
19
|
+
apiType: input.apiType ?? (hasApi ? "rest" : "unknown"),
|
|
20
|
+
docsUrl: input.docsUrl,
|
|
21
|
+
openapiSpecUrl: input.specUrl,
|
|
22
|
+
auth: input.auth,
|
|
23
|
+
tier,
|
|
24
|
+
alternatives: input.alternatives,
|
|
25
|
+
lastVerified: new Date().toISOString(),
|
|
26
|
+
notes: buildNotes(input),
|
|
27
|
+
};
|
|
28
|
+
writeMeta(workspaceRoot, slug, meta);
|
|
29
|
+
return {
|
|
30
|
+
slug,
|
|
31
|
+
meta,
|
|
32
|
+
cached: false,
|
|
33
|
+
nextStep: routeNextStep(meta),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function inputHasNewData(input) {
|
|
37
|
+
return !!(input.tier || input.hasApi !== undefined || input.docsUrl || input.specUrl || input.auth || input.apiType);
|
|
38
|
+
}
|
|
39
|
+
function inferHasApi(input) {
|
|
40
|
+
if (input.specUrl || input.docsUrl)
|
|
41
|
+
return true;
|
|
42
|
+
if (input.integrations?.zapier || input.integrations?.make || input.integrations?.n8n)
|
|
43
|
+
return true;
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
function inferTier(input) {
|
|
47
|
+
if (input.specUrl)
|
|
48
|
+
return 1;
|
|
49
|
+
if (input.docsUrl)
|
|
50
|
+
return 2;
|
|
51
|
+
return 3;
|
|
52
|
+
}
|
|
53
|
+
function buildNotes(input) {
|
|
54
|
+
const parts = [];
|
|
55
|
+
if (input.notes)
|
|
56
|
+
parts.push(input.notes);
|
|
57
|
+
const integrations = input.integrations;
|
|
58
|
+
if (integrations) {
|
|
59
|
+
const found = Object.entries(integrations)
|
|
60
|
+
.filter(([, v]) => v)
|
|
61
|
+
.map(([k]) => k);
|
|
62
|
+
if (found.length > 0) {
|
|
63
|
+
parts.push(`Integration platforms: ${found.join(", ")}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return parts.length > 0 ? parts.join("\n") : undefined;
|
|
67
|
+
}
|
|
68
|
+
function routeNextStep(meta) {
|
|
69
|
+
if (!meta.hasApi) {
|
|
70
|
+
return { action: "no-api", reason: `No API found for ${meta.name}` };
|
|
71
|
+
}
|
|
72
|
+
if (meta.openapiSpecUrl) {
|
|
73
|
+
return { action: "gather-from-spec", specUrl: meta.openapiSpecUrl };
|
|
74
|
+
}
|
|
75
|
+
if (meta.docsUrl) {
|
|
76
|
+
return { action: "gather-from-docs", docsUrl: meta.docsUrl };
|
|
77
|
+
}
|
|
78
|
+
return { action: "gather-from-har", reason: `${meta.name} has an API but no spec or docs URL — capture browser traffic as HAR` };
|
|
79
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface GatherResult {
|
|
2
|
+
slug: string;
|
|
3
|
+
endpointCount: number;
|
|
4
|
+
qualityScore: number;
|
|
5
|
+
warnings: string[];
|
|
6
|
+
specPath: string;
|
|
7
|
+
}
|
|
8
|
+
export interface NormalizeResult {
|
|
9
|
+
spec: Record<string, unknown>;
|
|
10
|
+
converted: boolean;
|
|
11
|
+
originalVersion?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface LintResult {
|
|
14
|
+
score: number;
|
|
15
|
+
warnings: string[];
|
|
16
|
+
errors: string[];
|
|
17
|
+
}
|
|
18
|
+
export declare function gatherFromSpec(workspaceRoot: string, slug: string, url: string): Promise<GatherResult>;
|
|
19
|
+
export declare function gatherFromFile(workspaceRoot: string, slug: string, filePath: string): Promise<GatherResult>;
|
|
20
|
+
export declare function gatherFromHar(workspaceRoot: string, slug: string, harPath: string): Promise<GatherResult>;
|
|
21
|
+
export declare function normalize(raw: Record<string, unknown>): Promise<NormalizeResult>;
|
|
22
|
+
export declare function lintSpec(spec: Record<string, unknown>): Promise<LintResult>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { parse as yamlParse } from "yaml";
|
|
3
|
+
import { writeSpec, ensureSpecDir } from "./spec-storage.js";
|
|
4
|
+
export async function gatherFromSpec(workspaceRoot, slug, url) {
|
|
5
|
+
const res = await fetch(url);
|
|
6
|
+
if (!res.ok) {
|
|
7
|
+
throw new Error(`Failed to fetch spec from ${url}: ${res.status} ${res.statusText}`);
|
|
8
|
+
}
|
|
9
|
+
const body = await res.text();
|
|
10
|
+
const raw = parseSpecString(body);
|
|
11
|
+
return processSpec(workspaceRoot, slug, raw);
|
|
12
|
+
}
|
|
13
|
+
export async function gatherFromFile(workspaceRoot, slug, filePath) {
|
|
14
|
+
const body = readFileSync(filePath, "utf-8");
|
|
15
|
+
const raw = parseSpecString(body);
|
|
16
|
+
return processSpec(workspaceRoot, slug, raw);
|
|
17
|
+
}
|
|
18
|
+
export async function gatherFromHar(workspaceRoot, slug, harPath) {
|
|
19
|
+
const { generateSpec } = await import("har-to-openapi");
|
|
20
|
+
const harContent = JSON.parse(readFileSync(harPath, "utf-8"));
|
|
21
|
+
const result = await generateSpec(harContent);
|
|
22
|
+
return processSpec(workspaceRoot, slug, result.spec);
|
|
23
|
+
}
|
|
24
|
+
async function processSpec(workspaceRoot, slug, raw) {
|
|
25
|
+
const { spec } = await normalize(raw);
|
|
26
|
+
const lint = await lintSpec(spec);
|
|
27
|
+
ensureSpecDir(workspaceRoot, slug);
|
|
28
|
+
writeSpec(workspaceRoot, slug, spec);
|
|
29
|
+
const paths = spec.paths;
|
|
30
|
+
const endpointCount = paths ? Object.keys(paths).length : 0;
|
|
31
|
+
const specPath = `src/connectors/.specs/${slug}/openapi.yaml`;
|
|
32
|
+
return {
|
|
33
|
+
slug,
|
|
34
|
+
endpointCount,
|
|
35
|
+
qualityScore: lint.score,
|
|
36
|
+
warnings: [...lint.errors, ...lint.warnings],
|
|
37
|
+
specPath,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export async function normalize(raw) {
|
|
41
|
+
const version = detectVersion(raw);
|
|
42
|
+
if (version === "2.0") {
|
|
43
|
+
const s2o = await import("swagger2openapi");
|
|
44
|
+
const result = await s2o.default.convertObj(raw, { patch: true, warnOnly: true });
|
|
45
|
+
return { spec: result.openapi, converted: true, originalVersion: "2.0" };
|
|
46
|
+
}
|
|
47
|
+
return { spec: resolveLocalRefs(raw), converted: false };
|
|
48
|
+
}
|
|
49
|
+
export async function lintSpec(spec) {
|
|
50
|
+
try {
|
|
51
|
+
const { Spectral } = await import("@stoplight/spectral-core");
|
|
52
|
+
const { oas } = await import("@stoplight/spectral-rulesets");
|
|
53
|
+
const spectral = new Spectral();
|
|
54
|
+
spectral.setRuleset(oas);
|
|
55
|
+
const results = await spectral.run(spec);
|
|
56
|
+
const errors = results.filter((r) => r.severity === 0).map((r) => r.message);
|
|
57
|
+
const warnings = results.filter((r) => r.severity === 1).map((r) => r.message);
|
|
58
|
+
const infos = results.filter((r) => r.severity >= 2);
|
|
59
|
+
const total = results.length || 1;
|
|
60
|
+
const errorPenalty = errors.length * 10;
|
|
61
|
+
const warnPenalty = warnings.length * 3;
|
|
62
|
+
const score = Math.max(0, Math.min(100, 100 - errorPenalty - warnPenalty));
|
|
63
|
+
return { score, warnings: dedupe(warnings), errors: dedupe(errors) };
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return { score: 50, warnings: ["Spectral lint unavailable — score estimated"], errors: [] };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function detectVersion(spec) {
|
|
70
|
+
if (typeof spec.swagger === "string")
|
|
71
|
+
return spec.swagger;
|
|
72
|
+
if (typeof spec.openapi === "string")
|
|
73
|
+
return spec.openapi;
|
|
74
|
+
return "unknown";
|
|
75
|
+
}
|
|
76
|
+
function resolveLocalRefs(spec) {
|
|
77
|
+
// For OpenAPI 3.x, return as-is — full $ref resolution happens during verify
|
|
78
|
+
return spec;
|
|
79
|
+
}
|
|
80
|
+
function parseSpecString(body) {
|
|
81
|
+
const trimmed = body.trim();
|
|
82
|
+
if (trimmed.startsWith("{")) {
|
|
83
|
+
return JSON.parse(trimmed);
|
|
84
|
+
}
|
|
85
|
+
return yamlParse(trimmed);
|
|
86
|
+
}
|
|
87
|
+
function dedupe(arr) {
|
|
88
|
+
return [...new Set(arr)];
|
|
89
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type ScaffoldResult } from "./scaffold.js";
|
|
2
|
+
export interface InitResult {
|
|
3
|
+
slug: string;
|
|
4
|
+
steps: StepResult[];
|
|
5
|
+
finalOutput?: ScaffoldResult;
|
|
6
|
+
stoppedAt?: string;
|
|
7
|
+
nextAction?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface StepResult {
|
|
10
|
+
step: string;
|
|
11
|
+
status: "done" | "skipped" | "stopped";
|
|
12
|
+
summary: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function init(workspaceRoot: string, product: string, sourceName?: string): Promise<InitResult>;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { readMeta, slugify } from "./spec-storage.js";
|
|
2
|
+
import { gatherFromSpec } from "./gather.js";
|
|
3
|
+
import { verify } from "./verify.js";
|
|
4
|
+
import { scaffold } from "./scaffold.js";
|
|
5
|
+
export async function init(workspaceRoot, product, sourceName) {
|
|
6
|
+
const slug = slugify(product);
|
|
7
|
+
const steps = [];
|
|
8
|
+
// Step 1 — Discover (check cache)
|
|
9
|
+
const meta = readMeta(workspaceRoot, slug);
|
|
10
|
+
if (!meta) {
|
|
11
|
+
steps.push({
|
|
12
|
+
step: "discover",
|
|
13
|
+
status: "stopped",
|
|
14
|
+
summary: `No metadata found for "${product}". Run discover first with research findings.`,
|
|
15
|
+
});
|
|
16
|
+
return {
|
|
17
|
+
slug,
|
|
18
|
+
steps,
|
|
19
|
+
stoppedAt: "discover",
|
|
20
|
+
nextAction: `mug source discover "${product}" --tier <n> --docs-url <url> [--spec-url <url>]`,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
steps.push({ step: "discover", status: "done", summary: `Tier ${meta.tier} — ${meta.apiType}` });
|
|
24
|
+
if (!meta.hasApi) {
|
|
25
|
+
steps.push({ step: "gather", status: "stopped", summary: `${product} has no API` });
|
|
26
|
+
return { slug, steps, stoppedAt: "gather", nextAction: "No API available — cannot build source" };
|
|
27
|
+
}
|
|
28
|
+
// Step 2 — Gather
|
|
29
|
+
if (meta.openapiSpecUrl) {
|
|
30
|
+
try {
|
|
31
|
+
const result = await gatherFromSpec(workspaceRoot, slug, meta.openapiSpecUrl);
|
|
32
|
+
steps.push({ step: "gather", status: "done", summary: `${result.endpointCount} endpoints, quality ${result.qualityScore}/100` });
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
steps.push({ step: "gather", status: "stopped", summary: `Fetch failed: ${err instanceof Error ? err.message : err}` });
|
|
36
|
+
return {
|
|
37
|
+
slug,
|
|
38
|
+
steps,
|
|
39
|
+
stoppedAt: "gather",
|
|
40
|
+
nextAction: `Spec URL failed. Try: mug source gather --slug ${slug} --from-file <agent-generated-spec>`,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
else if (meta.docsUrl) {
|
|
45
|
+
steps.push({
|
|
46
|
+
step: "gather",
|
|
47
|
+
status: "stopped",
|
|
48
|
+
summary: `Tier 2 — docs exist but no spec URL. Agent should generate spec from docs.`,
|
|
49
|
+
});
|
|
50
|
+
return {
|
|
51
|
+
slug,
|
|
52
|
+
steps,
|
|
53
|
+
stoppedAt: "gather",
|
|
54
|
+
nextAction: `Read docs at ${meta.docsUrl}, generate OpenAPI spec, then: mug source gather --slug ${slug} --from-file <spec-file>`,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
steps.push({
|
|
59
|
+
step: "gather",
|
|
60
|
+
status: "stopped",
|
|
61
|
+
summary: `Tier 3 — no spec or docs. Need HAR capture.`,
|
|
62
|
+
});
|
|
63
|
+
return {
|
|
64
|
+
slug,
|
|
65
|
+
steps,
|
|
66
|
+
stoppedAt: "gather",
|
|
67
|
+
nextAction: `Capture browser traffic as HAR, then: mug source gather --slug ${slug} --from-har <har-file>`,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
// Step 3 — Verify
|
|
71
|
+
if (sourceName) {
|
|
72
|
+
try {
|
|
73
|
+
const result = await verify(workspaceRoot, slug, sourceName);
|
|
74
|
+
const passCount = result.probes.filter((p) => p.status === "pass").length;
|
|
75
|
+
steps.push({ step: "verify", status: "done", summary: `${result.overallStatus} — ${passCount}/7 probes passed` });
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
steps.push({ step: "verify", status: "stopped", summary: `Verify failed: ${err instanceof Error ? err.message : err}` });
|
|
79
|
+
return {
|
|
80
|
+
slug,
|
|
81
|
+
steps,
|
|
82
|
+
stoppedAt: "verify",
|
|
83
|
+
nextAction: `Fix source config in mug.json, then: mug source verify --slug ${slug} --source ${sourceName}`,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
steps.push({
|
|
89
|
+
step: "verify",
|
|
90
|
+
status: "skipped",
|
|
91
|
+
summary: "No --source provided — skipping verification",
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
// Step 4 — Scaffold
|
|
95
|
+
try {
|
|
96
|
+
const result = scaffold(workspaceRoot, slug);
|
|
97
|
+
steps.push({ step: "scaffold", status: "done", summary: `${result.tableCount} tables → ${result.outputPath}` });
|
|
98
|
+
return { slug, steps, finalOutput: result };
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
steps.push({ step: "scaffold", status: "stopped", summary: `Scaffold failed: ${err instanceof Error ? err.message : err}` });
|
|
102
|
+
return {
|
|
103
|
+
slug,
|
|
104
|
+
steps,
|
|
105
|
+
stoppedAt: "scaffold",
|
|
106
|
+
nextAction: `mug source scaffold --slug ${slug}`,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface ScaffoldResult {
|
|
2
|
+
slug: string;
|
|
3
|
+
outputPath: string;
|
|
4
|
+
tableCount: number;
|
|
5
|
+
tables: TableInfo[];
|
|
6
|
+
syncStrategy: string;
|
|
7
|
+
}
|
|
8
|
+
interface TableInfo {
|
|
9
|
+
name: string;
|
|
10
|
+
endpoint: string;
|
|
11
|
+
primaryKey: string;
|
|
12
|
+
fieldCount: number;
|
|
13
|
+
pagination: string | null;
|
|
14
|
+
incrementalSync: boolean;
|
|
15
|
+
}
|
|
16
|
+
export declare function scaffold(workspaceRoot: string, slug: string): ScaffoldResult;
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { readSpec, readMeta } from "./spec-storage.js";
|
|
4
|
+
export function scaffold(workspaceRoot, slug) {
|
|
5
|
+
const spec = readSpec(workspaceRoot, slug);
|
|
6
|
+
if (!spec)
|
|
7
|
+
throw new Error(`No spec found for "${slug}" — run gather first`);
|
|
8
|
+
const meta = readMeta(workspaceRoot, slug);
|
|
9
|
+
const baseUrl = extractBaseUrl(spec);
|
|
10
|
+
const rateLimits = spec["x-mug-rate-limits"];
|
|
11
|
+
const endpoints = extractListEndpoints(spec);
|
|
12
|
+
if (endpoints.length === 0) {
|
|
13
|
+
throw new Error(`No list endpoints found in spec for "${slug}" — nothing to scaffold`);
|
|
14
|
+
}
|
|
15
|
+
const tables = endpoints.map((ep) => buildTableInfo(ep));
|
|
16
|
+
const code = generateCode(slug, baseUrl, meta?.auth?.type, rateLimits, endpoints, tables);
|
|
17
|
+
const sourcesDir = join(workspaceRoot, "src", "sources");
|
|
18
|
+
mkdirSync(sourcesDir, { recursive: true });
|
|
19
|
+
const outputPath = join(sourcesDir, `${slug}.ts`);
|
|
20
|
+
writeFileSync(outputPath, code);
|
|
21
|
+
const hasIncremental = tables.some((t) => t.incrementalSync);
|
|
22
|
+
const syncStrategy = hasIncremental ? "incremental where supported, full sync fallback" : "full sync";
|
|
23
|
+
return {
|
|
24
|
+
slug,
|
|
25
|
+
outputPath: `src/sources/${slug}.ts`,
|
|
26
|
+
tableCount: tables.length,
|
|
27
|
+
tables,
|
|
28
|
+
syncStrategy,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function extractBaseUrl(spec) {
|
|
32
|
+
const servers = spec.servers;
|
|
33
|
+
return servers?.[0]?.url ?? "https://api.example.com";
|
|
34
|
+
}
|
|
35
|
+
function extractListEndpoints(spec) {
|
|
36
|
+
const paths = spec.paths;
|
|
37
|
+
if (!paths)
|
|
38
|
+
return [];
|
|
39
|
+
const endpoints = [];
|
|
40
|
+
for (const [path, pathItem] of Object.entries(paths)) {
|
|
41
|
+
const op = pathItem.get;
|
|
42
|
+
if (!op)
|
|
43
|
+
continue;
|
|
44
|
+
if (path.match(/\{[^}]+\}$/))
|
|
45
|
+
continue;
|
|
46
|
+
const pagination = pathItem["x-mug-pagination"];
|
|
47
|
+
const sync = pathItem["x-mug-sync"];
|
|
48
|
+
const responses = op.responses;
|
|
49
|
+
let responseSchema;
|
|
50
|
+
const ok = responses?.["200"] ?? responses?.["201"];
|
|
51
|
+
if (ok?.content) {
|
|
52
|
+
const content = ok.content;
|
|
53
|
+
const json = content["application/json"] ?? content["application/xml"];
|
|
54
|
+
responseSchema = json?.schema;
|
|
55
|
+
}
|
|
56
|
+
endpoints.push({
|
|
57
|
+
path,
|
|
58
|
+
operationId: op.operationId,
|
|
59
|
+
responseSchema,
|
|
60
|
+
pagination,
|
|
61
|
+
sync,
|
|
62
|
+
parameters: op.parameters,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return endpoints;
|
|
66
|
+
}
|
|
67
|
+
function buildTableInfo(ep) {
|
|
68
|
+
const name = tableNameFromPath(ep.path);
|
|
69
|
+
const fields = countFields(ep.responseSchema);
|
|
70
|
+
return {
|
|
71
|
+
name,
|
|
72
|
+
endpoint: ep.path,
|
|
73
|
+
primaryKey: "id",
|
|
74
|
+
fieldCount: fields,
|
|
75
|
+
pagination: ep.pagination?.style ?? null,
|
|
76
|
+
incrementalSync: ep.sync?.supportsIncrementalSync ?? false,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function tableNameFromPath(path) {
|
|
80
|
+
return path
|
|
81
|
+
.replace(/^\//, "")
|
|
82
|
+
.replace(/\//g, "_")
|
|
83
|
+
.replace(/[^a-zA-Z0-9_]/g, "")
|
|
84
|
+
.toLowerCase();
|
|
85
|
+
}
|
|
86
|
+
function countFields(schema) {
|
|
87
|
+
if (!schema)
|
|
88
|
+
return 0;
|
|
89
|
+
if (schema.type === "array") {
|
|
90
|
+
const items = schema.items;
|
|
91
|
+
return countFields(items);
|
|
92
|
+
}
|
|
93
|
+
const props = schema.properties;
|
|
94
|
+
return props ? Object.keys(props).length : 0;
|
|
95
|
+
}
|
|
96
|
+
function generateCode(slug, baseUrl, authType, rateLimits, endpoints, tables) {
|
|
97
|
+
const lines = [];
|
|
98
|
+
lines.push(`import { source, type SourceContext } from "../source.js";`);
|
|
99
|
+
lines.push(``);
|
|
100
|
+
// Auth helper — still needed for the fetch fallback
|
|
101
|
+
const credName = authType === "bearer" || authType === "oauth2" ? "token"
|
|
102
|
+
: authType === "basic" ? "credentials" : "api_key";
|
|
103
|
+
const authHeader = authType === "basic"
|
|
104
|
+
? '`Basic ${btoa(cred)}`'
|
|
105
|
+
: '`Bearer ${cred}`';
|
|
106
|
+
lines.push(`async function authedFetch(url: string, ctx: SourceContext): Promise<Record<string, unknown>[]> {`);
|
|
107
|
+
lines.push(` const cred = await ctx.credential("${credName}");`);
|
|
108
|
+
lines.push(` const res = await fetch(url, { headers: { Authorization: ${authHeader} } });`);
|
|
109
|
+
lines.push(` if (!res.ok) throw new Error(\`\${res.status} \${res.statusText}: \${url}\`);`);
|
|
110
|
+
lines.push(` const body = await res.json() as any;`);
|
|
111
|
+
lines.push(` return Array.isArray(body) ? body : (body.data ?? body.results ?? body.records ?? body.items ?? []);`);
|
|
112
|
+
lines.push(`}`);
|
|
113
|
+
lines.push(``);
|
|
114
|
+
// Source definition — declarative style
|
|
115
|
+
lines.push(`export default source({`);
|
|
116
|
+
lines.push(` name: "${slug}",`);
|
|
117
|
+
lines.push(` database: "${slug}",`);
|
|
118
|
+
lines.push(` baseUrl: "${baseUrl}",`);
|
|
119
|
+
if (rateLimits) {
|
|
120
|
+
lines.push(` rateLimits: {`);
|
|
121
|
+
if (rateLimits.requestsPerSecond)
|
|
122
|
+
lines.push(` requestsPerSecond: ${rateLimits.requestsPerSecond},`);
|
|
123
|
+
if (rateLimits.requestsPerMinute)
|
|
124
|
+
lines.push(` requestsPerMinute: ${rateLimits.requestsPerMinute},`);
|
|
125
|
+
lines.push(` },`);
|
|
126
|
+
}
|
|
127
|
+
lines.push(` tables: [`);
|
|
128
|
+
for (let i = 0; i < endpoints.length; i++) {
|
|
129
|
+
const ep = endpoints[i];
|
|
130
|
+
const table = tables[i];
|
|
131
|
+
const wrapper = detectListWrapper(ep.responseSchema);
|
|
132
|
+
lines.push(` {`);
|
|
133
|
+
lines.push(` name: "${table.name}",`);
|
|
134
|
+
lines.push(` primaryKey: "${table.primaryKey}",`);
|
|
135
|
+
lines.push(` endpoint: "${ep.path}",`);
|
|
136
|
+
// Pagination config
|
|
137
|
+
if (ep.pagination) {
|
|
138
|
+
lines.push(` pagination: ${jsonLiteral(ep.pagination, 6)},`);
|
|
139
|
+
}
|
|
140
|
+
// Sync config
|
|
141
|
+
if (ep.sync?.supportsIncrementalSync) {
|
|
142
|
+
const syncConfig = {};
|
|
143
|
+
if (ep.sync.filterParam)
|
|
144
|
+
syncConfig.filterParam = ep.sync.filterParam;
|
|
145
|
+
if (ep.sync.filterFormat)
|
|
146
|
+
syncConfig.filterFormat = ep.sync.filterFormat;
|
|
147
|
+
if (ep.sync.updatedAtField)
|
|
148
|
+
syncConfig.updatedAtField = ep.sync.updatedAtField;
|
|
149
|
+
if (ep.sync.deletedAtField) {
|
|
150
|
+
syncConfig.deletedAtField = ep.sync.deletedAtField;
|
|
151
|
+
syncConfig.deletionStrategy = ep.sync.deletionStrategy ?? "soft-delete-field";
|
|
152
|
+
}
|
|
153
|
+
if (Object.keys(syncConfig).length > 0) {
|
|
154
|
+
lines.push(` sync: ${jsonLiteral(syncConfig, 6)},`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// Extract items — only if response wraps in a named field
|
|
158
|
+
if (wrapper) {
|
|
159
|
+
lines.push(` extractItems: (body: any) => body.${wrapper} ?? [],`);
|
|
160
|
+
}
|
|
161
|
+
// Fetch fallback — the sync runtime uses endpoint + pagination config,
|
|
162
|
+
// but fetch is the escape hatch for custom logic
|
|
163
|
+
lines.push(` async fetch(ctx) {`);
|
|
164
|
+
lines.push(` return authedFetch(\`${baseUrl}${ep.path}\`, ctx);`);
|
|
165
|
+
lines.push(` },`);
|
|
166
|
+
lines.push(` },`);
|
|
167
|
+
}
|
|
168
|
+
lines.push(` ],`);
|
|
169
|
+
lines.push(`});`);
|
|
170
|
+
lines.push(``);
|
|
171
|
+
return lines.join("\n");
|
|
172
|
+
}
|
|
173
|
+
function detectListWrapper(schema) {
|
|
174
|
+
if (!schema)
|
|
175
|
+
return null;
|
|
176
|
+
if (schema.type === "array")
|
|
177
|
+
return null;
|
|
178
|
+
const props = schema.properties;
|
|
179
|
+
if (!props)
|
|
180
|
+
return null;
|
|
181
|
+
for (const [key, value] of Object.entries(props)) {
|
|
182
|
+
if (value.type === "array")
|
|
183
|
+
return key;
|
|
184
|
+
}
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
function jsonLiteral(obj, indent) {
|
|
188
|
+
const pad = " ".repeat(indent);
|
|
189
|
+
const inner = " ".repeat(indent + 2);
|
|
190
|
+
const entries = Object.entries(obj)
|
|
191
|
+
.filter(([, v]) => v !== undefined)
|
|
192
|
+
.map(([k, v]) => `${inner}${k}: ${JSON.stringify(v)}`);
|
|
193
|
+
return `{\n${entries.join(",\n")},\n${pad}}`;
|
|
194
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ApiMetadata } from "./types.js";
|
|
2
|
+
export declare function ensureSpecDir(workspaceRoot: string, slug: string): string;
|
|
3
|
+
export declare function readMeta(workspaceRoot: string, slug: string): ApiMetadata | null;
|
|
4
|
+
export declare function writeMeta(workspaceRoot: string, slug: string, meta: ApiMetadata): void;
|
|
5
|
+
export declare function readSpec(workspaceRoot: string, slug: string): Record<string, unknown> | null;
|
|
6
|
+
export declare function writeSpec(workspaceRoot: string, slug: string, spec: Record<string, unknown>): void;
|
|
7
|
+
export declare function listSpecs(workspaceRoot: string): string[];
|
|
8
|
+
export declare function slugify(name: string): string;
|