edsger 0.69.0 → 0.71.0

package/dist/phases/discover-services/index.js

@@ -0,0 +1,528 @@
+/**
+ * Phase: discover-services (GitHub Discovery)
+ *
+ * Scans every repository in a team's connected GitHub organisation and infers
+ * a service for each one. The server mints a short-lived org-scoped token
+ * (`github/org_token`) and the CLI reads each repo's signal files directly via
+ * Octokit — a handful of root files (README, package manifests, Dockerfile) —
+ * so private repos are read over the API without cloning. It then derives a
+ * service identity (name, language, kind, description) heuristically and
+ * upserts it into the team-scoped service catalog (`services` + a `repo`
+ * component).
+ *
+ * Progress is recorded against a `discovery_runs` row: the run is flipped to
+ * `running`, heart-beaten on every repo, and finalized as `success`/`failed`.
+ * The desktop Discover Services page reads that row to render live status.
+ */
+import { Octokit } from '@octokit/rest';
+import { callMcpEndpoint } from '../../api/mcp-client.js';
+import { getSupabase, hasSupabaseSession } from '../../supabase/client.js';
+import { logDebug, logInfo, logSuccess, logWarning, } from '../../utils/logger.js';
+const ORG_NAME_RE = /^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?$/;
+const MAX_FILE_BYTES = 16_000;
+/** Signal files read from each repo's root to infer a service. */
+const SIGNAL_FILES = [
+    'README.md',
+    'readme.md',
+    'package.json',
+    'pyproject.toml',
+    'requirements.txt',
+    'go.mod',
+    'pom.xml',
+    'build.gradle',
+    'Cargo.toml',
+    'Gemfile',
+    'composer.json',
+    'Dockerfile',
+];
+/**
+ * Read a repo's root-level signal files with Octokit. Lists the root tree once,
+ * then fetches only the signal files that actually exist (in parallel) — avoids
+ * a 404 round-trip per candidate file. Returns a path → content map (capped).
+ */
+async function readRepoSignalFiles(octokit, owner, name, ref) {
+    const files = new Map();
+    let rootNames;
+    try {
+        const { data } = await octokit.repos.getContent({
+            owner,
+            repo: name,
+            path: '',
+            ...(ref ? { ref } : {}),
+        });
+        if (!Array.isArray(data)) {
+            return files;
+        }
+        rootNames = new Set(data.filter((e) => e.type === 'file').map((e) => e.name));
+    }
+    catch {
+        // Empty repo, no access, or missing ref — nothing to read.
+        return files;
+    }
+    const wanted = SIGNAL_FILES.filter((p) => rootNames.has(p));
+    await Promise.all(wanted.map(async (path) => {
+        try {
+            const { data } = await octokit.repos.getContent({
+                owner,
+                repo: name,
+                path,
+                ...(ref ? { ref } : {}),
+            });
+            if (!Array.isArray(data) &&
+                data.type === 'file' &&
+                typeof data.content === 'string') {
+                files.set(path, Buffer.from(data.content, 'base64')
+                    .toString('utf8')
+                    .slice(0, MAX_FILE_BYTES));
+            }
+        }
+        catch {
+            // Skip unreadable file.
+        }
+    }));
+    return files;
+}
+const SERVER_FRAMEWORKS = [
+    'express',
+    'fastify',
+    'koa',
+    '@nestjs/core',
+    'next',
+    'hapi',
+    'flask',
+    'django',
+    'fastapi',
+    'uvicorn',
+    'gunicorn',
+    'gin',
+    'echo',
+    'fiber',
+    'spring-boot',
+    'actix',
+    'rails',
+    'sinatra',
+];
+/** Read the first markdown heading or paragraph from a README as a fallback description. */
+function descriptionFromReadme(content) {
+    const lines = content.split('\n');
+    for (const raw of lines) {
+        const line = raw.trim();
+        if (!line || line.startsWith('![') || line.startsWith('<')) {
+            continue;
+        }
+        const heading = line.match(/^#{1,3}\s+(.*)$/);
+        if (heading) {
+            const text = heading[1].trim();
+            // Skip a bare title that just repeats the repo name — keep scanning for prose.
+            if (text.length > 0 && /\s/.test(text)) {
+                return text.slice(0, 280);
+            }
+            continue;
+        }
+        // First non-heading prose line.
+        return line.replace(/[*_`>#-]/g, '').trim().slice(0, 280) || null;
+    }
+    return null;
+}
+/** Map manifest presence to a language when GitHub doesn't report one. */
+function languageFromManifests(files) {
+    const byManifest = [
+        ['package.json', 'JavaScript'],
+        ['pyproject.toml', 'Python'],
+        ['requirements.txt', 'Python'],
+        ['go.mod', 'Go'],
+        ['pom.xml', 'Java'],
+        ['build.gradle', 'Java'],
+        ['Cargo.toml', 'Rust'],
+        ['Gemfile', 'Ruby'],
+        ['composer.json', 'PHP'],
+    ];
+    for (const [manifest, language] of byManifest) {
+        if (files.has(manifest)) {
+            return language;
+        }
+    }
+    return null;
+}
+/** Parse the subset of package.json fields we use for inference. */
+function parsePackageJson(raw) {
+    const empty = {
+        name: null,
+        deps: [],
+        hasStart: false,
+        isPrivate: null,
+        hasMain: false,
+    };
+    if (!raw) {
+        return empty;
+    }
+    try {
+        const pkg = JSON.parse(raw);
+        return {
+            name: typeof pkg.name === 'string' ? pkg.name.replace(/^@[^/]+\//, '') : null,
+            deps: [
+                ...Object.keys(pkg.dependencies ?? {}),
+                ...Object.keys(pkg.devDependencies ?? {}),
+            ],
+            hasStart: Boolean(pkg.scripts?.start),
+            isPrivate: pkg.private === true,
+            hasMain: Boolean(pkg.main || pkg.exports || pkg.module),
+        };
+    }
+    catch {
+        // Malformed package.json — fall back to empty.
+        return empty;
+    }
+}
+/** Infer a service identity from a repo + its signal files. */
+function inferService(repo, files) {
+    const fullName = repo.full_name;
+    const hasDockerfile = files.has('Dockerfile');
+    // Parse package.json if present.
+    const pkgRaw = files.get('package.json');
+    const pkg = parsePackageJson(pkgRaw);
+    const { name: pkgName, deps: pkgDeps, hasStart: pkgHasStart, isPrivate: pkgIsPrivate, hasMain: pkgHasMain, } = pkg;
+    // Detect language from repo metadata or manifest presence.
+    const language = repo.language ?? languageFromManifests(files);
+    // Detect server frameworks across manifests for kind + tags.
+    const manifestBlob = [
+        pkgDeps.join(' '),
+        files.get('requirements.txt') ?? '',
+        files.get('pyproject.toml') ?? '',
+        files.get('go.mod') ?? '',
+        files.get('pom.xml') ?? '',
+        files.get('build.gradle') ?? '',
+        files.get('Gemfile') ?? '',
+    ]
+        .join(' ')
+        .toLowerCase();
+    const detectedFrameworks = SERVER_FRAMEWORKS.filter((fw) => manifestBlob.includes(fw.toLowerCase()));
+    // Decide kind. Anything deployable (Dockerfile, start script, or a server
+    // framework) is a service; a published package manifest with a main/exports
+    // entry and no run target is a library; otherwise default to service.
+    const isDeployable = hasDockerfile || pkgHasStart || detectedFrameworks.length > 0;
+    const isLibrary = !isDeployable &&
+        Boolean(pkgRaw) &&
+        pkgHasMain &&
+        !pkgHasStart &&
+        pkgIsPrivate === false;
+    const kind = isLibrary ? 'library' : 'service';
+    let kindConfidence = 0.5;
+    if (isDeployable) {
+        kindConfidence = 0.8;
+    }
+    else if (isLibrary) {
+        kindConfidence = 0.7;
+    }
+    const name = (repo.name || pkgName || fullName.split('/')[1]).trim();
+    // Description: repo metadata wins, then README.
+    const readme = files.get('README.md') ?? files.get('readme.md');
+    const description = repo.description?.trim() ||
+        (readme ? descriptionFromReadme(readme) : null) ||
+        null;
+    const descriptionConfidence = repo.description?.trim() ? 0.9 : 0.5;
+    const tags = Array.from(new Set([
+        ...(language ? [language.toLowerCase()] : []),
+        ...detectedFrameworks.map((f) => f.replace(/^@[^/]+\//, '')),
+    ])).slice(0, 12);
+    const fieldSources = {
+        name: {
+            value: name,
+            confidence: 0.9,
+            source: 'github',
+            source_detail: fullName,
+        },
+        kind: { value: kind, confidence: kindConfidence, source: 'github' },
+    };
+    if (language) {
+        fieldSources.language = {
+            value: language,
+            confidence: repo.language ? 1.0 : 0.6,
+            source: 'github',
+        };
+    }
+    if (description) {
+        fieldSources.description = {
+            value: description,
+            confidence: descriptionConfidence,
+            source: 'github',
+        };
+    }
+    return {
+        name,
+        display_name: name,
+        description,
+        kind,
+        language,
+        tags,
+        field_sources: fieldSources,
+    };
+}
+/**
+ * Upsert one inferred service + its repo component, mirroring the
+ * `services.upsert_draft` semantics (insert when new, update only unlocked
+ * fields when it already exists). Returns whether the service was created or
+ * updated so the caller can keep run-level counters.
+ */
+async function upsertServiceForRepo(
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Supabase client type
+supabase, teamId, userId, repo, inferred) {
+    const { data: existing } = await supabase
+        .from('services')
+        .select('id, locked_fields')
+        .eq('team_id', teamId)
+        .eq('name', inferred.name)
+        .maybeSingle();
+    const fields = {
+        display_name: inferred.display_name,
+        description: inferred.description,
+        kind: inferred.kind,
+        language: inferred.language,
+        tags: inferred.tags,
+        source: 'github',
+        source_ref: `github:${repo.full_name}`,
+        field_sources: inferred.field_sources,
+    };
+    let serviceId;
+    let action;
+    if (existing) {
+        const locked = existing.locked_fields ?? [];
+        const updates = {};
+        for (const [k, v] of Object.entries(fields)) {
+            if (!locked.includes(k) && v !== null && v !== undefined) {
+                updates[k] = v;
+            }
+        }
+        const { error } = await supabase
+            .from('services')
+            .update(updates)
+            .eq('id', existing.id);
+        if (error) {
+            throw error;
+        }
+        serviceId = existing.id;
+        action = 'updated';
+    }
+    else {
+        const { data: ins, error } = await supabase
+            .from('services')
+            .insert({
+            team_id: teamId,
+            name: inferred.name,
+            ...fields,
+            needs_review: true,
+            created_by: userId,
+        })
+            .select('id')
+            .single();
+        if (error) {
+            throw error;
+        }
+        serviceId = ins?.id ?? null;
+        action = 'created';
+    }
+    if (serviceId) {
+        await supabase.from('service_components').upsert({
+            service_id: serviceId,
+            kind: 'repo',
+            provider: 'github',
+            external_id: repo.full_name,
+            url: repo.html_url,
+            metadata: {
+                default_branch: repo.default_branch,
+                private: repo.private,
+                language: repo.language,
+            },
+        }, { onConflict: 'service_id,kind,external_id' });
+    }
+    return action;
+}
+export async function discoverServices(opts) {
+    const { teamId, verbose } = opts;
+    const err = (message) => ({
+        status: 'error',
+        message,
+        reposScanned: 0,
+        servicesCreated: 0,
+        servicesUpdated: 0,
+    });
+    if (!hasSupabaseSession()) {
+        return err('Supabase session unavailable. Sign in to the Edsger desktop app.');
+    }
+    const supabase = getSupabase();
+    const { data: { user }, } = await supabase.auth.getUser();
+    if (!user) {
+        return err('Not authenticated');
+    }
+    // Resolve the team's GitHub org.
+    let orgLogin = opts.org;
+    if (!orgLogin) {
+        const { data: team, error: teamErr } = await supabase
+            .from('teams')
+            .select('github_org')
+            .eq('id', teamId)
+            .single();
+        if (teamErr || !team) {
+            return err(`Team not found: ${teamId}`);
+        }
+        orgLogin = team.github_org ?? undefined;
+    }
+    if (!orgLogin) {
+        return err('No GitHub organization configured for this team. Set it in team settings first.');
+    }
+    if (!ORG_NAME_RE.test(orgLogin) || orgLogin.length > 39) {
+        return err(`Invalid GitHub organization name: "${orgLogin}"`);
+    }
+    // Resolve / create the discovery_runs row and flip it to running.
+    let runId = opts.runId ?? null;
+    if (runId) {
+        const { error: updErr } = await supabase
+            .from('discovery_runs')
+            .update({
+            status: 'running',
+            last_heartbeat_at: new Date().toISOString(),
+        })
+            .eq('id', runId)
+            .eq('team_id', teamId);
+        if (updErr) {
+            logWarning(`Could not mark discovery run as running: ${updErr.message}`);
+        }
+    }
+    else {
+        const { data: inserted, error: insErr } = await supabase
+            .from('discovery_runs')
+            .insert({
+            team_id: teamId,
+            scope: 'org',
+            scope_ref: orgLogin,
+            status: 'running',
+            created_by: user.id,
+            last_heartbeat_at: new Date().toISOString(),
+        })
+            .select('id')
+            .single();
+        if (insErr) {
+            logWarning(`Could not create discovery run row: ${insErr.message}`);
+        }
+        else {
+            runId = inserted?.id ?? null;
+        }
+    }
+    const finalize = async (status, counts, error) => {
+        if (!runId) {
+            return;
+        }
+        await supabase
+            .from('discovery_runs')
+            .update({
+            status,
+            repos_scanned: counts.repos,
+            services_found: counts.created,
+            services_updated: counts.updated,
+            error: error ?? null,
+            completed_at: new Date().toISOString(),
+            last_heartbeat_at: new Date().toISOString(),
+        })
+            .eq('id', runId)
+            .eq('team_id', teamId);
+    };
+    // List the org's repos via the Edsger GitHub App (server-side).
+    logInfo(`Scanning GitHub org "${orgLogin}" for services...`);
+    let repos;
+    try {
+        const res = (await callMcpEndpoint('github/org_repos', {
+            team_id: teamId,
+            org: orgLogin,
+        }));
+        if (!res.configured) {
+            const message = res.message || `The Edsger GitHub App is not configured for "${orgLogin}".`;
+            await finalize('failed', { repos: 0, created: 0, updated: 0 }, message);
+            return err(message);
+        }
+        repos = res.repos ?? [];
+    }
+    catch (e) {
+        const message = `Failed to list repos: ${e instanceof Error ? e.message : String(e)}`;
+        await finalize('failed', { repos: 0, created: 0, updated: 0 }, message);
+        return err(message);
+    }
+    if (repos.length === 0) {
+        await finalize('success', { repos: 0, created: 0, updated: 0 });
+        logSuccess(`No repositories found in "${orgLogin}"`);
+        return {
+            status: 'success',
+            message: `No repositories found in ${orgLogin}`,
+            reposScanned: 0,
+            servicesCreated: 0,
+            servicesUpdated: 0,
+        };
+    }
+    // Mint an org-scoped GitHub token so the CLI can read repo files itself.
+    let octokit;
+    try {
+        const tokenRes = (await callMcpEndpoint('github/org_token', {
+            team_id: teamId,
+            org: orgLogin,
+        }));
+        if (!tokenRes.configured || !tokenRes.token) {
+            const message = tokenRes.message ||
+                `Could not obtain a GitHub token for "${orgLogin}".`;
+            await finalize('failed', { repos: 0, created: 0, updated: 0 }, message);
+            return err(message);
+        }
+        octokit = new Octokit({ auth: tokenRes.token });
+    }
+    catch (e) {
+        const message = `Failed to obtain GitHub token: ${e instanceof Error ? e.message : String(e)}`;
+        await finalize('failed', { repos: 0, created: 0, updated: 0 }, message);
+        return err(message);
+    }
+    logInfo(`Found ${repos.length} repos. Inferring services...`);
+    let scanned = 0;
+    let created = 0;
+    let updated = 0;
+    for (const repo of repos) {
+        scanned++;
+        try {
+            // Read the repo's signal files directly via Octokit.
+            const [owner, name] = repo.full_name.split('/');
+            const files = await readRepoSignalFiles(octokit, owner, name, repo.default_branch ?? undefined);
+            const inferred = inferService(repo, files);
+            const action = await upsertServiceForRepo(supabase, teamId, user.id, repo, inferred);
+            if (action === 'created') {
+                created++;
+            }
+            else {
+                updated++;
+            }
+            if (verbose) {
+                logDebug(`  ${repo.full_name} → ${inferred.name} (${inferred.kind}${inferred.language ? `, ${inferred.language}` : ''})`);
+            }
+        }
+        catch (e) {
+            logWarning(`Skipped ${repo.full_name}: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        // Heartbeat + running counters so the UI can show live progress.
+        if (runId && (scanned % 3 === 0 || scanned === repos.length)) {
+            await supabase
+                .from('discovery_runs')
+                .update({
+                repos_scanned: scanned,
+                services_found: created,
+                services_updated: updated,
+                last_heartbeat_at: new Date().toISOString(),
+            })
+                .eq('id', runId)
+                .eq('team_id', teamId);
+        }
+    }
+    await finalize('success', { repos: scanned, created, updated });
+    logSuccess(`Discovery completed: scanned ${scanned} repos, ${created} services created, ${updated} updated`);
+    return {
+        status: 'success',
+        message: `Discovered ${created + updated} services from ${scanned} repos`,
+        reposScanned: scanned,
+        servicesCreated: created,
+        servicesUpdated: updated,
+    };
+}

package/dist/phases/er-diagram/index.d.ts

@@ -0,0 +1,28 @@
+/**
+ * er-diagram phase: clone the product's repo, ask Claude to map every
+ * persistence entity (table / view / enum / junction) and the relationships
+ * between them into a structured ErDiagramExtraction, then persist the result
+ * to diagrams / diagram_nodes / diagram_edges (rows tagged `type = 'er'`) via the
+ * Supabase SDK.
+ *
+ * Companion to data-flow / screen-flow: same generation pattern (workspace
+ * clone + Claude Agent SDK + in-process MCP server), same storage tables,
+ * different domain.
+ */
+export interface ErDiagramPhaseOptions {
+    /** Product-scoped diagram. Mutually exclusive with `repoId`. */
+    productId?: string;
+    /** Repo-only diagram: a single repositories row, no product context. */
+    repoId?: string;
+    diagramId: string;
+    guidance?: string;
+    verbose?: boolean;
+}
+export interface ErDiagramPhaseResult {
+    status: 'success' | 'error';
+    message: string;
+    nodesCreated?: number;
+    edgesCreated?: number;
+    summary?: string;
+}
+export declare function runErDiagramPhase(options: ErDiagramPhaseOptions): Promise<ErDiagramPhaseResult>;