modular-studio 1.0.5 → 1.1.0

package/dist-server/server/routes/connectors/confluence.js

@@ -0,0 +1,176 @@
+/**
+ * Confluence Connector — Pages + Spaces
+ * Issue #97
+ */
+import { Router } from 'express';
+import { rateLimitedFetch, fetchPaginated, htmlToMarkdown, connectorError } from './shared.js';
+const router = Router();
+const sessionKeys = new Map();
+function confHeaders(email, token) {
+    return {
+        Authorization: `Basic ${Buffer.from(`${email}:${token}`).toString('base64')}`,
+        Accept: 'application/json',
+    };
+}
+// ── Test ──
+router.post('/test', async (req, res) => {
+    const { apiKey, email, domain } = req.body;
+    if (!apiKey || !email || !domain) {
+        res.status(400).json({ status: 'error', error: 'Missing apiKey, email, or domain (e.g., yoursite.atlassian.net)' });
+        return;
+    }
+    try {
+        const resp = await rateLimitedFetch(`https://${domain}/wiki/api/v2/spaces?limit=1`, { headers: confHeaders(email, apiKey) });
+        if (!resp.ok) {
+            res.status(401).json({ status: 'error', error: 'Invalid Confluence credentials' });
+            return;
+        }
+        sessionKeys.set('confluence', JSON.stringify({ apiKey, email, domain }));
+        res.json({ status: 'ok', data: { connected: true } });
+    }
+    catch (err) {
+        connectorError(res, 'Confluence', err);
+    }
+});
+// ── List Spaces ──
+router.post('/spaces', async (req, res) => {
+    const body = req.body;
+    const creds = getConfCreds(body);
+    if (!creds) {
+        res.status(401).json({ status: 'error', error: 'No Confluence credentials.' });
+        return;
+    }
+    try {
+        const resp = await rateLimitedFetch(`https://${creds.domain}/wiki/api/v2/spaces?limit=50`, { headers: confHeaders(creds.email, creds.apiKey) });
+        if (!resp.ok)
+            throw new Error(`Confluence API ${resp.status}`);
+        const data = await resp.json();
+        res.json({ status: 'ok', data: data.results });
+    }
+    catch (err) {
+        connectorError(res, 'Confluence', err);
+    }
+});
+// ── Fetch Pages ──
+router.post('/fetch', async (req, res) => {
+    const body = req.body;
+    const creds = getConfCreds(body);
+    if (!creds) {
+        res.status(401).json({ status: 'error', error: 'No Confluence credentials.' });
+        return;
+    }
+    const spaceKey = body.spaceKey;
+    const pageIds = body.pageIds;
+    try {
+        const items = [];
+        if (pageIds?.length) {
+            for (const id of pageIds.slice(0, 20)) {
+                const page = await fetchConfPage(id, creds);
+                if (page)
+                    items.push(page);
+            }
+        }
+        else if (spaceKey) {
+            const pages = await fetchPaginated({
+                maxPages: 5,
+                maxItems: 50,
+                fetchPage: async (cursor) => {
+                    const params = new URLSearchParams({ limit: '25', 'body-format': 'storage' });
+                    if (cursor)
+                        params.set('cursor', cursor);
+                    const resp = await rateLimitedFetch(`https://${creds.domain}/wiki/api/v2/spaces/${spaceKey}/pages?${params}`, { headers: confHeaders(creds.email, creds.apiKey) });
+                    if (!resp.ok)
+                        throw new Error(`Confluence API ${resp.status}`);
+                    const data = await resp.json();
+                    const nextCursor = data._links?.next
+                        ? new URL(data._links.next, `https://${creds.domain}`).searchParams.get('cursor') ?? undefined
+                        : undefined;
+                    return {
+                        items: data.results.map((p) => ({
+                            id: p.id,
+                            title: p.title,
+                            body: p.body?.storage?.value ?? '',
+                        })),
+                        nextCursor,
+                    };
+                },
+            });
+            for (const p of pages) {
+                const md = htmlToMarkdown(p.body);
+                items.push({ id: p.id, title: p.title, markdown: md, tokens: Math.ceil(md.length / 4) });
+            }
+        }
+        const fullMarkdown = items.map(i => `# ${i.title}\n\n${i.markdown}`).join('\n\n---\n\n');
+        res.json({
+            status: 'ok',
+            data: { items, markdown: fullMarkdown, count: items.length, tokens: Math.ceil(fullMarkdown.length / 4) },
+        });
+    }
+    catch (err) {
+        connectorError(res, 'Confluence', err);
+    }
+});
+// ── Search ──
+router.post('/search', async (req, res) => {
+    const body = req.body;
+    const creds = getConfCreds(body);
+    if (!creds) {
+        res.status(401).json({ status: 'error', error: 'No Confluence credentials.' });
+        return;
+    }
+    const query = body.query;
+    const spaceKey = body.spaceKey;
+    if (!query) {
+        res.status(400).json({ status: 'error', error: 'query required' });
+        return;
+    }
+    try {
+        const cql = spaceKey
+            ? `type=page AND space="${spaceKey}" AND text~"${query}"`
+            : `type=page AND text~"${query}"`;
+        const resp = await rateLimitedFetch(`https://${creds.domain}/wiki/rest/api/search?cql=${encodeURIComponent(cql)}&limit=20`, { headers: confHeaders(creds.email, creds.apiKey) });
+        if (!resp.ok)
+            throw new Error(`Confluence search ${resp.status}`);
+        const data = await resp.json();
+        res.json({
+            status: 'ok',
+            data: data.results.map(r => ({
+                id: r.content.id,
+                title: r.content.title,
+                type: r.content.type,
+            })),
+        });
+    }
+    catch (err) {
+        connectorError(res, 'Confluence', err);
+    }
+});
+function getConfCreds(body) {
+    const stored = sessionKeys.get('confluence');
+    if (stored) {
+        const parsed = JSON.parse(stored);
+        return {
+            apiKey: body.apiKey ?? parsed.apiKey,
+            email: body.email ?? parsed.email,
+            domain: body.domain ?? parsed.domain,
+        };
+    }
+    if (body.apiKey && body.email && body.domain) {
+        return { apiKey: body.apiKey, email: body.email, domain: body.domain };
+    }
+    return null;
+}
+async function fetchConfPage(id, creds) {
+    try {
+        const resp = await rateLimitedFetch(`https://${creds.domain}/wiki/api/v2/pages/${id}?body-format=storage`, { headers: confHeaders(creds.email, creds.apiKey) });
+        if (!resp.ok)
+            return null;
+        const page = await resp.json();
+        const md = htmlToMarkdown(page.body?.storage?.value ?? '');
+        return { id: page.id, title: page.title, markdown: md, tokens: Math.ceil(md.length / 4) };
+    }
+    catch {
+        return null;
+    }
+}
+export default router;

package/dist-server/server/routes/connectors/github.d.ts

@@ -0,0 +1,7 @@
+/**
+ * GitHub Connector — Issues + PRs
+ * Issues #90, #99
+ */
+declare const router: import("express-serve-static-core").Router;
+export default router;
+//# sourceMappingURL=github.d.ts.map

package/dist-server/server/routes/connectors/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../../../server/routes/connectors/github.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,QAAA,MAAM,MAAM,4CAAW,CAAC;AAqMxB,eAAe,MAAM,CAAC"}

package/dist-server/server/routes/connectors/github.js

@@ -0,0 +1,195 @@
+/**
+ * GitHub Connector — Issues + PRs
+ * Issues #90, #99
+ */
+import { Router } from 'express';
+import { rateLimitedFetch, fetchPaginated, connectorError, getApiKey, formatTimestamp } from './shared.js';
+const router = Router();
+const sessionKeys = new Map();
+const GH_API = 'https://api.github.com';
+function ghHeaders(token) {
+    return {
+        Authorization: `Bearer ${token}`,
+        Accept: 'application/vnd.github+json',
+        'X-GitHub-Api-Version': '2022-11-28',
+    };
+}
+// ── Test ──
+router.post('/test', async (req, res) => {
+    const { apiKey } = req.body;
+    if (!apiKey) {
+        res.status(400).json({ status: 'error', error: 'Missing apiKey' });
+        return;
+    }
+    try {
+        const resp = await rateLimitedFetch(`${GH_API}/user`, { headers: ghHeaders(apiKey) });
+        if (!resp.ok) {
+            res.status(401).json({ status: 'error', error: 'Invalid GitHub token' });
+            return;
+        }
+        const user = await resp.json();
+        sessionKeys.set('github', apiKey);
+        res.json({ status: 'ok', data: { user: user.name ?? user.login } });
+    }
+    catch (err) {
+        connectorError(res, 'GitHub', err);
+    }
+});
+// ── Fetch Issues ──
+router.post('/issues', async (req, res) => {
+    const body = req.body;
+    const token = getApiKey('github', body, sessionKeys);
+    if (!token) {
+        res.status(401).json({ status: 'error', error: 'No GitHub token. Test connection first.' });
+        return;
+    }
+    const repo = body.repo;
+    const state = body.state ?? 'open';
+    const labels = body.labels;
+    if (!repo || !repo.includes('/')) {
+        res.status(400).json({ status: 'error', error: 'repo must be owner/name format' });
+        return;
+    }
+    try {
+        const items = await fetchPaginated({
+            maxPages: 5,
+            maxItems: 200,
+            fetchPage: async (cursor) => {
+                const params = new URLSearchParams({ state, per_page: '100', sort: 'updated', direction: 'desc' });
+                if (labels?.length)
+                    params.set('labels', labels.join(','));
+                if (cursor)
+                    params.set('page', cursor);
+                const resp = await rateLimitedFetch(`${GH_API}/repos/${repo}/issues?${params}`, { headers: ghHeaders(token) });
+                if (!resp.ok)
+                    throw new Error(`GitHub API ${resp.status}`);
+                const data = await resp.json();
+                // Filter out pull requests (GitHub API returns PRs as issues)
+                const issues = data.filter((d) => !d.pull_request);
+                // Check for next page via Link header
+                const link = resp.headers.get('Link') ?? '';
+                const nextMatch = link.match(/page=(\d+)>; rel="next"/);
+                const nextCursor = nextMatch ? nextMatch[1] : undefined;
+                return {
+                    items: issues.map((issue) => ({
+                        number: issue.number,
+                        title: issue.title,
+                        body: issue.body ?? '',
+                        state: issue.state,
+                        labels: (issue.labels ?? []).map((l) => l.name),
+                        assignee: issue.assignee?.login ?? 'unassigned',
+                        author: issue.user?.login ?? '',
+                        createdAt: issue.created_at,
+                        updatedAt: issue.updated_at,
+                        comments: issue.comments ?? 0,
+                        url: issue.html_url,
+                    })),
+                    nextCursor,
+                };
+            },
+        });
+        // Convert to markdown
+        const markdown = items.map((issue) => `## #${issue.number}: ${issue.title}\n` +
+            `**Status:** ${issue.state} · **Assignee:** ${issue.assignee} · **Author:** ${issue.author}\n` +
+            `**Labels:** ${issue.labels.join(', ') || 'none'} · **Comments:** ${issue.comments}\n` +
+            `**Created:** ${formatTimestamp(issue.createdAt)} · **Updated:** ${formatTimestamp(issue.updatedAt)}\n\n` +
+            `${issue.body}\n`).join('\n---\n\n');
+        res.json({
+            status: 'ok',
+            data: {
+                items,
+                markdown,
+                count: items.length,
+                tokens: Math.ceil(markdown.length / 4),
+            },
+        });
+    }
+    catch (err) {
+        connectorError(res, 'GitHub', err);
+    }
+});
+// ── Fetch PRs (#99) ──
+router.post('/pulls', async (req, res) => {
+    const body = req.body;
+    const token = getApiKey('github', body, sessionKeys);
+    if (!token) {
+        res.status(401).json({ status: 'error', error: 'No GitHub token.' });
+        return;
+    }
+    const repo = body.repo;
+    const state = body.state ?? 'open';
+    if (!repo?.includes('/')) {
+        res.status(400).json({ status: 'error', error: 'repo must be owner/name format' });
+        return;
+    }
+    try {
+        const params = new URLSearchParams({ state, per_page: '50', sort: 'updated', direction: 'desc' });
+        const resp = await rateLimitedFetch(`${GH_API}/repos/${repo}/pulls?${params}`, { headers: ghHeaders(token) });
+        if (!resp.ok)
+            throw new Error(`GitHub API ${resp.status}`);
+        const prs = await resp.json();
+        const items = prs.map((pr) => ({
+            number: pr.number,
+            title: pr.title,
+            body: pr.body ?? '',
+            state: pr.state,
+            author: pr.user?.login ?? '',
+            draft: pr.draft ?? false,
+            additions: pr.additions ?? 0,
+            deletions: pr.deletions ?? 0,
+            changedFiles: pr.changed_files ?? 0,
+            createdAt: pr.created_at,
+            updatedAt: pr.updated_at,
+            mergedAt: pr.merged_at,
+            url: pr.html_url,
+        }));
+        const markdown = items.map((pr) => `## PR #${pr.number}: ${pr.title}${pr.draft ? ' [DRAFT]' : ''}\n` +
+            `**Status:** ${pr.mergedAt ? 'merged' : pr.state} · **Author:** ${pr.author}\n` +
+            `**Changes:** +${pr.additions} -${pr.deletions} in ${pr.changedFiles} files\n` +
+            `**Created:** ${formatTimestamp(pr.createdAt)}${pr.mergedAt ? ` · **Merged:** ${formatTimestamp(pr.mergedAt)}` : ''}\n\n` +
+            `${pr.body}\n`).join('\n---\n\n');
+        res.json({
+            status: 'ok',
+            data: { items, markdown, count: items.length, tokens: Math.ceil(markdown.length / 4) },
+        });
+    }
+    catch (err) {
+        connectorError(res, 'GitHub', err);
+    }
+});
+// ── Search ──
+router.post('/search', async (req, res) => {
+    const body = req.body;
+    const token = getApiKey('github', body, sessionKeys);
+    if (!token) {
+        res.status(401).json({ status: 'error', error: 'No GitHub token.' });
+        return;
+    }
+    const query = body.query;
+    const repo = body.repo;
+    if (!query) {
+        res.status(400).json({ status: 'error', error: 'query required' });
+        return;
+    }
+    try {
+        const q = repo ? `${query}+repo:${repo}` : query;
+        const resp = await rateLimitedFetch(`${GH_API}/search/issues?q=${encodeURIComponent(q)}&per_page=20`, { headers: ghHeaders(token) });
+        if (!resp.ok)
+            throw new Error(`GitHub search ${resp.status}`);
+        const data = await resp.json();
+        res.json({
+            status: 'ok',
+            data: data.items.map((i) => ({
+                number: i.number,
+                title: i.title,
+                type: i.pull_request ? 'pr' : 'issue',
+                state: i.state,
+                repo: i.repository_url?.split('/').slice(-2).join('/'),
+            })),
+        });
+    }
+    catch (err) {
+        connectorError(res, 'GitHub', err);
+    }
+});
+export default router;

package/dist-server/server/routes/connectors/gmail.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Gmail Connector — Messages via search
+ * Issue #100
+ */
+declare const router: import("express-serve-static-core").Router;
+export default router;
+//# sourceMappingURL=gmail.d.ts.map

package/dist-server/server/routes/connectors/gmail.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gmail.d.ts","sourceRoot":"","sources":["../../../../server/routes/connectors/gmail.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,QAAA,MAAM,MAAM,4CAAW,CAAC;AAmIxB,eAAe,MAAM,CAAC"}

package/dist-server/server/routes/connectors/gmail.js

@@ -0,0 +1,115 @@
+/**
+ * Gmail Connector — Messages via search
+ * Issue #100
+ */
+import { Router } from 'express';
+import { rateLimitedFetch, connectorError, getApiKey } from './shared.js';
+const router = Router();
+const sessionKeys = new Map();
+const GMAIL_API = 'https://gmail.googleapis.com/gmail/v1/users/me';
+function gHeaders(token) {
+    return { Authorization: `Bearer ${token}` };
+}
+// ── Test ──
+router.post('/test', async (req, res) => {
+    const { apiKey } = req.body;
+    if (!apiKey) {
+        res.status(400).json({ status: 'error', error: 'Missing OAuth token' });
+        return;
+    }
+    try {
+        const resp = await rateLimitedFetch(`${GMAIL_API}/profile`, { headers: gHeaders(apiKey) });
+        if (!resp.ok) {
+            res.status(401).json({ status: 'error', error: 'Invalid Gmail token' });
+            return;
+        }
+        const data = await resp.json();
+        sessionKeys.set('gmail', apiKey);
+        res.json({ status: 'ok', data: { email: data.emailAddress } });
+    }
+    catch (err) {
+        connectorError(res, 'Gmail', err);
+    }
+});
+// ── Fetch Messages ──
+router.post('/fetch', async (req, res) => {
+    const body = req.body;
+    const token = getApiKey('gmail', body, sessionKeys);
+    if (!token) {
+        res.status(401).json({ status: 'error', error: 'No Gmail token.' });
+        return;
+    }
+    const query = body.query ?? 'in:inbox';
+    const maxResults = Math.min(body.maxResults ?? 20, 50);
+    try {
+        // List message IDs
+        const listResp = await rateLimitedFetch(`${GMAIL_API}/messages?q=${encodeURIComponent(query)}&maxResults=${maxResults}`, { headers: gHeaders(token) });
+        if (!listResp.ok)
+            throw new Error(`Gmail list ${listResp.status}`);
+        const listData = await listResp.json();
+        const messageIds = (listData.messages ?? []).map(m => m.id);
+        const items = [];
+        // Fetch each message (batch would be better but simpler this way)
+        for (const msgId of messageIds.slice(0, 20)) {
+            try {
+                const msgResp = await rateLimitedFetch(`${GMAIL_API}/messages/${msgId}?format=full`, { headers: gHeaders(token) });
+                if (!msgResp.ok)
+                    continue;
+                const msg = await msgResp.json();
+                const headers = msg.payload?.headers ?? [];
+                const getHeader = (name) => headers.find(h => h.name.toLowerCase() === name.toLowerCase())?.value ?? '';
+                items.push({
+                    id: msgId,
+                    subject: getHeader('Subject'),
+                    from: getHeader('From'),
+                    date: getHeader('Date'),
+                    snippet: msg.snippet ?? '',
+                    body: extractGmailBody(msg.payload),
+                });
+            }
+            catch { /* skip */ }
+        }
+        const markdown = items.map(m => `## ${m.subject}\n` +
+            `**From:** ${m.from}\n` +
+            `**Date:** ${m.date}\n\n` +
+            `${m.body}\n`).join('\n---\n\n');
+        res.json({
+            status: 'ok',
+            data: { items, markdown, count: items.length, tokens: Math.ceil(markdown.length / 4) },
+        });
+    }
+    catch (err) {
+        connectorError(res, 'Gmail', err);
+    }
+});
+function extractGmailBody(payload) {
+    if (!payload)
+        return '';
+    // Try to find text/plain first, then text/html
+    const plain = findPart(payload, 'text/plain');
+    if (plain?.body?.data) {
+        return Buffer.from(plain.body.data, 'base64url').toString('utf-8');
+    }
+    const html = findPart(payload, 'text/html');
+    if (html?.body?.data) {
+        const raw = Buffer.from(html.body.data, 'base64url').toString('utf-8');
+        // Simple HTML strip (shared htmlToMarkdown is overkill for email)
+        return raw.replace(/<[^>]+>/g, '').replace(/&nbsp;/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
+    }
+    // Direct body
+    if (payload.body?.data) {
+        return Buffer.from(payload.body.data, 'base64url').toString('utf-8');
+    }
+    return '';
+}
+function findPart(part, mimeType) {
+    if (part.mimeType === mimeType)
+        return part;
+    for (const child of part.parts ?? []) {
+        const found = findPart(child, mimeType);
+        if (found)
+            return found;
+    }
+    return undefined;
+}
+export default router;

package/dist-server/server/routes/connectors/google-docs.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Google Docs Connector
+ * Issue #91
+ *
+ * Requires OAuth token (from Google Drive OAuth flow in connectors.ts).
+ * Fetches document structural elements and converts to markdown.
+ */
+declare const router: import("express-serve-static-core").Router;
+export default router;
+//# sourceMappingURL=google-docs.d.ts.map

package/dist-server/server/routes/connectors/google-docs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"google-docs.d.ts","sourceRoot":"","sources":["../../../../server/routes/connectors/google-docs.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,QAAA,MAAM,MAAM,4CAAW,CAAC;AA8LxB,eAAe,MAAM,CAAC"}