@hubblecommerce/overmind-core 0.1.0

package/README.md

@@ -0,0 +1,40 @@
+# @hubblecommerce/overmind-core
+
+Shared infrastructure package for the **Overmind** AI agent system. Contains all foundational code used across the agent server and webhook worker — API clients, vectorstore, embeddings, LLM providers, document processors, and reusable LangChain tools.
+
+## Contents
+
+| Module | Import path | Description |
+|---|---|---|
+| Config | `@hubblecommerce/overmind-core/config` | Centralized app configuration from env vars |
+| Confluence client | `@hubblecommerce/overmind-core/integrations/confluence` | Confluence REST API v2 client |
+| Jira client | `@hubblecommerce/overmind-core/integrations/jira` | Jira REST API v3 client |
+| GitLab client | `@hubblecommerce/overmind-core/integrations/gitlab` | GitLab API client |
+| Vectorstore | `@hubblecommerce/overmind-core/vectorstore` | pgvector provider + interface |
+| Embeddings | `@hubblecommerce/overmind-core/embeddings` | Hugging Face Transformers provider + interface |
+| LLM | `@hubblecommerce/overmind-core/llm` | Anthropic/Claude provider |
+| LLM retry | `@hubblecommerce/overmind-core/llm/retry` | 429 rate limit retry wrapper |
+| Processors | `@hubblecommerce/overmind-core/processors` | Confluence HTML parser + document processor |
+| Confluence search tool | `@hubblecommerce/overmind-core/tools/confluence-search` | LangChain tool for semantic Confluence search |
+| Date tool | `@hubblecommerce/overmind-core/tools/get-current-date` | LangChain tool returning current date |
+
+## Usage
+
+```typescript
+import { config } from '@hubblecommerce/overmind-core/config';
+import { ConfluenceClient } from '@hubblecommerce/overmind-core/integrations/confluence';
+import { createPostgresVectorStore } from '@hubblecommerce/overmind-core/vectorstore';
+import { createConfluenceSearchTool } from '@hubblecommerce/overmind-core/tools/confluence-search';
+```
+
+## Requirements
+
+- Node.js 20+
+- PostgreSQL 15+ with `pgvector` and `uuid-ossp` extensions enabled
+- `ANTHROPIC_API_KEY` env var set
+
+## Publishing
+
+```bash
+npm publish --access restricted
+```

package/dist/src/config/app.config.js

@@ -0,0 +1,52 @@
+export const config = {
+    llm: {
+        provider: 'anthropic',
+        modelName: 'claude-sonnet-4-6',
+        temperature: 0,
+        timeoutMs: parseInt(process.env.CLAUDE_TIMEOUT_MS || '60000'),
+        maxRetries: 2, // LangChain built-in retry for network errors
+    },
+    queue: {
+        pollIntervalMs: parseInt(process.env.QUEUE_POLL_INTERVAL_MS || '5000'),
+        maxRetries: parseInt(process.env.QUEUE_MAX_RETRIES || '3'),
+    },
+    embedding: {
+        provider: 'transformers',
+        modelName: 'Xenova/jina-embeddings-v2-base-de',
+        pooling: 'mean',
+        normalize: true,
+    },
+    vectorStore: {
+        postgres: {
+            host: process.env.POSTGRES_HOST || 'localhost',
+            port: parseInt(process.env.POSTGRES_PORT || '5432'),
+            user: process.env.POSTGRES_USER || 'postgres',
+            password: process.env.POSTGRES_PASSWORD || 'postgres',
+            database: process.env.POSTGRES_DB || 'rag_db',
+            tableName: 'confluence',
+        },
+    },
+    gitlab: {
+        baseUrl: process.env.GITLAB_BASE_URL || 'https://gitlab.com',
+        token: process.env.GITLAB_ACCESS_TOKEN || '',
+        rateLimit: 10, // requests per second
+        chunking: {
+            chunkSize: 1000,
+            chunkOverlap: 200,
+        },
+    },
+    confluence: {
+        baseUrl: process.env.CONFLUENCE_URL || '',
+        email: process.env.CONFLUENCE_EMAIL || '',
+        apiToken: process.env.CONFLUENCE_API_TOKEN || '',
+        spaces: process.env.CONFLUENCE_SPACES?.split(',').map(s => s.trim()).filter(Boolean) || [],
+        rateLimit: 10, // requests per second
+    },
+    jira: {
+        baseUrl: process.env.JIRA_BASE_URL || '',
+        email: process.env.JIRA_EMAIL || '',
+        apiToken: process.env.JIRA_API_TOKEN || '',
+        suggestionsSpaceId: process.env.CONFLUENCE_SUGGESTIONS_SPACE_ID || '',
+        suggestionsParentId: process.env.CONFLUENCE_SUGGESTIONS_PARENT_ID || '',
+    },
+};

package/dist/src/embeddings/embeddings.interface.js

@@ -0,0 +1 @@
+export {};

package/dist/src/embeddings/transformers.provider.js

@@ -0,0 +1,19 @@
+import { pipeline } from '@xenova/transformers';
+export async function createTransformersEmbeddings(modelName, pooling, normalize) {
+    console.log('Loading local embedding model (first time may take a moment to download)...');
+    const embedder = await pipeline('feature-extraction', modelName);
+    return {
+        async embedDocuments(texts) {
+            const embeddings = [];
+            for (const text of texts) {
+                const output = await embedder(text, { pooling, normalize });
+                embeddings.push(Array.from(output.data));
+            }
+            return embeddings;
+        },
+        async embedQuery(text) {
+            const output = await embedder(text, { pooling, normalize });
+            return Array.from(output.data);
+        },
+    };
+}

package/dist/src/integrations/confluence/confluence.client.js

@@ -0,0 +1,230 @@
+export class ConfluenceClient {
+    baseUrl;
+    authHeader;
+    rateLimit;
+    lastRequestTime = 0;
+    constructor(config) {
+        this.baseUrl = config.baseUrl.replace(/\/$/, ''); // Remove trailing slash
+        this.rateLimit = config.rateLimit || 10; // Default: 10 req/sec
+        // Create Basic Auth header
+        const credentials = Buffer.from(`${config.email}:${config.apiToken}`).toString('base64');
+        this.authHeader = `Basic ${credentials}`;
+    }
+    /**
+     * Rate limiting: Ensure we don't exceed configured requests per second
+     */
+    async enforceRateLimit() {
+        const now = Date.now();
+        const timeSinceLastRequest = now - this.lastRequestTime;
+        const minInterval = 1000 / this.rateLimit;
+        if (timeSinceLastRequest < minInterval) {
+            const delay = minInterval - timeSinceLastRequest;
+            await new Promise(resolve => setTimeout(resolve, delay));
+        }
+        this.lastRequestTime = Date.now();
+    }
+    /**
+     * Make HTTP request to Confluence API with retries
+     */
+    async request(endpoint, options = {}) {
+        await this.enforceRateLimit();
+        const url = `${this.baseUrl}${endpoint}`;
+        const maxRetries = 3;
+        let lastError = null;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+            try {
+                const response = await fetch(url, {
+                    ...options,
+                    headers: {
+                        'Authorization': this.authHeader,
+                        'Accept': 'application/json',
+                        'Content-Type': 'application/json',
+                        ...options.headers,
+                    },
+                });
+                if (!response.ok) {
+                    const errorText = await response.text();
+                    const error = {
+                        statusCode: response.status,
+                        message: `Confluence API error: ${response.statusText}`,
+                        reason: errorText,
+                    };
+                    throw new Error(JSON.stringify(error));
+                }
+                const data = await response.json();
+                return data;
+            }
+            catch (error) {
+                lastError = error;
+                // Don't retry on 4xx errors (client errors)
+                if (error instanceof Error && error.message.includes('"statusCode":4')) {
+                    throw error;
+                }
+                // Retry on network errors or 5xx errors
+                if (attempt < maxRetries) {
+                    const backoffDelay = Math.pow(2, attempt) * 1000; // Exponential backoff
+                    console.warn(`Request failed (attempt ${attempt}/${maxRetries}), retrying in ${backoffDelay}ms...`);
+                    await new Promise(resolve => setTimeout(resolve, backoffDelay));
+                }
+                else {
+                    throw lastError;
+                }
+            }
+        }
+        throw lastError || new Error('Request failed after max retries');
+    }
+    /**
+     * Get all spaces (with pagination)
+     */
+    async getSpaces(limit = 25) {
+        const spaces = [];
+        let nextUrl = `/wiki/api/v2/spaces?limit=${limit}`;
+        while (nextUrl) {
+            const response = await this.request(nextUrl);
+            const results = response.results;
+            spaces.push(...results);
+            // Extract cursor from next link if exists
+            const links = response._links;
+            const nextLink = links.next;
+            nextUrl = nextLink ? nextLink.replace(this.baseUrl, '') : undefined;
+        }
+        return spaces;
+    }
+    /**
+     * Get all pages in a space (with pagination)
+     * Note: Accepts space key and automatically resolves to space ID for API v2
+     */
+    async getSpacePages(spaceKey, limit = 25) {
+        // First, resolve space key to space ID (required by API v2)
+        const space = await this.getSpaceByKey(spaceKey);
+        const spaceId = space.id;
+        const pages = [];
+        let nextUrl = `/wiki/api/v2/spaces/${spaceId}/pages?limit=${limit}&status=current`;
+        while (nextUrl) {
+            const response = await this.request(nextUrl);
+            const results = response.results;
+            pages.push(...results);
+            // Extract cursor from next link if exists
+            const links = response._links;
+            nextUrl = links.next || undefined;
+            if (nextUrl) {
+                // Convert full URL to relative path
+                nextUrl = nextUrl.replace(this.baseUrl, '');
+            }
+        }
+        return pages;
+    }
+    /**
+     * Get a single page by ID with content
+     */
+    async getPageById(pageId, bodyFormat = 'storage') {
+        const endpoint = `/wiki/api/v2/pages/${pageId}?body-format=${bodyFormat}`;
+        return await this.request(endpoint);
+    }
+    /**
+     * Get page content only (HTML in storage format)
+     */
+    async getPageContent(pageId) {
+        const page = await this.getPageById(pageId, 'storage');
+        if (!page.body?.storage?.value) {
+            throw new Error(`Page ${pageId} has no content`);
+        }
+        return page.body.storage.value;
+    }
+    /**
+     * Get space by key
+     */
+    async getSpaceByKey(spaceKey) {
+        const endpoint = `/wiki/api/v2/spaces?keys=${spaceKey}&limit=1`;
+        const response = await this.request(endpoint);
+        if (response.results.length === 0) {
+            throw new Error(`Space with key "${spaceKey}" not found`);
+        }
+        return response.results[0];
+    }
+    /**
+     * Get recent pages across all spaces or specific spaces
+     */
+    async getRecentPages(spaceKeys, limit = 25) {
+        let endpoint = `/wiki/api/v2/pages?limit=${limit}&sort=-modified-date&status=current`;
+        if (spaceKeys && spaceKeys.length > 0) {
+            const spaceFilter = spaceKeys.map(key => `space-key=${key}`).join('&');
+            endpoint += `&${spaceFilter}`;
+        }
+        const response = await this.request(endpoint);
+        return response.results;
+    }
+    /**
+     * Get labels for a page (using v1 API as v2 doesn't support this yet)
+     * Returns only label names as strings
+     */
+    async getPageLabels(pageId) {
+        const labels = [];
+        let start = 0;
+        const limit = 200;
+        // Use v1 API endpoint for labels
+        let hasMore = true;
+        while (hasMore) {
+            const endpoint = `/wiki/rest/api/content/${pageId}/label?start=${start}&limit=${limit}`;
+            const response = await this.request(endpoint);
+            labels.push(...response.results);
+            // Check if there are more results
+            if (response.results.length < limit) {
+                hasMore = false;
+            }
+            else {
+                start += limit;
+            }
+        }
+        // Return only the label names (without prefix)
+        return labels.map(label => label.name);
+    }
+    /**
+     * Get numeric space ID from space key
+     * Helper method to convert space key to numeric ID required by v2 API
+     * @param spaceKey - The space key (e.g., "TEST")
+     * @returns The numeric space ID
+     */
+    async getSpaceIdByKey(spaceKey) {
+        const space = await this.getSpaceByKey(spaceKey);
+        return space.id;
+    }
+    /**
+     * Create a new page in Confluence using REST API v2 with ADF (Atlassian Document Format)
+     * This is the modern format for the new Confluence editor
+     * @param spaceId - The numeric space ID (not space key) - use getSpaceIdByKey() if you have a space key
+     * @param title - The page title
+     * @param adfBody - The page content in ADF format (from @atlaskit/editor-markdown-transformer)
+     * @param parentId - Optional parent page ID to create as child page
+     * @returns The created page
+     */
+    async createPageWithADF(params) {
+        const payload = {
+            spaceId: params.spaceId,
+            status: 'current',
+            title: params.title,
+            body: {
+                representation: 'atlas_doc_format',
+                value: JSON.stringify(params.adfBody), // ADF must be JSON-encoded string
+            },
+        };
+        // If parentId provided, add to payload
+        if (params.parentId) {
+            payload.parentId = params.parentId;
+        }
+        const response = await this.request('/wiki/api/v2/pages', {
+            method: 'POST',
+            body: JSON.stringify(payload),
+        });
+        return {
+            id: response.id,
+            status: response.status,
+            title: response.title,
+            spaceId: response.spaceId,
+            authorId: response.authorId,
+            createdAt: response.createdAt,
+            version: response.version,
+            _links: response._links,
+        };
+    }
+}

package/dist/src/integrations/confluence/confluence.types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/integrations/gitlab/gitlab.client.js

@@ -0,0 +1,15 @@
+import { Gitlab } from '@gitbeaker/rest';
+import { config } from '../../config/app.config.js';
+let gitlabClient = null;
+export function getGitlabClient() {
+    if (!gitlabClient) {
+        if (!config.gitlab.token) {
+            throw new Error('GitLab access token not configured. Set GITLAB_ACCESS_TOKEN in .env');
+        }
+        gitlabClient = new Gitlab({
+            host: config.gitlab.baseUrl,
+            token: config.gitlab.token,
+        });
+    }
+    return gitlabClient;
+}

package/dist/src/integrations/gitlab/gitlab.types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/integrations/jira/jira.client.js

@@ -0,0 +1,150 @@
+export class JiraClient {
+    baseUrl;
+    authHeader;
+    constructor(config) {
+        this.baseUrl = config.baseUrl;
+        this.authHeader = 'Basic ' + Buffer.from(`${config.email}:${config.apiToken}`).toString('base64');
+    }
+    /**
+     * Parse Atlassian Document Format (ADF) to plain text
+     */
+    parseADFToText(node) {
+        if (node.type === 'text') {
+            return node.text || '';
+        }
+        if (node.type === 'mention' && node.attrs?.text) {
+            return node.attrs.text;
+        }
+        if (node.content) {
+            const childText = node.content.map(child => this.parseADFToText(child)).join('');
+            // Add spacing/formatting based on node type
+            switch (node.type) {
+                case 'paragraph':
+                    return childText + '\n';
+                case 'blockquote':
+                    return '> ' + childText + '\n';
+                case 'heading':
+                    return '## ' + childText + '\n';
+                case 'listItem':
+                    return '- ' + childText + '\n';
+                case 'codeBlock':
+                    return '```\n' + childText + '```\n';
+                default:
+                    return childText;
+            }
+        }
+        return '';
+    }
+    /**
+     * Convert ADF comment body to readable text
+     */
+    parseCommentBody(body) {
+        if (typeof body === 'string') {
+            return body;
+        }
+        if (typeof body === 'object' && body !== null) {
+            const adfDoc = body;
+            if (adfDoc.type === 'doc' && adfDoc.content) {
+                return adfDoc.content.map(node => this.parseADFToText(node)).join('').trim();
+            }
+        }
+        return '';
+    }
+    /**
+     * Fetch a Jira issue by key or ID, including comments
+     */
+    async getIssue(issueKeyOrId) {
+        try {
+            // Fetch issue details
+            const issueResponse = await fetch(`${this.baseUrl}/rest/api/3/issue/${issueKeyOrId}`, {
+                headers: {
+                    'Authorization': this.authHeader,
+                    'Content-Type': 'application/json',
+                    'Accept': 'application/json',
+                },
+            });
+            if (!issueResponse.ok) {
+                throw new Error(`Failed to fetch issue: ${issueResponse.status} ${issueResponse.statusText}`);
+            }
+            const issue = await issueResponse.json();
+            // Fetch comments
+            const commentsResponse = await fetch(`${this.baseUrl}/rest/api/3/issue/${issueKeyOrId}/comment`, {
+                headers: {
+                    'Authorization': this.authHeader,
+                    'Content-Type': 'application/json',
+                    'Accept': 'application/json',
+                },
+            });
+            if (!commentsResponse.ok) {
+                throw new Error(`Failed to fetch comments: ${commentsResponse.status} ${commentsResponse.statusText}`);
+            }
+            const commentsData = await commentsResponse.json();
+            // Parse and filter comments (only 'doc' type, latest version only)
+            const commentsMap = new Map();
+            commentsData.comments?.forEach(comment => {
+                // Only include ADF doc type comments
+                if (typeof comment.body === 'object' && comment.body !== null) {
+                    const adfDoc = comment.body;
+                    if (adfDoc.type !== 'doc') {
+                        return;
+                    }
+                    const version = adfDoc.version || 1;
+                    const existing = commentsMap.get(comment.id);
+                    // Only keep the latest version of each comment
+                    if (!existing || version > existing.version) {
+                        commentsMap.set(comment.id, {
+                            author: comment.author.displayName,
+                            body: this.parseCommentBody(comment.body),
+                            updated: comment.updated,
+                            version,
+                        });
+                    }
+                }
+            });
+            // Convert map to array (without version field)
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const parsedComments = Array.from(commentsMap.values()).map(({ version, ...comment }) => comment);
+            // Transform to our interface
+            return {
+                id: issue.id,
+                key: issue.key,
+                self: issue.self,
+                fields: {
+                    summary: issue.fields.summary,
+                    description: issue.fields.description,
+                    status: {
+                        name: issue.fields.status.name,
+                        id: issue.fields.status.id,
+                    },
+                    issuetype: {
+                        name: issue.fields.issuetype.name,
+                        id: issue.fields.issuetype.id,
+                    },
+                    project: {
+                        key: issue.fields.project.key,
+                        id: issue.fields.project.id,
+                        name: issue.fields.project.name,
+                    },
+                    assignee: issue.fields.assignee ? {
+                        accountId: issue.fields.assignee.accountId,
+                        displayName: issue.fields.assignee.displayName,
+                    } : null,
+                    reporter: issue.fields.reporter ? {
+                        accountId: issue.fields.reporter.accountId,
+                        displayName: issue.fields.reporter.displayName,
+                    } : null,
+                    created: issue.fields.created,
+                    updated: issue.fields.updated,
+                    resolutiondate: issue.fields.resolutiondate,
+                    labels: issue.fields.labels || [],
+                    components: issue.fields.components || [],
+                },
+                comments: parsedComments,
+            };
+        }
+        catch (error) {
+            console.error(`[JiraClient] Error fetching issue ${issueKeyOrId}:`, error);
+            throw error;
+        }
+    }
+}

package/dist/src/llm/anthropic-retry-wrapper.js

@@ -0,0 +1,110 @@
+/**
+ * Retry wrapper for Anthropic API calls with 429 rate limit handling
+ * Respects retry-after headers and implements exponential backoff
+ */
+/**
+ * Custom error for rate limit responses
+ * Includes retry-after information from API headers
+ */
+export class RateLimitError extends Error {
+    retryAfterSeconds;
+    constructor(message, retryAfterSeconds) {
+        super(message);
+        this.retryAfterSeconds = retryAfterSeconds;
+        this.name = 'RateLimitError';
+    }
+}
+/**
+ * Invoke an agent with automatic retry on 429 errors
+ * @param agent LangChain agent instance
+ * @param input Agent input (messages, etc.)
+ * @param maxRetries Maximum number of retry attempts (default: 3)
+ * @returns Agent response
+ * @throws RateLimitError if all retries exhausted due to 429
+ * @throws Original error if non-429 error occurs
+ */
+export async function invokeWithRetry(agent, input, maxRetries = 3) {
+    let lastError = new Error('Unknown error');
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const result = await agent.invoke(input);
+            return result;
+        }
+        catch (error) {
+            if (!(error instanceof Error)) {
+                throw new Error('Unknown error occurred');
+            }
+            lastError = error;
+            const apiError = error;
+            // Check if it's a 429 rate limit error
+            const is429 = apiError.status === 429 ||
+                apiError.message.includes('429') ||
+                apiError.message.includes('rate limit') ||
+                apiError.message.toLowerCase().includes('too many requests');
+            if (is429) {
+                // Extract retry-after header (in seconds)
+                const retryAfter = extractRetryAfter(apiError);
+                const waitMs = retryAfter
+                    ? retryAfter * 1000
+                    : Math.pow(2, attempt) * 1000; // Exponential fallback: 2s, 4s, 8s
+                console.log(`[AnthropicRetry] Attempt ${attempt}/${maxRetries} failed with 429`, {
+                    retryAfterHeader: retryAfter,
+                    waitMs,
+                    willRetry: attempt < maxRetries,
+                });
+                if (attempt < maxRetries) {
+                    // Wait and retry
+                    await sleep(waitMs);
+                    continue;
+                }
+                else {
+                    // Last attempt failed - throw with retry info for queue worker
+                    throw new RateLimitError(apiError.message || 'Rate limit exceeded', retryAfter);
+                }
+            }
+            // Non-429 errors - throw immediately (don't retry)
+            console.error('[AnthropicRetry] Non-retryable error:', apiError.message);
+            throw error;
+        }
+    }
+    // Should never reach here, but TypeScript requires it
+    throw lastError;
+}
+/**
+ * Extract retry-after header from error response
+ * Tries multiple possible error structures from different API clients
+ * @param error Error object from API call
+ * @returns Retry-after value in seconds, or null if not found
+ */
+function extractRetryAfter(error) {
+    // Try multiple paths to find retry-after header
+    let retryAfter;
+    if (error.response?.headers) {
+        const headers = error.response.headers;
+        if (typeof headers === 'object' && 'get' in headers && typeof headers.get === 'function') {
+            retryAfter = headers.get('retry-after');
+        }
+        else if (typeof headers === 'object') {
+            retryAfter = headers['retry-after'];
+        }
+    }
+    if (!retryAfter && error.headers) {
+        retryAfter = error.headers['retry-after'];
+    }
+    if (!retryAfter && error.error?.headers) {
+        retryAfter = error.error.headers['retry-after'];
+    }
+    if (!retryAfter) {
+        return null;
+    }
+    // Parse as integer (seconds)
+    const parsed = parseInt(retryAfter, 10);
+    return isNaN(parsed) ? null : parsed;
+}
+/**
+ * Sleep utility for async delays
+ * @param ms Milliseconds to sleep
+ */
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}

package/dist/src/llm/anthropic.provider.js

@@ -0,0 +1,11 @@
+import { ChatAnthropic } from '@langchain/anthropic';
+export function createAnthropicProvider(config) {
+    return new ChatAnthropic({
+        modelName: config.modelName,
+        temperature: config.temperature,
+        maxRetries: config.maxRetries || 2, // LangChain built-in retry for network errors
+        clientOptions: {
+            timeout: config.timeoutMs || 60000, // 60 second timeout default
+        },
+    });
+}