@jagilber-org/index-server 1.22.0

package/dist/dashboard/server/routes/api.instructions.routes.js

@@ -0,0 +1,213 @@
+"use strict";
+/**
+ * API Instructions Routes — Endpoint configuration and management.
+ *
+ * Manages APIEndpoint CRUD, rate limiting, and request queuing infrastructure
+ * used by the APIIntegration coordinator.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EndpointManager = void 0;
+// ── EndpointManager ──────────────────────────────────────────────────────────────────
+class EndpointManager {
+    endpoints = new Map();
+    requestQueue = new Map();
+    responseCache = new Map();
+    rateLimiters = new Map();
+    constructor() {
+        this.initializeDefaultEndpoints();
+    }
+    initializeDefaultEndpoints() {
+        const defaultEndpoints = [
+            {
+                id: 'system_health',
+                name: 'System Health Check',
+                url: 'http://localhost:8989/api/health',
+                method: 'GET',
+                protocol: 'rest',
+                authentication: { type: 'none', config: {} },
+                headers: { 'Content-Type': 'application/json' },
+                timeout: 5000,
+                retryConfig: {
+                    enabled: true,
+                    maxAttempts: 3,
+                    backoffStrategy: 'exponential',
+                    baseDelay: 1000,
+                    maxDelay: 5000,
+                    retryOn: ['timeout', 'network_error', 'server_error']
+                },
+                rateLimit: {
+                    enabled: true,
+                    requestsPerMinute: 60,
+                    burstLimit: 10,
+                    queueMaxSize: 100,
+                    dropOnFull: false
+                },
+                dataMapping: { fieldMappings: [] },
+                validation: { validateRequest: false, validateResponse: true, strictMode: false },
+                monitoring: {
+                    logRequests: true,
+                    logResponses: false,
+                    logErrors: true,
+                    collectMetrics: true,
+                    alertOnFailure: true,
+                    alertThreshold: 5
+                }
+            },
+            {
+                id: 'external_metrics',
+                name: 'External Metrics API',
+                url: 'https://api.example.com/metrics',
+                method: 'POST',
+                protocol: 'rest',
+                authentication: {
+                    type: 'api_key',
+                    config: {
+                        keyName: 'X-API-Key',
+                        keyValue: '${API_KEY}'
+                    }
+                },
+                headers: {
+                    'Content-Type': 'application/json',
+                    'User-Agent': 'Index-Server/1.0'
+                },
+                timeout: 10000,
+                retryConfig: {
+                    enabled: true,
+                    maxAttempts: 5,
+                    backoffStrategy: 'exponential',
+                    baseDelay: 2000,
+                    maxDelay: 30000,
+                    retryOn: ['timeout', 'network_error', 'server_error', 'rate_limit']
+                },
+                rateLimit: {
+                    enabled: true,
+                    requestsPerMinute: 30,
+                    burstLimit: 5,
+                    queueMaxSize: 50,
+                    dropOnFull: true
+                },
+                dataMapping: {
+                    requestTransform: `
+            function transform(data) {
+              return {
+                timestamp: Date.now(),
+                source: 'index',
+                metrics: data
+              };
+            }
+          `,
+                    responseTransform: `
+            function transform(response) {
+              return {
+                success: response.status === 'ok',
+                processed: response.count || 0,
+                errors: response.errors || []
+              };
+            }
+          `,
+                    fieldMappings: [
+                        { source: 'cpu_usage', target: 'cpu', transform: 'Math.round(value * 100) / 100' },
+                        { source: 'memory_usage', target: 'memory', transform: 'Math.round(value)' }
+                    ]
+                },
+                validation: {
+                    validateRequest: true,
+                    validateResponse: true,
+                    strictMode: true,
+                    requestSchema: {
+                        type: 'object',
+                        properties: {
+                            metrics: { type: 'array' },
+                            timestamp: { type: 'number' }
+                        },
+                        required: ['metrics', 'timestamp']
+                    }
+                },
+                monitoring: {
+                    logRequests: true,
+                    logResponses: true,
+                    logErrors: true,
+                    collectMetrics: true,
+                    alertOnFailure: true,
+                    alertThreshold: 3
+                }
+            }
+        ];
+        defaultEndpoints.forEach(endpoint => {
+            this.endpoints.set(endpoint.id, endpoint);
+            this.initializeRateLimit(endpoint.id, endpoint.rateLimit);
+        });
+    }
+    initializeRateLimit(endpointId, config) {
+        if (config.enabled) {
+            this.rateLimiters.set(endpointId, { requests: [], lastReset: Date.now() });
+        }
+    }
+    checkRateLimit(endpointId) {
+        const endpoint = this.endpoints.get(endpointId);
+        if (!endpoint?.rateLimit.enabled)
+            return true;
+        const limiter = this.rateLimiters.get(endpointId);
+        if (!limiter)
+            return true;
+        const now = Date.now();
+        const config = endpoint.rateLimit;
+        limiter.requests = limiter.requests.filter(time => now - time < 60000);
+        if (limiter.requests.length >= config.requestsPerMinute)
+            return false;
+        const recentRequests = limiter.requests.filter(time => now - time < 1000);
+        if (recentRequests.length >= config.burstLimit)
+            return false;
+        limiter.requests.push(now);
+        return true;
+    }
+    createEndpoint(endpoint) {
+        const id = `endpoint_${Date.now()}`;
+        const fullEndpoint = { id, ...endpoint };
+        this.endpoints.set(id, fullEndpoint);
+        this.initializeRateLimit(id, fullEndpoint.rateLimit);
+        return id;
+    }
+    getEndpoint(id) {
+        return this.endpoints.get(id);
+    }
+    listEndpoints() {
+        return Array.from(this.endpoints.values());
+    }
+    updateEndpoint(id, updates) {
+        const endpoint = this.endpoints.get(id);
+        if (!endpoint)
+            return false;
+        Object.assign(endpoint, updates);
+        if (updates.rateLimit) {
+            this.initializeRateLimit(id, endpoint.rateLimit);
+        }
+        return true;
+    }
+    deleteEndpoint(id) {
+        this.rateLimiters.delete(id);
+        this.requestQueue.delete(id);
+        return this.endpoints.delete(id);
+    }
+    getConfiguredEndpoints() {
+        return Array.from(this.endpoints.values());
+    }
+    cleanupCache() {
+        const now = Date.now();
+        for (const [key, cached] of Array.from(this.responseCache.entries())) {
+            if (cached.expiry <= now) {
+                this.responseCache.delete(key);
+            }
+        }
+    }
+    resetRateLimiters() {
+        const now = Date.now();
+        this.rateLimiters.forEach(limiter => {
+            if (now - limiter.lastReset > 60000) {
+                limiter.requests = [];
+                limiter.lastReset = now;
+            }
+        });
+    }
+}
+exports.EndpointManager = EndpointManager;

package/dist/dashboard/server/routes/api.usage.routes.d.ts

@@ -0,0 +1,57 @@
+/**
+ * API Usage Routes — Request execution and monitoring.
+ *
+ * Manages API request execution (retry, rate-limit enforcement, auth, transforms)
+ * and the monitoring event bus. Depends on EndpointManager for endpoint config.
+ */
+import type { EndpointManager, APIAuthentication, RetryConfig, DataMapping } from './api.instructions.routes.js';
+export interface APIRequest {
+    id: string;
+    endpointId: string;
+    method: string;
+    url: string;
+    headers: Record<string, string>;
+    body?: unknown;
+    timestamp: number;
+    timeout: number;
+}
+export interface APIResponse {
+    requestId: string;
+    status: number;
+    statusText: string;
+    headers: Record<string, string>;
+    body: unknown;
+    responseTime: number;
+    timestamp: number;
+    error?: string;
+}
+export interface APIMonitoringEvent {
+    type: string;
+    timestamp: number;
+    data: Record<string, unknown>;
+}
+export declare class UsageManager {
+    private readonly endpointMgr;
+    private readonly monitoringCallbacks;
+    constructor(endpointMgr: EndpointManager);
+    executeRequest(endpointId: string, data?: unknown, overrides?: Partial<APIRequest>): Promise<APIResponse>;
+    executeWithRetry(request: APIRequest, retryConfig: RetryConfig): Promise<APIResponse>;
+    performHTTPRequest(request: APIRequest): Promise<APIResponse>;
+    private simulateHTTPRequest;
+    private shouldRetry;
+    private shouldRetryError;
+    private calculateRetryDelay;
+    applyAuthentication(request: APIRequest, auth: APIAuthentication): void;
+    resolveConfigValue(value: string): string;
+    applySafeTransform(data: any, transformSpec: string): any;
+    applySafeFieldTransform(value: any, transform: string): any;
+    applyFieldMappings(data: unknown, mappings: DataMapping['fieldMappings']): unknown;
+    private getNestedValue;
+    private setNestedValue;
+    private validateData;
+    private collectMetrics;
+    logAPIEvent(type: string, data: Record<string, unknown>): void;
+    onAPIEvent(callback: (event: APIMonitoringEvent) => void): void;
+    processRequestQueues(): void;
+    sleep(ms: number): Promise<void>;
+}

package/dist/dashboard/server/routes/api.usage.routes.js

@@ -0,0 +1,374 @@
+"use strict";
+/**
+ * API Usage Routes — Request execution and monitoring.
+ *
+ * Manages API request execution (retry, rate-limit enforcement, auth, transforms)
+ * and the monitoring event bus. Depends on EndpointManager for endpoint config.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UsageManager = void 0;
+// ── UsageManager ─────────────────────────────────────────────────────────────────────
+class UsageManager {
+    endpointMgr;
+    monitoringCallbacks = [];
+    constructor(endpointMgr) {
+        this.endpointMgr = endpointMgr;
+    }
+    async executeRequest(endpointId, data, overrides) {
+        const endpoint = this.endpointMgr.getEndpoint(endpointId);
+        if (!endpoint)
+            throw new Error(`Endpoint not found: ${endpointId}`);
+        if (!this.endpointMgr.checkRateLimit(endpointId)) {
+            throw new Error(`Rate limit exceeded for endpoint: ${endpointId}`);
+        }
+        const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+        const request = {
+            id: requestId,
+            endpointId,
+            method: endpoint.method,
+            url: endpoint.url,
+            headers: { ...endpoint.headers, ...(overrides?.headers || {}) },
+            body: data,
+            timestamp: Date.now(),
+            timeout: overrides?.timeout || endpoint.timeout
+        };
+        this.applyAuthentication(request, endpoint.authentication);
+        if (data && endpoint.dataMapping.requestTransform) {
+            try {
+                request.body = this.applySafeTransform(data, endpoint.dataMapping.requestTransform);
+            }
+            catch (error) {
+                console.error('Request transform error:', error);
+            }
+        }
+        if (endpoint.validation.validateRequest && endpoint.validation.requestSchema) {
+            const valid = this.validateData(request.body, endpoint.validation.requestSchema);
+            if (!valid && endpoint.validation.strictMode) {
+                throw new Error('Request validation failed');
+            }
+        }
+        if (endpoint.monitoring.logRequests) {
+            this.logAPIEvent('request', { endpointId, requestId, url: request.url, method: request.method });
+        }
+        const response = await this.executeWithRetry(request, endpoint.retryConfig);
+        if (endpoint.dataMapping.responseTransform) {
+            try {
+                response.body = this.applySafeTransform(response.body, endpoint.dataMapping.responseTransform);
+            }
+            catch (error) {
+                console.error('Response transform error:', error);
+            }
+        }
+        if (endpoint.dataMapping.fieldMappings.length > 0) {
+            response.body = this.applyFieldMappings(response.body, endpoint.dataMapping.fieldMappings);
+        }
+        if (endpoint.validation.validateResponse && endpoint.validation.responseSchema) {
+            const valid = this.validateData(response.body, endpoint.validation.responseSchema);
+            if (!valid && endpoint.validation.strictMode) {
+                console.warn('Response validation failed for endpoint:', endpointId);
+            }
+        }
+        if (endpoint.monitoring.logResponses) {
+            this.logAPIEvent('response', {
+                endpointId,
+                requestId,
+                status: response.status,
+                responseTime: response.responseTime
+            });
+        }
+        if (response.error && endpoint.monitoring.logErrors) {
+            this.logAPIEvent('error', { endpointId, requestId, error: response.error, status: response.status });
+        }
+        if (endpoint.monitoring.collectMetrics) {
+            this.collectMetrics(endpointId, response);
+        }
+        return response;
+    }
+    async executeWithRetry(request, retryConfig) {
+        let lastError = null;
+        let attempt = 0;
+        while (attempt <= retryConfig.maxAttempts) {
+            try {
+                const response = await this.performHTTPRequest(request);
+                if (this.shouldRetry(response, retryConfig)) {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                }
+                return response;
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                attempt++;
+                if (attempt > retryConfig.maxAttempts)
+                    break;
+                if (!this.shouldRetryError(lastError, retryConfig))
+                    break;
+                const delay = this.calculateRetryDelay(attempt, retryConfig);
+                await this.sleep(delay);
+            }
+        }
+        return {
+            requestId: request.id,
+            status: 0,
+            statusText: 'Request Failed',
+            headers: {},
+            body: null,
+            responseTime: 0,
+            timestamp: Date.now(),
+            error: lastError?.message || 'Unknown error'
+        };
+    }
+    async performHTTPRequest(request) {
+        const startTime = Date.now();
+        try {
+            const response = await this.simulateHTTPRequest(request);
+            return {
+                requestId: request.id,
+                status: response.status,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+                responseTime: Date.now() - startTime,
+                timestamp: Date.now()
+            };
+        }
+        catch (error) {
+            throw new Error(`Network error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    async simulateHTTPRequest(request) {
+        await this.sleep(Math.random() * 100 + 50);
+        const scenarios = [
+            { weight: 0.8, status: 200, statusText: 'OK', body: { success: true, data: 'Response data' } },
+            { weight: 0.1, status: 429, statusText: 'Too Many Requests', body: { error: 'Rate limit exceeded' } },
+            { weight: 0.05, status: 500, statusText: 'Internal Server Error', body: { error: 'Server error' } },
+            { weight: 0.05, status: 503, statusText: 'Service Unavailable', body: { error: 'Service unavailable' } }
+        ];
+        const random = Math.random();
+        let cumulative = 0;
+        for (const scenario of scenarios) {
+            cumulative += scenario.weight;
+            if (random <= cumulative) {
+                return { status: scenario.status, statusText: scenario.statusText, headers: { 'Content-Type': 'application/json' }, body: scenario.body };
+            }
+        }
+        return { status: 200, statusText: 'OK', headers: { 'Content-Type': 'application/json' }, body: { success: true, data: request.body } };
+    }
+    shouldRetry(response, retryConfig) {
+        if (!retryConfig.enabled)
+            return false;
+        if (response.status >= 500 || response.status === 429) {
+            return retryConfig.retryOn.includes('server_error') || retryConfig.retryOn.includes('rate_limit');
+        }
+        return false;
+    }
+    shouldRetryError(error, retryConfig) {
+        if (!retryConfig.enabled)
+            return false;
+        const message = error.message.toLowerCase();
+        if (message.includes('timeout') && retryConfig.retryOn.includes('timeout'))
+            return true;
+        if ((message.includes('network') || message.includes('fetch')) && retryConfig.retryOn.includes('network_error'))
+            return true;
+        return false;
+    }
+    calculateRetryDelay(attempt, retryConfig) {
+        let delay;
+        switch (retryConfig.backoffStrategy) {
+            case 'linear':
+                delay = retryConfig.baseDelay * attempt;
+                break;
+            case 'exponential':
+                delay = retryConfig.baseDelay * Math.pow(2, attempt - 1);
+                break;
+            case 'fixed':
+            default:
+                delay = retryConfig.baseDelay;
+                break;
+        }
+        return Math.min(delay, retryConfig.maxDelay);
+    }
+    applyAuthentication(request, auth) {
+        switch (auth.type) {
+            case 'api_key': {
+                const keyName = auth.config.keyName;
+                const keyValue = this.resolveConfigValue(auth.config.keyValue);
+                request.headers[keyName] = keyValue;
+                break;
+            }
+            case 'bearer_token': {
+                const token = this.resolveConfigValue(auth.config.token);
+                request.headers['Authorization'] = `Bearer ${token}`;
+                break;
+            }
+            case 'basic_auth': {
+                const username = this.resolveConfigValue(auth.config.username);
+                const password = this.resolveConfigValue(auth.config.password);
+                const credentials = Buffer.from(`${username}:${password}`).toString('base64');
+                request.headers['Authorization'] = `Basic ${credentials}`;
+                break;
+            }
+            case 'oauth2':
+                console.warn('OAuth2 authentication not fully implemented');
+                break;
+            case 'custom': {
+                const customHeaders = auth.config.headers;
+                Object.assign(request.headers, customHeaders);
+                break;
+            }
+            case 'none':
+            default:
+                break;
+        }
+    }
+    resolveConfigValue(value) {
+        if (value.startsWith('${') && value.endsWith('}')) {
+            const envVar = value.slice(2, -1);
+            return process.env[envVar] || value;
+        }
+        return value;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    applySafeTransform(data, transformSpec) {
+        if (!data || !transformSpec)
+            return data;
+        const dotPath = transformSpec.trim().match(/^(?:data|response|value)\.(.+)$/);
+        if (dotPath) {
+            const parts = dotPath[1].split('.');
+            let result = data;
+            for (const part of parts) {
+                if (result == null || typeof result !== 'object')
+                    return data;
+                result = result[part];
+            }
+            return result;
+        }
+        return data;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    applySafeFieldTransform(value, transform) {
+        const t = transform.trim();
+        if (t === 'String(value)' || t === 'value.toString()')
+            return String(value);
+        if (t === 'Number(value)' || t === 'parseInt(value)')
+            return Number(value);
+        if (t === 'Boolean(value)')
+            return Boolean(value);
+        if (t === 'JSON.parse(value)') {
+            try {
+                return JSON.parse(value);
+            }
+            catch {
+                return value;
+            }
+        }
+        if (t === 'JSON.stringify(value)')
+            return JSON.stringify(value);
+        if (t === 'value.toLowerCase()')
+            return typeof value === 'string' ? value.toLowerCase() : value;
+        if (t === 'value.toUpperCase()')
+            return typeof value === 'string' ? value.toUpperCase() : value;
+        if (t === 'value.trim()')
+            return typeof value === 'string' ? value.trim() : value;
+        const dotPath = t.match(/^value\.(.+)$/);
+        if (dotPath) {
+            let result = value;
+            for (const part of dotPath[1].split('.')) {
+                if (result == null || typeof result !== 'object')
+                    return value;
+                result = result[part];
+            }
+            return result;
+        }
+        return value;
+    }
+    applyFieldMappings(data, mappings) {
+        if (!data || typeof data !== 'object' || !Array.isArray(mappings))
+            return data;
+        const result = { ...data };
+        mappings.forEach(mapping => {
+            const sourceValue = this.getNestedValue(result, mapping.source);
+            let targetValue = sourceValue;
+            if (mapping.transform) {
+                try {
+                    targetValue = this.applySafeFieldTransform(sourceValue, mapping.transform);
+                }
+                catch (error) {
+                    console.error('Field mapping transform error:', error);
+                }
+            }
+            this.setNestedValue(result, mapping.target, targetValue);
+        });
+        return result;
+    }
+    getNestedValue(obj, path) {
+        return path.split('.').reduce((current, key) => {
+            return current && typeof current === 'object' ? current[key] : undefined;
+        }, obj);
+    }
+    setNestedValue(obj, path, value) {
+        const keys = path.split('.');
+        const lastKey = keys.pop();
+        const target = keys.reduce((current, key) => {
+            if (!current[key] || typeof current[key] !== 'object')
+                current[key] = {};
+            return current[key];
+        }, obj);
+        target[lastKey] = value;
+    }
+    validateData(data, _schema) {
+        try {
+            if (typeof data !== 'object' || data === null)
+                return false;
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    collectMetrics(endpointId, response) {
+        this.logAPIEvent('metrics', {
+            endpointId,
+            responseTime: response.responseTime,
+            status: response.status,
+            success: response.status >= 200 && response.status < 300
+        });
+    }
+    logAPIEvent(type, data) {
+        const event = { type, timestamp: Date.now(), data };
+        this.monitoringCallbacks.forEach(callback => {
+            try {
+                callback(event);
+            }
+            catch (error) {
+                console.error('API monitoring callback error:', error);
+            }
+        });
+    }
+    onAPIEvent(callback) {
+        this.monitoringCallbacks.push(callback);
+    }
+    processRequestQueues() {
+        this.endpointMgr.requestQueue.forEach((requests, endpointId) => {
+            if (requests.length === 0)
+                return;
+            const endpoint = this.endpointMgr.getEndpoint(endpointId);
+            if (!endpoint?.rateLimit.enabled)
+                return;
+            if (this.endpointMgr.checkRateLimit(endpointId)) {
+                const request = requests.shift();
+                if (request) {
+                    this.executeRequest(endpointId, request.body, {
+                        headers: request.headers,
+                        timeout: request.timeout
+                    }).catch(error => {
+                        console.error('Queued request execution error:', error);
+                    });
+                }
+            }
+        });
+    }
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+exports.UsageManager = UsageManager;

package/dist/dashboard/server/routes/embeddings.routes.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Embeddings Routes
+ * Route: GET /embeddings/projection — PCA-project embeddings to 2D for visualization.
+ */
+import { Router } from 'express';
+export declare function createEmbeddingsRoutes(embeddingPathOverride?: string): Router;