@robosystems/client 0.1.17 → 0.1.18

package/extensions/CopyClient.d.ts

@@ -0,0 +1,97 @@
+import type { DataFrameCopyRequest, S3CopyRequest, UrlCopyRequest } from '../types.gen';
+export type CopySourceType = 's3' | 'url' | 'dataframe';
+export interface CopyOptions {
+    onProgress?: (message: string, progressPercent?: number) => void;
+    onQueueUpdate?: (position: number, estimatedWait: number) => void;
+    onWarning?: (warning: string) => void;
+    timeout?: number;
+    testMode?: boolean;
+}
+export interface CopyResult {
+    status: 'completed' | 'failed' | 'partial' | 'accepted';
+    rowsImported?: number;
+    rowsSkipped?: number;
+    bytesProcessed?: number;
+    executionTimeMs?: number;
+    warnings?: string[];
+    error?: string;
+    operationId?: string;
+    sseUrl?: string;
+    message?: string;
+}
+export interface CopyStatistics {
+    totalRows: number;
+    importedRows: number;
+    skippedRows: number;
+    bytesProcessed: number;
+    duration: number;
+    throughput: number;
+}
+export declare class CopyClient {
+    private sseClient?;
+    private config;
+    constructor(config: {
+        baseUrl: string;
+        credentials?: 'include' | 'same-origin' | 'omit';
+        headers?: Record<string, string>;
+    });
+    /**
+     * Copy data from S3 to graph database
+     */
+    copyFromS3(graphId: string, request: S3CopyRequest, options?: CopyOptions): Promise<CopyResult>;
+    /**
+     * Copy data from URL to graph database (when available)
+     */
+    copyFromUrl(graphId: string, request: UrlCopyRequest, options?: CopyOptions): Promise<CopyResult>;
+    /**
+     * Copy data from DataFrame to graph database (when available)
+     */
+    copyFromDataFrame(graphId: string, request: DataFrameCopyRequest, options?: CopyOptions): Promise<CopyResult>;
+    /**
+     * Execute copy operation with automatic SSE monitoring for long-running operations
+     */
+    private executeCopy;
+    /**
+     * Monitor a copy operation using SSE
+     */
+    private monitorCopyOperation;
+    /**
+     * Build copy result from response data
+     */
+    private buildCopyResult;
+    /**
+     * Calculate copy statistics from result
+     */
+    calculateStatistics(result: CopyResult): CopyStatistics | null;
+    /**
+     * Convenience method for simple S3 copy with default options
+     */
+    copyS3(graphId: string, tableName: string, s3Uri: string, accessKeyId: string, secretAccessKey: string, options?: {
+        region?: string;
+        fileFormat?: 'csv' | 'parquet' | 'json' | 'delta' | 'iceberg';
+        ignoreErrors?: boolean;
+    }): Promise<CopyResult>;
+    /**
+     * Monitor multiple copy operations concurrently
+     */
+    monitorMultipleCopies(operationIds: string[], options?: CopyOptions): Promise<Map<string, CopyResult>>;
+    /**
+     * Batch copy multiple tables from S3
+     */
+    batchCopyFromS3(graphId: string, copies: Array<{
+        request: S3CopyRequest;
+        options?: CopyOptions;
+    }>): Promise<CopyResult[]>;
+    /**
+     * Copy with retry logic for transient failures
+     */
+    copyWithRetry(graphId: string, request: S3CopyRequest | UrlCopyRequest | DataFrameCopyRequest, sourceType: CopySourceType, maxRetries?: number, options?: CopyOptions): Promise<CopyResult>;
+    /**
+     * Check if an error is retryable
+     */
+    private isRetryableError;
+    /**
+     * Cancel any active SSE connections
+     */
+    close(): void;
+}

package/extensions/CopyClient.js

@@ -0,0 +1,287 @@
+'use client';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CopyClient = void 0;
+/**
+ * Enhanced Copy Client with SSE support
+ * Provides intelligent data copy operations with progress monitoring
+ */
+const sdk_gen_1 = require("../sdk.gen");
+const SSEClient_1 = require("./SSEClient");
+class CopyClient {
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Copy data from S3 to graph database
+     */
+    async copyFromS3(graphId, request, options = {}) {
+        return this.executeCopy(graphId, request, 's3', options);
+    }
+    /**
+     * Copy data from URL to graph database (when available)
+     */
+    async copyFromUrl(graphId, request, options = {}) {
+        return this.executeCopy(graphId, request, 'url', options);
+    }
+    /**
+     * Copy data from DataFrame to graph database (when available)
+     */
+    async copyFromDataFrame(graphId, request, options = {}) {
+        return this.executeCopy(graphId, request, 'dataframe', options);
+    }
+    /**
+     * Execute copy operation with automatic SSE monitoring for long-running operations
+     */
+    async executeCopy(graphId, request, _sourceType, options = {}) {
+        const startTime = Date.now();
+        const data = {
+            url: '/v1/{graph_id}/copy',
+            path: { graph_id: graphId },
+            body: request,
+        };
+        try {
+            // Execute the copy request
+            const response = await (0, sdk_gen_1.copyDataToGraph)(data);
+            const responseData = response.data;
+            // Check if this is an accepted (async) operation
+            if (responseData.status === 'accepted' && responseData.operation_id) {
+                // This is a long-running operation with SSE monitoring
+                options.onProgress?.(`Copy operation started. Monitoring progress...`);
+                // If SSE URL is provided, use it for monitoring
+                if (responseData.sse_url) {
+                    return this.monitorCopyOperation(responseData.operation_id, options, startTime);
+                }
+                // Otherwise return the accepted response
+                return {
+                    status: 'accepted',
+                    operationId: responseData.operation_id,
+                    sseUrl: responseData.sse_url,
+                    message: responseData.message,
+                };
+            }
+            // This is a synchronous response - operation completed immediately
+            return this.buildCopyResult(responseData, Date.now() - startTime);
+        }
+        catch (error) {
+            return {
+                status: 'failed',
+                error: error instanceof Error ? error.message : String(error),
+                executionTimeMs: Date.now() - startTime,
+            };
+        }
+    }
+    /**
+     * Monitor a copy operation using SSE
+     */
+    async monitorCopyOperation(operationId, options, startTime) {
+        return new Promise((resolve, reject) => {
+            const sseClient = new SSEClient_1.SSEClient(this.config);
+            const timeoutMs = options.timeout || 3600000; // Default 1 hour for copy operations
+            const timeoutHandle = setTimeout(() => {
+                sseClient.close();
+                reject(new Error(`Copy operation timeout after ${timeoutMs}ms`));
+            }, timeoutMs);
+            sseClient
+                .connect(operationId)
+                .then(() => {
+                let result = { status: 'failed' };
+                const warnings = [];
+                // Listen for queue updates
+                sseClient.on(SSEClient_1.EventType.QUEUE_UPDATE, (data) => {
+                    options.onQueueUpdate?.(data.position || data.queue_position, data.estimated_wait_seconds || 0);
+                });
+                // Listen for progress updates
+                sseClient.on(SSEClient_1.EventType.OPERATION_PROGRESS, (data) => {
+                    const message = data.message || data.status || 'Processing...';
+                    const progressPercent = data.progress_percent || data.progress;
+                    options.onProgress?.(message, progressPercent);
+                    // Check for warnings in progress updates
+                    if (data.warnings) {
+                        warnings.push(...data.warnings);
+                        data.warnings.forEach((warning) => {
+                            options.onWarning?.(warning);
+                        });
+                    }
+                });
+                // Listen for completion
+                sseClient.on(SSEClient_1.EventType.OPERATION_COMPLETED, (data) => {
+                    clearTimeout(timeoutHandle);
+                    const completionData = data.result || data;
+                    result = {
+                        status: completionData.status || 'completed',
+                        rowsImported: completionData.rows_imported,
+                        rowsSkipped: completionData.rows_skipped,
+                        bytesProcessed: completionData.bytes_processed,
+                        executionTimeMs: Date.now() - startTime,
+                        warnings: warnings.length > 0 ? warnings : completionData.warnings,
+                        message: completionData.message,
+                    };
+                    sseClient.close();
+                    resolve(result);
+                });
+                // Listen for errors
+                sseClient.on(SSEClient_1.EventType.OPERATION_ERROR, (error) => {
+                    clearTimeout(timeoutHandle);
+                    result = {
+                        status: 'failed',
+                        error: error.message || error.error || 'Copy operation failed',
+                        executionTimeMs: Date.now() - startTime,
+                        warnings: warnings.length > 0 ? warnings : undefined,
+                    };
+                    sseClient.close();
+                    resolve(result); // Resolve with error result, not reject
+                });
+                // Listen for cancellation
+                sseClient.on(SSEClient_1.EventType.OPERATION_CANCELLED, () => {
+                    clearTimeout(timeoutHandle);
+                    result = {
+                        status: 'failed',
+                        error: 'Copy operation cancelled',
+                        executionTimeMs: Date.now() - startTime,
+                        warnings: warnings.length > 0 ? warnings : undefined,
+                    };
+                    sseClient.close();
+                    resolve(result);
+                });
+            })
+                .catch((error) => {
+                clearTimeout(timeoutHandle);
+                reject(error);
+            });
+        });
+    }
+    /**
+     * Build copy result from response data
+     */
+    buildCopyResult(responseData, executionTimeMs) {
+        return {
+            status: responseData.status,
+            rowsImported: responseData.rows_imported || undefined,
+            rowsSkipped: responseData.rows_skipped || undefined,
+            bytesProcessed: responseData.bytes_processed || undefined,
+            executionTimeMs: responseData.execution_time_ms || executionTimeMs,
+            warnings: responseData.warnings || undefined,
+            message: responseData.message,
+            error: responseData.error_details ? String(responseData.error_details) : undefined,
+        };
+    }
+    /**
+     * Calculate copy statistics from result
+     */
+    calculateStatistics(result) {
+        if (result.status === 'failed' || !result.rowsImported) {
+            return null;
+        }
+        const totalRows = (result.rowsImported || 0) + (result.rowsSkipped || 0);
+        const duration = (result.executionTimeMs || 0) / 1000; // Convert to seconds
+        const throughput = duration > 0 ? (result.rowsImported || 0) / duration : 0;
+        return {
+            totalRows,
+            importedRows: result.rowsImported || 0,
+            skippedRows: result.rowsSkipped || 0,
+            bytesProcessed: result.bytesProcessed || 0,
+            duration,
+            throughput,
+        };
+    }
+    /**
+     * Convenience method for simple S3 copy with default options
+     */
+    async copyS3(graphId, tableName, s3Uri, accessKeyId, secretAccessKey, options) {
+        const request = {
+            table_name: tableName,
+            source_type: 's3',
+            s3_path: s3Uri,
+            s3_access_key_id: accessKeyId,
+            s3_secret_access_key: secretAccessKey,
+            s3_region: options?.region || 'us-east-1',
+            file_format: options?.fileFormat,
+            ignore_errors: options?.ignoreErrors || false,
+        };
+        return this.copyFromS3(graphId, request);
+    }
+    /**
+     * Monitor multiple copy operations concurrently
+     */
+    async monitorMultipleCopies(operationIds, options = {}) {
+        const results = await Promise.all(operationIds.map(async (id) => {
+            const result = await this.monitorCopyOperation(id, options, Date.now());
+            return [id, result];
+        }));
+        return new Map(results);
+    }
+    /**
+     * Batch copy multiple tables from S3
+     */
+    async batchCopyFromS3(graphId, copies) {
+        return Promise.all(copies.map(({ request, options }) => this.copyFromS3(graphId, request, options || {})));
+    }
+    /**
+     * Copy with retry logic for transient failures
+     */
+    async copyWithRetry(graphId, request, sourceType, maxRetries = 3, options = {}) {
+        let lastError;
+        let attempt = 0;
+        while (attempt < maxRetries) {
+            attempt++;
+            try {
+                const result = await this.executeCopy(graphId, request, sourceType, options);
+                // If successful or partially successful, return
+                if (result.status === 'completed' || result.status === 'partial') {
+                    return result;
+                }
+                // If failed, check if it's retryable
+                if (result.status === 'failed') {
+                    const isRetryable = this.isRetryableError(result.error);
+                    if (!isRetryable || attempt === maxRetries) {
+                        return result;
+                    }
+                    // Wait before retry with exponential backoff
+                    const waitTime = Math.min(1000 * Math.pow(2, attempt - 1), 30000);
+                    options.onProgress?.(`Retrying copy operation (attempt ${attempt}/${maxRetries}) in ${waitTime}ms...`);
+                    await new Promise((resolve) => setTimeout(resolve, waitTime));
+                }
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (attempt === maxRetries) {
+                    throw lastError;
+                }
+                // Wait before retry
+                const waitTime = Math.min(1000 * Math.pow(2, attempt - 1), 30000);
+                options.onProgress?.(`Retrying after error (attempt ${attempt}/${maxRetries}) in ${waitTime}ms...`);
+                await new Promise((resolve) => setTimeout(resolve, waitTime));
+            }
+        }
+        throw lastError || new Error('Copy operation failed after all retries');
+    }
+    /**
+     * Check if an error is retryable
+     */
+    isRetryableError(error) {
+        if (!error)
+            return false;
+        const retryablePatterns = [
+            'timeout',
+            'network',
+            'connection',
+            'temporary',
+            'unavailable',
+            'rate limit',
+            'throttl',
+        ];
+        const lowerError = error.toLowerCase();
+        return retryablePatterns.some((pattern) => lowerError.includes(pattern));
+    }
+    /**
+     * Cancel any active SSE connections
+     */
+    close() {
+        if (this.sseClient) {
+            this.sseClient.close();
+            this.sseClient = undefined;
+        }
+    }
+}
+exports.CopyClient = CopyClient;