@robosystems/client 0.1.17 → 0.1.19

package/sdk-extensions/CopyClient.ts

@@ -0,0 +1,438 @@
+'use client'
+
+/**
+ * Enhanced Copy Client with SSE support
+ * Provides intelligent data copy operations with progress monitoring
+ */
+
+import { copyDataToGraph } from '../sdk/sdk.gen'
+import type {
+  CopyDataToGraphData,
+  CopyResponse,
+  DataFrameCopyRequest,
+  S3CopyRequest,
+  UrlCopyRequest,
+} from '../sdk/types.gen'
+import { EventType, SSEClient } from './SSEClient'
+
+export type CopySourceType = 's3' | 'url' | 'dataframe'
+
+export interface CopyOptions {
+  onProgress?: (message: string, progressPercent?: number) => void
+  onQueueUpdate?: (position: number, estimatedWait: number) => void
+  onWarning?: (warning: string) => void
+  timeout?: number
+  testMode?: boolean
+}
+
+export interface CopyResult {
+  status: 'completed' | 'failed' | 'partial' | 'accepted'
+  rowsImported?: number
+  rowsSkipped?: number
+  bytesProcessed?: number
+  executionTimeMs?: number
+  warnings?: string[]
+  error?: string
+  operationId?: string
+  sseUrl?: string
+  message?: string
+}
+
+export interface CopyStatistics {
+  totalRows: number
+  importedRows: number
+  skippedRows: number
+  bytesProcessed: number
+  duration: number
+  throughput: number // rows per second
+}
+
+export class CopyClient {
+  private sseClient?: SSEClient
+  private config: {
+    baseUrl: string
+    credentials?: 'include' | 'same-origin' | 'omit'
+    headers?: Record<string, string>
+  }
+
+  constructor(config: {
+    baseUrl: string
+    credentials?: 'include' | 'same-origin' | 'omit'
+    headers?: Record<string, string>
+  }) {
+    this.config = config
+  }
+
+  /**
+   * Copy data from S3 to graph database
+   */
+  async copyFromS3(
+    graphId: string,
+    request: S3CopyRequest,
+    options: CopyOptions = {}
+  ): Promise<CopyResult> {
+    return this.executeCopy(graphId, request, 's3', options)
+  }
+
+  /**
+   * Copy data from URL to graph database (when available)
+   */
+  async copyFromUrl(
+    graphId: string,
+    request: UrlCopyRequest,
+    options: CopyOptions = {}
+  ): Promise<CopyResult> {
+    return this.executeCopy(graphId, request, 'url', options)
+  }
+
+  /**
+   * Copy data from DataFrame to graph database (when available)
+   */
+  async copyFromDataFrame(
+    graphId: string,
+    request: DataFrameCopyRequest,
+    options: CopyOptions = {}
+  ): Promise<CopyResult> {
+    return this.executeCopy(graphId, request, 'dataframe', options)
+  }
+
+  /**
+   * Execute copy operation with automatic SSE monitoring for long-running operations
+   */
+  private async executeCopy(
+    graphId: string,
+    request: S3CopyRequest | UrlCopyRequest | DataFrameCopyRequest,
+    _sourceType: CopySourceType,
+    options: CopyOptions = {}
+  ): Promise<CopyResult> {
+    const startTime = Date.now()
+
+    const data: CopyDataToGraphData = {
+      url: '/v1/{graph_id}/copy' as const,
+      path: { graph_id: graphId },
+      body: request,
+    }
+
+    try {
+      // Execute the copy request
+      const response = await copyDataToGraph(data)
+      const responseData = response.data as CopyResponse
+
+      // Check if this is an accepted (async) operation
+      if (responseData.status === 'accepted' && responseData.operation_id) {
+        // This is a long-running operation with SSE monitoring
+        options.onProgress?.(`Copy operation started. Monitoring progress...`)
+
+        // If SSE URL is provided, use it for monitoring
+        if (responseData.sse_url) {
+          return this.monitorCopyOperation(responseData.operation_id, options, startTime)
+        }
+
+        // Otherwise return the accepted response
+        return {
+          status: 'accepted',
+          operationId: responseData.operation_id,
+          sseUrl: responseData.sse_url,
+          message: responseData.message,
+        }
+      }
+
+      // This is a synchronous response - operation completed immediately
+      return this.buildCopyResult(responseData, Date.now() - startTime)
+    } catch (error) {
+      return {
+        status: 'failed',
+        error: error instanceof Error ? error.message : String(error),
+        executionTimeMs: Date.now() - startTime,
+      }
+    }
+  }
+
+  /**
+   * Monitor a copy operation using SSE
+   */
+  private async monitorCopyOperation(
+    operationId: string,
+    options: CopyOptions,
+    startTime: number
+  ): Promise<CopyResult> {
+    return new Promise((resolve, reject) => {
+      const sseClient = new SSEClient(this.config)
+      const timeoutMs = options.timeout || 3600000 // Default 1 hour for copy operations
+
+      const timeoutHandle = setTimeout(() => {
+        sseClient.close()
+        reject(new Error(`Copy operation timeout after ${timeoutMs}ms`))
+      }, timeoutMs)
+
+      sseClient
+        .connect(operationId)
+        .then(() => {
+          let result: CopyResult = { status: 'failed' }
+          const warnings: string[] = []
+
+          // Listen for queue updates
+          sseClient.on(EventType.QUEUE_UPDATE, (data) => {
+            options.onQueueUpdate?.(
+              data.position || data.queue_position,
+              data.estimated_wait_seconds || 0
+            )
+          })
+
+          // Listen for progress updates
+          sseClient.on(EventType.OPERATION_PROGRESS, (data) => {
+            const message = data.message || data.status || 'Processing...'
+            const progressPercent = data.progress_percent || data.progress
+
+            options.onProgress?.(message, progressPercent)
+
+            // Check for warnings in progress updates
+            if (data.warnings) {
+              warnings.push(...data.warnings)
+              data.warnings.forEach((warning: string) => {
+                options.onWarning?.(warning)
+              })
+            }
+          })
+
+          // Listen for completion
+          sseClient.on(EventType.OPERATION_COMPLETED, (data) => {
+            clearTimeout(timeoutHandle)
+
+            const completionData = data.result || data
+            result = {
+              status: completionData.status || 'completed',
+              rowsImported: completionData.rows_imported,
+              rowsSkipped: completionData.rows_skipped,
+              bytesProcessed: completionData.bytes_processed,
+              executionTimeMs: Date.now() - startTime,
+              warnings: warnings.length > 0 ? warnings : completionData.warnings,
+              message: completionData.message,
+            }
+
+            sseClient.close()
+            resolve(result)
+          })
+
+          // Listen for errors
+          sseClient.on(EventType.OPERATION_ERROR, (error) => {
+            clearTimeout(timeoutHandle)
+
+            result = {
+              status: 'failed',
+              error: error.message || error.error || 'Copy operation failed',
+              executionTimeMs: Date.now() - startTime,
+              warnings: warnings.length > 0 ? warnings : undefined,
+            }
+
+            sseClient.close()
+            resolve(result) // Resolve with error result, not reject
+          })
+
+          // Listen for cancellation
+          sseClient.on(EventType.OPERATION_CANCELLED, () => {
+            clearTimeout(timeoutHandle)
+
+            result = {
+              status: 'failed',
+              error: 'Copy operation cancelled',
+              executionTimeMs: Date.now() - startTime,
+              warnings: warnings.length > 0 ? warnings : undefined,
+            }
+
+            sseClient.close()
+            resolve(result)
+          })
+        })
+        .catch((error) => {
+          clearTimeout(timeoutHandle)
+          reject(error)
+        })
+    })
+  }
+
+  /**
+   * Build copy result from response data
+   */
+  private buildCopyResult(responseData: CopyResponse, executionTimeMs: number): CopyResult {
+    return {
+      status: responseData.status,
+      rowsImported: responseData.rows_imported || undefined,
+      rowsSkipped: responseData.rows_skipped || undefined,
+      bytesProcessed: responseData.bytes_processed || undefined,
+      executionTimeMs: responseData.execution_time_ms || executionTimeMs,
+      warnings: responseData.warnings || undefined,
+      message: responseData.message,
+      error: responseData.error_details ? String(responseData.error_details) : undefined,
+    }
+  }
+
+  /**
+   * Calculate copy statistics from result
+   */
+  calculateStatistics(result: CopyResult): CopyStatistics | null {
+    if (result.status === 'failed' || !result.rowsImported) {
+      return null
+    }
+
+    const totalRows = (result.rowsImported || 0) + (result.rowsSkipped || 0)
+    const duration = (result.executionTimeMs || 0) / 1000 // Convert to seconds
+    const throughput = duration > 0 ? (result.rowsImported || 0) / duration : 0
+
+    return {
+      totalRows,
+      importedRows: result.rowsImported || 0,
+      skippedRows: result.rowsSkipped || 0,
+      bytesProcessed: result.bytesProcessed || 0,
+      duration,
+      throughput,
+    }
+  }
+
+  /**
+   * Convenience method for simple S3 copy with default options
+   */
+  async copyS3(
+    graphId: string,
+    tableName: string,
+    s3Uri: string,
+    accessKeyId: string,
+    secretAccessKey: string,
+    options?: {
+      region?: string
+      fileFormat?: 'csv' | 'parquet' | 'json' | 'delta' | 'iceberg'
+      ignoreErrors?: boolean
+    }
+  ): Promise<CopyResult> {
+    const request: S3CopyRequest = {
+      table_name: tableName,
+      source_type: 's3',
+      s3_path: s3Uri,
+      s3_access_key_id: accessKeyId,
+      s3_secret_access_key: secretAccessKey,
+      s3_region: options?.region || 'us-east-1',
+      file_format: options?.fileFormat,
+      ignore_errors: options?.ignoreErrors || false,
+    }
+
+    return this.copyFromS3(graphId, request)
+  }
+
+  /**
+   * Monitor multiple copy operations concurrently
+   */
+  async monitorMultipleCopies(
+    operationIds: string[],
+    options: CopyOptions = {}
+  ): Promise<Map<string, CopyResult>> {
+    const results = await Promise.all(
+      operationIds.map(async (id) => {
+        const result = await this.monitorCopyOperation(id, options, Date.now())
+        return [id, result] as [string, CopyResult]
+      })
+    )
+    return new Map(results)
+  }
+
+  /**
+   * Batch copy multiple tables from S3
+   */
+  async batchCopyFromS3(
+    graphId: string,
+    copies: Array<{
+      request: S3CopyRequest
+      options?: CopyOptions
+    }>
+  ): Promise<CopyResult[]> {
+    return Promise.all(
+      copies.map(({ request, options }) => this.copyFromS3(graphId, request, options || {}))
+    )
+  }
+
+  /**
+   * Copy with retry logic for transient failures
+   */
+  async copyWithRetry(
+    graphId: string,
+    request: S3CopyRequest | UrlCopyRequest | DataFrameCopyRequest,
+    sourceType: CopySourceType,
+    maxRetries: number = 3,
+    options: CopyOptions = {}
+  ): Promise<CopyResult> {
+    let lastError: Error | undefined
+    let attempt = 0
+
+    while (attempt < maxRetries) {
+      attempt++
+
+      try {
+        const result = await this.executeCopy(graphId, request, sourceType, options)
+
+        // If successful or partially successful, return
+        if (result.status === 'completed' || result.status === 'partial') {
+          return result
+        }
+
+        // If failed, check if it's retryable
+        if (result.status === 'failed') {
+          const isRetryable = this.isRetryableError(result.error)
+          if (!isRetryable || attempt === maxRetries) {
+            return result
+          }
+
+          // Wait before retry with exponential backoff
+          const waitTime = Math.min(1000 * Math.pow(2, attempt - 1), 30000)
+          options.onProgress?.(
+            `Retrying copy operation (attempt ${attempt}/${maxRetries}) in ${waitTime}ms...`
+          )
+          await new Promise((resolve) => setTimeout(resolve, waitTime))
+        }
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error))
+
+        if (attempt === maxRetries) {
+          throw lastError
+        }
+
+        // Wait before retry
+        const waitTime = Math.min(1000 * Math.pow(2, attempt - 1), 30000)
+        options.onProgress?.(
+          `Retrying after error (attempt ${attempt}/${maxRetries}) in ${waitTime}ms...`
+        )
+        await new Promise((resolve) => setTimeout(resolve, waitTime))
+      }
+    }
+
+    throw lastError || new Error('Copy operation failed after all retries')
+  }
+
+  /**
+   * Check if an error is retryable
+   */
+  private isRetryableError(error?: string): boolean {
+    if (!error) return false
+
+    const retryablePatterns = [
+      'timeout',
+      'network',
+      'connection',
+      'temporary',
+      'unavailable',
+      'rate limit',
+      'throttl',
+    ]
+
+    const lowerError = error.toLowerCase()
+    return retryablePatterns.some((pattern) => lowerError.includes(pattern))
+  }
+
+  /**
+   * Cancel any active SSE connections
+   */
+  close(): void {
+    if (this.sseClient) {
+      this.sseClient.close()
+      this.sseClient = undefined
+    }
+  }
+}

package/sdk-extensions/README.md

@@ -11,6 +11,7 @@ The RoboSystems Typescript Client Extensions provide production-ready enhancemen
 
 - **Server-Sent Events (SSE)** with automatic reconnection and event replay
 - **Smart Query Execution** with automatic queueing and progress monitoring
+- **Data Copy Operations** with S3 import and real-time progress tracking
 - **Operation Monitoring** for long-running tasks with real-time updates
 - **Connection Management** with rate limiting and circuit breaker patterns
 - **React Hooks** for seamless UI integration
@@ -91,6 +92,53 @@ const result = await queryClient.executeWithProgress(
 console.log(`Query completed with ${result.rowCount} results`)
 ```
 
+### Data Copy Operations with Progress Monitoring
+
+```typescript
+import { CopyClient } from '@robosystems/client/extensions'
+
+const copyClient = new CopyClient({
+  baseUrl: 'https://api.robosystems.ai',
+  headers: { 'X-API-Key': 'your-api-key' },
+})
+
+// Copy data from S3 with real-time progress
+const result = await copyClient.copyFromS3(
+  'your-graph-id',
+  {
+    table_name: 'companies',
+    source_type: 's3',
+    s3_path: 's3://my-bucket/data/companies.csv',
+    s3_access_key_id: 'AWS_ACCESS_KEY',
+    s3_secret_access_key: 'AWS_SECRET_KEY',
+    s3_region: 'us-east-1',
+    file_format: 'csv',
+    ignore_errors: false, // Stop on first error
+  },
+  {
+    onProgress: (message, percent) => {
+      console.log(`Copy progress: ${message} (${percent}%)`)
+      updateProgressBar(percent)
+    },
+    onQueueUpdate: (position, estimatedWait) => {
+      console.log(`Queue position: ${position}, ETA: ${estimatedWait}s`)
+    },
+    onWarning: (warning) => {
+      console.warn(`Copy warning: ${warning}`)
+    },
+  }
+)
+
+// Check results
+if (result.status === 'completed') {
+  console.log(`Successfully imported ${result.rowsImported} rows`)
+  const stats = copyClient.calculateStatistics(result)
+  console.log(`Throughput: ${stats.throughput.toFixed(2)} rows/second`)
+} else if (result.status === 'partial') {
+  console.log(`Imported ${result.rowsImported} rows, skipped ${result.rowsSkipped}`)
+}
+```
+
 ## 📊 SSE Event Types
 
 The SDK supports all RoboSystems SSE event types:
@@ -181,6 +229,113 @@ try {
 }
 ```
 
+## 📤 Data Copy Operations
+
+### CopyClient for S3 Data Import
+
+```typescript
+import { CopyClient, CopySourceType } from '@robosystems/client/extensions'
+
+const copyClient = new CopyClient({
+  baseUrl: 'https://api.robosystems.ai',
+  headers: { 'X-API-Key': 'your-api-key' },
+})
+
+// Batch copy multiple tables
+const results = await copyClient.batchCopyFromS3('graph-id', [
+  {
+    request: {
+      table_name: 'companies',
+      source_type: 's3',
+      s3_path: 's3://bucket/companies.csv',
+      s3_access_key_id: 'KEY',
+      s3_secret_access_key: 'SECRET',
+      file_format: 'csv',
+    },
+  },
+  {
+    request: {
+      table_name: 'transactions',
+      source_type: 's3',
+      s3_path: 's3://bucket/transactions.parquet',
+      s3_access_key_id: 'KEY',
+      s3_secret_access_key: 'SECRET',
+      file_format: 'parquet',
+      ignore_errors: true, // Continue on errors
+    },
+  },
+])
+
+results.forEach((result, index) => {
+  console.log(`Table ${index}: ${result.status}`)
+  if (result.rowsImported) {
+    console.log(`  Imported: ${result.rowsImported} rows`)
+  }
+})
+```
+
+### Copy with Retry Logic
+
+```typescript
+// Retry failed copy operations with exponential backoff
+const result = await copyClient.copyWithRetry(
+  'graph-id',
+  {
+    table_name: 'large_dataset',
+    source_type: 's3',
+    s3_path: 's3://bucket/large-dataset.csv',
+    s3_access_key_id: 'KEY',
+    s3_secret_access_key: 'SECRET',
+    max_file_size_gb: 50,
+    extended_timeout: true,
+  },
+  CopySourceType.S3,
+  3, // Max retries
+  {
+    onProgress: (message, percent) => {
+      console.log(`Progress: ${message} (${percent}%)`)
+    },
+  }
+)
+```
+
+### Monitor Multiple Copy Operations
+
+```typescript
+// Start multiple copy operations
+const operations = await Promise.all([
+  copyClient.copyFromS3('graph1', request1),
+  copyClient.copyFromS3('graph2', request2),
+])
+
+// Extract operation IDs for monitoring
+const operationIds = operations
+  .filter((op) => op.status === 'accepted')
+  .map((op) => op.operationId!)
+
+// Monitor all operations concurrently
+const results = await copyClient.monitorMultipleCopies(operationIds, {
+  onProgress: (message) => console.log(`Progress: ${message}`),
+})
+```
+
+### Copy Statistics
+
+```typescript
+const result = await copyClient.copyFromS3('graph-id', request)
+
+// Calculate performance statistics
+const stats = copyClient.calculateStatistics(result)
+if (stats) {
+  console.log(`Total Rows: ${stats.totalRows}`)
+  console.log(`Imported: ${stats.importedRows}`)
+  console.log(`Skipped: ${stats.skippedRows}`)
+  console.log(`Throughput: ${stats.throughput.toFixed(2)} rows/sec`)
+  console.log(`Duration: ${stats.duration.toFixed(2)} seconds`)
+  console.log(`Bytes Processed: ${(stats.bytesProcessed / 1024 / 1024).toFixed(2)} MB`)
+}
+```
+
 ## 🎯 Operation Monitoring
 
 ### OperationClient for Long-Running Tasks
@@ -366,6 +521,68 @@ function QueryRunner() {
 }
 ```
 
+### useCopy Hook for Data Import
+
+```typescript
+import { useCopy } from '@robosystems/client/extensions/hooks'
+
+function DataImporter({ graphId }: { graphId: string }) {
+  const {
+    copyFromS3,
+    loading,
+    progress,
+    error,
+    result,
+    queuePosition
+  } = useCopy(graphId)
+
+  const handleImport = async () => {
+    const importResult = await copyFromS3({
+      table_name: 'products',
+      source_type: 's3',
+      s3_path: 's3://data-bucket/products.csv',
+      s3_access_key_id: process.env.NEXT_PUBLIC_AWS_KEY!,
+      s3_secret_access_key: process.env.NEXT_PUBLIC_AWS_SECRET!,
+      file_format: 'csv',
+    })
+
+    if (importResult?.status === 'completed') {
+      alert(`Successfully imported ${importResult.rowsImported} products`)
+    }
+  }
+
+  return (
+    <div>
+      <button onClick={handleImport} disabled={loading}>
+        {loading ? 'Importing...' : 'Import Products'}
+      </button>
+
+      {progress && (
+        <div>
+          <p>{progress.message}</p>
+          {progress.percent && (
+            <progress value={progress.percent} max={100} />
+          )}
+        </div>
+      )}
+
+      {queuePosition && (
+        <p>Queue position: {queuePosition}</p>
+      )}
+
+      {error && <p style={{ color: 'red' }}>Error: {error.message}</p>}
+
+      {result && result.status === 'completed' && (
+        <div style={{ color: 'green' }}>
+          <p>✅ Successfully imported {result.rowsImported} rows</p>
+          <p>Execution time: {(result.executionTimeMs / 1000).toFixed(2)}s</p>
+        </div>
+      )}
+    </div>
+  )
+}
+```
+
 ## 🛡️ Error Handling & Resilience
 
 ### Circuit Breaker Pattern
@@ -572,6 +789,7 @@ describe('SSE Integration', () => {
 
 - **`SSEClient`** - Server-Sent Events client with auto-reconnection
 - **`QueryClient`** - Enhanced query execution with SSE support
+- **`CopyClient`** - Data copy operations with S3 import and progress monitoring
 - **`OperationClient`** - Long-running operation monitoring
 - **`StreamProcessor`** - Efficient stream processing for large datasets
 
@@ -586,6 +804,7 @@ describe('SSE Integration', () => {
 
 - **`useSSE`** - Hook for SSE connection management
 - **`useQueryWithSSE`** - Hook for queries with progress
+- **`useCopy`** - Hook for data copy operations with progress
 - **`useOperation`** - Hook for operation monitoring
 
 ### Utilities