@robosystems/client 0.2.23 → 0.2.25

package/extensions/GraphClient.ts

@@ -2,13 +2,25 @@
 
 /**
  * Graph Management Client
- * Provides high-level graph management operations with automatic operation monitoring
+ * Provides high-level graph management operations with automatic operation monitoring.
+ * Supports SSE (Server-Sent Events) for real-time updates with polling fallback.
  */
 
 import { createGraph, getGraphs, getOperationStatus } from '../sdk.gen'
 import type { GraphMetadata, InitialEntityData } from '../types.gen'
 import { OperationClient } from './OperationClient'
 
+/**
+ * Error thrown when a graph operation fails (as reported by the server)
+ * This is distinct from connection/SSE errors
+ */
+export class GraphOperationError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'GraphOperationError'
+  }
+}
+
 // API Response Types
 interface GraphCreateResponse {
   graph_id?: string
@@ -70,6 +82,7 @@ export interface CreateGraphOptions {
   timeout?: number
   pollInterval?: number
   onProgress?: (message: string) => void
+  useSSE?: boolean
 }
 
 export class GraphClient {
@@ -94,6 +107,9 @@ export class GraphClient {
   /**
    * Create a graph and wait for completion
    *
+   * Uses SSE (Server-Sent Events) for real-time progress updates with
+   * automatic fallback to polling if SSE connection fails.
+   *
    * @param metadata - Graph metadata (name, description, etc.)
    * @param initialEntity - Optional initial entity to create
    * @param options - Additional options including:
@@ -102,8 +118,9 @@ export class GraphClient {
    *                   create graph without populating entity data (useful for file-based ingestion).
    *                   Defaults to true.
    *   - timeout: Maximum time to wait in milliseconds (default: 60000)
-   *   - pollInterval: Time between status checks in milliseconds (default: 2000)
+   *   - pollInterval: Time between status checks in milliseconds (default: 2000, for polling fallback)
    *   - onProgress: Callback for progress updates
+   *   - useSSE: Whether to try SSE first (default: true). Falls back to polling on failure.
    * @returns The graph ID when creation completes
    */
   async createGraphAndWait(
@@ -111,7 +128,13 @@ export class GraphClient {
     initialEntity?: InitialEntityInput,
     options: CreateGraphOptions = {}
   ): Promise<string> {
-    const { createEntity = true, timeout = 60000, pollInterval = 2000, onProgress } = options
+    const {
+      createEntity = true,
+      timeout = 60000,
+      pollInterval = 2000,
+      onProgress,
+      useSSE = true,
+    } = options
 
     if (!this.config.token) {
       throw new Error('No API key provided. Set token in config.')
@@ -160,51 +183,118 @@ export class GraphClient {
     }
 
     // Otherwise, we have an operation_id to monitor
-    if (responseData?.operation_id) {
-      const operationId = responseData.operation_id
-
-      if (onProgress) {
-        onProgress(`Graph creation queued (operation: ${operationId})`)
-      }
-
-      // Poll operation status
-      const maxAttempts = Math.floor(timeout / pollInterval)
-      for (let attempt = 0; attempt < maxAttempts; attempt++) {
-        await new Promise((resolve) => setTimeout(resolve, pollInterval))
+    if (!responseData?.operation_id) {
+      throw new Error('No graph_id or operation_id in response')
+    }
 
-        const statusResponse = await getOperationStatus({
-          path: { operation_id: operationId },
-        })
+    const operationId = responseData.operation_id
 
-        const statusData = statusResponse.data as OperationStatusResponse
-        const status = statusData?.status
+    if (onProgress) {
+      onProgress(`Graph creation queued (operation: ${operationId})`)
+    }
 
+    // Try SSE first, fall back to polling
+    if (useSSE) {
+      try {
+        return await this.waitWithSSE(operationId, timeout, onProgress)
+      } catch (error) {
+        // Only fall back to polling for SSE connection failures
+        // If it's a GraphOperationError, the operation actually failed - don't retry with polling
+        if (error instanceof GraphOperationError) {
+          throw error
+        }
+        // SSE connection failed, fall back to polling
         if (onProgress) {
-          onProgress(`Status: ${status} (attempt ${attempt + 1}/${maxAttempts})`)
+          onProgress('SSE unavailable, using polling...')
         }
+      }
+    }
 
-        if (status === 'completed') {
-          const result = statusData?.result
-          const graphId = result?.graph_id
+    // Fallback to polling
+    return await this.waitWithPolling(operationId, timeout, pollInterval, onProgress)
+  }
 
-          if (graphId) {
-            if (onProgress) {
-              onProgress(`Graph created: ${graphId}`)
-            }
-            return graphId
+  /**
+   * Wait for operation completion using SSE stream
+   */
+  private async waitWithSSE(
+    operationId: string,
+    timeout: number,
+    onProgress?: (message: string) => void
+  ): Promise<string> {
+    const result = await this.operationClient.monitorOperation<{ graph_id?: string }>(operationId, {
+      timeout,
+      onProgress: (progress) => {
+        if (onProgress) {
+          if (progress.progressPercent !== undefined) {
+            onProgress(`${progress.message} (${Math.round(progress.progressPercent)}%)`)
           } else {
-            throw new Error('Operation completed but no graph_id in result')
+            onProgress(progress.message)
           }
-        } else if (status === 'failed') {
-          const error = statusData?.error || statusData?.message || 'Unknown error'
-          throw new Error(`Graph creation failed: ${error}`)
         }
+      },
+    })
+
+    if (!result.success) {
+      throw new GraphOperationError(result.error || 'Graph creation failed')
+    }
+
+    const graphId = result.result?.graph_id
+    if (!graphId) {
+      throw new GraphOperationError('Operation completed but no graph_id in result')
+    }
+
+    if (onProgress) {
+      onProgress(`Graph created: ${graphId}`)
+    }
+
+    return graphId
+  }
+
+  /**
+   * Wait for operation completion using polling
+   */
+  private async waitWithPolling(
+    operationId: string,
+    timeout: number,
+    pollInterval: number,
+    onProgress?: (message: string) => void
+  ): Promise<string> {
+    const maxAttempts = Math.floor(timeout / pollInterval)
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      await new Promise((resolve) => setTimeout(resolve, pollInterval))
+
+      const statusResponse = await getOperationStatus({
+        path: { operation_id: operationId },
+      })
+
+      const statusData = statusResponse.data as OperationStatusResponse
+      const status = statusData?.status
+
+      if (onProgress) {
+        onProgress(`Status: ${status} (attempt ${attempt + 1}/${maxAttempts})`)
       }
 
-      throw new Error(`Graph creation timed out after ${timeout}ms`)
+      if (status === 'completed') {
+        const result = statusData?.result
+        const graphId = result?.graph_id
+
+        if (graphId) {
+          if (onProgress) {
+            onProgress(`Graph created: ${graphId}`)
+          }
+          return graphId
+        } else {
+          throw new GraphOperationError('Operation completed but no graph_id in result')
+        }
+      } else if (status === 'failed') {
+        const error = statusData?.error || statusData?.message || 'Unknown error'
+        throw new GraphOperationError(`Graph creation failed: ${error}`)
+      }
     }
 
-    throw new Error('No graph_id or operation_id in response')
+    throw new GraphOperationError(`Graph creation timed out after ${timeout}ms`)
   }
 
   /**

package/extensions/MaterializationClient.d.ts

@@ -2,6 +2,7 @@ export interface MaterializationOptions {
     ignoreErrors?: boolean;
     rebuild?: boolean;
     force?: boolean;
+    timeout?: number;
     onProgress?: (message: string) => void;
 }
 export interface MaterializationResult {
@@ -27,6 +28,7 @@ export interface MaterializationStatus {
 }
 export declare class MaterializationClient {
     private config;
+    private operationClient;
     constructor(config: {
         baseUrl: string;
         credentials?: 'include' | 'same-origin' | 'omit';
@@ -36,11 +38,15 @@ export declare class MaterializationClient {
     /**
      * Materialize graph from DuckDB staging tables
      *
-     * Rebuilds the complete graph database from the current state of DuckDB
-     * staging tables. Automatically discovers all tables, materializes them in
-     * the correct order (nodes before relationships), and clears the staleness flag.
+     * Submits a materialization job to Dagster and monitors progress via SSE.
+     * The operation runs asynchronously on the server but this method waits
+     * for completion and returns the final result.
      */
     materialize(graphId: string, options?: MaterializationOptions): Promise<MaterializationResult>;
+    /**
+     * Convert SSE operation result to MaterializationResult
+     */
+    private convertOperationResult;
     /**
      * Get current materialization status for the graph
      *
@@ -48,4 +54,8 @@ export declare class MaterializationClient {
      * when it was last materialized, and how long since last materialization.
      */
     status(graphId: string): Promise<MaterializationStatus | null>;
+    /**
+     * Close any active SSE connections
+     */
+    close(): void;
 }

package/extensions/MaterializationClient.js

@@ -6,23 +6,25 @@ exports.MaterializationClient = void 0;
  * Materialization Client for RoboSystems API
  *
  * Manages graph materialization from DuckDB staging tables.
- * Treats the graph database as a materialized view of the mutable DuckDB data lake.
+ * Submits materialization jobs to Dagster and monitors progress via SSE.
  */
 const sdk_gen_1 = require("../sdk.gen");
+const OperationClient_1 = require("./OperationClient");
 class MaterializationClient {
     constructor(config) {
         this.config = config;
+        this.operationClient = new OperationClient_1.OperationClient(config);
     }
     /**
      * Materialize graph from DuckDB staging tables
      *
-     * Rebuilds the complete graph database from the current state of DuckDB
-     * staging tables. Automatically discovers all tables, materializes them in
-     * the correct order (nodes before relationships), and clears the staleness flag.
+     * Submits a materialization job to Dagster and monitors progress via SSE.
+     * The operation runs asynchronously on the server but this method waits
+     * for completion and returns the final result.
      */
     async materialize(graphId, options = {}) {
         try {
-            options.onProgress?.('Starting graph materialization...');
+            options.onProgress?.('Submitting materialization job...');
             const request = {
                 ignore_errors: options.ignoreErrors ?? true,
                 rebuild: options.rebuild ?? false,
@@ -44,19 +46,24 @@ class MaterializationClient {
                     error: `Failed to materialize graph: ${response.error}`,
                 };
             }
-            const result = response.data;
-            options.onProgress?.(`✅ Materialization complete: ${result.tables_materialized?.length || 0} tables, ` +
-                `${result.total_rows?.toLocaleString() || 0} rows in ${result.execution_time_ms?.toFixed(2) || 0}ms`);
-            return {
-                status: result.status,
-                wasStale: result.was_stale,
-                staleReason: result.stale_reason,
-                tablesMaterialized: result.tables_materialized || [],
-                totalRows: result.total_rows || 0,
-                executionTimeMs: result.execution_time_ms || 0,
-                message: result.message || 'Materialization complete',
-                success: true,
-            };
+            const queuedResponse = response.data;
+            const operationId = queuedResponse.operation_id;
+            options.onProgress?.(`Materialization queued (operation: ${operationId})`);
+            // Monitor the operation via SSE until completion
+            const opResult = await this.operationClient.monitorOperation(operationId, {
+                timeout: (options.timeout ?? 600) * 1000, // Convert to milliseconds, 10 minute default
+                onProgress: (progress) => {
+                    if (options.onProgress) {
+                        let msg = progress.message;
+                        if (progress.progressPercent !== undefined) {
+                            msg += ` (${progress.progressPercent.toFixed(0)}%)`;
+                        }
+                        options.onProgress(msg);
+                    }
+                },
+            });
+            // Convert operation result to materialization result
+            return this.convertOperationResult(opResult, options);
         }
         catch (error) {
             return {
@@ -71,6 +78,43 @@ class MaterializationClient {
             };
         }
     }
+    /**
+     * Convert SSE operation result to MaterializationResult
+     */
+    convertOperationResult(opResult, options) {
+        if (opResult.success) {
+            // Extract details from SSE completion event result
+            const sseResult = opResult.result || {};
+            const tables = sseResult.tables_materialized || [];
+            const rows = sseResult.total_rows || 0;
+            const timeMs = sseResult.execution_time_ms || 0;
+            options.onProgress?.(`✅ Materialization complete: ${tables.length} tables, ` +
+                `${rows.toLocaleString()} rows in ${timeMs.toFixed(2)}ms`);
+            return {
+                status: 'success',
+                wasStale: sseResult.was_stale || false,
+                staleReason: sseResult.stale_reason,
+                tablesMaterialized: tables,
+                totalRows: rows,
+                executionTimeMs: timeMs,
+                message: sseResult.message || 'Graph materialized successfully',
+                success: true,
+            };
+        }
+        else {
+            // Operation failed or was cancelled
+            return {
+                status: 'failed',
+                wasStale: false,
+                tablesMaterialized: [],
+                totalRows: 0,
+                executionTimeMs: 0,
+                message: opResult.error || 'Operation failed',
+                success: false,
+                error: opResult.error,
+            };
+        }
+    }
     /**
      * Get current materialization status for the graph
      *
@@ -103,5 +147,11 @@ class MaterializationClient {
             return null;
         }
     }
+    /**
+     * Close any active SSE connections
+     */
+    close() {
+        this.operationClient.closeAll();
+    }
 }
 exports.MaterializationClient = MaterializationClient;

package/extensions/MaterializationClient.ts

@@ -4,16 +4,18 @@
  * Materialization Client for RoboSystems API
  *
  * Manages graph materialization from DuckDB staging tables.
- * Treats the graph database as a materialized view of the mutable DuckDB data lake.
+ * Submits materialization jobs to Dagster and monitors progress via SSE.
  */
 
 import { getMaterializationStatus, materializeGraph } from '../sdk.gen'
 import type { MaterializeRequest } from '../types.gen'
+import { OperationClient, type OperationResult } from './OperationClient'
 
 export interface MaterializationOptions {
   ignoreErrors?: boolean
   rebuild?: boolean
   force?: boolean
+  timeout?: number // Default 10 minutes
   onProgress?: (message: string) => void
 }
 
@@ -47,6 +49,7 @@ export class MaterializationClient {
     headers?: Record<string, string>
     token?: string
   }
+  private operationClient: OperationClient
 
   constructor(config: {
     baseUrl: string
@@ -55,21 +58,22 @@ export class MaterializationClient {
     token?: string
   }) {
     this.config = config
+    this.operationClient = new OperationClient(config)
   }
 
   /**
    * Materialize graph from DuckDB staging tables
    *
-   * Rebuilds the complete graph database from the current state of DuckDB
-   * staging tables. Automatically discovers all tables, materializes them in
-   * the correct order (nodes before relationships), and clears the staleness flag.
+   * Submits a materialization job to Dagster and monitors progress via SSE.
+   * The operation runs asynchronously on the server but this method waits
+   * for completion and returns the final result.
    */
   async materialize(
     graphId: string,
     options: MaterializationOptions = {}
   ): Promise<MaterializationResult> {
     try {
-      options.onProgress?.('Starting graph materialization...')
+      options.onProgress?.('Submitting materialization job...')
 
       const request: MaterializeRequest = {
         ignore_errors: options.ignoreErrors ?? true,
@@ -95,33 +99,82 @@ export class MaterializationClient {
         }
       }
 
-      const result = response.data as any
+      const queuedResponse = response.data as { operation_id: string; message: string }
+      const operationId = queuedResponse.operation_id
+
+      options.onProgress?.(`Materialization queued (operation: ${operationId})`)
+
+      // Monitor the operation via SSE until completion
+      const opResult = await this.operationClient.monitorOperation(operationId, {
+        timeout: (options.timeout ?? 600) * 1000, // Convert to milliseconds, 10 minute default
+        onProgress: (progress) => {
+          if (options.onProgress) {
+            let msg = progress.message
+            if (progress.progressPercent !== undefined) {
+              msg += ` (${progress.progressPercent.toFixed(0)}%)`
+            }
+            options.onProgress(msg)
+          }
+        },
+      })
+
+      // Convert operation result to materialization result
+      return this.convertOperationResult(opResult, options)
+    } catch (error) {
+      return {
+        status: 'failed',
+        wasStale: false,
+        tablesMaterialized: [],
+        totalRows: 0,
+        executionTimeMs: 0,
+        message: error instanceof Error ? error.message : String(error),
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+      }
+    }
+  }
+
+  /**
+   * Convert SSE operation result to MaterializationResult
+   */
+  private convertOperationResult(
+    opResult: OperationResult,
+    options: MaterializationOptions
+  ): MaterializationResult {
+    if (opResult.success) {
+      // Extract details from SSE completion event result
+      const sseResult = opResult.result || {}
+
+      const tables = sseResult.tables_materialized || []
+      const rows = sseResult.total_rows || 0
+      const timeMs = sseResult.execution_time_ms || 0
 
       options.onProgress?.(
-        `✅ Materialization complete: ${result.tables_materialized?.length || 0} tables, ` +
-          `${result.total_rows?.toLocaleString() || 0} rows in ${result.execution_time_ms?.toFixed(2) || 0}ms`
+        `✅ Materialization complete: ${tables.length} tables, ` +
+          `${rows.toLocaleString()} rows in ${timeMs.toFixed(2)}ms`
       )
 
       return {
-        status: result.status,
-        wasStale: result.was_stale,
-        staleReason: result.stale_reason,
-        tablesMaterialized: result.tables_materialized || [],
-        totalRows: result.total_rows || 0,
-        executionTimeMs: result.execution_time_ms || 0,
-        message: result.message || 'Materialization complete',
+        status: 'success',
+        wasStale: sseResult.was_stale || false,
+        staleReason: sseResult.stale_reason,
+        tablesMaterialized: tables,
+        totalRows: rows,
+        executionTimeMs: timeMs,
+        message: sseResult.message || 'Graph materialized successfully',
         success: true,
       }
-    } catch (error) {
+    } else {
+      // Operation failed or was cancelled
       return {
         status: 'failed',
         wasStale: false,
         tablesMaterialized: [],
         totalRows: 0,
         executionTimeMs: 0,
-        message: error instanceof Error ? error.message : String(error),
+        message: opResult.error || 'Operation failed',
         success: false,
-        error: error instanceof Error ? error.message : String(error),
+        error: opResult.error,
       }
     }
   }
@@ -160,4 +213,11 @@ export class MaterializationClient {
       return null
     }
   }
+
+  /**
+   * Close any active SSE connections
+   */
+  close(): void {
+    this.operationClient.closeAll()
+  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@robosystems/client",
-  "version": "0.2.23",
+  "version": "0.2.25",
   "description": "TypeScript client library for RoboSystems Financial Knowledge Graph API",
   "main": "index.js",
   "types": "index.d.ts",
@@ -70,11 +70,11 @@
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "typecheck": "tsc --noEmit",
-    "validate": "npm run format:check && npm run lint && npm run typecheck",
+    "validate": "npm run validate:fix && npm run lint && npm run typecheck",
     "feature:create": "./bin/create-feature",
     "release:create": "./bin/create-release",
     "pr:create": "./bin/create-pr",
-    "validate:fix": "npm run format && npm run lint:fix && npm run typecheck",
+    "validate:fix": "npm run format && npm run lint:fix",
     "test": "vitest run",
     "test:watch": "vitest",
     "test:all": "npm run validate && npm run test && npm run build",

package/sdk/sdk.gen.d.ts

@@ -433,7 +433,7 @@ export declare const listAgents: <ThrowOnError extends boolean = false>(options:
  * **Execution Strategies (automatic):**
  * - Fast operations (<5s): Immediate synchronous response
  * - Medium operations (5-30s): SSE streaming with progress updates
- * - Long operations (>30s): Async Celery worker with operation tracking
+ * - Long operations (>30s): Background queue with operation tracking
  *
  * **Response Mode Override:**
  * Use query parameter `?mode=sync|async` to override automatic strategy selection.
@@ -494,7 +494,7 @@ export declare const getAgentMetadata: <ThrowOnError extends boolean = false>(op
  * **Execution Strategies (automatic):**
  * - Fast operations (<5s): Immediate synchronous response
  * - Medium operations (5-30s): SSE streaming with progress updates
- * - Long operations (>30s): Async Celery worker with operation tracking
+ * - Long operations (>30s): Background queue with operation tracking
  *
  * **Response Mode Override:**
  * Use query parameter `?mode=sync|async` to override automatic strategy selection.
@@ -1708,7 +1708,7 @@ export declare const getFile: <ThrowOnError extends boolean = false>(options: Op
  * **What Happens (status='uploaded'):**
  * 1. File validated in S3
  * 2. Row count calculated
- * 3. DuckDB staging triggered immediately (Celery task)
+ * 3. DuckDB staging triggered immediately (background task)
  * 4. If ingest_to_graph=true, graph ingestion queued
  * 5. File queryable in DuckDB within seconds
  *

package/sdk/sdk.gen.js

@@ -1079,7 +1079,7 @@ exports.listAgents = listAgents;
  * **Execution Strategies (automatic):**
  * - Fast operations (<5s): Immediate synchronous response
  * - Medium operations (5-30s): SSE streaming with progress updates
- * - Long operations (>30s): Async Celery worker with operation tracking
+ * - Long operations (>30s): Background queue with operation tracking
  *
  * **Response Mode Override:**
  * Use query parameter `?mode=sync|async` to override automatic strategy selection.
@@ -1176,7 +1176,7 @@ exports.getAgentMetadata = getAgentMetadata;
  * **Execution Strategies (automatic):**
  * - Fast operations (<5s): Immediate synchronous response
  * - Medium operations (5-30s): SSE streaming with progress updates
- * - Long operations (>30s): Async Celery worker with operation tracking
+ * - Long operations (>30s): Background queue with operation tracking
  *
  * **Response Mode Override:**
  * Use query parameter `?mode=sync|async` to override automatic strategy selection.
@@ -3130,7 +3130,7 @@ exports.getFile = getFile;
  * **What Happens (status='uploaded'):**
  * 1. File validated in S3
  * 2. Row count calculated
- * 3. DuckDB staging triggered immediately (Celery task)
+ * 3. DuckDB staging triggered immediately (background task)
  * 4. If ingest_to_graph=true, graph ingestion queued
  * 5. File queryable in DuckDB within seconds
  *

package/sdk/sdk.gen.ts

@@ -1092,7 +1092,7 @@ export const listAgents = <ThrowOnError extends boolean = false>(options: Option
  * **Execution Strategies (automatic):**
  * - Fast operations (<5s): Immediate synchronous response
  * - Medium operations (5-30s): SSE streaming with progress updates
- * - Long operations (>30s): Async Celery worker with operation tracking
+ * - Long operations (>30s): Background queue with operation tracking
  *
  * **Response Mode Override:**
  * Use query parameter `?mode=sync|async` to override automatic strategy selection.
@@ -1189,7 +1189,7 @@ export const getAgentMetadata = <ThrowOnError extends boolean = false>(options:
  * **Execution Strategies (automatic):**
  * - Fast operations (<5s): Immediate synchronous response
  * - Medium operations (5-30s): SSE streaming with progress updates
- * - Long operations (>30s): Async Celery worker with operation tracking
+ * - Long operations (>30s): Background queue with operation tracking
  *
  * **Response Mode Override:**
  * Use query parameter `?mode=sync|async` to override automatic strategy selection.
@@ -3143,7 +3143,7 @@ export const getFile = <ThrowOnError extends boolean = false>(options: Options<G
  * **What Happens (status='uploaded'):**
  * 1. File validated in S3
  * 2. Row count calculated
- * 3. DuckDB staging triggered immediately (Celery task)
+ * 3. DuckDB staging triggered immediately (background task)
  * 4. If ingest_to_graph=true, graph ingestion queued
  * 5. File queryable in DuckDB within seconds
  *