@things-factory/integration-label-studio 9.1.19

package/server/service/external-data-source-service.ts

@@ -0,0 +1,438 @@
+import { Resolver, Mutation, Query, Arg, Ctx, Int, Directive } from 'type-graphql'
+import { Field, InputType, ObjectType } from 'type-graphql'
+import axios from 'axios'
+import { labelStudioApi } from '../utils/label-studio-api-client.js'
+
+/**
+ * External Data Source Service
+ *
+ * Import data from external sources into Label Studio
+ * Supports:
+ * - REST API endpoints
+ * - S3/Cloud Storage URLs
+ * - Database queries (via Things Factory connections)
+ * - CSV/JSON file URLs
+ */
+
+// ============================================================================
+// Type Definitions
+// ============================================================================
+
+@InputType()
+class ExternalDataSourceConfig {
+  @Field({ description: 'Data source type' })
+  sourceType: 'api' | 'url' | 's3' | 'database' | 'csv' | 'json'
+
+  @Field({ description: 'Source URL or endpoint' })
+  sourceUrl: string
+
+  @Field({ nullable: true, description: 'Authentication header if required' })
+  authHeader?: string
+
+  @Field({ nullable: true, description: 'HTTP method for API calls' })
+  httpMethod?: string
+
+  @Field({ nullable: true, description: 'Request body for POST/PUT' })
+  requestBody?: string
+
+  @Field({ nullable: true, description: 'JSONPath or XPath for data extraction' })
+  dataPath?: string
+}
+
+@InputType()
+class ImportFromExternalSourceRequest {
+  @Field(type => Int, { description: 'Label Studio project ID' })
+  projectId: number
+
+  @Field(type => ExternalDataSourceConfig, { description: 'External data source configuration' })
+  source: ExternalDataSourceConfig
+
+  @Field({ nullable: true, description: 'Field mapping JSON (source -> Label Studio)' })
+  fieldMapping?: string
+
+  @Field({ nullable: true, description: 'Image field name in source data' })
+  imageField?: string
+
+  @Field({ nullable: true, defaultValue: false, description: 'Auto-generate AI predictions' })
+  autoGeneratePredictions?: boolean
+
+  @Field({ nullable: true, description: 'Maximum number of items to import' })
+  limit?: number
+}
+
+@ObjectType()
+class ExternalImportResult {
+  @Field(type => Int, { description: 'Total items fetched from source' })
+  totalFetched: number
+
+  @Field(type => Int, { description: 'Tasks successfully imported' })
+  tasksImported: number
+
+  @Field(type => Int, { description: 'Tasks failed to import' })
+  tasksFailed: number
+
+  @Field(type => [Int], { description: 'Imported task IDs' })
+  taskIds: number[]
+
+  @Field({ nullable: true, description: 'Error message if any' })
+  error?: string
+}
+
+@ObjectType()
+class DataSourcePreview {
+  @Field(type => Int, { description: 'Number of items available' })
+  itemCount: number
+
+  @Field({ description: 'Sample data (first item)' })
+  sampleData: string
+
+  @Field({ description: 'Detected schema/fields' })
+  schema: string
+
+  @Field({ nullable: true, description: 'Preview error if any' })
+  error?: string
+}
+
+// ============================================================================
+// Resolver
+// ============================================================================
+
+@Resolver()
+export class ExternalDataSourceService {
+  /**
+   * Preview external data source before import
+   */
+  @Query(returns => DataSourcePreview, {
+    description: 'Preview external data source to verify connection and data structure'
+  })
+  @Directive('@privilege(category: "label-studio", privilege: "query")')
+  async previewExternalDataSource(
+    @Arg('source', type => ExternalDataSourceConfig) source: ExternalDataSourceConfig,
+    @Ctx() context: ResolverContext
+  ): Promise<DataSourcePreview> {
+    console.log(`[External Data] Previewing source: ${source.sourceType} - ${source.sourceUrl}`)
+
+    try {
+      const data = await this.fetchFromSource(source, 1)
+
+      if (!data || data.length === 0) {
+        return {
+          itemCount: 0,
+          sampleData: '{}',
+          schema: '{}',
+          error: 'No data found'
+        }
+      }
+
+      const firstItem = data[0]
+      const schema = this.detectSchema(firstItem)
+
+      return {
+        itemCount: data.length,
+        sampleData: JSON.stringify(firstItem, null, 2),
+        schema: JSON.stringify(schema, null, 2)
+      }
+    } catch (error) {
+      console.error('[External Data] Preview failed:', error)
+      return {
+        itemCount: 0,
+        sampleData: '{}',
+        schema: '{}',
+        error: error.message
+      }
+    }
+  }
+
+  /**
+   * Import data from external source to Label Studio
+   */
+  @Mutation(returns => ExternalImportResult, {
+    description: 'Import data from external source into Label Studio project'
+  })
+  @Directive('@privilege(category: "label-studio", privilege: "mutation")')
+  async importFromExternalSource(
+    @Arg('input') input: ImportFromExternalSourceRequest,
+    @Ctx() context: ResolverContext
+  ): Promise<ExternalImportResult> {
+    console.log(`[External Data] Importing from ${input.source.sourceType}: ${input.source.sourceUrl}`)
+
+    try {
+      // 1. Fetch data from external source
+      const rawData = await this.fetchFromSource(input.source, input.limit)
+
+      console.log(`[External Data] Fetched ${rawData.length} items`)
+
+      if (rawData.length === 0) {
+        return {
+          totalFetched: 0,
+          tasksImported: 0,
+          tasksFailed: 0,
+          taskIds: [],
+          error: 'No data fetched from source'
+        }
+      }
+
+      // 2. Transform data to Label Studio format
+      const fieldMapping = input.fieldMapping ? JSON.parse(input.fieldMapping) : {}
+      const imageField = input.imageField || 'image'
+
+      const tasks = rawData.map(item => this.transformToLabelStudioTask(item, fieldMapping, imageField))
+
+      // 3. Import tasks to Label Studio
+      const result: ExternalImportResult = {
+        totalFetched: rawData.length,
+        tasksImported: 0,
+        tasksFailed: 0,
+        taskIds: []
+      }
+
+      // Import in batches
+      const BATCH_SIZE = 50
+      for (let i = 0; i < tasks.length; i += BATCH_SIZE) {
+        const batch = tasks.slice(i, i + BATCH_SIZE)
+
+        try {
+          const importResult = await labelStudioApi.importTasks(input.projectId, batch)
+
+          // Label Studio import returns task IDs
+          if (importResult.task_count) {
+            result.tasksImported += importResult.task_count
+          } else if (Array.isArray(importResult)) {
+            result.tasksImported += importResult.length
+            result.taskIds.push(...importResult.map((t: any) => t.id))
+          }
+
+          console.log(`[External Data] Imported batch ${i / BATCH_SIZE + 1}: ${batch.length} tasks`)
+        } catch (batchError) {
+          console.error(`[External Data] Batch import failed:`, batchError)
+          result.tasksFailed += batch.length
+        }
+      }
+
+      console.log(
+        `[External Data] Import completed: ${result.tasksImported} imported, ${result.tasksFailed} failed`
+      )
+
+      return result
+    } catch (error) {
+      console.error('[External Data] Import failed:', error)
+      throw new Error(`Failed to import from external source: ${error.message}`)
+    }
+  }
+
+  /**
+   * Fetch data from external source
+   */
+  private async fetchFromSource(
+    source: ExternalDataSourceConfig,
+    limit?: number
+  ): Promise<any[]> {
+    switch (source.sourceType) {
+      case 'api':
+        return await this.fetchFromApi(source, limit)
+
+      case 'url':
+      case 'json':
+        return await this.fetchFromUrl(source, limit)
+
+      case 'csv':
+        return await this.fetchFromCsv(source, limit)
+
+      case 's3':
+        throw new Error('S3 source not yet implemented. Use URL with S3 presigned URL instead.')
+
+      case 'database':
+        throw new Error('Database source not yet implemented. Use API endpoint instead.')
+
+      default:
+        throw new Error(`Unsupported source type: ${source.sourceType}`)
+    }
+  }
+
+  /**
+   * Fetch from REST API
+   */
+  private async fetchFromApi(source: ExternalDataSourceConfig, limit?: number): Promise<any[]> {
+    const method = (source.httpMethod || 'GET').toUpperCase()
+    const headers: any = {
+      'Content-Type': 'application/json'
+    }
+
+    if (source.authHeader) {
+      headers['Authorization'] = source.authHeader
+    }
+
+    const config: any = {
+      method,
+      url: source.sourceUrl,
+      headers,
+      timeout: 30000
+    }
+
+    if (method === 'POST' || method === 'PUT') {
+      config.data = source.requestBody ? JSON.parse(source.requestBody) : {}
+    }
+
+    const response = await axios(config)
+
+    let data = response.data
+
+    // Extract data using JSONPath if specified
+    if (source.dataPath) {
+      data = this.extractDataByPath(data, source.dataPath)
+    }
+
+    // Ensure data is an array
+    if (!Array.isArray(data)) {
+      data = [data]
+    }
+
+    // Apply limit
+    if (limit && data.length > limit) {
+      data = data.slice(0, limit)
+    }
+
+    return data
+  }
+
+  /**
+   * Fetch from URL (JSON file)
+   */
+  private async fetchFromUrl(source: ExternalDataSourceConfig, limit?: number): Promise<any[]> {
+    const headers: any = {}
+
+    if (source.authHeader) {
+      headers['Authorization'] = source.authHeader
+    }
+
+    const response = await axios.get(source.sourceUrl, { headers })
+
+    let data = response.data
+
+    // Extract data using JSONPath if specified
+    if (source.dataPath) {
+      data = this.extractDataByPath(data, source.dataPath)
+    }
+
+    // Ensure data is an array
+    if (!Array.isArray(data)) {
+      data = [data]
+    }
+
+    // Apply limit
+    if (limit && data.length > limit) {
+      data = data.slice(0, limit)
+    }
+
+    return data
+  }
+
+  /**
+   * Fetch from CSV
+   */
+  private async fetchFromCsv(source: ExternalDataSourceConfig, limit?: number): Promise<any[]> {
+    const headers: any = {}
+
+    if (source.authHeader) {
+      headers['Authorization'] = source.authHeader
+    }
+
+    const response = await axios.get(source.sourceUrl, { headers })
+    const csvText = response.data
+
+    // Simple CSV parsing (for production, use a proper CSV library)
+    const lines = csvText.trim().split('\n')
+    const headerLine = lines[0]
+    const headers_csv = headerLine.split(',').map((h: string) => h.trim())
+
+    const data = []
+    const maxLines = limit ? Math.min(lines.length, limit + 1) : lines.length
+
+    for (let i = 1; i < maxLines; i++) {
+      const line = lines[i]
+      const values = line.split(',').map((v: string) => v.trim())
+
+      const obj: any = {}
+      headers_csv.forEach((header, index) => {
+        obj[header] = values[index]
+      })
+
+      data.push(obj)
+    }
+
+    return data
+  }
+
+  /**
+   * Extract data by JSON path (simple implementation)
+   */
+  private extractDataByPath(data: any, path: string): any {
+    const parts = path.split('.')
+    let current = data
+
+    for (const part of parts) {
+      if (current && typeof current === 'object' && part in current) {
+        current = current[part]
+      } else {
+        throw new Error(`Path "${path}" not found in data`)
+      }
+    }
+
+    return current
+  }
+
+  /**
+   * Transform raw data item to Label Studio task format
+   */
+  private transformToLabelStudioTask(
+    item: any,
+    fieldMapping: Record<string, string>,
+    imageField: string
+  ): any {
+    // Apply field mapping
+    const mappedData: any = {}
+
+    if (Object.keys(fieldMapping).length > 0) {
+      // Use explicit mapping
+      for (const [sourceField, targetField] of Object.entries(fieldMapping)) {
+        if (item[sourceField] !== undefined) {
+          mappedData[targetField] = item[sourceField]
+        }
+      }
+    } else {
+      // No mapping, use data as-is
+      Object.assign(mappedData, item)
+    }
+
+    // Ensure image field exists
+    const imageUrl = mappedData[imageField] || item[imageField] || item.url || item.image_url
+
+    if (!imageUrl) {
+      console.warn('[External Data] No image URL found in item:', item)
+    }
+
+    return {
+      data: {
+        image: imageUrl,
+        ...mappedData
+      },
+      meta: {
+        source: 'external-import',
+        originalData: JSON.stringify(item)
+      }
+    }
+  }
+
+  /**
+   * Detect schema from data item
+   */
+  private detectSchema(item: any): Record<string, string> {
+    const schema: Record<string, string> = {}
+
+    for (const [key, value] of Object.entries(item)) {
+      schema[key] = typeof value
+    }
+
+    return schema
+  }
+}

package/server/service/index.ts

@@ -0,0 +1,24 @@
+import { UserSyncMutation } from './user-provisioning/user-sync-mutation.js'
+import { ProjectManagement } from './project/project-management.js'
+import { TaskManagement } from './task/task-management.js'
+import { WebhookManagement } from './webhook/webhook-management.js'
+import { MLBackendService } from './ml/ml-backend-service.js'
+import { PredictionManagement } from './prediction/prediction-management.js'
+import { LabelStudioAIPredictionService } from './ai-prediction-service.js'
+import { DatasetLabelingIntegration } from './dataset-labeling-integration.js'
+import { ExternalDataSourceService } from './external-data-source-service.js'
+
+export const schema = {
+  resolverClasses: [
+    /* RESOLVER CLASSES */
+    UserSyncMutation,
+    ProjectManagement,
+    TaskManagement,
+    WebhookManagement,
+    MLBackendService,
+    PredictionManagement,
+    LabelStudioAIPredictionService,
+    DatasetLabelingIntegration,
+    ExternalDataSourceService
+  ]
+}

package/server/service/label-studio-sso-service.ts

@@ -0,0 +1,108 @@
+import axios from 'axios'
+import { config } from '@things-factory/env'
+
+/**
+ * Label Studio SSO Service
+ *
+ * Handles JWT token acquisition from Label Studio for SSO authentication.
+ * The client application requests JWT tokens from Label Studio's token API,
+ * then uses cookie-based authentication to automatically log users in.
+ */
+export class LabelStudioSSOService {
+  /**
+   * Get Label Studio SSO configuration
+   */
+  private static getConfig() {
+    const labelStudioConfig = config.get('labelStudio', {
+      serverUrl: 'http://localhost:8080',
+      apiToken: ''
+    })
+
+    return {
+      serverUrl: labelStudioConfig.serverUrl,
+      apiToken: labelStudioConfig.apiToken
+    }
+  }
+
+  /**
+   * Request JWT token from Label Studio for SSO authentication
+   *
+   * @param email User email address
+   * @returns JWT token and expiry time, or null if failed
+   *
+   * @example
+   * ```typescript
+   * const tokenData = await LabelStudioSSOService.getSSOToken('user@example.com')
+   * if (tokenData) {
+   *   console.log('Token:', tokenData.token)
+   *   console.log('Expires in:', tokenData.expires_in, 'seconds')
+   * }
+   * ```
+   */
+  static async getSSOToken(email: string): Promise<{ token: string; expires_in: number } | null> {
+    const { serverUrl, apiToken } = this.getConfig()
+
+    if (!apiToken) {
+      console.error('[LSS SSO] Label Studio API token not configured')
+      return null
+    }
+
+    if (!email) {
+      console.error('[LSS SSO] Email is required')
+      return null
+    }
+
+    try {
+      console.log(`[LSS SSO] Requesting JWT token for: ${email}`)
+
+      const response = await axios.post(
+        `${serverUrl}/api/sso/token`,
+        { email },
+        {
+          headers: {
+            'Authorization': `Token ${apiToken}`,
+            'Content-Type': 'application/json'
+          },
+          timeout: 5000
+        }
+      )
+
+      const { token, expires_in } = response.data
+
+      console.log(`[LSS SSO] JWT token acquired for ${email} (expires in ${expires_in}s)`)
+
+      return { token, expires_in }
+    } catch (error: any) {
+      if (error.response) {
+        console.error(`[LSS SSO] Label Studio API error: ${error.response.status}`, error.response.data)
+      } else if (error.request) {
+        console.error('[LSS SSO] No response from Label Studio:', error.message)
+      } else {
+        console.error('[LSS SSO] Request error:', error.message)
+      }
+
+      return null
+    }
+  }
+
+  /**
+   * Verify SSO configuration is properly set up
+   *
+   * @returns True if configuration is valid
+   */
+  static verifyConfig(): boolean {
+    const { serverUrl, apiToken } = this.getConfig()
+
+    if (!serverUrl) {
+      console.error('[LSS SSO] Label Studio server URL not configured')
+      return false
+    }
+
+    if (!apiToken) {
+      console.error('[LSS SSO] Label Studio API token not configured')
+      return false
+    }
+
+    return true
+  }
+}