@elevasis/core 0.1.0

package/src/execution/engine/tools/integration/server/adapters/apify/__tests__/apify-run-actor.integration.test.ts

@@ -0,0 +1,293 @@
+import { describe, it, expect, beforeAll, vi } from 'vitest'
+import { ApifyAdapter } from '../apify-adapter'
+import type { ExecutionContext } from '../../../../../../base/types'
+import { createClient } from '@supabase/supabase-js'
+import type { Database } from '../../../../../../../../supabase/database.types'
+import { decryptCredentialValue } from '../../../../../../../../auth/multi-tenancy/credentials/server/service'
+import type { RunActorResult } from '../fetch/run-actor'
+
+/**
+ * Apify Actor Integration Test Suite
+ *
+ * Tests the complete runActor flow:
+ * 1. Start actor with input
+ * 2. Poll for completion
+ * 3. Fetch dataset results
+ * 4. Handle various actor statuses
+ *
+ * Prerequisites:
+ * - Supabase database with credentials table
+ * - Credential 'elevasis-apify' with valid Apify API token
+ * - Organization ID: f9aa5a56-8c13-4cd1-9161-8827ae7b452b
+ * - SUPABASE_URL and SUPABASE_SERVICE_KEY env vars set
+ * - SECRETS_ENCRYPTION_KEY env var set
+ * - Apify account accessible with provided credentials
+ *
+ * Run: pnpm test apify-run-actor.integration.test.ts
+ */
+
+const SKIP_TESTS = !process.env.SUPABASE_URL || !process.env.SUPABASE_SERVICE_KEY || !process.env.SECRETS_ENCRYPTION_KEY
+
+describe.skipIf(SKIP_TESTS)('Apify Run Actor Integration Tests', () => {
+  const adapter = new ApifyAdapter()
+  const organizationId = 'f9aa5a56-8c13-4cd1-9161-8827ae7b452b'
+  const credentialName = 'elevasis-apify'
+
+  const context: ExecutionContext = {
+    organizationId,
+    executionId: 'apify-integration-test',
+    resourceId: 'apify-test-agent',
+    resourceType: 'agent',
+    logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child: vi.fn().mockReturnThis() }
+  }
+
+  let supabase: ReturnType<typeof createClient<Database>>
+  let credentials: Record<string, unknown>
+
+  beforeAll(async () => {
+    console.log('\n=== Apify Actor Integration Test Suite ===')
+    console.log(`Organization: ${organizationId}`)
+    console.log(`Credential: ${credentialName}\n`)
+
+    // Initialize Supabase
+    supabase = createClient<Database>(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_KEY!)
+
+    // Fetch credential from database
+    const { data: credRow, error } = await supabase
+      .from('credentials')
+      .select('encrypted_value')
+      .eq('organization_id', organizationId)
+      .eq('name', credentialName)
+      .single()
+
+    if (error || !credRow) {
+      throw new Error(`Credential '${credentialName}' not found for org ${organizationId}. Error: ${error?.message}`)
+    }
+
+    // Decrypt credentials
+    credentials = decryptCredentialValue(credRow.encrypted_value)
+
+    console.log('✓ Credentials loaded and decrypted')
+    console.log('Credential fields:', Object.keys(credentials))
+    console.log(
+      'Credential values (masked):',
+      Object.keys(credentials).reduce(
+        (acc, key) => {
+          acc[key] = typeof credentials[key] === 'string' ? credentials[key].substring(0, 10) + '...' : credentials[key]
+          return acc
+        },
+        {} as Record<string, unknown>
+      )
+    )
+    const tokenPresent = !!(credentials.apiToken || credentials.apiKey || credentials.api_token || credentials.token)
+    console.log(`✓ API Token present: ${tokenPresent}`)
+
+    // Validate credentials
+    const isValid = adapter.validateCredentials(credentials)
+    if (!isValid) {
+      throw new Error('Credential validation failed')
+    }
+    console.log('✓ Credentials validated')
+    console.log('\n✓ Ready to run tests')
+  })
+
+  it('should run a simple actor and return results', { timeout: 60000 }, async () => {
+    console.log('\n[Test] Run simple hello-world actor')
+
+    // Using apify/hello-world - a simple actor that runs quickly
+    const result = (await adapter.call(
+      'runActor',
+      {
+        actorId: 'apify/hello-world',
+        input: {
+          message: 'Integration test from Elevasis'
+        },
+        timeoutSecs: 60,
+        pollIntervalSecs: 5
+      },
+      credentials,
+      context
+    )) as RunActorResult
+
+    console.log(`✓ Actor completed with status: ${result.status}`)
+    console.log(`✓ Run ID: ${result.runId}`)
+    console.log(`✓ Dataset ID: ${result.datasetId}`)
+    console.log(`✓ Items returned: ${result.totalCount}`)
+    console.log(`✓ Execution time: ${result.executionTimeMs}ms`)
+
+    expect(result).toBeDefined()
+    expect(result.status).toBe('SUCCEEDED')
+    expect(result.runId).toBeDefined()
+    expect(result.datasetId).toBeDefined()
+    expect(result.items).toBeInstanceOf(Array)
+    expect(result.totalCount).toBeGreaterThanOrEqual(0)
+    expect(result.executionTimeMs).toBeGreaterThan(0)
+  })
+
+  it('should handle actor with custom input', { timeout: 60000 }, async () => {
+    console.log('\n[Test] Run actor with custom input')
+
+    const result = (await adapter.call(
+      'runActor',
+      {
+        actorId: 'apify/hello-world',
+        input: {
+          message: 'Custom test message',
+          outputDatasetItems: 5
+        },
+        timeoutSecs: 60,
+        pollIntervalSecs: 5
+      },
+      credentials,
+      context
+    )) as RunActorResult
+
+    console.log(`✓ Actor completed: ${result.status}`)
+    console.log(`✓ Items in dataset: ${result.totalCount}`)
+
+    expect(result).toBeDefined()
+    expect(result.status).toBe('SUCCEEDED')
+    expect(result.items).toBeInstanceOf(Array)
+  })
+
+  it('should handle maxItems parameter', { timeout: 60000 }, async () => {
+    console.log('\n[Test] Run actor with maxItems limit')
+
+    const maxItems = 3
+
+    const result = (await adapter.call(
+      'runActor',
+      {
+        actorId: 'apify/hello-world',
+        input: {
+          outputDatasetItems: 10
+        },
+        maxItems: maxItems,
+        timeoutSecs: 60,
+        pollIntervalSecs: 5
+      },
+      credentials,
+      context
+    )) as RunActorResult
+
+    console.log(`✓ Actor completed: ${result.status}`)
+    console.log(`✓ Items returned (limited): ${result.totalCount}`)
+
+    expect(result).toBeDefined()
+    expect(result.status).toBe('SUCCEEDED')
+    expect(result.items).toBeInstanceOf(Array)
+    expect(result.items.length).toBeLessThanOrEqual(maxItems)
+  })
+
+  it('should handle timeout gracefully', { timeout: 15000 }, async () => {
+    console.log('\n[Test] Handle actor timeout')
+
+    // Use very short timeout to trigger TIMED_OUT status
+    const result = (await adapter.call(
+      'runActor',
+      {
+        actorId: 'apify/hello-world',
+        input: {},
+        timeoutSecs: 1, // Very short timeout
+        pollIntervalSecs: 1
+      },
+      credentials,
+      context
+    )) as RunActorResult
+
+    console.log(`✓ Actor status: ${result.status}`)
+    console.log(`✓ Timeout handled gracefully`)
+
+    expect(result).toBeDefined()
+    // Actor might succeed, timeout, or fail depending on external state
+    expect(['SUCCEEDED', 'TIMED_OUT', 'FAILED']).toContain(result.status)
+
+    if (result.status === 'TIMED_OUT') {
+      expect(result.items).toEqual([])
+      expect(result.totalCount).toBe(0)
+      console.log('✓ TIMED_OUT status returns empty items as expected')
+    }
+  })
+
+  it('should handle invalid actor ID gracefully', async () => {
+    console.log('\n[Test] Handle invalid actor ID')
+
+    await expect(
+      adapter.call(
+        'runActor',
+        {
+          actorId: 'invalid/nonexistent-actor-xyz-123',
+          input: {},
+          timeoutSecs: 30,
+          pollIntervalSecs: 5
+        },
+        credentials,
+        context
+      )
+    ).rejects.toThrow()
+
+    console.log('✓ Invalid actor ID rejected as expected')
+  })
+
+  it('should handle missing actorId parameter', async () => {
+    console.log('\n[Test] Handle missing actorId')
+
+    await expect(
+      adapter.call(
+        'runActor',
+        {
+          // Missing actorId
+          input: {},
+          timeoutSecs: 30
+        },
+        credentials,
+        context
+      )
+    ).rejects.toThrow('Missing required parameter: actorId')
+
+    console.log('✓ Missing actorId parameter rejected as expected')
+  })
+
+  it('should handle invalid credentials gracefully', async () => {
+    console.log('\n[Test] Handle invalid credentials')
+
+    const invalidCreds = { apiToken: 'invalid-token-12345' }
+
+    await expect(
+      adapter.call(
+        'runActor',
+        {
+          actorId: 'apify/hello-world',
+          input: {},
+          timeoutSecs: 30
+        },
+        invalidCreds,
+        context
+      )
+    ).rejects.toThrow()
+
+    console.log('✓ Invalid credentials rejected as expected')
+  })
+
+  it('should validate credentials correctly', () => {
+    console.log('\n[Test] Validate credentials')
+
+    // Valid credentials
+    const validCreds = { apiToken: 'test-token' }
+    expect(adapter.validateCredentials(validCreds)).toBe(true)
+
+    // Invalid credentials (missing apiToken)
+    const invalidCreds1 = {}
+    expect(adapter.validateCredentials(invalidCreds1)).toBe(false)
+
+    // Invalid credentials (empty apiToken)
+    const invalidCreds2 = { apiToken: '' }
+    expect(adapter.validateCredentials(invalidCreds2)).toBe(false)
+
+    // Invalid credentials (wrong type)
+    const invalidCreds3 = { apiToken: 123 }
+    expect(adapter.validateCredentials(invalidCreds3)).toBe(false)
+
+    console.log('✓ Credential validation working correctly')
+  })
+})

package/src/execution/engine/tools/integration/server/adapters/apify/apify-adapter.ts

@@ -0,0 +1,100 @@
+import type { BaseIntegrationAdapter } from '../../../base-integration-adapter'
+import { ToolingError } from '../../../../types'
+import type { ExecutionContext } from '../../../../../base/types'
+import { runActor } from './fetch/run-actor/index'
+import { getDatasetItems } from './fetch/get-dataset-items/index'
+import { startActor } from './fetch/start-actor/index'
+import type { StartActorParams } from '../../../../integration/types/apify'
+
+interface ApifyCredentials {
+  apiToken?: string
+  apiKey?: string
+  api_token?: string
+  token?: string
+}
+
+interface RunActorParams {
+  actorId: string
+  input?: Record<string, unknown>
+  timeoutSecs?: number
+  pollIntervalSecs?: number
+  maxItems?: number
+}
+
+interface GetDatasetItemsParams {
+  datasetId: string
+  maxItems?: number
+  offset?: number
+}
+
+/**
+ * Apify integration adapter
+ *
+ * Provides access to Apify actor automation platform for web scraping
+ * and browser automation tasks.
+ *
+ * @example
+ * const adapter = new ApifyAdapter()
+ * const result = await adapter.call(
+ *   'runActor',
+ *   { actorId: 'curious_coder/clutch-scraper', input: { category: 'digital-marketing' } },
+ *   { apiToken: 'apify_api_...' },
+ *   context
+ * )
+ */
+export class ApifyAdapter implements BaseIntegrationAdapter {
+  readonly name = 'apify'
+
+  async call(
+    method: string,
+    params: unknown,
+    credentials: Record<string, unknown>,
+    context?: ExecutionContext
+  ): Promise<unknown> {
+    // Validate credentials first
+    if (!this.validateCredentials(credentials)) {
+      throw new ToolingError('credentials_invalid', 'Invalid Apify credentials', {
+        integration: this.name,
+        organizationId: context?.organizationId
+      })
+    }
+
+    // Normalize credentials to consistent field name
+    const normalizedCreds = this.normalizeCredentials(credentials as unknown as ApifyCredentials)
+
+    // Route to method handler
+    switch (method) {
+      case 'runActor':
+        return runActor(normalizedCreds, params as RunActorParams, context)
+      case 'getDatasetItems':
+        return getDatasetItems(normalizedCreds, params as GetDatasetItemsParams, context)
+      case 'startActor':
+        return startActor(normalizedCreds, params as StartActorParams, context)
+      default:
+        throw new ToolingError('method_not_found', `Unknown Apify method: ${method}`, {
+          integration: this.name,
+          method,
+          organizationId: context?.organizationId
+        })
+    }
+  }
+
+  validateCredentials(credentials: Record<string, unknown>): boolean {
+    const creds = credentials as unknown as ApifyCredentials
+    const token = creds.apiToken || creds.apiKey || creds.api_token || creds.token
+    return !!token && typeof token === 'string' && token.length > 0
+  }
+
+  /**
+   * Normalize credentials to use consistent field name
+   */
+  private normalizeCredentials(credentials: ApifyCredentials): { apiToken: string } {
+    const token = credentials.apiToken || credentials.apiKey || credentials.api_token || credentials.token
+    if (!token) {
+      throw new ToolingError('credentials_invalid', 'Missing Apify API token', {
+        integration: this.name
+      })
+    }
+    return { apiToken: token }
+  }
+}

package/src/execution/engine/tools/integration/server/adapters/apify/apify-tools.ts

@@ -0,0 +1,217 @@
+import { z } from 'zod'
+import { createIntegrationTool } from '../../../tool'
+import type { Tool } from '../../../../types'
+
+/**
+ * Create Apify run actor tool
+ *
+ * Runs an Apify actor, waits for completion, and returns dataset results.
+ *
+ * @param credentialName - Name of credential in credentials table
+ * @returns Tool that runs Apify actors
+ *
+ * @example
+ * const clutchScraper = createApifyRunActorTool('apify-elevasis')
+ */
+export function createApifyRunActorTool(credentialName: string): Tool {
+  return createIntegrationTool({
+    name: 'apify_run_actor',
+    description: `Run an Apify actor and get results.
+
+USE CASES:
+- Run a web scraper or automation actor with custom input
+- Wait for completion and get dataset items
+- Automate lead generation, data extraction, and web automation tasks
+
+COMMON ACTORS:
+- curious_coder/clutch-scraper - Scrape marketing agencies from Clutch.co
+- code_crafter/leads-finder - Find leads with ICP filtering
+- compass/crawler-google-places - Scrape Google Maps businesses
+
+REQUIRED PARAMETERS:
+- actorId: Actor ID (e.g., "curious_coder/clutch-scraper")
+
+OPTIONAL PARAMETERS:
+- input: Actor input configuration (object, varies by actor)
+- timeoutSecs: Max wait time in seconds (default: 300)
+- pollIntervalSecs: Poll interval in seconds (default: 10)
+- maxItems: Limit dataset items returned
+
+EXAMPLE:
+{
+  "actorId": "curious_coder/clutch-scraper",
+  "input": {
+    "category": "digital-marketing",
+    "location": "USA",
+    "maxItems": 1000
+  },
+  "timeoutSecs": 300,
+  "pollIntervalSecs": 10
+}
+
+OUTPUT:
+{
+  "status": "SUCCEEDED",
+  "runId": "abc123",
+  "datasetId": "xyz789",
+  "items": [...scraped data...],
+  "totalCount": 847,
+  "executionTimeMs": 45000
+}
+
+TIMEOUT BEHAVIOR:
+If actor exceeds timeoutSecs, returns status='TIMED_OUT' with empty items array.
+Handle by increasing timeoutSecs or retrying with exponential backoff.`,
+
+    inputSchema: z.object({
+      actorId: z.string().describe('Actor ID (e.g., "curious_coder/clutch-scraper")'),
+      input: z.record(z.string(), z.unknown()).optional().describe('Actor input configuration'),
+      timeoutSecs: z.number().optional().describe('Max wait time in seconds (default: 300)'),
+      pollIntervalSecs: z.number().optional().describe('Poll interval in seconds (default: 10)'),
+      maxItems: z.number().optional().describe('Limit dataset items returned')
+    }),
+
+    outputSchema: z.object({
+      status: z.enum(['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED_OUT']).describe('Run status'),
+      runId: z.string().describe('Apify run ID'),
+      datasetId: z.string().describe('Dataset ID containing results'),
+      items: z.array(z.unknown()).describe('Dataset items (scraped data)'),
+      totalCount: z.number().describe('Total items returned'),
+      executionTimeMs: z.number().describe('Total execution time')
+    }),
+
+    integration: 'apify' as const,
+    method: 'runActor' as const,
+    credentialName
+  })
+}
+
+/**
+ * Create Apify get dataset items tool
+ *
+ * Fetches items from an Apify dataset by ID. Useful for recovering data
+ * after a runActor timeout or fetching results from a previous run.
+ *
+ * @param credentialName - Name of credential in credentials table
+ * @returns Tool that fetches Apify dataset items
+ *
+ * @example
+ * const datasetFetcher = createApifyGetDatasetItemsTool('apify-elevasis')
+ */
+export function createApifyGetDatasetItemsTool(credentialName: string): Tool {
+  return createIntegrationTool({
+    name: 'apify_get_dataset_items',
+    description: `Fetch items from an Apify dataset by ID.
+
+USE CASES:
+- Recover data after a runActor timeout (actor keeps running on Apify)
+- Fetch results from a previously completed actor run
+- Paginate through large datasets with offset/maxItems
+
+REQUIRED PARAMETERS:
+- datasetId: Dataset ID from a previous runActor result
+
+OPTIONAL PARAMETERS:
+- maxItems: Limit number of items returned
+- offset: Skip first N items (for pagination)
+
+EXAMPLE:
+{
+  "datasetId": "xyz789",
+  "maxItems": 100,
+  "offset": 0
+}
+
+OUTPUT:
+{
+  "items": [...dataset items...],
+  "totalCount": 100
+}`,
+
+    inputSchema: z.object({
+      datasetId: z.string().describe('Dataset ID from a previous Apify run'),
+      maxItems: z.number().optional().describe('Limit number of items returned'),
+      offset: z.number().optional().describe('Skip first N items (for pagination)')
+    }),
+
+    outputSchema: z.object({
+      items: z.array(z.unknown()).describe('Dataset items'),
+      totalCount: z.number().describe('Total items returned')
+    }),
+
+    integration: 'apify' as const,
+    method: 'getDatasetItems' as const,
+    credentialName
+  })
+}
+
+/**
+ * Create Apify start actor tool
+ *
+ * Starts an Apify actor run and returns immediately (async, no polling).
+ * Results are delivered via webhook when the actor completes.
+ *
+ * @param credentialName - Name of credential in credentials table
+ * @returns Tool that starts Apify actors asynchronously
+ *
+ * @example
+ * const starter = createApifyStartActorTool('apify-elevasis')
+ */
+export function createApifyStartActorTool(credentialName: string): Tool {
+  return createIntegrationTool({
+    name: 'apify_start_actor',
+    description: `Start an Apify actor run and return immediately (async, no polling).
+
+USE CASES:
+- Start a long-running web scraper without blocking the workflow
+- Configure webhook to receive results when actor completes
+- Fire-and-forget pattern for actors that take 10+ minutes
+
+REQUIRED PARAMETERS:
+- actorId: Actor ID (e.g., "apify/website-content-crawler")
+
+OPTIONAL PARAMETERS:
+- input: Actor input configuration
+- webhooks: Array of webhook configs -- { eventTypes, requestUrl }
+- timeoutSecs: Actor timeout on Apify's side (not workflow timeout)
+- maxItems: Limit dataset items
+
+EXAMPLE:
+{
+  "actorId": "apify/website-content-crawler",
+  "input": { "startUrls": [{"url": "https://example.com"}] },
+  "webhooks": [{ "eventTypes": ["ACTOR.RUN.SUCCEEDED"], "requestUrl": "https://api.elevasis.io/webhooks/apify?org=..." }]
+}
+
+OUTPUT:
+{
+  "status": "STARTED",
+  "runId": "abc123",
+  "datasetId": "xyz789"
+}`,
+    inputSchema: z.object({
+      actorId: z.string().describe('Actor ID'),
+      input: z.record(z.string(), z.unknown()).optional(),
+      webhooks: z
+        .array(
+          z.object({
+            eventTypes: z.array(z.string()),
+            requestUrl: z.string(),
+            payloadTemplate: z.string().optional(),
+            headersTemplate: z.string().optional()
+          })
+        )
+        .optional(),
+      timeoutSecs: z.number().optional(),
+      maxItems: z.number().optional()
+    }),
+    outputSchema: z.object({
+      status: z.literal('STARTED'),
+      runId: z.string(),
+      datasetId: z.string()
+    }),
+    integration: 'apify' as const,
+    method: 'startActor' as const,
+    credentialName
+  })
+}

package/src/execution/engine/tools/integration/server/adapters/apify/fetch/get-dataset-items/index.ts

@@ -0,0 +1,92 @@
+import { ToolingError } from '../../../../../../types'
+import { withRetry, createHttpError, DEFAULT_RETRY_POLICY } from '../../../../../../../../../platform/resilience'
+import type { ExecutionContext } from '../../../../../../../base/types'
+
+interface ApifyCredentials {
+  apiToken: string
+}
+
+export interface GetDatasetItemsParams {
+  datasetId: string
+  maxItems?: number
+  offset?: number
+}
+
+export interface GetDatasetItemsResult {
+  items: unknown[]
+  totalCount: number
+}
+
+const APIFY_API_BASE = 'https://api.apify.com/v2'
+
+/**
+ * Fetch items from an Apify dataset
+ *
+ * Retrieves items from a dataset by ID. Useful for:
+ * - Recovering data after a runActor timeout (actor keeps running on Apify)
+ * - Fetching results from a previously completed run
+ * - Paginating through large datasets with offset/maxItems
+ *
+ * @param credentials - Apify API token
+ * @param params - Dataset ID and optional pagination
+ * @param context - Execution context for logging
+ * @returns Dataset items and total count
+ */
+export async function getDatasetItems(
+  credentials: ApifyCredentials,
+  params: GetDatasetItemsParams,
+  context?: ExecutionContext
+): Promise<GetDatasetItemsResult> {
+  const { apiToken } = credentials
+  const { datasetId, maxItems, offset } = params
+
+  if (!datasetId) {
+    throw new ToolingError('validation_error', 'Missing required parameter: datasetId', {
+      integration: 'apify',
+      method: 'getDatasetItems'
+    })
+  }
+
+  let datasetUrl = `${APIFY_API_BASE}/datasets/${datasetId}/items?format=json`
+  if (maxItems) {
+    datasetUrl += `&limit=${maxItems}`
+  }
+  if (offset) {
+    datasetUrl += `&offset=${offset}`
+  }
+
+  if (context?.logger) {
+    context.logger.info(
+      `[ApifyAdapter] Fetching dataset items: organizationId=${context.organizationId} executionId=${context.executionId} datasetId=${datasetId} maxItems=${maxItems} offset=${offset}`
+    )
+  }
+
+  const datasetResponse = await withRetry(async () => {
+    const response = await fetch(datasetUrl, {
+      headers: { Authorization: `Bearer ${apiToken}` }
+    })
+
+    if (!response.ok) {
+      throw await createHttpError(response, {
+        integration: 'apify',
+        method: 'getDatasetItems',
+        datasetId
+      })
+    }
+
+    return response.json()
+  }, DEFAULT_RETRY_POLICY)
+
+  const items = Array.isArray(datasetResponse) ? datasetResponse : []
+
+  if (context?.logger) {
+    context.logger.info(
+      `[ApifyAdapter] Dataset items fetched: organizationId=${context.organizationId} executionId=${context.executionId} datasetId=${datasetId} itemCount=${items.length}`
+    )
+  }
+
+  return {
+    items,
+    totalCount: items.length
+  }
+}