@membranehq/cli 0.1.1

package/src/testing/test-suites/base.ts

@@ -0,0 +1,92 @@
+import { extractIntegrationAppErrorData, setValueAtLocator } from '@integration-app/sdk'
+import chalk from 'chalk'
+import merge from 'lodash/merge.js'
+
+import { handleTemplate } from '../../utils/templating'
+import { TestEnvironment } from '../environment'
+import { BaseTester } from '../testers/base'
+
+export class BaseTestSuite {
+  private result: any
+  protected environment: TestEnvironment
+
+  constructor({ environment }: { environment: TestEnvironment }) {
+    this.environment = environment
+    this.result = {}
+  }
+
+  async run() {}
+
+  getResult() {
+    return this.result
+  }
+
+  protected async runTest(tester: BaseTester) {
+    console.debug(`${chalk.bold.cyan('[start]')} ${chalk.yellow(tester.path)}`)
+
+    let done = false
+    let triedToFix = false
+
+    let result = {}
+    let testCase = {}
+    let processedTestCase
+
+    while (!done) {
+      try {
+        testCase = await tester.readTestCase()
+
+        // Create test file if it doesn't exist
+        if (!testCase) {
+          testCase = await tester.generateConfig()
+          await this.environment.writeYaml(tester.getTestCasePath(), testCase)
+          console.debug(`${chalk.bold.yellow('[initialized]')} ${chalk.yellow(tester.path)}`)
+        }
+
+        processedTestCase = handleTemplate(testCase, this.environment.state)
+
+        await tester.run(processedTestCase)
+
+        console.debug(`${chalk.bold.green('[success]')} ${chalk.yellow(tester.path)}`)
+        done = true
+        result = tester.getResult()
+
+        setValueAtLocator(this.result, tester.path, result)
+      } catch (e) {
+        console.error(`${chalk.bold.red('[error]')} ${chalk.yellow(tester.path)}: ${e}`)
+
+        if (this.environment.options.fix && !triedToFix) {
+          triedToFix = true
+
+          try {
+            await tester.fix(e as Error)
+            continue
+          } catch (e) {
+            console.error(`${chalk.bold.red('[fix fail]')} ${chalk.yellow(tester.path)}: ${e}`)
+          }
+        }
+
+        setValueAtLocator(this.result, tester.path, {
+          error: extractIntegrationAppErrorData(e),
+        })
+
+        done = true
+        result = {
+          error: extractIntegrationAppErrorData(e),
+        }
+      }
+    }
+
+    const updatedTestCase = {
+      ...processedTestCase,
+      result,
+    }
+
+    await this.environment.writeYaml(tester.getTestCasePath(), updatedTestCase)
+  }
+
+  protected async runTestSuite(suite: BaseTestSuite) {
+    await suite.run()
+
+    merge(this.result, suite.getResult())
+  }
+}

package/src/testing/test-suites/data-collection.ts

@@ -0,0 +1,128 @@
+import { DataCollectionSpec } from '@integration-app/sdk'
+
+import { TestEnvironment } from '../environment'
+import { BaseTestSuite } from './base'
+import { Logger } from '../../utils/logger'
+import { DataCollectionCreateTester } from '../testers/create'
+import { DataCollectionDeleteTester } from '../testers/delete'
+import { DataCollectionFindByIdTester } from '../testers/find-by-id'
+import { DataCollectionListTester } from '../testers/list'
+import { DataCollectionMatchTester } from '../testers/match'
+import { DataCollectionSearchTester } from '../testers/search'
+import { DataCollectionSpecTester } from '../testers/spec'
+import { DataCollectionUpdateTester } from '../testers/update'
+
+interface TesterConfig {
+  testerClass: any
+  operationKey?: string
+}
+
+const DATA_COLLECTION_TESTERS = {
+  spec: { testerClass: DataCollectionSpecTester },
+  create: { testerClass: DataCollectionCreateTester, operationKey: 'create' },
+  'find-by-id': { testerClass: DataCollectionFindByIdTester, operationKey: 'findById' },
+  list: { testerClass: DataCollectionListTester, operationKey: 'list' },
+  match: { testerClass: DataCollectionMatchTester, operationKey: 'match' },
+  search: { testerClass: DataCollectionSearchTester, operationKey: 'search' },
+  update: { testerClass: DataCollectionUpdateTester, operationKey: 'update' },
+  delete: { testerClass: DataCollectionDeleteTester, operationKey: 'delete' },
+} as const
+
+type TestMethod = keyof typeof DATA_COLLECTION_TESTERS
+
+export class DataCollectionTestSuite extends BaseTestSuite {
+  private readonly dataCollectionKey: string
+  private readonly testMethod?: string
+
+  constructor({
+    environment,
+    dataCollectionKey,
+    testMethod,
+  }: {
+    environment: TestEnvironment
+    dataCollectionKey: string
+    testMethod?: string
+  }) {
+    super({ environment })
+    this.dataCollectionKey = dataCollectionKey
+    this.testMethod = testMethod
+  }
+
+  async run(): Promise<void> {
+    const dataCollection = await this.fetchDataCollection()
+    const testersToRun = this.testMethod
+      ? [this.createTester(this.testMethod as TestMethod, dataCollection)]
+      : this.createAllAvailableTesters(dataCollection)
+
+    Logger.info('🚀 Running tests...')
+
+    for (const tester of testersToRun) {
+      await this.runTest(tester)
+    }
+
+    this.displayTestSummary(testersToRun.length)
+  }
+
+  private async fetchDataCollection() {
+    return this.environment.client
+      .connection(this.environment.connectionId)
+      .dataCollection(this.dataCollectionKey)
+      .get()
+  }
+
+  private isOperationAvailable(dataCollection: DataCollectionSpec, operationKey?: string): boolean {
+    // If no operation is required (e.g., spec validation), tester is always available
+    if (!operationKey) return true
+
+    return !!dataCollection[operationKey]
+  }
+
+  private createTester(testMethod: TestMethod, dataCollection: any) {
+    const config = DATA_COLLECTION_TESTERS[testMethod]
+
+    if (!config) {
+      throw new Error(
+        `Unknown test method "${testMethod}". Available: ${this.getAvailableTestMethods(dataCollection).join(', ')}`,
+      )
+    }
+
+    const operationKey = 'operationKey' in config ? config.operationKey : undefined
+    if (!this.isOperationAvailable(dataCollection, operationKey)) {
+      throw new Error(`${operationKey} operation not available for data collection "${this.dataCollectionKey}"`)
+    }
+
+    return this.instantiateTester(config, dataCollection)
+  }
+
+  private createAllAvailableTesters(dataCollection: any) {
+    return Object.entries(DATA_COLLECTION_TESTERS)
+      .filter(([, config]) => {
+        const operationKey = 'operationKey' in config ? config.operationKey : undefined
+        return this.isOperationAvailable(dataCollection, operationKey)
+      })
+      .map(([, config]) => this.instantiateTester(config, dataCollection))
+  }
+
+  private instantiateTester(config: TesterConfig, dataCollection: any) {
+    const testerConfig = {
+      environment: this.environment,
+      dataCollectionKey: this.dataCollectionKey,
+      dataCollection,
+    }
+
+    return new config.testerClass(testerConfig)
+  }
+
+  private getAvailableTestMethods(dataCollection: any): string[] {
+    return Object.entries(DATA_COLLECTION_TESTERS)
+      .filter(([, config]) => {
+        const operationKey = 'operationKey' in config ? config.operationKey : undefined
+        return this.isOperationAvailable(dataCollection, operationKey)
+      })
+      .map(([method]) => method)
+  }
+
+  private displayTestSummary(testCount: number): void {
+    Logger.info(`📊 ${testCount} test${testCount === 1 ? '' : 's'} executed`)
+  }
+}

package/src/testing/testers/base.ts

@@ -0,0 +1,115 @@
+import { extractIntegrationAppErrorData } from '@integration-app/sdk'
+import chalk from 'chalk'
+
+import { TestEnvironment } from '../environment'
+
+export type TestRunnerOptions = {
+  environment: TestEnvironment
+  level: number
+}
+
+export type TestRunnerFixConfigParams<ConfigClass> = {
+  config: ConfigClass
+  error: Error
+}
+
+export type TestRunAssertion = {
+  message: string
+  result: boolean
+  details?: any
+}
+
+export type TestRunnerResult = {
+  logs: any[]
+  assertions: TestRunAssertion[]
+}
+
+export abstract class BaseTester<ConfigClass = any> {
+  protected environment: TestEnvironment
+  protected level: number
+  protected logs: any[]
+  protected assertions: TestRunAssertion[]
+  protected resultsLocator: string
+  public path: string
+
+  constructor({ environment, path }: { environment: TestEnvironment; path: string }) {
+    this.environment = environment
+    this.logs = []
+    this.assertions = []
+    this.path = path
+    this.level = 0
+  }
+
+  abstract run(config: ConfigClass): Promise<void>
+
+  async fix(error: Error) {
+    const testCase = await this.readTestCase()
+
+    if (!testCase) {
+      throw new Error(`No config found for test ${this.path}`)
+    }
+
+    const fixedTestCase = await this.fixTestCase({
+      config: testCase,
+      error,
+    })
+
+    console.debug(chalk.bold.yellow('[auto-fix]'), this.path)
+    await this.environment.writeYaml(this.getTestCasePath(), fixedTestCase)
+  }
+
+  async readTestCase(): Promise<ConfigClass | undefined> {
+    const configPath = this.getTestCasePath()
+    return this.environment.readYaml<ConfigClass>(configPath)
+  }
+
+  async generateConfig(): Promise<ConfigClass> {
+    return {} as ConfigClass
+  }
+
+  protected async fixTestCase(_params: TestRunnerFixConfigParams<ConfigClass>): Promise<ConfigClass> {
+    throw new Error(`Auto-fix is not implemented for test ${this.path}`)
+  }
+
+  getResult(): TestRunnerResult {
+    return {
+      logs: this.logs,
+      assertions: this.assertions,
+    }
+  }
+
+  protected async assert(condition: () => Promise<boolean> | boolean, message: string, details?: any) {
+    try {
+      const result = await condition()
+      if (result) {
+        this.logMsg(`✅ ${message}`)
+        this.assertions.push({
+          message,
+          result,
+        })
+      } else {
+        this.logMsg(`❌ ${message}`)
+        this.assertions.push({
+          message,
+          result: false,
+          details,
+        })
+      }
+    } catch (e) {
+      this.assertions.push({
+        message,
+        result: false,
+        details: extractIntegrationAppErrorData(e),
+      })
+      this.logMsg(`❌ ${message}: ${(e as Error).message}`)
+    }
+  }
+
+  protected logMsg(message: string) {
+    console.debug(`${' '.repeat(this.level * 2)}${message}`)
+  }
+
+  public getTestCasePath() {
+    return `${this.path}.test.yml`
+  }
+}

package/src/testing/testers/create.ts

@@ -0,0 +1,273 @@
+import {
+  DataCollectionSpec,
+  excludeWriteOnlyFieldsFromSchema,
+  getDataCollectionCreateFields,
+  valueToSchema,
+  extractIntegrationAppErrorData,
+  makeDataLocationPath,
+  getRequiredFieldsFromSchema,
+  getValueAtLocator,
+  setValueAtLocator,
+  walkSchema,
+} from '@integration-app/sdk'
+import chalk from 'chalk'
+
+import { TestEnvironment } from '../environment'
+import { BaseTester } from './base'
+import { getNotMatchingSubFields } from '../../utils/fields'
+
+interface DataCollectionCreateTestConfig {
+  input: {
+    fields: Record<string, any>
+  }
+}
+
+interface ReferenceCollection {
+  key: string
+  parameters?: Record<string, any>
+}
+
+interface ExampleRecord {
+  id: string
+  fields: Record<string, any>
+}
+
+export class DataCollectionCreateTester extends BaseTester<DataCollectionCreateTestConfig> {
+  private dataCollectionKey: string
+  private dataCollection: DataCollectionSpec
+
+  constructor({
+    environment,
+    dataCollectionKey,
+    dataCollection,
+  }: {
+    environment: TestEnvironment
+    dataCollectionKey: string
+    dataCollection: DataCollectionSpec
+  }) {
+    super({
+      environment,
+      path: `data/${dataCollectionKey}/create`,
+    })
+
+    this.dataCollectionKey = dataCollectionKey
+    this.dataCollection = dataCollection
+  }
+
+  async run(config: DataCollectionCreateTestConfig): Promise<void> {
+    const createResponse = await this.environment.client
+      .connection(this.environment.connectionId)
+      .dataCollection(this.dataCollectionKey)
+      .create(config.input)
+
+    await this.assert(() => !!createResponse.id, 'Returned ID of a created record')
+
+    if (createResponse.id) {
+      this.environment.state[this.dataCollectionKey] = {
+        createdRecordId: createResponse.id,
+      }
+    }
+
+    const createFieldsSchema = getDataCollectionCreateFields(this.dataCollection)
+
+    if (this.dataCollection.findById) {
+      const findByIdResponse = await this.environment.client
+        .connection(this.environment.connectionId)
+        .dataCollection(this.dataCollectionKey)
+        .findById({
+          id: createResponse.id,
+        })
+
+      await this.assert(() => !!findByIdResponse.record, 'Record is returned from findById')
+
+      if (findByIdResponse.record) {
+        this.environment.state[this.dataCollectionKey] = {
+          createdRecordId: createResponse.id,
+          createdRecord: findByIdResponse.record.fields,
+        }
+      }
+
+      const schemaToCompare = excludeWriteOnlyFieldsFromSchema(createFieldsSchema)
+      const sentFieldsToCheck = valueToSchema(config.input.fields, schemaToCompare, {
+        skipUnknownFields: true,
+      })
+      const receivedFieldsToCheck = valueToSchema(findByIdResponse.record.fields, schemaToCompare)
+
+      const diff = getNotMatchingSubFields(receivedFieldsToCheck, sentFieldsToCheck)
+
+      await this.assert(() => !diff, 'Returned fields match created fields', {
+        difference: diff,
+        sentFields: sentFieldsToCheck,
+        receivedFields: receivedFieldsToCheck,
+      })
+    }
+
+    const requiredFields = getRequiredFieldsFromSchema(createFieldsSchema)
+
+    if (requiredFields.length > 0) {
+      const requiredPayload: Record<string, any> = {}
+
+      requiredFields.forEach((path) => {
+        const value = getValueAtLocator(config.input.fields, path)
+        if (typeof value !== 'undefined') {
+          setValueAtLocator(requiredPayload, path, value)
+        }
+      })
+
+      const result = await this.environment.client
+        .connection(this.environment.connectionId)
+        .dataCollection(this.dataCollectionKey)
+        .create({ fields: requiredPayload })
+
+      await this.assert(() => !!result.id, 'Record can be created with schema-defined required fields only')
+    }
+  }
+
+  async generateConfig(): Promise<DataCollectionCreateTestConfig> {
+    const createFieldsSchema = getDataCollectionCreateFields(this.dataCollection)
+
+    if (!createFieldsSchema?.properties) {
+      throw new Error('No fields schema found for data collection')
+    }
+
+    const fields = await this.generateFieldsWithLLM(createFieldsSchema)
+
+    return {
+      input: {
+        fields,
+      },
+    }
+  }
+
+  protected async fixTestCase({
+    config,
+    error,
+  }: {
+    config: DataCollectionCreateTestConfig
+    error: any
+  }): Promise<DataCollectionCreateTestConfig> {
+    const errorData = extractIntegrationAppErrorData(error)
+
+    const createFieldsSchema = getDataCollectionCreateFields(this.dataCollection)
+
+    const examples = await this.getExampleRecordsForSchema(createFieldsSchema)
+
+    const prompt = `I'm trying to create a data record in a data collection with the following fields schema:
+
+${JSON.stringify(createFieldsSchema, null, 2)}
+
+I tried to create a record with these fields:
+
+${JSON.stringify(config.input?.fields, null, 2)}
+
+But got this error:
+
+${JSON.stringify(errorData)}
+
+${this.getExampleRecordsPrompt(examples)}
+
+Please analyze the error and provide:
+1. A fixed fields object that would work (as a JSON object)
+2. A short explanation of what was wrong with the original fields
+
+Format your response as a JSON object with two fields:
+{
+  "explanation": "short explanation of what was wrong and how you fixed it",
+  "fields": { ... fixed fields ... }
+}.
+
+Only return the JSON object, no other text or wrapping (like \`\`\`json or \`\`\`).`
+
+    const response = await this.environment.llm.complete({
+      prompt,
+      maxTokens: 10000,
+    })
+
+    const result = JSON.parse(response.trim())
+    console.warn(chalk.bold.yellow('[auto-fix]'), `${this.path}:`, result.explanation)
+
+    return {
+      input: {
+        fields: result.fields,
+      },
+    }
+  }
+
+  private async generateFieldsWithLLM(schema: any): Promise<Record<string, any>> {
+    const examples = await this.getExampleRecordsForSchema(schema)
+
+    const prompt = `Generate a valid JSON object that matches this JSONSchema. Return only the JSON object, no other text.
+
+${this.getExampleRecordsPrompt(examples)}
+
+JSONSchema:
+${JSON.stringify(schema, null, 2)}.
+
+Do not populate fields with null or empty strings, unless the schema explicitly requires it.
+
+Only return the JSON object, no other text or wrapping (like \`\`\`json or \`\`\`).`
+
+    const response = await this.environment.llm.complete({
+      prompt,
+      maxTokens: 10000,
+    })
+
+    const fields = JSON.parse(response.trim())
+    return fields
+  }
+
+  private getExampleRecordsPrompt(examples: Record<string, ExampleRecord[]>): string {
+    const examplesText = Object.entries(examples)
+      .map(([key, records]) => {
+        return `Example records from collection "${key}":\n${JSON.stringify(records, null, 2)}`
+      })
+      .join('\n\n')
+
+    return `When a field has a "referenceCollection" property, its value should be an ID of a record from that collection. Here are some example records you can use as references:
+
+${examplesText}.
+
+If you don't have an example for a given collection, leave the field empty instead of coming up with a fake record id.`
+  }
+
+  private async findReferenceCollections(schema: any): Promise<ReferenceCollection[]> {
+    const references = new Set<string>()
+    const result: ReferenceCollection[] = []
+
+    walkSchema(schema, (obj) => {
+      if (obj.referenceCollection) {
+        const key = obj.referenceCollection.key
+        const parameters = obj.referenceCollection.parameters
+        const refKey = `${key}:${JSON.stringify(parameters || {})}`
+
+        if (!references.has(refKey)) {
+          references.add(refKey)
+          result.push({ key, parameters })
+        }
+      }
+      return obj
+    })
+
+    return result
+  }
+
+  private async fetchExampleRecords(collection: ReferenceCollection): Promise<ExampleRecord[]> {
+    const response = await this.environment.client
+      .connection(this.environment.connectionId)
+      .dataCollection(collection.key)
+      .list({
+        parameters: collection.parameters,
+      })
+    return response.records.map((r) => ({ id: r.id, fields: r.fields }))
+  }
+
+  private async getExampleRecordsForSchema(schema: any): Promise<Record<string, ExampleRecord[]>> {
+    const collections = await this.findReferenceCollections(schema)
+    const result: Record<string, ExampleRecord[]> = {}
+    for (const collection of collections) {
+      const locationPath = makeDataLocationPath(collection)
+      result[locationPath] = await this.fetchExampleRecords(collection)
+    }
+    return result
+  }
+}