@membranehq/cli 0.1.1

package/src/testing/testers/spec.ts

@@ -0,0 +1,30 @@
+import { DataCollectionSpec } from '@integration-app/sdk'
+
+import { TestEnvironment } from '../environment'
+import { BaseTester } from './base'
+
+export class DataCollectionSpecTester extends BaseTester<undefined> {
+  private dataCollection: DataCollectionSpec
+
+  constructor({
+    environment,
+    dataCollectionKey,
+    dataCollection,
+  }: {
+    environment: TestEnvironment
+    dataCollectionKey: string
+    dataCollection: DataCollectionSpec
+  }) {
+    super({
+      environment,
+      path: `data/${dataCollectionKey}/spec`,
+    })
+
+    this.dataCollection = dataCollection
+  }
+
+  async run(): Promise<void> {
+    const fieldCount = Object.keys(this.dataCollection.fieldsSchema).length
+    await this.assert(() => fieldCount > 0, 'Fields schema has field definitions')
+  }
+}

package/src/testing/testers/update.ts

@@ -0,0 +1,284 @@
+import {
+  DataCollectionSpec,
+  excludeWriteOnlyFieldsFromSchema,
+  getDataCollectionUpdateFields,
+  valueToSchema,
+  extractIntegrationAppErrorData,
+  walkSchema,
+  makeDataLocationPath,
+} from '@integration-app/sdk'
+import chalk from 'chalk'
+
+import { TestEnvironment } from '../environment'
+import { BaseTester } from './base'
+import { getNotMatchingSubFields } from '../../utils/fields'
+
+interface DataCollectionUpdateTestConfig {
+  input: {
+    id: string
+    fields: Record<string, any>
+  }
+}
+
+interface ReferenceCollection {
+  key: string
+  parameters?: Record<string, any>
+}
+
+interface ExampleRecord {
+  id: string
+  fields: Record<string, any>
+}
+
+export class DataCollectionUpdateTester extends BaseTester<DataCollectionUpdateTestConfig> {
+  private dataCollectionKey: string
+  private dataCollection: DataCollectionSpec
+
+  constructor({
+    environment,
+    dataCollectionKey,
+    dataCollection,
+  }: {
+    environment: TestEnvironment
+    dataCollectionKey: string
+    dataCollection: DataCollectionSpec
+  }) {
+    super({
+      environment,
+      path: `data/${dataCollectionKey}/update`,
+    })
+
+    this.dataCollectionKey = dataCollectionKey
+    this.dataCollection = dataCollection
+  }
+
+  async run(config: DataCollectionUpdateTestConfig): Promise<void> {
+    const id =
+      (
+        this.environment.state[this.dataCollectionKey] as {
+          createdRecordId: string
+        }
+      )?.createdRecordId || config?.input?.id
+    if (!id) {
+      throw new Error(`No ID found in state or config for ${this.dataCollectionKey} update`)
+    }
+
+    const updateResponse = await this.environment.client
+      .connection(this.environment.connectionId)
+      .dataCollection(this.dataCollectionKey)
+      .update({ ...config.input, id })
+
+    await this.assert(() => !!updateResponse.id, 'Returned ID of updated record')
+
+    if (!this.environment.state[this.dataCollectionKey]) {
+      this.environment.state[this.dataCollectionKey] = {}
+    }
+    ;(
+      this.environment.state[this.dataCollectionKey] as {
+        createdRecordId?: string
+      }
+    ).createdRecordId = updateResponse.id
+
+    if (this.dataCollection.findById) {
+      const findByIdResponse = await this.environment.client
+        .connection(this.environment.connectionId)
+        .dataCollection(this.dataCollectionKey)
+        .findById({
+          id: updateResponse.id,
+        })
+
+      await this.assert(() => !!findByIdResponse.record, 'Record is returned from findById')
+
+      const updateFieldsSchema = getDataCollectionUpdateFields(this.dataCollection)
+      const schemaToCompare = excludeWriteOnlyFieldsFromSchema(updateFieldsSchema)
+      const sentFieldsToCheck = valueToSchema(config.input.fields, schemaToCompare, {
+        skipUnknownFields: true,
+      })
+      const receivedFieldsToCheck = valueToSchema(findByIdResponse.record.fields, schemaToCompare)
+
+      const diff = getNotMatchingSubFields(receivedFieldsToCheck, sentFieldsToCheck)
+
+      await this.assert(() => !diff, 'Returned fields match updated fields', {
+        difference: diff,
+        sentFields: sentFieldsToCheck,
+        receivedFields: receivedFieldsToCheck,
+      })
+    }
+  }
+
+  async generateConfig(): Promise<DataCollectionUpdateTestConfig> {
+    let id = (
+      this.environment.state[this.dataCollectionKey] as {
+        createdRecordId: string
+      }
+    )?.createdRecordId
+
+    if (!id) {
+      if (!this.dataCollection.list) {
+        throw new Error(
+          `Can't find a record to test update operation for ${this.dataCollectionKey}. List operation is not implemented for this data collection`,
+        )
+      }
+
+      const listResponse = await this.environment.client
+        .connection(this.environment.connectionId)
+        .dataCollection(this.dataCollectionKey)
+        .list({})
+
+      if (!listResponse?.records?.length) {
+        throw new Error(`No records found to test update for ${this.dataCollectionKey}`)
+      }
+
+      id = listResponse.records[0].id
+    }
+
+    const updateFieldsSchema = getDataCollectionUpdateFields(this.dataCollection)
+
+    if (!updateFieldsSchema?.properties) {
+      throw new Error('No fields schema found for data collection')
+    }
+
+    const fields = await this.generateFieldsWithLLM(updateFieldsSchema)
+
+    return {
+      input: {
+        id,
+        fields,
+      },
+    }
+  }
+
+  protected async fixTestCase({
+    config,
+    error,
+  }: {
+    config: DataCollectionUpdateTestConfig
+    error: any
+  }): Promise<DataCollectionUpdateTestConfig> {
+    const errorData = extractIntegrationAppErrorData(error)
+
+    const updateFieldsSchema = getDataCollectionUpdateFields(this.dataCollection)
+
+    const examples = await this.getExampleRecordsForSchema(updateFieldsSchema)
+
+    const prompt = `I'm trying to update a data record in a data collection with the following fields schema:
+
+${JSON.stringify(updateFieldsSchema, null, 2)}
+
+I tried to update a record with ID "${config.input?.id}" and these fields:
+
+${JSON.stringify(config.input?.fields, null, 2)}
+
+But got this error:
+
+${JSON.stringify(errorData)}
+
+${this.getExampleRecordsPrompt(examples)}
+
+Please analyze the error and provide:
+1. A fixed fields object that would work (as a JSON object)
+2. A short explanation of what was wrong with the original fields
+
+Format your response as a JSON object with two fields:
+{
+  "explanation": "short explanation of what was wrong and how you fixed it",
+  "fields": { ... fixed fields ... }
+}.
+
+Only return the JSON object, no other text or wrapping (like \`\`\`json or \`\`\`).`
+
+    const response = await this.environment.llm.complete({
+      prompt,
+      maxTokens: 10000,
+    })
+
+    const result = JSON.parse(response.trim())
+
+    console.warn(chalk.bold.yellow('[auto-fix]'), `${this.path}:`, result.explanation)
+
+    return {
+      input: {
+        id: config.input.id,
+        fields: result.fields,
+      },
+    }
+  }
+
+  private async generateFieldsWithLLM(schema: any): Promise<Record<string, any>> {
+    const examples = await this.getExampleRecordsForSchema(schema)
+
+    const prompt = `Generate a valid JSON object that matches this JSONSchema. Return only the JSON object, no other text.
+
+${this.getExampleRecordsPrompt(examples)}
+
+JSONSchema:
+${JSON.stringify(schema, null, 2)}.
+
+Do not populate fields with null or empty strings, unless the schema explicitly requires it.
+
+Only return the JSON object, no other text or wrapping (like \`\`\`json or \`\`\`).`
+
+    const response = await this.environment.llm.complete({
+      prompt,
+      maxTokens: 10000,
+    })
+
+    const fields = JSON.parse(response.trim())
+    return fields
+  }
+
+  private getExampleRecordsPrompt(examples: Record<string, ExampleRecord[]>): string {
+    const examplesText = Object.entries(examples)
+      .map(([key, records]) => {
+        return `Example records from collection "${key}":\n${JSON.stringify(records, null, 2)}`
+      })
+      .join('\n\n')
+
+    return `When a field has a "referenceCollection" property, its value should be an ID of a record from that collection. Here are some example records you can use as references:
+
+${examplesText}.
+
+If you don't have an example for a given collection, leave the field empty instead of coming up with a fake record id.`
+  }
+
+  private async findReferenceCollections(schema: any): Promise<ReferenceCollection[]> {
+    const references = new Set<string>()
+    const result: ReferenceCollection[] = []
+
+    walkSchema(schema, (obj) => {
+      if (obj.referenceCollection) {
+        const key = obj.referenceCollection.key
+        const parameters = obj.referenceCollection.parameters
+        const refKey = `${key}:${JSON.stringify(parameters || {})}`
+
+        if (!references.has(refKey)) {
+          references.add(refKey)
+          result.push({ key, parameters })
+        }
+      }
+      return obj
+    })
+
+    return result
+  }
+
+  private async fetchExampleRecords(collection: ReferenceCollection): Promise<ExampleRecord[]> {
+    const response = await this.environment.client
+      .connection(this.environment.connectionId)
+      .dataCollection(collection.key)
+      .list({
+        parameters: collection.parameters,
+      })
+    return response.records.map((r) => ({ id: r.id, fields: r.fields }))
+  }
+
+  private async getExampleRecordsForSchema(schema: any): Promise<Record<string, ExampleRecord[]>> {
+    const collections = await this.findReferenceCollections(schema)
+    const result: Record<string, ExampleRecord[]> = {}
+    for (const collection of collections) {
+      const locationPath = makeDataLocationPath(collection)
+      result[locationPath] = await this.fetchExampleRecords(collection)
+    }
+    return result
+  }
+}

package/src/utils/auth.ts

@@ -0,0 +1,19 @@
+import jwt from 'jsonwebtoken'
+
+export async function generateAccessToken(key: string, secret: string): Promise<string> {
+  try {
+    return jwt.sign(
+      {
+        name: 'Membrane CLI',
+        isAdmin: true,
+        exp: Math.floor(Date.now() / 1000) + 3600,
+        iss: key,
+      },
+      secret,
+      { algorithm: 'HS512' },
+    )
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error'
+    throw new Error(`Failed to generate access token: ${errorMessage}`)
+  }
+}

package/src/utils/constants.ts

@@ -0,0 +1,27 @@
+import * as path from 'node:path'
+
+export const MEMBRANE_DIR_NAME = 'membrane'
+
+export const CONFIG_FILE_NAME = 'membrane.config.yml'
+
+export function getPaths() {
+  const baseDir = process.cwd()
+
+  const membraneDirPath = path.join(baseDir, MEMBRANE_DIR_NAME)
+  const configPath = path.join(baseDir, CONFIG_FILE_NAME)
+
+  const payloadDirPath = membraneDirPath
+  const workspaceDataFilePath = path.join(payloadDirPath, 'workspace.yaml')
+
+  return {
+    membraneDirPath,
+    configPath,
+    payloadDirPath,
+    workspaceDataFilePath,
+  }
+}
+
+export function getBasePath() {
+  const paths = getPaths()
+  return paths.membraneDirPath
+}

package/src/utils/fields.ts

@@ -0,0 +1,83 @@
+/**
+ * Returns a subset of subFields that have values that do not match the values in allFields.
+ * For nested objects and arrays, returns only items that do not match.
+ */
+export function getNotMatchingSubFields(
+  allFields: Record<string, any>,
+  subFields: Record<string, any>,
+): Record<string, any> | null {
+  const result: Record<string, any> = {}
+  for (const key in subFields) {
+    if (!(key in allFields)) {
+      result[key] = subFields[key]
+      continue
+    }
+    const origVal = allFields[key]
+    const subVal = subFields[key]
+    if (origVal && subVal && typeof origVal === 'object' && typeof subVal === 'object') {
+      if (Array.isArray(origVal) && Array.isArray(subVal)) {
+        const notMatchingValues = subVal.filter(
+          (subItem) =>
+            !origVal.some((origItem) => {
+              if (
+                typeof origItem === 'object' &&
+                typeof subItem === 'object' &&
+                origItem !== null &&
+                subItem !== null &&
+                !Array.isArray(origItem) &&
+                !Array.isArray(subItem)
+              ) {
+                for (const subKey in subItem) {
+                  if (!(subKey in origItem) || !softCompare(origItem[subKey], subItem[subKey])) {
+                    return false
+                  }
+                }
+                return true
+              } else {
+                return JSON.stringify(origItem) === JSON.stringify(subItem)
+              }
+            }),
+        )
+        if (notMatchingValues.length > 0) {
+          result[key] = notMatchingValues
+        }
+      } else if (!Array.isArray(origVal) && !Array.isArray(subVal)) {
+        const subResult = getNotMatchingSubFields(origVal, subVal)
+        if (subResult !== null) {
+          result[key] = subResult
+        }
+      } else {
+        result[key] = subVal
+      }
+      continue
+    }
+    if (!softCompare(origVal, subVal)) {
+      result[key] = subVal
+    }
+  }
+  return Object.keys(result).length === 0 ? null : result
+}
+
+// it shouldnt fail because one is string and other is number in the account create etc.
+// more details in the match test operation.
+
+/**
+ * Compare two values in a way that is forgiving for typical API quirks.
+ */
+export function softCompare(a: any, b: any) {
+  if (a == b) {
+    return true
+  }
+
+  if (a?.toString?.() === b?.toString?.()) {
+    return true
+  }
+
+  const dateA = new Date(a)
+  const dateB = new Date(b)
+  if (!isNaN(dateA.getTime()) && !isNaN(dateB.getTime())) {
+    return dateA.getTime() === dateB.getTime()
+  }
+
+  return false
+}

package/src/utils/logger.ts

@@ -0,0 +1,106 @@
+import chalk from 'chalk'
+
+type LogLevel = 'info' | 'success' | 'warning' | 'error' | 'debug'
+
+interface LogOptions {
+  prefix?: string
+  suffix?: string
+  timestamp?: boolean
+  error?: string
+  color?: string
+}
+
+const LOG_ICONS = {
+  info: '📝',
+  success: '✨',
+  warning: '⚠️',
+  error: '❌',
+  debug: '🔍',
+}
+
+const LOG_STYLES = {
+  info: chalk.blue,
+  success: chalk.green,
+  warning: chalk.yellow,
+  error: chalk.red,
+  debug: chalk.gray,
+}
+
+export class Logger {
+  private static formatMessage(message: string, level: LogLevel, options: LogOptions = {}) {
+    const timestamp = options.timestamp ? `${chalk.gray(new Date().toISOString())} ` : ''
+    const prefix = options.prefix ? `${chalk.gray(options.prefix)} ` : ''
+    const suffix = options.suffix ? ` ${chalk.gray(options.suffix)}` : ''
+    const icon = LOG_ICONS[level]
+    const style = LOG_STYLES[level]
+
+    return `${timestamp}${icon} ${prefix}${style(message)}${suffix}`
+  }
+
+  static info(message: string, options?: LogOptions) {
+    let chalkMethod = chalk.blue
+
+    if (options?.color) {
+      chalkMethod = chalk[options.color.toLowerCase()] || chalk.blue
+    }
+
+    if (options?.timestamp) {
+      const timestamp = new Date().toLocaleTimeString()
+      console.debug(`${chalk.gray(`[${timestamp}]`)} ${chalkMethod(message)}`)
+    } else {
+      console.debug(chalkMethod(message))
+    }
+  }
+
+  static group(_message: string, _options?: LogOptions) {}
+
+  static groupEnd() {}
+  static newLine() {}
+
+  static success(message: string, options?: LogOptions) {
+    console.debug(this.formatMessage(message, 'success', options))
+  }
+
+  static warning(message: string, options?: LogOptions) {
+    console.debug(this.formatMessage(message, 'warning', options))
+  }
+
+  static error(message: string, options?: LogOptions) {
+    console.error(this.formatMessage(message, 'error', options))
+  }
+
+  static debug(message: string, options?: LogOptions) {
+    if (options?.prefix) {
+      console.debug(chalk.gray(`[${options.prefix}] ${message}`), options.error ? `\n${chalk.red(options.error)}` : '')
+    } else {
+      console.debug(chalk.gray(message), options?.error ? `\n${chalk.red(options.error)}` : '')
+    }
+  }
+
+  static step(_stepNumber: number, _message: string) {}
+
+  static header(message: string) {
+    console.debug()
+    console.debug(chalk.bold.cyan(`▶ ${message}`))
+    console.debug()
+  }
+
+  static table(data: Record<string, any>[], columns?: string[]) {
+    if (data.length === 0) {
+      return
+    }
+
+    const columnsToShow = columns || Object.keys(data[0])
+    const _formattedData = data.map((item) => {
+      const formatted: Record<string, any> = {}
+      columnsToShow.forEach((col) => {
+        formatted[col] = item[col]
+      })
+      return formatted
+    })
+    // eslint-disable-next-line no-console
+    console.table(_formattedData, columnsToShow)
+  }
+
+  static divider() {}
+}

package/src/utils/templating.ts

@@ -0,0 +1,50 @@
+import { faker } from '@faker-js/faker'
+import template from 'lodash/template.js'
+import templateSettings from 'lodash/templateSettings.js'
+
+templateSettings.interpolate = /{{([\s\S]+?)}}/g
+
+function processNode(node: any, state: any): any {
+  if (typeof node === 'string') {
+    const compiled = template(node)
+    const templateData = {
+      state,
+      random: {
+        number: () => faker.number.int(),
+        alphaNumeric: (count: number) => faker.string.alphanumeric(count),
+      },
+      faker: {
+        company: {
+          name: () => faker.company.name(),
+          catchPhrase: () => faker.company.catchPhrase(),
+        },
+        internet: {
+          email: () => faker.internet.email(),
+        },
+      },
+      entries: state['journal-entries'],
+      orders: state['purchase-orders'] || state['sales-orders'],
+      bills: state['vendor-bills'],
+    }
+
+    const result = compiled(templateData)
+
+    if (node.includes('{{') && node.includes('}}')) {
+      console.debug(`[TEMPLATE] Input: ${node}`)
+      console.debug(`[TEMPLATE] Output: ${result}`)
+    }
+
+    return result
+  }
+  if (Array.isArray(node)) {
+    return node.map((item) => processNode(item, state))
+  }
+  if (typeof node === 'object' && node !== null) {
+    return Object.fromEntries(Object.entries(node).map(([key, value]) => [key, processNode(value, state)]))
+  }
+  return node
+}
+
+export function handleTemplate(config: any, state: any) {
+  return processNode(config, state)
+}

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "types": ["node", "vitest/globals"],
+    "allowImportingTsExtensions": true,
+    "allowJs": true,
+    "strict": true,
+    "noEmit": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "jsx": "react-jsx",
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src/**/*.ts", "src/**/*.tsx"],
+  "exclude": ["node_modules"]
+}