@membranehq/cli 0.1.1

package/src/components/WorkspaceStatus.tsx

@@ -0,0 +1,61 @@
+import { Box, Text } from 'ink'
+import useSWRImmutable from 'swr/immutable'
+
+import { RecentChanges } from './RecentChanges'
+import { useStatus } from '../modules/status/useStatus'
+
+import type { OrgWorkspace } from '@integration-app/sdk'
+
+function truncateWorkspaceName(name: string, maxLength: number = 30): string {
+  if (name.length <= maxLength) {
+    return name
+  }
+  return name.slice(0, maxLength - 3) + '...'
+}
+
+export const WorkspaceSync = () => {
+  const { workspaceKey, cwd } = useStatus()
+  const width = Math.min(100, process.stdout.columns || 100)
+
+  // Fetch workspace data to get the workspace name
+  const { data } = useSWRImmutable('/account')
+  const workspaces = (data as any)?.workspaces as OrgWorkspace[]
+
+  // Find the current workspace by key to get its name
+  const currentWorkspace = workspaces?.find((ws: any) => ws.key === workspaceKey)
+  const workspaceName = currentWorkspace?.name
+  const displayName = workspaceName ? truncateWorkspaceName(workspaceName) : workspaceKey
+
+  return (
+    <Box flexDirection='column' paddingX={1} borderStyle='round' borderTop width={width}>
+      {/* Header row with sync emoji and status */}
+      <Box marginTop={-1} marginBottom={1}>
+        <Text bold>
+          🔄 Workspaces — <Text color='green'>in sync</Text>
+        </Text>
+      </Box>
+      {/* Local row */}
+      <Box>
+        <Box width={12}>
+          <Text color='grey'>Local:</Text>
+        </Box>
+        <Text color='grey'>{cwd}</Text>
+      </Box>
+      {/* Remote row (always shown) */}
+      <Box>
+        <Box width={12}>
+          <Text color='grey'>Remote:</Text>
+        </Box>
+        {workspaceKey ? (
+          <Text color='grey'>{displayName} [o: open in console] [w: change]</Text>
+        ) : (
+          <Text>
+            <Text color='yellow'>not selected</Text> [w: select]
+          </Text>
+        )}
+      </Box>
+
+      <RecentChanges />
+    </Box>
+  )
+}

package/src/contexts/FileWatcherContext.tsx

@@ -0,0 +1,81 @@
+import React, { createContext, useContext, useEffect, useState } from 'react'
+import { WorkspaceElementType } from '@integration-app/sdk'
+
+import { FileChangeEventType } from '../modules/file-watcher/types'
+import { getTypeAndKeyFromPath } from '../modules/workspace-element-service/getTypeAndKeyFromPath'
+
+import type { FileWatcher } from '../modules/file-watcher'
+import type { FileChangeEvent } from '../modules/file-watcher/types'
+
+interface FileChangeWithType extends FileChangeEvent {
+  readonly type: FileChangeEventType
+  readonly elementKey: string
+  readonly elementType: WorkspaceElementType
+  readonly timestamp: number
+}
+
+interface FileWatcherContextValue {
+  readonly fileWatcher: FileWatcher | null
+  readonly recentChanges: readonly FileChangeWithType[]
+}
+
+const FileWatcherContext = createContext<FileWatcherContextValue | null>(null)
+
+interface FileWatcherProviderProps {
+  readonly children: React.ReactNode
+  readonly fileWatcher: FileWatcher
+}
+
+export function FileWatcherProvider({ children, fileWatcher }: FileWatcherProviderProps) {
+  const [recentChanges, setRecentChanges] = useState<FileChangeWithType[]>([])
+
+  useEffect(() => {
+    const handleFileChange = (type: FileChangeEventType) => (event: FileChangeEvent) => {
+      const elementRef = getTypeAndKeyFromPath(event.relativePath)
+      if (!elementRef) {
+        return
+      }
+
+      const changeWithType: FileChangeWithType = {
+        ...event,
+        elementType: elementRef.type,
+        elementKey: elementRef.key,
+        type,
+        timestamp: Date.now(),
+      }
+
+      setRecentChanges((prev) => [changeWithType, ...prev.slice(0, 2)]) // keep only last 3 changes
+    }
+
+    const handleAdded = handleFileChange('added')
+    const handleChanged = handleFileChange('changed')
+    const handleDeleted = handleFileChange('deleted')
+
+    fileWatcher.on('added', handleAdded)
+    fileWatcher.on('changed', handleChanged)
+    fileWatcher.on('deleted', handleDeleted)
+
+    return () => {
+      fileWatcher.off('added', handleAdded)
+      fileWatcher.off('changed', handleChanged)
+      fileWatcher.off('deleted', handleDeleted)
+    }
+  }, [fileWatcher])
+
+  const value: FileWatcherContextValue = {
+    fileWatcher,
+    recentChanges,
+  }
+
+  return <FileWatcherContext.Provider value={value}>{children}</FileWatcherContext.Provider>
+}
+
+export function useFileWatcher(): FileWatcherContextValue {
+  const context = useContext(FileWatcherContext)
+
+  if (!context) {
+    throw new Error('useFileWatcher must be used within a FileWatcherProvider')
+  }
+
+  return context
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+
+import { runAgent } from './agent'
+import { runCLI } from './cli'
+
+const AGENT_ARGUMENTS = ['--cwd']
+
+function hasCLIArguments(): boolean {
+  const args = process.argv.slice(2)
+  return args.some((arg, index) => {
+    // Skip the value of an agent argument
+    if (index > 0 && AGENT_ARGUMENTS.includes(args[index - 1])) {
+      return false
+    }
+    // Check if current argument is not an agent argument
+    return !AGENT_ARGUMENTS.includes(arg)
+  })
+}
+
+// Check if there are any arguments that are not agent arguments
+if (1 || hasCLIArguments()) {
+  // Run single-command CLI mode
+  runCLI()
+} else {
+  // Run in agent mode (interactive)
+  runAgent()
+}

package/src/legacy/commands/pullWorkspace.ts

@@ -0,0 +1,70 @@
+import fs from 'fs'
+import path from 'path'
+
+import { IntegrationAppClient, WorkspaceElementType } from '@integration-app/sdk'
+import YAML from 'js-yaml'
+
+import { getWorkspaceData, coloredLog } from '../util'
+import { getWorkspaceElementTypePath } from '../workspace-elements'
+import { pullConnectors } from '../workspace-elements/connectors'
+
+interface ExportPackageOptions {
+  outputPath?: string
+  allConnectors?: boolean
+  client?: IntegrationAppClient
+}
+
+export async function pullWorkspace(options: ExportPackageOptions): Promise<void> {
+  const { outputPath, allConnectors, client } = options
+
+  await pullConnectors({
+    client,
+    basePath: outputPath,
+    allConnectors,
+  })
+
+  const workspaceData = await getWorkspaceData(client)
+
+  for (const elementType of Object.keys(workspaceData)) {
+    if (workspaceData[elementType].length > 0) {
+      try {
+        fs.readdirSync(outputPath)
+      } catch (_err) {
+        fs.mkdirSync(outputPath, { recursive: true })
+      }
+
+      const elementTypePath = getWorkspaceElementTypePath(elementType as WorkspaceElementType)
+
+      for (const element of workspaceData[elementType]) {
+        try {
+          fs.readdirSync(path.join(outputPath, elementTypePath, `${element.key}`))
+        } catch (_err) {
+          fs.mkdirSync(path.join(outputPath, elementTypePath, `${element.key}`), {
+            recursive: true,
+          })
+        }
+        if (element.integration || element.integrationKey) {
+          const integrationKey = element.integration ? element.integration.key : element.integrationKey
+          try {
+            fs.readdirSync(path.join(outputPath, elementTypePath, `${element.key}`, `${integrationKey}`))
+          } catch (_err) {
+            fs.mkdirSync(path.join(outputPath, elementTypePath, `${element.key}`, `${integrationKey}`), {
+              recursive: true,
+            })
+          }
+          fs.writeFileSync(
+            path.join(outputPath, elementTypePath, `${element.key}`, `${integrationKey}`, `${integrationKey}.yaml`),
+            YAML.dump(element),
+          )
+        } else {
+          fs.writeFileSync(
+            path.join(outputPath, elementTypePath, element.key, `${element.key}.yaml`),
+            YAML.dump(element),
+          )
+        }
+      }
+    }
+  }
+
+  coloredLog(`Data written to ${outputPath} successfully.`, 'BgGreen')
+}

package/src/legacy/commands/pushWorkspace.ts

@@ -0,0 +1,246 @@
+import { WorkspaceElementSpecs } from '@integration-app/sdk'
+import { toCamelCase } from 'js-convert-case'
+
+import { Logger } from '../../utils/logger'
+import { INTEGRATION_ELEMENTS } from '../integrationElements'
+import { PushContext, WorkspaceData } from '../push/types'
+import { readIntegrationLevelElements, readTopLevelElements } from '../reader/index'
+import { getWorkspaceData, hasParent, splitWorkspaceData, coloredLog } from '../util'
+import { pushConnectors } from '../workspace-elements/connectors'
+import { mergeWorkspaceData } from '../workspace-elements/index'
+
+export async function pushWorkspace({ basePath, client }) {
+  const response = await client.get('org-workspace-id')
+  const workspaceId = response.id
+
+  const workspaceData = await getWorkspaceData(client, 'minified')
+
+  const data: WorkspaceData = {}
+  Logger.info(`Loading local workspace data`)
+
+  const topLevelElements = await readTopLevelElements()
+  mergeWorkspaceData(data, topLevelElements)
+
+  const integrationLevelElements = await readIntegrationLevelElements()
+  mergeWorkspaceData(data, integrationLevelElements)
+
+  const { universalElements, integrationSpecificElements } = splitWorkspaceData(data)
+
+  const pushContext: PushContext = {
+    connectorsMapping: {},
+  }
+
+  await syncIntegrations({
+    basePath,
+    sourceData: data,
+    workspaceData,
+    pushContext,
+    iApp: client,
+    workspaceId,
+  })
+
+  // Re-fetch latest integrations
+  workspaceData.integration = await client.integrations.findAll()
+
+  coloredLog(`Syncing Universal Elements`, 'BgBlue')
+  Logger.group('Syncing Universal Elements')
+  await syncElements(universalElements, workspaceData, client)
+  Logger.groupEnd()
+
+  coloredLog(`Syncing Integration Specific Elements`, 'BgBlue')
+  Logger.group('Syncing Integration Specific Elements')
+
+  await syncElements(integrationSpecificElements, workspaceData, client)
+  Logger.groupEnd()
+
+  Logger.success(`Data written successfully.`)
+
+  return pushContext
+}
+
+async function syncIntegrations({
+  basePath,
+  sourceData,
+  workspaceData,
+  pushContext,
+  iApp,
+  workspaceId,
+}: {
+  basePath: string
+  sourceData: any
+  workspaceData: WorkspaceData
+  pushContext: PushContext
+  iApp: any
+  workspaceId: string
+}) {
+  coloredLog('Matching imported integrations and connectors with existing ones', 'BgBlue')
+
+  await pushConnectors({
+    client: iApp,
+    basePath,
+    workspaceData,
+    pushContext,
+    workspaceId,
+  })
+
+  const integrationMissmatchErrors = []
+  for (const integration of sourceData.integration) {
+    const matchedIntegration = workspaceData.integration.find((item) => item.key == integration.key)
+
+    const integrationPayload = {
+      connectorId: integration.connectorId,
+      connectorVersion: integration.connectorVersion,
+      name: integration.name,
+      key: integration.key,
+    }
+
+    if (pushContext.connectorsMapping[integration.connectorId]) {
+      integrationPayload.connectorId = pushContext.connectorsMapping[integration.connectorId]
+    }
+
+    if (!matchedIntegration) {
+      console.debug(`[Push] Creating integration ${integration.key}`)
+      await iApp.integrations.create(integrationPayload)
+    } else {
+      console.debug(`[Push] Updating integration ${integration.key}`)
+      await iApp.integration((matchedIntegration as any).id).patch(integrationPayload)
+    }
+
+    coloredLog(`Imported integration ${integration.name} (${integration.key})`, 'Green')
+  }
+  if (integrationMissmatchErrors.length > 0) {
+    Logger.table(
+      integrationMissmatchErrors.map((item) => ({
+        key: item.key,
+        name: item.name,
+      })),
+    )
+    coloredLog(
+      'Integration missmatch errors. Make sure you have those applications in your destination workspace',
+      'BgRed',
+    )
+    throw new Error('Integration missmatch errors')
+  }
+}
+
+async function syncElements(data, workspaceData, iApp) {
+  for (const elementType of Object.keys(data)) {
+    const isExportable = INTEGRATION_ELEMENTS[elementType]?.exportable
+
+    if (isExportable === false) continue
+
+    for (const element of data[elementType]) {
+      // Cleanup
+      delete element.integrationId
+
+      const destinationElement = matchElement(element, elementType, workspaceData)
+      const singleElementAccessorKey = toCamelCase(WorkspaceElementSpecs[elementType].name)
+      const pluralElementAccessorKey = toCamelCase(WorkspaceElementSpecs[elementType].namePlural)
+
+      if (destinationElement) {
+        if (!hasParent(element) || (elementIsCustomized(element) && hasParent(element))) {
+          // Update the element without parent OR the integration-specific element that should be customized
+          await iApp[singleElementAccessorKey](destinationElement.id).put(element)
+          coloredLog(
+            `Updated ${element.integrationKey || 'universal'} ${INTEGRATION_ELEMENTS[elementType].element} ${element.key}`,
+            'Cyan',
+          )
+        } else if (hasParent(element)) {
+          // Reset Integration-specific element if it has universal parent
+          try {
+            await iApp[singleElementAccessorKey](destinationElement.id).reset()
+          } catch (error) {
+            console.debug(destinationElement, element)
+            throw error
+          }
+          coloredLog(
+            `Customization reset ${element.integrationKey || 'universal'} ${INTEGRATION_ELEMENTS[elementType].element} ${element.key}`,
+            'Magenta',
+          )
+        } else {
+          coloredLog(
+            `Corrupted. Migrate Manually ${element.integrationKey || 'universal'} ${INTEGRATION_ELEMENTS[elementType].element} ${element.key}`,
+            'Red',
+          )
+        }
+      } else {
+        // Create the element
+
+        if (hasParent(element)) {
+          try {
+            await iApp[singleElementAccessorKey]({
+              key: element.key,
+            }).apply([element.integrationKey])
+          } catch (_error) {
+            try {
+              await iApp[singleElementAccessorKey]({
+                key: element.key,
+                integrationKey: element.integrationKey,
+              }).put(element)
+            } catch (_error) {
+              await iApp[pluralElementAccessorKey].create({
+                ...element,
+                integrationId: workspaceData.integration.find((item) => item.key == element.integrationKey).id,
+              })
+            }
+          }
+          if (elementIsCustomized(element)) {
+            try {
+              await iApp[singleElementAccessorKey]({
+                key: element.key,
+                integrationKey: element.integrationKey,
+              }).put(element)
+            } catch (_error) {
+              console.debug(destinationElement, element)
+              throw _error
+            }
+            coloredLog(
+              `Applied & Customized ${element.integrationKey || 'universal'} ${INTEGRATION_ELEMENTS[elementType].element} ${element.key}`,
+              'Green',
+            )
+          } else {
+            coloredLog(
+              `Applied universal ${INTEGRATION_ELEMENTS[elementType].element} ${element.key} to ${element.integrationKey} `,
+              'Green',
+            )
+          }
+        } else {
+          try {
+            if (element.integrationKey) {
+              delete element.integration
+              element.integrationId = workspaceData.integration.find((item) => item.key == element.integrationKey).id
+            }
+            await iApp[pluralElementAccessorKey].create(element)
+          } catch (_error) {
+            console.debug(`Error creating element ${elementType}`, hasParent(element), element.integrationKey, element)
+            throw _error
+          }
+          coloredLog(
+            `Created ${element.integrationKey || 'universal'} ${INTEGRATION_ELEMENTS[elementType].element} ${element.key}`,
+            'Green',
+          )
+        }
+      }
+      //console.debug(elementType, element.key, element.integrationKey || "universal")
+    }
+  }
+}
+
+function elementIsCustomized(element) {
+  return element.integrationKey && (element.customized || element.isCustomized)
+}
+
+function matchElement(element, elementType, workspaceData) {
+  const matchedElements = workspaceData[elementType]?.filter(
+    (item) => item.key == element.key && item.integrationKey == element.integrationKey && !item.archivedAt,
+  )
+  if (matchedElements.length > 1) {
+    throw new Error(
+      `More than one ${element.integrationKey || 'universal'} ${elementType} with key ${element.key} found in the workspace`,
+    )
+  }
+  if (matchedElements.length == 1) {
+    return matchedElements[0]
+  }
+  return
+}

package/src/legacy/integrationElements.ts

@@ -0,0 +1,78 @@
+import { WorkspaceElementType } from '@integration-app/sdk'
+
+export const INTEGRATION_ELEMENTS = {
+  [WorkspaceElementType.Integration]: {
+    element: 'integration',
+    elements: 'integrations',
+    exportable: false,
+    exportCleanup: (el) => {
+      return {
+        id: el.id,
+        key: el.key,
+        name: el.name,
+        connectorId: el.connectorId,
+        baseUri: el.baseUri,
+        connectorVersion: el.connectorVersion,
+      }
+    },
+  },
+  [WorkspaceElementType.Connector]: {
+    element: 'connector',
+    elements: 'connectors',
+    exportable: false,
+  },
+  [WorkspaceElementType.Action]: {
+    element: 'action',
+    elements: 'actions',
+    integrationSpecific: true,
+    exportCleanup: (el) => {
+      delete el.integration
+      return el
+    },
+  },
+  [WorkspaceElementType.AppDataSchema]: {
+    element: 'appDataSchema',
+    elements: 'appDataSchemas',
+  },
+  [WorkspaceElementType.AppEventType]: {
+    element: 'appEventType',
+    elements: 'appEventTypes',
+  },
+  [WorkspaceElementType.DataLinkTable]: {
+    element: 'dataLinkTable',
+    elements: 'dataLinkTables',
+  },
+  [WorkspaceElementType.DataSource]: {
+    element: 'dataSource',
+    elements: 'dataSources',
+    integrationSpecific: true,
+  },
+  [WorkspaceElementType.FieldMapping]: {
+    element: 'fieldMapping',
+    elements: 'fieldMappings',
+    integrationSpecific: true,
+    exportCleanup: (fieldMapping) => {
+      delete fieldMapping.dataSourceId
+      return fieldMapping
+    },
+  },
+  [WorkspaceElementType.Flow]: {
+    element: 'flow',
+    elements: 'flows',
+    integrationSpecific: true,
+  },
+}
+
+export function baseExportCleanup(element) {
+  delete element.workspaceId
+  delete element.createdAt
+  delete element.updatedAt
+  delete element.revision
+  delete element.parentRevision
+  Object.keys(element).map((key) => {
+    if (key.match(/universal.*Revision/g)) {
+      delete element[key]
+    }
+  })
+  return element
+}

package/src/legacy/push/types.ts

@@ -0,0 +1,17 @@
+import { WorkspaceElementType } from '@integration-app/sdk'
+
+export type ConnectorsMapping = Record<string, string>
+
+export interface PushContext {
+  connectorsMapping: ConnectorsMapping
+}
+
+interface WorkspaceElement {
+  key: string
+  integration?: {
+    key: string
+  }
+  integrationKey?: string
+}
+
+export type WorkspaceData = Partial<Record<WorkspaceElementType, WorkspaceElement[]>>

package/src/legacy/reader/index.ts

@@ -0,0 +1,113 @@
+import fs from 'fs'
+import path from 'path'
+
+import { WorkspaceElementType, WorkspaceElement, WorkspaceElementSpecs } from '@integration-app/sdk'
+import { toCamelCase } from 'js-convert-case'
+import yaml from 'js-yaml'
+
+import { getPaths } from '../../utils/constants'
+import { Logger } from '../../utils/logger'
+import { WorkspaceData } from '../push/types'
+import { getWorkspaceElementTypePath, PUSHABLE_ELEMENT_TYPES } from '../workspace-elements'
+
+export async function readTopLevelElements(): Promise<WorkspaceData> {
+  const result: WorkspaceData = {}
+
+  for (const elementType of PUSHABLE_ELEMENT_TYPES) {
+    Logger.info(`Loading ${WorkspaceElementSpecs[elementType].namePlural}`)
+
+    const elements: WorkspaceElement[] = []
+
+    const paths = getPaths()
+    const basePath = paths.membraneDirPath
+
+    const elementsDirs = [path.join(basePath, getWorkspaceElementTypePath(elementType))]
+
+    const legacyDir = path.join(basePath, toCamelCase(getWorkspaceElementTypePath(elementType))) // Legacy: we used `dataSources` instead of `data-sources`.
+
+    if (!elementsDirs.includes(legacyDir)) {
+      elementsDirs.push(legacyDir)
+    }
+
+    for (const elementsDir of elementsDirs) {
+      if (!fs.existsSync(elementsDir)) {
+        continue
+      }
+
+      elements.push(...(await readWorkspaceElementsFromDir(elementsDir)))
+    }
+
+    result[elementType] = [...(result[elementType] || []), ...elements]
+  }
+
+  return result
+}
+
+export async function readIntegrationLevelElements(): Promise<WorkspaceData> {
+  const result: WorkspaceData = {}
+  const paths = getPaths()
+  const basePath = paths.membraneDirPath
+  const integrationsPath = path.join(basePath, getWorkspaceElementTypePath(WorkspaceElementType.Integration))
+
+  const integrationItems = fs.readdirSync(integrationsPath)
+
+  for (const integrationPath of integrationItems) {
+    if (fs.statSync(path.join(integrationsPath, integrationPath)).isDirectory()) {
+      const integration = await readWorkspaceElementFromDirItem(integrationsPath, integrationPath)
+
+      for (const elementType of PUSHABLE_ELEMENT_TYPES) {
+        const elementTypeDir = path.join(integrationsPath, integrationPath, getWorkspaceElementTypePath(elementType))
+        const elements = await readWorkspaceElementsFromDir(elementTypeDir)
+        elements.forEach((element) => {
+          ;(element as any).integrationKey = integration?.key
+        })
+        result[elementType] = [...(result[elementType] || []), ...elements]
+      }
+    }
+  }
+
+  return result
+}
+
+export async function readWorkspaceElementsFromDir(dirPath: string): Promise<WorkspaceElement[]> {
+  const elements: WorkspaceElement[] = []
+
+  if (!fs.existsSync(dirPath)) {
+    return elements
+  }
+
+  Logger.info(`Reading ${dirPath}`)
+
+  const elementItems = fs.readdirSync(dirPath)
+
+  for (const elementPath of elementItems) {
+    const element = await readWorkspaceElementFromDirItem(dirPath, elementPath)
+
+    if (element) {
+      elements.push(element)
+
+      Logger.success(`Loaded ${elementPath}`)
+    }
+  }
+
+  return elements
+}
+
+export async function readWorkspaceElementFromDirItem(
+  dirPath: string,
+  elementPath: string,
+): Promise<WorkspaceElement | undefined> {
+  const fullElementPath = path.join(dirPath, elementPath)
+
+  const elementPathIsDirectory = fs.statSync(fullElementPath).isDirectory()
+
+  const elementSpecPath = elementPathIsDirectory ? path.join(fullElementPath, `${elementPath}.yaml`) : fullElementPath
+
+  if (fs.existsSync(elementSpecPath)) {
+    const elementSpec = yaml.load(fs.readFileSync(elementSpecPath, 'utf8'))
+
+    return elementSpec as WorkspaceElement
+  }
+
+  return undefined
+}