@things-factory/integration-sftp 4.3.653 → 4.3.667

package/server/storage/providers/sftp-storage.provider.ts

@@ -0,0 +1,157 @@
+import Client from 'ssh2-sftp-client'
+import { StorageProvider, StorageConfig } from '../storage-provider.interface'
+import { logger } from '@things-factory/env'
+
+export class SftpStorageProvider implements StorageProvider {
+  private sftp: Client
+  private config: StorageConfig
+  private connected: boolean = false
+
+  constructor(config: StorageConfig) {
+    this.config = config
+    this.sftp = new Client()
+  }
+
+  async connect(): Promise<void> {
+    try {
+      const connectionConfig = {
+        host: this.config.host!,
+        port: this.config.port || 22,
+        username: this.config.username!,
+        password: this.config.password,
+        privateKey: this.config.privateKey,
+        readyTimeout: this.config.timeout || 20000,
+        retries: this.config.retries || 3
+      }
+
+      await this.sftp.connect(connectionConfig)
+      this.connected = true
+      logger.info('SFTP Storage Provider connected successfully')
+    } catch (error) {
+      this.connected = false
+      throw new Error(`Failed to connect to SFTP: ${error}`)
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    if (this.connected) {
+      await this.sftp.end()
+      this.connected = false
+      logger.info('SFTP Storage Provider disconnected')
+    }
+  }
+
+  isConnected(): boolean {
+    return this.connected
+  }
+
+  async readFile(path: string, encoding?: string): Promise<any> {
+    const fullPath = this.getFullPath(path)
+    const buffer = await this.sftp.get(fullPath)
+
+    if (encoding) {
+      return buffer.toString(encoding)
+    }
+
+    return buffer
+  }
+
+  async writeFile(path: string, content: any, encoding?: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    const buffer = Buffer.from(content)
+
+    await this.sftp.put(buffer, fullPath)
+  }
+
+  async uploadFile(params: {
+    stream: any
+    filename: string
+    uploadPath: string
+  }): Promise<{ id: string; path: string; size: number }> {
+    const fullPath = this.getFullPath(`${params.uploadPath}/${params.filename}`)
+    let size = 0
+
+    return new Promise<{ id: string; path: string; size: number }>((resolve, reject) => {
+      const uploadStream = this.sftp.createWriteStream(fullPath)
+
+      // Safety timeout to avoid hanging indefinitely
+      const timeoutMs = (this.config.timeout as number) || 30000
+      const timer = setTimeout(() => {
+        reject(new Error(`SFTP upload timed out for ${fullPath}`))
+      }, timeoutMs)
+
+      const onResolve = () => {
+        clearTimeout(timer)
+        resolve({ id: Date.now().toString(), path: params.uploadPath, size })
+      }
+
+      uploadStream.on('error', (error: any) => {
+        clearTimeout(timer)
+        reject(error)
+      })
+      uploadStream.on('close', onResolve)
+      uploadStream.on('finish', onResolve)
+
+      params.stream
+        .on('error', (error: any) => {
+          clearTimeout(timer)
+          reject(error)
+        })
+        .on('data', (chunk: any) => (size += chunk.length))
+        .pipe(uploadStream)
+    })
+  }
+
+  async deleteFile(path: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    await this.sftp.delete(fullPath)
+  }
+
+  async moveFile(params: { source: string; destination: string }): Promise<void> {
+    const sourcePath = this.getFullPath(params.source)
+    const destPath = this.getFullPath(params.destination)
+
+    await this.sftp.rename(sourcePath, destPath)
+  }
+
+  async readFolders(params: { path: string }): Promise<any[]> {
+    const fullPath = this.getFullPath(params.path)
+    const list = await this.sftp.list(fullPath)
+
+    return list.filter(item => item.type === '-') // Only files, not directories
+  }
+
+  async createDirectory(path: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    await this.sftp.mkdir(fullPath, true) // true for recursive
+  }
+
+  async deleteDirectory(path: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    await this.sftp.rmdir(fullPath, true) // true for recursive
+  }
+
+  async fileExists(path: string): Promise<boolean> {
+    try {
+      const fullPath = this.getFullPath(path)
+      await this.sftp.stat(fullPath)
+      return true
+    } catch (error) {
+      return false
+    }
+  }
+
+  async getFileSize(path: string): Promise<number> {
+    const fullPath = this.getFullPath(path)
+    const stats = await this.sftp.stat(fullPath)
+    return stats.size
+  }
+
+  private getFullPath(path: string): string {
+    if (this.config.basePath) {
+      const base = this.config.basePath.startsWith('/') ? this.config.basePath : `/${this.config.basePath}`
+      return `${base}/${path}`.replace(/\/+/g, '/')
+    }
+    return path.startsWith('/') ? path : `/${path}`
+  }
+}

package/server/storage/storage-factory.ts

@@ -0,0 +1,62 @@
+import type { StorageProvider, StorageConfig } from './storage-provider.interface'
+import { S3StorageProvider } from './providers/s3-storage.provider'
+import { SftpStorageProvider } from './providers/sftp-storage.provider'
+import { FtpStorageProvider } from './providers/ftp-storage.provider'
+import { LocalStorageProvider } from './providers/local-storage.provider'
+
+export class StorageFactory {
+  private static providers: Map<string, StorageProvider> = new Map()
+
+  static createProvider(config: StorageConfig): StorageProvider {
+    const key = this.generateConfigKey(config)
+
+    if (this.providers.has(key)) {
+      return this.providers.get(key)!
+    }
+
+    let provider: StorageProvider
+
+    switch (config.type) {
+      case 's3':
+        provider = new S3StorageProvider(config)
+        break
+      case 'sftp':
+        provider = new SftpStorageProvider(config)
+        break
+      case 'ftp':
+        provider = new FtpStorageProvider(config)
+        break
+      case 'local':
+        provider = new LocalStorageProvider(config)
+        break
+      default:
+        throw new Error(`Unsupported storage type: ${config.type}`)
+    }
+
+    this.providers.set(key, provider)
+    return provider
+  }
+
+  static async getProvider(config: StorageConfig): Promise<StorageProvider> {
+    const provider = this.createProvider(config)
+
+    if (!provider.isConnected()) {
+      await provider.connect()
+    }
+
+    return provider
+  }
+
+  static async disconnectAll(): Promise<void> {
+    for (const provider of this.providers.values()) {
+      if (provider.isConnected()) {
+        await provider.disconnect()
+      }
+    }
+    this.providers.clear()
+  }
+
+  private static generateConfigKey(config: StorageConfig): string {
+    return `${config.type}-${config.host || config.bucketName || 'local'}-${config.username || 'default'}`
+  }
+}

package/server/storage/storage-manager.ts

@@ -0,0 +1,103 @@
+import { StorageFactory } from './storage-factory'
+import { StorageProvider, StorageConfig } from './storage-provider.interface'
+import { logger } from '@things-factory/env'
+
+export class StorageManager {
+  private static instance: StorageManager
+  private providers: Map<string, StorageProvider> = new Map()
+
+  private constructor() {}
+
+  static getInstance(): StorageManager {
+    if (!StorageManager.instance) {
+      StorageManager.instance = new StorageManager()
+    }
+    return StorageManager.instance
+  }
+
+  async getProvider(config: StorageConfig): Promise<StorageProvider> {
+    const key = this.generateConfigKey(config)
+
+    if (!this.providers.has(key)) {
+      const provider = StorageFactory.createProvider(config)
+      await provider.connect()
+      this.providers.set(key, provider)
+      logger.info(`Storage provider created for: ${key}`)
+    }
+
+    const provider = this.providers.get(key)!
+    // Reconnect if the cached provider has been disconnected (e.g., idle timeout)
+    if (!provider.isConnected()) {
+      logger.info(`Storage provider re-connecting: ${key}`)
+      await provider.connect()
+    }
+    return provider
+  }
+
+  async readFile(config: StorageConfig, path: string, encoding?: string): Promise<any> {
+    const provider = await this.getProvider(config)
+    return await provider.readFile(path, encoding)
+  }
+
+  async writeFile(config: StorageConfig, path: string, content: any, encoding?: string): Promise<void> {
+    const provider = await this.getProvider(config)
+    await provider.writeFile(path, content, encoding)
+  }
+
+  async uploadFile(
+    config: StorageConfig,
+    params: { stream: any; filename: string; uploadPath: string }
+  ): Promise<{ id: string; path: string; size: number }> {
+    const provider = await this.getProvider(config)
+    return await provider.uploadFile(params)
+  }
+
+  async deleteFile(config: StorageConfig, path: string): Promise<void> {
+    const provider = await this.getProvider(config)
+    await provider.deleteFile(path)
+  }
+
+  async moveFile(config: StorageConfig, params: { source: string; destination: string }): Promise<void> {
+    const provider = await this.getProvider(config)
+    await provider.moveFile(params)
+  }
+
+  async readFolders(config: StorageConfig, params: { path: string }): Promise<any[]> {
+    const provider = await this.getProvider(config)
+    return await provider.readFolders(params)
+  }
+
+  async createDirectory(config: StorageConfig, path: string): Promise<void> {
+    const provider = await this.getProvider(config)
+    await provider.createDirectory(path)
+  }
+
+  async deleteDirectory(config: StorageConfig, path: string): Promise<void> {
+    const provider = await this.getProvider(config)
+    await provider.deleteDirectory(path)
+  }
+
+  async fileExists(config: StorageConfig, path: string): Promise<boolean> {
+    const provider = await this.getProvider(config)
+    return await provider.fileExists(path)
+  }
+
+  async getFileSize(config: StorageConfig, path: string): Promise<number> {
+    const provider = await this.getProvider(config)
+    return await provider.getFileSize(path)
+  }
+
+  async disconnectAll(): Promise<void> {
+    for (const [key, provider] of this.providers.entries()) {
+      if (provider.isConnected()) {
+        await provider.disconnect()
+        logger.info(`Storage provider disconnected: ${key}`)
+      }
+    }
+    this.providers.clear()
+  }
+
+  private generateConfigKey(config: StorageConfig): string {
+    return `${config.type}-${config.host || config.bucketName || 'local'}-${config.username || 'default'}`
+  }
+}

package/server/storage/storage-provider.interface.ts

@@ -0,0 +1,42 @@
+export interface StorageProvider {
+  // File operations
+  readFile(path: string, encoding?: string): Promise<any>
+  writeFile(path: string, content: any, encoding?: string): Promise<void>
+  uploadFile(params: {
+    stream: any
+    filename: string
+    uploadPath: string
+  }): Promise<{ id: string; path: string; size: number }>
+  deleteFile(path: string): Promise<void>
+  moveFile(params: { source: string; destination: string }): Promise<void>
+
+  // Directory operations
+  readFolders(params: { path: string }): Promise<any[]>
+  createDirectory(path: string): Promise<void>
+  deleteDirectory(path: string): Promise<void>
+
+  // File info
+  fileExists(path: string): Promise<boolean>
+  getFileSize(path: string): Promise<number>
+
+  // Connection management
+  connect(): Promise<void>
+  disconnect(): Promise<void>
+  isConnected(): boolean
+}
+
+export interface StorageConfig {
+  type: 's3' | 'sftp' | 'ftp' | 'local'
+  host?: string
+  port?: number
+  username?: string
+  password?: string
+  privateKey?: string
+  bucketName?: string
+  accessKeyId?: string
+  secretAccessKey?: string
+  region?: string
+  basePath?: string
+  timeout?: number
+  retries?: number
+}

package/server/util/file-formatters.ts

@@ -0,0 +1,97 @@
+import { xml2js, js2xml } from 'xml-js'
+
+function formatDateYMD(date: Date): string {
+  const y = date.getFullYear()
+  const m = String(date.getMonth() + 1).padStart(2, '0')
+  const d = String(date.getDate()).padStart(2, '0')
+  return `${y}-${m}-${d}`
+}
+
+function normalizeDateLikeValue(value: any): any {
+  if (value instanceof Date) {
+    return formatDateYMD(value)
+  }
+  if (typeof value === 'string') {
+    // Heuristic: ISO-like or timezone string → format as YYYY-MM-DD
+    if (/^(\d{4}-\d{2}-\d{2})([T\s].*)?$/.test(value) || /GMT/.test(value)) {
+      const t = new Date(value)
+      if (!isNaN(t.getTime())) return formatDateYMD(t)
+    }
+  }
+  return value
+}
+
+export function formatAsCSV(data: any[]): string {
+  if (!data || data.length === 0) return ''
+  const headers = Object.keys(data[0])
+  const lines = [headers.join(',')]
+  for (const row of data) {
+    const line = headers
+      .map(h => {
+        const raw = row[h]
+        const value = normalizeDateLikeValue(raw)
+        if (value === null || value === undefined) return ''
+        return typeof value === 'string' ? `"${value}"` : value
+      })
+      .join(',')
+    lines.push(line)
+  }
+  return lines.join('\n')
+}
+
+export function formatAsXML(data: any[], rootName = 'rows', rowName = 'row'): string {
+  // Normalize date-like values in a deep-copy manner
+  const normalized = data.map(row => {
+    const obj: any = {}
+    for (const [k, v] of Object.entries(row)) {
+      obj[k] = normalizeDateLikeValue(v)
+    }
+    return obj
+  })
+
+  const wrapped = {
+    [rootName]: {
+      _attributes: {},
+      [rowName]: normalized
+    }
+  }
+  return js2xml(wrapped, { compact: true, spaces: 2 })
+}
+
+export type SupportedContentType = 'csv' | 'xml' | 'xlsx'
+
+export function formatData(data: any[], type: SupportedContentType = 'csv'): string | Buffer {
+  switch (type) {
+    case 'csv':
+      return formatAsCSV(data)
+    case 'xml':
+      return formatAsXML(data)
+    case 'xlsx':
+      // TODO: implement excel generation using a lightweight lib if needed
+      // returning CSV as a placeholder to avoid new deps
+      return formatAsCSV(data)
+    default:
+      return formatAsCSV(data)
+  }
+}
+
+// Simple CSV parser (does not handle embedded commas/newlines/escaped quotes)
+export function parseCSV(csvContent: string): any[] {
+  const lines = csvContent.trim().split('\n')
+  if (lines.length < 2) return []
+  const headers = lines[0].split(',').map(h => h.trim())
+  const result = []
+  for (let i = 1; i < lines.length; i++) {
+    const values = lines[i].split(',').map(v => v.trim())
+    const row: any = {}
+    headers.forEach((header, index) => {
+      let value = values[index] || ''
+      if (value.startsWith('"') && value.endsWith('"')) {
+        value = value.slice(1, -1)
+      }
+      row[header] = value
+    })
+    result.push(row)
+  }
+  return result
+}

package/server/util/generate-files.ts

@@ -1,33 +1,95 @@
-import '../sftp-s3'
+import { StorageManager } from '../storage/storage-manager'
+import { StorageConfig } from '../storage/storage-provider.interface'
+import { logger } from '@things-factory/env'
 
-import { SFTPFILESTORAGE } from '../sftp-const'
+export async function generateFiles(params: any[], storageConfig?: StorageConfig) {
+  const storageManager = StorageManager.getInstance()
 
-const fs = require('fs')
+  // Primary target config (SFTP or caller-provided)
+  const primaryConfig: StorageConfig = storageConfig || require('@things-factory/env').config.get('sftpFileStorage')
+  // Optional archive config to S3 (uses sftpFileStorage in config when type is 's3')
+  const archiveConfig: StorageConfig | null = (() => {
+    try {
+      const cfg = require('@things-factory/env').config.get('sftpFileStorage')
+      return cfg && cfg.type === 's3' ? cfg : null
+    } catch (_) {
+      return null
+    }
+  })()
+
+  logger.info(
+    `generateFiles: primary=${primaryConfig.type} archive=${
+      archiveConfig ? archiveConfig.type + ':' + archiveConfig.bucketName : 'disabled'
+    }`
+  )
+
+  const sleep = (ms: number) => new Promise(res => setTimeout(res, ms))
 
-export async function generateFiles(params: any[]) {
-  const fileDirectory = './uploaded-files'
   for (let i = 0; i < params.length; i++) {
     let param: any = params[i]
     const { uploadPath, content, title } = param
 
-    if (!fs.existsSync(fileDirectory)) {
-      fs.mkdirSync(fileDirectory)
+    // Ensure remote directory exists before upload (skip when empty string or undefined)
+    if (uploadPath) {
+      let attempts = 0
+      while (true) {
+        try {
+          await storageManager.createDirectory(primaryConfig, uploadPath)
+          break
+        } catch (e) {
+          attempts++
+          if (attempts >= 3) {
+            logger.warn(`createDirectory failed for ${uploadPath}: ${e?.message || e}`)
+            break
+          }
+          await sleep(200 * attempts)
+        }
+      }
     }
 
-    const filePath = fileDirectory + '/' + title
+    // Compose remote relative path (under basePath when configured)
+    const remotePath = `${uploadPath ? uploadPath + '/' : ''}${title}`.replace(/\/+/, '/').replace(/\/+/, '/')
 
-    fs.writeFile(filePath, content, function (err) {
-      if (err) throw err
-      console.log('File is created successfully.')
-    })
+    // Upload to primary
+    if (primaryConfig.type === 'sftp') {
+      // Stream upload with retries to reduce connection issues
+      const { Readable } = require('stream')
+      let attempts = 0
+      while (true) {
+        try {
+          const dataStream = Readable.from([content])
+          await storageManager.uploadFile(primaryConfig, {
+            stream: dataStream,
+            filename: title,
+            uploadPath: uploadPath || ''
+          })
+          break
+        } catch (e) {
+          attempts++
+          if (attempts >= 3) {
+            logger.warn(`SFTP upload failed for ${title} to ${uploadPath || '(root)'}: ${e?.message || e}`)
+            throw e
+          }
+          await sleep(300 * attempts)
+        }
+      }
+    } else {
+      // Direct write for non-SFTP providers
+      await storageManager.writeFile(primaryConfig, remotePath, content, 'utf-8')
+    }
 
-    if (uploadPath) {
-      const stream = fs.createReadStream(filePath)
-      await SFTPFILESTORAGE.uploadFile({ stream, filename: title, uploadPath })
+    // Optional: archive a copy to S3 under <platform>/
+    if (archiveConfig) {
+      const platformPrefix: string = (param.platform || '').toString().trim()
+      const archiveKey = `${platformPrefix}/${title}`
+      logger.info(`generateFiles: archiving to S3 key=${archiveKey}`)
+      try {
+        await storageManager.writeFile(archiveConfig, archiveKey, content, 'utf-8')
+      } catch (e) {
+        logger.warn(`S3 archive failed for ${title}: ${e?.message || e}`)
+      }
     }
   }
 
-  fs.rm(fileDirectory, { recursive: true })
-
   return true
 }

package/server/util/get-permitted-directories.ts

@@ -1,15 +1,19 @@
-import '../sftp-s3'
+import { StorageManager } from '../storage/storage-manager'
+import { StorageConfig } from '../storage/storage-provider.interface'
+import { getSftpStorageConfig } from '../sftp-const'
 
-import { SFTPFILESTORAGE } from '../sftp-const'
-
-export async function getPermittedDirectories(params: any, context: any) {
-  const sftpDirectories: any[] = await SFTPFILESTORAGE.readFolders(params)
+export async function getPermittedDirectories(params: any, context: any, storageConfig?: StorageConfig) {
+  const storageManager = StorageManager.getInstance()
+  const config = storageConfig || getSftpStorageConfig()
+  const sftpDirectories: any[] = await storageManager.readFolders(config, params)
 
   return sftpDirectories
 }
 
-export async function readFile(fileKey: any) {
-  const file: any = await SFTPFILESTORAGE.readFile(fileKey, 'utf-8')
+export async function readFile(fileKey: any, storageConfig?: StorageConfig) {
+  const storageManager = StorageManager.getInstance()
+  const config = storageConfig || getSftpStorageConfig()
+  const file: any = await storageManager.readFile(config, fileKey, 'utf-8')
 
   return file
 }