@things-factory/integration-sftp 4.3.660 → 4.3.668

package/server/service/sftp/sftp.ts

@@ -60,6 +60,22 @@ export class Sftp {
   @Field({ nullable: true })
   username?: string
 
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  schedule?: string
+
+  // Store password securely; not exposed via GraphQL Field decorator
+  @Column({ nullable: true })
+  password?: string
+
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  host?: string
+
+  @Column({ type: 'int', nullable: true })
+  @Field({ nullable: true })
+  port?: number
+
   @Column({
     nullable: true
   })
@@ -78,6 +94,10 @@ export class Sftp {
   @Field({ nullable: true })
   folderPath?: string
 
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  basePath?: string
+
   @Column({
     nullable: true
   })
@@ -111,6 +131,20 @@ export class Sftp {
   @Field({ nullable: true })
   responseFileTypes: string
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  sftpType: string
+
+  @Column({ type: 'int', nullable: true })
+  @Field({ nullable: true })
+  timeout?: number
+
+  @Column({ type: 'int', nullable: true })
+  @Field({ nullable: true })
+  retries?: number
+
   @Column({ nullable: true })
   @Field({ nullable: true })
   lastTimeSync?: Date

package/server/sftp-const.ts

@@ -1,6 +1,8 @@
 import { config } from '@things-factory/env'
+import { StorageManager } from './storage/storage-manager'
+import { StorageConfig } from './storage/storage-provider.interface'
 
-export var SFTPFILESTORAGE: any = config.get('sftpFileStorage')
+// Path constants
 export var SUBMITDATAPATH: string = '/sob/data'
 export var SUCCESSDATAPATH: string = '/sob/success'
 export var FAILEDDATAPATH: string = '/sob/failed'
@@ -11,3 +13,41 @@ export var SNSUBMITDATAPATH: string = '/sn/data'
 export var SNSUCCESSDATAPATH: string = '/sn/success'
 export var SNFAILEDDATAPATH: string = '/sn/failed'
 export var BACKUPPATH: string = 'hatiosea/'
+
+// Storage configuration helper
+export function getSftpStorageConfig(): StorageConfig {
+  return config.get('sftpFileStorage')
+}
+
+// Legacy compatibility - deprecated, use StorageManager instead
+export var SFTPFILESTORAGE: any = {
+  // This is kept for backward compatibility but should not be used in new code
+  // Use StorageManager.getInstance() instead
+  get config() {
+    return getSftpStorageConfig()
+  },
+
+  async readFile(path: string, encoding?: string) {
+    const storageManager = StorageManager.getInstance()
+    const storageConfig = getSftpStorageConfig()
+    return await storageManager.readFile(storageConfig, path, encoding)
+  },
+
+  async uploadFile(params: { stream: any; filename: string; uploadPath: string }) {
+    const storageManager = StorageManager.getInstance()
+    const storageConfig = getSftpStorageConfig()
+    return await storageManager.uploadFile(storageConfig, params)
+  },
+
+  async readFolders(params: { path: string }) {
+    const storageManager = StorageManager.getInstance()
+    const storageConfig = getSftpStorageConfig()
+    return await storageManager.readFolders(storageConfig, params)
+  },
+
+  async moveFile(params: { source: string; destination: string }) {
+    const storageManager = StorageManager.getInstance()
+    const storageConfig = getSftpStorageConfig()
+    return await storageManager.moveFile(storageConfig, params)
+  }
+}

package/server/storage/providers/ftp-storage.provider.ts

@@ -0,0 +1,177 @@
+import * as ftp from 'basic-ftp'
+import { FileType } from 'basic-ftp'
+import { StorageProvider, StorageConfig } from '../storage-provider.interface'
+import { logger } from '@things-factory/env'
+
+export class FtpStorageProvider implements StorageProvider {
+  private client: ftp.Client
+  private config: StorageConfig
+  private connected: boolean = false
+
+  constructor(config: StorageConfig) {
+    this.config = config
+    this.client = new ftp.Client()
+    this.client.ftp.verbose = false
+  }
+
+  async connect(): Promise<void> {
+    try {
+      await this.client.access({
+        host: this.config.host!,
+        port: this.config.port || 21,
+        user: this.config.username!,
+        password: this.config.password!,
+        secure: false // Set to true for FTPS
+      })
+
+      this.connected = true
+      logger.info('FTP Storage Provider connected successfully')
+    } catch (error) {
+      this.connected = false
+      throw new Error(`Failed to connect to FTP: ${error}`)
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    if (this.connected) {
+      this.client.close()
+      this.connected = false
+      logger.info('FTP Storage Provider disconnected')
+    }
+  }
+
+  isConnected(): boolean {
+    return this.connected
+  }
+
+  async readFile(path: string, encoding?: string): Promise<any> {
+    const fullPath = this.getFullPath(path)
+
+    // Create a temporary file to download to
+    const tempFile = `/tmp/temp_${Date.now()}.tmp`
+
+    try {
+      await this.client.downloadTo(tempFile, fullPath)
+      const fs = require('fs')
+      const content = fs.readFileSync(tempFile)
+      fs.unlinkSync(tempFile) // Clean up temp file
+
+      if (encoding) {
+        return content.toString(encoding as any)
+      }
+
+      return content
+    } catch (error) {
+      // Clean up temp file if it exists
+      const fs = require('fs')
+      if (fs.existsSync(tempFile)) {
+        fs.unlinkSync(tempFile)
+      }
+      throw error
+    }
+  }
+
+  async writeFile(path: string, content: any, encoding?: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+
+    if (typeof content === 'string') {
+      await this.client.uploadFrom(content, fullPath)
+    } else {
+      // Convert to string if it's a buffer or other type
+      const contentString = content.toString(encoding || 'utf8')
+      await this.client.uploadFrom(contentString, fullPath)
+    }
+  }
+
+  async uploadFile(params: {
+    stream: any
+    filename: string
+    uploadPath: string
+  }): Promise<{ id: string; path: string; size: number }> {
+    const fullPath = this.getFullPath(`${params.uploadPath}/${params.filename}`)
+    let size = 0
+
+    return new Promise<{ id: string; path: string; size: number }>((resolve, reject) => {
+      params.stream
+        .on('data', (chunk: any) => (size += chunk.length))
+        .on('error', (error: any) => reject(error))
+        .on('end', async () => {
+          try {
+            await this.client.uploadFrom(params.stream, fullPath)
+            resolve({
+              id: Date.now().toString(),
+              path: params.uploadPath,
+              size
+            })
+          } catch (error) {
+            reject(error)
+          }
+        })
+    })
+  }
+
+  async deleteFile(path: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    await this.client.remove(fullPath)
+  }
+
+  async moveFile(params: { source: string; destination: string }): Promise<void> {
+    const sourcePath = this.getFullPath(params.source)
+    const destPath = this.getFullPath(params.destination)
+
+    // FTP doesn't have native move, so we copy then delete
+    const tempFile = `/tmp/temp_move_${Date.now()}.tmp`
+
+    try {
+      await this.client.downloadTo(tempFile, sourcePath)
+      await this.client.uploadFrom(tempFile, destPath)
+      await this.client.remove(sourcePath)
+    } finally {
+      // Clean up temp file
+      const fs = require('fs')
+      if (fs.existsSync(tempFile)) {
+        fs.unlinkSync(tempFile)
+      }
+    }
+  }
+
+  async readFolders(params: { path: string }): Promise<any[]> {
+    const fullPath = this.getFullPath(params.path)
+    const list = await this.client.list(fullPath)
+
+    // Filter for files only, checking if it's not a directory
+    return list.filter(item => item.type !== FileType.Directory)
+  }
+
+  async createDirectory(path: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    await this.client.ensureDir(fullPath)
+  }
+
+  async deleteDirectory(path: string): Promise<void> {
+    const fullPath = this.getFullPath(path)
+    await this.client.removeDir(fullPath)
+  }
+
+  async fileExists(path: string): Promise<boolean> {
+    try {
+      const fullPath = this.getFullPath(path)
+      await this.client.size(fullPath)
+      return true
+    } catch (error) {
+      return false
+    }
+  }
+
+  async getFileSize(path: string): Promise<number> {
+    const fullPath = this.getFullPath(path)
+    return await this.client.size(fullPath)
+  }
+
+  private getFullPath(path: string): string {
+    if (this.config.basePath) {
+      return `${this.config.basePath}/${path}`.replace(/\/+/g, '/')
+    }
+    return path
+  }
+}

package/server/storage/providers/local-storage.provider.ts

@@ -0,0 +1,159 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import { StorageProvider, StorageConfig } from '../storage-provider.interface'
+import { logger } from '@things-factory/env'
+
+export class LocalStorageProvider implements StorageProvider {
+  private config: StorageConfig
+  private connected: boolean = false
+
+  constructor(config: StorageConfig) {
+    this.config = config
+  }
+
+  async connect(): Promise<void> {
+    try {
+      const basePath = this.config.basePath || './storage'
+
+      // Ensure base directory exists
+      if (!fs.existsSync(basePath)) {
+        fs.mkdirSync(basePath, { recursive: true })
+      }
+
+      this.connected = true
+      logger.info('Local Storage Provider connected successfully')
+    } catch (error) {
+      this.connected = false
+      throw new Error(`Failed to connect to Local Storage: ${error}`)
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    this.connected = false
+    logger.info('Local Storage Provider disconnected')
+  }
+
+  isConnected(): boolean {
+    return this.connected
+  }
+
+  async readFile(filePath: string, encoding?: string): Promise<any> {
+    const fullPath = this.getFullPath(filePath)
+
+    if (encoding) {
+      return fs.readFileSync(fullPath, encoding as BufferEncoding)
+    }
+
+    return fs.readFileSync(fullPath)
+  }
+
+  async writeFile(filePath: string, content: any, encoding?: string): Promise<void> {
+    const fullPath = this.getFullPath(filePath)
+
+    // Ensure directory exists
+    const dir = path.dirname(fullPath)
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true })
+    }
+
+    if (encoding) {
+      fs.writeFileSync(fullPath, content, encoding as BufferEncoding)
+    } else {
+      fs.writeFileSync(fullPath, content)
+    }
+  }
+
+  async uploadFile(params: {
+    stream: any
+    filename: string
+    uploadPath: string
+  }): Promise<{ id: string; path: string; size: number }> {
+    const fullPath = this.getFullPath(`${params.uploadPath}/${params.filename}`)
+
+    // Ensure directory exists
+    const dir = path.dirname(fullPath)
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true })
+    }
+
+    let size = 0
+
+    return new Promise<{ id: string; path: string; size: number }>((resolve, reject) => {
+      const writeStream = fs.createWriteStream(fullPath)
+
+      params.stream
+        .pipe(writeStream)
+        .on('error', (error: any) => reject(error))
+        .on('data', (chunk: any) => (size += chunk.length))
+        .on('finish', () => {
+          resolve({
+            id: Date.now().toString(),
+            path: params.uploadPath,
+            size
+          })
+        })
+    })
+  }
+
+  async deleteFile(filePath: string): Promise<void> {
+    const fullPath = this.getFullPath(filePath)
+    fs.unlinkSync(fullPath)
+  }
+
+  async moveFile(params: { source: string; destination: string }): Promise<void> {
+    const sourcePath = this.getFullPath(params.source)
+    const destPath = this.getFullPath(params.destination)
+
+    // Ensure destination directory exists
+    const dir = path.dirname(destPath)
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true })
+    }
+
+    fs.renameSync(sourcePath, destPath)
+  }
+
+  async readFolders(params: { path: string }): Promise<any[]> {
+    const fullPath = this.getFullPath(params.path)
+
+    if (!fs.existsSync(fullPath)) {
+      return []
+    }
+
+    const items = fs.readdirSync(fullPath, { withFileTypes: true })
+    return items
+      .filter(item => item.isFile())
+      .map(item => ({
+        Key: path.join(params.path, item.name),
+        Size: fs.statSync(path.join(fullPath, item.name)).size
+      }))
+  }
+
+  async createDirectory(dirPath: string): Promise<void> {
+    const fullPath = this.getFullPath(dirPath)
+    fs.mkdirSync(fullPath, { recursive: true })
+  }
+
+  async deleteDirectory(dirPath: string): Promise<void> {
+    const fullPath = this.getFullPath(dirPath)
+    fs.rmSync(fullPath, { recursive: true, force: true })
+  }
+
+  async fileExists(filePath: string): Promise<boolean> {
+    const fullPath = this.getFullPath(filePath)
+    return fs.existsSync(fullPath) && fs.statSync(fullPath).isFile()
+  }
+
+  async getFileSize(filePath: string): Promise<number> {
+    const fullPath = this.getFullPath(filePath)
+    const stats = fs.statSync(fullPath)
+    return stats.size
+  }
+
+  private getFullPath(filePath: string): string {
+    if (this.config.basePath) {
+      return path.resolve(this.config.basePath, filePath)
+    }
+    return path.resolve(filePath)
+  }
+}

package/server/storage/providers/s3-storage.provider.ts

@@ -0,0 +1,214 @@
+import AWS from 'aws-sdk'
+import { v4 as uuidv4 } from 'uuid'
+import { StorageProvider, StorageConfig } from '../storage-provider.interface'
+import { logger } from '@things-factory/env'
+
+const { PassThrough } = require('stream')
+
+export class S3StorageProvider implements StorageProvider {
+  private s3: AWS.S3
+  private config: StorageConfig
+  private connected: boolean = false
+
+  constructor(config: StorageConfig) {
+    this.config = config
+    this.s3 = new AWS.S3({
+      accessKeyId: config.accessKeyId,
+      secretAccessKey: config.secretAccessKey,
+      region: config.region
+    })
+  }
+
+  async connect(): Promise<void> {
+    try {
+      // Assume client ready; defer validation to first real operation (write/read)
+      this.connected = true
+      logger.info('S3 Storage Provider ready')
+    } catch (error) {
+      this.connected = false
+      throw new Error(`Failed to initialize S3 client: ${error}`)
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    this.connected = false
+    logger.info('S3 Storage Provider disconnected')
+  }
+
+  isConnected(): boolean {
+    return this.connected
+  }
+
+  async readFile(path: string, encoding?: string): Promise<any> {
+    const result = await this.s3
+      .getObject({
+        Bucket: this.config.bucketName!,
+        Key: path
+      })
+      .promise()
+
+    const body = result.Body
+
+    if (encoding) {
+      return body.toString(encoding as BufferEncoding)
+    }
+
+    return body
+  }
+
+  async writeFile(path: string, content: any, encoding?: string): Promise<void> {
+    let body: Buffer
+
+    if (typeof content === 'string' && encoding) {
+      body = Buffer.from(content, encoding as any)
+    } else {
+      body = Buffer.from(content)
+    }
+
+    const putParams: AWS.S3.PutObjectRequest = {
+      Bucket: this.config.bucketName!,
+      Key: path,
+      Body: body
+    }
+
+    // Optional SSE if configured via env/config
+    const sse = (this.config as any).serverSideEncryption
+    const kmsKeyId = (this.config as any).sseKmsKeyId
+    if (sse) {
+      putParams.ServerSideEncryption = sse
+      if (sse === 'aws:kms' && kmsKeyId) {
+        putParams.SSEKMSKeyId = kmsKeyId
+      }
+    }
+
+    await this.s3.putObject(putParams).promise()
+  }
+
+  async uploadFile(params: {
+    stream: any
+    filename: string
+    uploadPath: string
+  }): Promise<{ id: string; path: string; size: number }> {
+    const id = uuidv4()
+    let size: number = 0
+
+    return new Promise<{ id: string; path: string; size: number }>((resolve, reject) =>
+      params.stream
+        .pipe(
+          (() => {
+            var pass = new PassThrough()
+
+            this.s3.upload(
+              {
+                Bucket: this.config.bucketName!,
+                Key: params.uploadPath + '/' + params.filename,
+                Body: pass,
+                ServerSideEncryption: (this.config as any).serverSideEncryption,
+                SSEKMSKeyId: (this.config as any).sseKmsKeyId
+              },
+              (err, data) => (err ? reject(err) : resolve({ id, path: params.uploadPath, size }))
+            )
+
+            return pass
+          })()
+        )
+        .on('error', error => reject(error))
+        .on('data', chunk => (size += chunk.length))
+    )
+  }
+
+  async deleteFile(path: string): Promise<void> {
+    await this.s3
+      .deleteObject({
+        Bucket: this.config.bucketName!,
+        Key: path
+      })
+      .promise()
+  }
+
+  async moveFile(params: { source: string; destination: string }): Promise<void> {
+    try {
+      // Copy file to new location
+      await this.s3
+        .copyObject({
+          Bucket: this.config.bucketName!,
+          CopySource: this.config.bucketName + '/' + params.source,
+          Key: params.destination
+        })
+        .promise()
+
+      // Delete original file
+      await this.deleteFile(params.source)
+    } catch (error) {
+      throw new Error(`Failed to move file: ${error}`)
+    }
+  }
+
+  async readFolders(params: { path: string }): Promise<any[]> {
+    const s3Params = {
+      Bucket: this.config.bucketName!,
+      Delimiter: '/',
+      Prefix: params.path
+    }
+
+    const result = await this.s3.listObjects(s3Params).promise()
+    return result.Contents || []
+  }
+
+  async createDirectory(path: string): Promise<void> {
+    // S3 doesn't have real directories, but we can create an empty object
+    await this.s3
+      .putObject({
+        Bucket: this.config.bucketName!,
+        Key: path.endsWith('/') ? path : path + '/',
+        Body: ''
+      })
+      .promise()
+  }
+
+  async deleteDirectory(path: string): Promise<void> {
+    // List all objects with the prefix and delete them
+    const objects = await this.s3
+      .listObjects({
+        Bucket: this.config.bucketName!,
+        Prefix: path
+      })
+      .promise()
+
+    if (objects.Contents && objects.Contents.length > 0) {
+      const deleteParams = {
+        Bucket: this.config.bucketName!,
+        Delete: {
+          Objects: objects.Contents.map(obj => ({ Key: obj.Key! }))
+        }
+      }
+
+      await this.s3.deleteObjects(deleteParams).promise()
+    }
+  }
+
+  async fileExists(path: string): Promise<boolean> {
+    try {
+      await this.s3
+        .headObject({
+          Bucket: this.config.bucketName!,
+          Key: path
+        })
+        .promise()
+      return true
+    } catch (error) {
+      return false
+    }
+  }
+
+  async getFileSize(path: string): Promise<number> {
+    const result = await this.s3
+      .headObject({
+        Bucket: this.config.bucketName!,
+        Key: path
+      })
+      .promise()
+
+    return result.ContentLength || 0
+  }
+}