kalai-attach 1.0.0

package/srv/basic.js

@@ -0,0 +1,331 @@
+const cds = require('@sap/cds')
+const LOG = cds.log('attachments')
+const { computeHash, traverseEntity } = require('../lib/helper')
+
+class AttachmentsService extends cds.Service {
+
+  init() {
+    this.on('DeleteAttachment', async msg => {
+      await this.delete(msg.data.url, msg.data.target)
+    })
+
+    this.on('DeleteInfectedAttachment', async msg => {
+      const { target, hash, keys } = msg.data
+      const attachment = await SELECT.one.from(target).where(Object.assign({ hash }, keys)).columns('url')
+      if (attachment) { //Might happen that a draft object is the target
+        await this.delete(attachment.url, target)
+      } else {
+        LOG.warn(`Cannot delete malware file with the hash ${hash} for attachment ${target}, keys: ${keys}`)
+      }
+    })
+    return super.init()
+  }
+
+  /**
+   * Uploads attachments to the database and initiates malware scans for database-stored files
+   * @param {cds.Entity} attachments - Attachments entity definition
+   * @param {Array|Object} data - The attachment data to be uploaded
+   * @returns {Promise<Array>} - Result of the upsert operation
+   */
+  async put(attachments, data) {
+    if (!Array.isArray(data)) {
+      data = [data]
+    }
+
+    LOG.debug('Starting database attachment upload', {
+      attachmentEntity: attachments.name,
+      fileCount: data.length,
+      filenames: data.map((d) => d.filename || 'unknown'),
+    })
+
+    let res
+
+    try {
+      res = await Promise.all(
+        data.map(async (d) => {
+          const res = await UPSERT(d).into(attachments)
+          const attachmentForHash = await this.get(attachments, { ID: d.ID })
+          // If this is just the PUT for metadata, there is not yet any file to retrieve
+          if (attachmentForHash) {
+            const hash = await computeHash(attachmentForHash)
+            await this.update(attachments, { ID: d.ID }, { hash })
+          }
+          return res
+        })
+      )
+
+      LOG.debug('Attachment records upserted to database successfully', {
+        attachmentEntity: attachments.name,
+        recordCount: data.length
+      })
+
+    } catch (error) {
+      LOG.error(
+        'Failed to upsert attachment records to database', error,
+        'Check database connectivity and attachment entity configuration',
+        { attachmentEntity: attachments.name, recordCount: data.length, errorMessage: error.message })
+      throw error
+    }
+
+    // Initiate malware scanning for database-stored files
+    LOG.debug('Initiating malware scans for database-stored files', {
+      fileCount: data.length,
+      fileIds: data.map(d => d.ID)
+    })
+
+    const MalwareScanner = await cds.connect.to('malwareScanner')
+    await Promise.all(
+      data.map(async (d) => {
+        await MalwareScanner.emit('ScanAttachmentsFile', { target: attachments.name, keys: { ID: d.ID } })
+      })
+    )
+
+    return res
+  }
+
+  /**
+   * Registers attachment handlers for the given service and entity
+   * @param {cds.Entity} attachments - The attachment service instance
+   * @param {string} keys - The keys to identify the attachment
+   * @param {import('@sap/cds').Request} req - The request object
+   * @returns {Buffer|Stream|null} - The content of the attachment or null if not found
+   */
+  async get(attachments, keys) {
+    LOG.debug("Downloading attachment for", {
+      attachmentName: attachments.name,
+      attachmentKeys: keys
+    })
+    let result = await SELECT.from(attachments, keys).columns("content")
+    if (!result && attachments.isDraft) {
+      attachments = attachments.actives
+      result = await SELECT.from(attachments, keys).columns("content")
+    }
+    return (result?.content) ? result.content : null
+  }
+  /**
+   * Returns a handler to copy updated attachments content from draft to active / object store
+   * @param {cds.Entity} attachments - Attachments entity definition
+   * @returns {Function} - The draft save handler function
+   */
+  draftSaveHandler(attachments) {
+    const queryFields = this.getFields(attachments)
+
+    return async (_, req) => {
+      // The below query loads the attachments into streams
+      const cqn = SELECT(queryFields)
+        .from(attachments.drafts)
+        .where([
+          ...req.subject.ref[0].where.map((x) =>
+            x.ref ? { ref: ["up_", ...x.ref] } : x
+          )
+          // NOTE: needs skip LargeBinary fix to Lean Draft
+        ])
+      cqn.where({ content: { '!=': null } })
+      const draftAttachments = await cqn
+
+      if (draftAttachments.length)
+        await this.put(attachments, draftAttachments)
+    }
+  }
+  /**
+   * Returns the fields to be selected from Attachments entity definition
+   * including the association keys if Attachments entity definition is associated to another entity
+   * @param {cds.Entity} attachments - Attachments entity definition
+   * @returns {Array} - Array of fields to be selected
+   */
+  getFields(attachments) {
+    const attachmentFields = ["filename", "mimeType", "content", "url", "ID"]
+    const { up_ } = attachments.keys
+    if (up_)
+      return up_.keys
+        .map((k) => "up__" + k.ref[0])
+        .concat(...attachmentFields)
+        .map((k) => ({ ref: [k] }))
+    else return Object.keys(attachments.keys)
+  }
+
+  /**
+   * Registers handlers for attachment entities in the service
+   * @param {cds.Service} srv - The CDS service instance
+   */
+  registerHandlers(srv) {
+    if (!cds.env.fiori.move_media_data_in_db) {
+      srv.after("SAVE", async function saveDraftAttachments(res, req) {
+        if (
+          req.target.isDraft ||
+          !req.target.drafts ||
+          !req.target._attachments.hasAttachmentsComposition ||
+          !req.target._attachments.attachmentCompositions
+        ) {
+          return
+        }
+        await Promise.all(
+          req.target._attachments.attachmentCompositions.map(attachmentsEle =>{
+            const target = traverseEntity(req.target, attachmentsEle)
+            if (!target) {
+              LOG.error(`Could not resolve target for attachment composition: ${attachmentsEle}`)
+              return
+            }
+            return this.draftSaveHandler(target)(res, req)
+          })
+        )
+      }.bind(this))
+    }
+  }
+
+  /**
+   * Updates attachment metadata in the database
+   * @param {cds.Entity} Attachments - Attachments entity definition
+   * @param {string} key - The key of the attachment to update
+   * @param {*} data - The data to update the attachment with
+   * @returns {Promise} - Result of the update operation
+   */
+  async update(Attachments, key, data) {
+    LOG.debug("Updating attachment for", {
+      attachmentName: Attachments.name,
+      attachmentKey: key
+    })
+
+    return await UPDATE(Attachments, key).with(data)
+  }
+
+  /**
+   * Retrieves the malware scan status of an attachment
+   * @param {cds.Entity} Attachments - Attachments entity definition
+   * @param {string} key - The key of the attachment to retrieve the status for
+   * @returns {string} - The malware scan status of the attachment
+   */
+  async getStatus(Attachments, key) {
+    const result = await SELECT.from(Attachments, key).columns('status')
+    return result?.status
+  }
+
+  /**
+   * Registers attachment handlers for the given service and entity
+   * @param {*} records - The records to process
+   * @param {import('@sap/cds').Request} req - The request object
+   */
+  async deleteAttachmentsWithKeys(records, req) {
+    req.attachmentsToDelete?.forEach(async (attachment) => {
+      if (attachment.url) {
+        const attachmentsSrv = await cds.connect.to('attachments')
+        await attachmentsSrv.emit('DeleteAttachment', { url: attachment.url, target: attachment.target })
+      } else {
+        LOG.warn(`Attachment cannot be deleted because URL is missing`, attachment)
+      }
+    })
+  }
+
+  /**
+   * Traverses nested data by a given path array.
+   * @param {Object} root - The root object or array to traverse.
+   * @param {Array} path - The array of keys representing the path.
+   * @returns {*} - The value found at the path, or [] if not found.
+   */
+  traverseDataByPath(root, path) {
+    let current = root
+    for (let i = 0; i < path.length; i++) {
+      const part = path[i]
+      if (Array.isArray(current)) {
+        return current.flatMap(item => this.traverseDataByPath(item, path.slice(i)))
+      }
+      if (!current || !(part in current)) return []
+      current = current[part]
+    }
+    return current
+  }
+
+  /**
+   * Registers attachment handlers for the given service and entity
+   * @param {import('@sap/cds').Request} req - The request object
+   */
+  async attachDeletionData(req) {
+    const attachmentCompositions = req?.target?._attachments.attachmentCompositions
+    if (attachmentCompositions.length > 0) {
+      const diffData = await req.diff()
+      if (!diffData || Object.keys(diffData).length === 0) {
+        return
+      }
+      const queries = []
+      const queryTargets = []
+      for (const attachmentsComp of attachmentCompositions) {
+        const leaf = this.traverseDataByPath(diffData, attachmentsComp)
+        const deletedAttachments = Array.isArray(leaf) ? leaf.filter(obj => obj._op === "delete").map(obj => obj.ID) : []
+
+        const entityTarget = traverseEntity(req.target, attachmentsComp)
+        if (deletedAttachments.length) {
+          queries.push(
+            SELECT.from(entityTarget).columns("url").where({ ID: { in: [...deletedAttachments] } })
+          )
+          queryTargets.push(entityTarget.name)
+        }
+      }
+      if (queries.length > 0) {
+        const attachmentsToDelete = (await Promise.all(queries)).reduce((acc, attachments, idx) => {
+          attachments.forEach(attachment => attachment.target = queryTargets[idx])
+          acc = acc.concat(attachments)
+          return acc;
+        }, [])
+        if (attachmentsToDelete.length > 0) {
+          req.attachmentsToDelete = attachmentsToDelete
+        }
+      }
+    }
+  }
+
+  /**
+   * Registers attachment handlers for the given service and entity
+   * @param {{draftEntity: string, activeEntity:cds.Entity, id:string}} param0 - The service and entities
+   * @returns 
+   */
+  async getAttachmentsToDelete({ draftEntity, activeEntity, whereXpr }) {
+    const [draftAttachments, activeAttachments] = await Promise.all([
+      SELECT.from(draftEntity).columns("url").where(whereXpr),
+      SELECT.from(activeEntity).columns("url").where(whereXpr)
+    ])
+
+    const activeUrls = new Set(activeAttachments.map(a => a.url))
+    return draftAttachments
+      .filter(({ url }) => !activeUrls.has(url))
+      .map(({ url }) => ({ url, target: draftEntity.name }))
+  }
+
+  /**
+   * Add draft attachment deletion data to the request
+   * @param {import('@sap/cds').Request} req - The request object
+   */
+  async attachDraftDeletionData(req) {
+    const draftEntity = cds.model.definitions[req?.target?.name]
+    const name = req?.target?.name
+    const activeEntity = name ? cds.model.definitions?.[name.split(".").slice(0, -1).join(".")] : undefined
+
+    if (!draftEntity || !activeEntity) return
+
+    const diff = await req.diff()
+    if (diff._op !== "delete" || !diff.ID) return
+
+    const attachmentsToDelete = await this.getAttachmentsToDelete({
+      draftEntity,
+      activeEntity,
+      whereXpr: { ID: diff.ID }
+    })
+
+    if (attachmentsToDelete.length) {
+      req.attachmentsToDelete = attachmentsToDelete
+    }
+  }
+
+  /**
+   * Deletes a file from the database. Does not delete metadata
+   * @param {string} url - The url of the file to delete
+   * @returns {Promise} - Promise resolving when deletion is complete
+   */
+  async delete(url, target) {
+    return await UPDATE(target).where({ url }).with({ content: null })
+  }
+}
+
+
+AttachmentsService.prototype._is_queueable = true
+
+module.exports = AttachmentsService

package/srv/gcp.js

@@ -0,0 +1,226 @@
+const { Storage } = require('@google-cloud/storage')
+const cds = require("@sap/cds")
+const LOG = cds.log('attachments')
+const utils = require('../lib/helper')
+
+module.exports = class GoogleAttachmentsService extends require("./object-store") {
+
+  /**
+   * Creates or retrieves a cached Google Cloud Platform client for the given tenant
+   * @returns {Promise<{bucket: import('@google-cloud/storage').Bucket}>}
+   */
+  async retrieveClient() {
+    const tenantID = this.separateObjectStore ? cds.context.tenant : 'shared'
+    LOG.debug('Retrieving tenant-specific Google Cloud Platform client', { tenantID })
+    const existingClient = this.clientsCache.get(tenantID)
+    if (existingClient) {
+      LOG.debug('Using cached GCP client', {
+        tenantID,
+        bucketName: existingClient.bucket.name
+      })
+      return existingClient
+    }
+
+    try {
+      LOG.debug(`Fetching object store credentials for tenant ${tenantID}. Using ${this.separateObjectStore ? 'shared' : 'tenant-specific'} object store.`)
+      const credentials = this.separateObjectStore
+        ? (await utils.getObjectStoreCredentials(tenantID))?.credentials
+        : cds.env.requires?.objectStore?.credentials
+
+      if (!credentials) {
+        throw new Error("SAP Object Store instance is not bound.")
+      }
+
+      // Validate required credentials
+      const requiredFields = ['bucket', 'projectId', 'base64EncodedPrivateKeyData']
+      const missingFields = requiredFields.filter(field => !credentials[field])
+
+      if (missingFields.length > 0) {
+        if (credentials.access_key_id) {
+          throw new Error('AWS S3 credentials found where Google Cloud Platform credentials expected, please check your service bindings.')
+        } else if (credentials.container_name) {
+          throw new Error('Azure credentials found where Google Cloud Platform credentials expected, please check your service bindings.')
+        }
+        throw new Error(`Missing Google Cloud Platform credentials: ${missingFields.join(', ')}`)
+      }
+
+      LOG.debug('Creating Google Cloud Platform client for tenant', {
+        tenantID,
+        bucketName: credentials.bucket
+      })
+
+      const storageClient = new Storage({
+        projectId: credentials.projectId,
+        credentials: JSON.parse(Buffer.from(credentials.base64EncodedPrivateKeyData, 'base64').toString('utf8'))
+      })
+
+      const newGoogleClient = {
+        bucket: storageClient.bucket(credentials.bucket),
+      }
+
+      this.clientsCache.set(tenantID, newGoogleClient)
+
+      LOG.debug('Google Cloud Platform client has been created successful', {
+        tenantID,
+        bucketName: newGoogleClient.bucket.name
+      })
+
+      return newGoogleClient
+
+    } catch (error) {
+      LOG.error(
+        'Failed to create tenant-specific Google Cloud Platform client', error,
+        'Check Service Manager and Google Cloud Platform instance configuration',
+        { tenantID })
+      throw error
+    }
+  }
+
+  /**
+  * @inheritdoc
+  */
+  async put(attachments, data) {
+    if (Array.isArray(data)) {
+      LOG.debug('Processing bulk file upload', {
+        fileCount: data.length,
+        filenames: data.map(d => d.filename)
+      })
+      return Promise.all(
+        data.map((d) => this.put(attachments, d))
+      )
+    }
+
+    const startTime = Date.now()
+
+    LOG.debug('Starting file upload to Google Cloud Platform', {
+      attachmentEntity: attachments.name,
+      tenant: cds.context.tenant
+    })
+
+    const { bucket } = await this.retrieveClient()
+
+    try {
+      const { content, ...metadata } = data
+      const blobName = metadata.url
+
+      if (!blobName) {
+        LOG.error(
+          'File key/URL is required for Google Cloud Platform upload', null,
+          'Ensure attachment data includes a valid URL/key',
+          { metadata: { ...metadata, content: !!content } })
+        throw new Error('File key is required for upload')
+      }
+
+      if (!content) {
+        LOG.error(
+          'File content is required for Google Cloud Platform upload', null,
+          'Ensure attachment data includes file content',
+          { key: blobName, hasContent: !!content })
+        throw new Error('File content is required for upload')
+      }
+
+      const file = bucket.file(blobName)
+
+      LOG.debug('Uploading file to Google Cloud Platform', {
+        bucketName: bucket.name,
+        blobName,
+        filename: metadata.filename,
+        contentSize: content.length || content.size || 'unknown'
+      })
+
+      // The file upload has to be done first, so super.put can compute the hash and trigger malware scan
+      await file.save(content)
+      await super.put(attachments, metadata)
+
+      const duration = Date.now() - startTime
+      LOG.debug('File upload to Google Cloud Platform completed successfully', {
+        filename: metadata.filename,
+        fileId: metadata.ID,
+        bucketName: bucket.name,
+        blobName,
+        duration
+      })
+    } catch (err) {
+      const duration = Date.now() - startTime
+      LOG.error(
+        'File upload to Google Cloud Platform failed', err,
+        'Check Google Cloud Platform connectivity, credentials, and container permissions',
+        { filename: data?.filename, fileId: data?.ID, bucketName: bucket.name, blobName: data?.url, duration })
+      throw err
+    }
+  }
+
+  /**
+  * @inheritdoc
+  */
+  async get(attachments, keys) {
+    const startTime = Date.now()
+    LOG.debug('Starting stream from Google Cloud Platform', {
+      attachmentEntity: attachments.name,
+      keys,
+      tenant: cds.context.tenant
+    })
+    const { bucket } = await this.retrieveClient()
+
+    try {
+      LOG.debug('Fetching attachment metadata', { keys })
+      const response = await SELECT.from(attachments, keys).columns("url")
+
+      if (!response?.url) {
+        LOG.warn(
+          'File URL not found in database', null,
+          'Check if the attachment exists and has been properly uploaded',
+          { keys, hasResponse: !!response })
+        return null
+      }
+
+      const blobName = response.url
+
+      LOG.debug('Streaming file from Google Cloud Platform', {
+        bucketName: bucket.name,
+        blobName
+      })
+
+      const file = bucket.file(blobName)
+      const readStream = await file.createReadStream()
+
+      const duration = Date.now() - startTime
+      LOG.debug('File streamed from Google Cloud Platform successfully', {
+        fileId: keys.ID,
+        bucketName: bucket.name,
+        blobName,
+        duration
+      })
+
+      return readStream
+    } catch (error) {
+      const duration = Date.now() - startTime
+      const suggestion = error.code === 'BlobNotFound' ?
+        'File may have been deleted from Google Cloud Platform or URL is incorrect' :
+        error.code === 'AuthenticationFailed' ?
+          'Check Google Cloud Platform credentials and SAS token' :
+          'Check Google Cloud Platform connectivity and configuration'
+
+      LOG.error(
+        'File download from Google Cloud Platform failed', error,
+        suggestion,
+        { fileId: keys?.ID, bucketName: bucket.name, attachmentName: attachments.name, duration })
+
+      throw error
+    }
+  }
+
+  /**
+   * Deletes a file from Google Cloud Platform
+   * @param {string} Key - The key of the file to delete
+   * @returns {Promise} - Promise resolving when deletion is complete
+   */
+  async delete(blobName) {
+    const { bucket } = await this.retrieveClient()
+    LOG.debug(`[GCP] Executing delete for file ${blobName} in bucket ${bucket.name}`)
+
+    const file = bucket.file(blobName)
+    const response = await file.delete()
+    return response[0]?.statusCode === 204
+  }
+}

package/srv/malwareScanner-mocked.cds

@@ -0,0 +1,3 @@
+using {malwareScanner} from './malwareScanner';
+
+annotate malwareScanner with @impl : './malwareScanner-mocked';

package/srv/malwareScanner-mocked.js

@@ -0,0 +1,127 @@
+const cds = require('@sap/cds')
+const LOG = cds.log('attachments')
+const crypto = require('crypto');
+
+class MockedMalwareScanner extends cds.ApplicationService {
+  init() {
+    this.on('ScanAttachmentsFile', this.scanAttachmentsFile)
+    this.on('scan', this.scanFile)
+
+    return super.init()
+  }
+
+  /**
+   * Scans the "content" property of the given entity, specified by the CSN name (target) and 
+   * the keys object, for malware. 
+   * Updates the status property on the given entity to reflect if the file is clean.
+   * Triggers an attachments service delete event to remove the malware.
+   * @param {{data: {target: string, keys: object}}} msg The target is the CSN Entity name, which is used to lookup entity via cds.model.definitions[<target>].
+   */
+  async scanAttachmentsFile(msg) {
+    const { target, keys } = msg.data
+    const scanEnabled = cds.env.requires?.attachments?.scan ?? true
+    if (!scanEnabled) {
+      LOG.warn(`Malware scanner is disabled! Please consider enabling it`)
+      return
+    }
+
+    LOG.debug(`Initiating malware scan request for ${target}, ${JSON.stringify(keys)} `)
+
+    const AttachmentsSrv = await cds.connect.to("attachments")
+
+    const model = cds.context.model ?? cds.model
+    //Make sure its the active target
+    const _target = model.definitions[target].actives ?? model.definitions[target]
+
+    if (!_target) {
+      LOG.error(`Could not scan ${target}, ${JSON.stringify(keys)} for malware as no CSN entity definition was found for the name!`)
+      return
+    }
+
+    await this.updateStatus(_target, keys, "Scanning")
+
+    LOG.debug(`Fetching file content for scanning for ${target}, ${JSON.stringify(keys)}`)
+    const contentStream = await AttachmentsSrv.get(model.definitions[target], keys)
+
+    if (!contentStream) {
+      LOG.warn(`Cannot fetch file content for malware scanning for ${target}, ${JSON.stringify(keys)}! Check if the file exists.`)
+      await this.updateStatus(_target, keys, "Failed")
+      return
+    }
+
+    let res;
+    try {
+      res = await this.scan(contentStream)
+    } catch (err) {
+      LOG.error(`Request to malware scanner failed for ${target}, ${JSON.stringify(keys)}`, err)
+      await this.updateStatus(target, keys, "Failed")
+      throw err;
+    }
+
+    let status = res.isMalware ? "Infected" : "Clean"
+    const hash = res.hash
+
+    if (status === "Infected") {
+      LOG.warn(`Malware scan completed for ${target}, ${keys} - file is infected. Triggering delete of the file.`)
+      await AttachmentsSrv.emit('DeleteInfectedAttachment', { target: target, keys, hash })
+      if (_target.drafts?.name === target) {
+        await AttachmentsSrv.emit('DeleteInfectedAttachment', { target: _target.drafts.name, keys, hash })
+      }
+    } else {
+      LOG.debug(`Malware scan completed for ${target}, ${keys} - file is clean`)
+    }
+
+    // Assign hash as another condition to ensure the correct file is marked as fine
+    await this.updateStatus(_target, Object.assign({ hash }, keys), status)
+  }
+
+  /**
+   * Mocks scanning the file. Always returns true!
+   * @param {import('@sap/cds').Request} the request object
+   */
+  async scanFile(req) {
+    const { file } = req.data;
+
+    LOG.info(`Setting scan status to Clean (development mode)!`)
+
+    let fileSize = 0;
+    const hash = crypto.createHash('sha256');
+
+    if (file) {
+      for await (const chunk of file) {
+        fileSize += chunk.length;
+        hash.update(chunk);
+      }
+    }
+
+    const sha256Hash = hash.digest('hex');
+    file?.destroy();
+
+    return {
+      isMalware: false,
+      encryptedContentDetected: false,
+      scanSize: fileSize,
+      finding: undefined,
+      mimeType: 'empty',
+      hash: sha256Hash,
+    };
+  }
+
+  async getFileInformation(_target, keys) {
+    const dbResult = await SELECT.one.from(_target.drafts || _target).columns('mimeType').where(keys)
+    return dbResult
+  }
+
+  async updateStatus(_target, keys, status) {
+    if (_target.drafts) {
+      await Promise.all([
+        UPDATE.entity(_target).where(keys).set({ status }),
+        UPDATE.entity(_target.drafts).where(keys).set({ status })
+      ])
+    } else {
+      await UPDATE.entity(_target).where(keys).set({ status })
+    }
+  }
+}
+
+module.exports = MockedMalwareScanner