@defra-fish/pocl-job 1.55.0 → 1.56.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defra-fish/pocl-job",
-  "version": "1.55.0",
+  "version": "1.56.0",
   "description": "Post Office Counter Licence sales processor",
   "type": "module",
   "engines": {
@@ -35,16 +35,15 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@defra-fish/business-rules-lib": "1.55.0",
-    "@defra-fish/connectors-lib": "1.55.0",
+    "@defra-fish/business-rules-lib": "1.56.0",
+    "@defra-fish/connectors-lib": "1.56.0",
     "commander": "^7.2.0",
     "debug": "^4.3.3",
     "filesize": "^6.4.0",
     "md5-file": "^5.0.0",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.34",
-    "sax-stream": "^1.3.0",
-    "ssh2-sftp-client": "^6.0.1"
+    "sax-stream": "^1.3.0"
   },
-  "gitHead": "3405322b81a2016ba22e530de8d096a49be0bea4"
+  "gitHead": "4edcdd349b077ad0bdb3e7b4029df3aba7c04b49"
 }

package/src/__tests__/config.spec.js

@@ -11,11 +11,6 @@ describe('config', () => {
     process.env.POCL_RECORD_STAGING_TABLE = 'test-record-staging-table'
     process.env.POCL_STAGING_TTL = 1234
 
-    process.env.POCL_FTP_HOST = 'test-host'
-    process.env.POCL_FTP_PORT = 2222
-    process.env.POCL_FTP_PATH = '/remote/share'
-    process.env.POCL_FTP_USERNAME = 'test-user'
-    process.env.POCL_FTP_KEY_SECRET_ID = 'test-secret-id'
     process.env.POCL_S3_BUCKET = 'test-bucket'
     await config.initialise()
   })
@@ -38,32 +33,6 @@ describe('config', () => {
     })
   })
 
-  describe('ftp', () => {
-    it('provides properties relating the use of SFTP', async () => {
-      expect(config.ftp).toEqual(
-        expect.objectContaining({
-          host: 'test-host',
-          port: '2222',
-          path: '/remote/share',
-          username: 'test-user',
-          privateKey: 'test-ssh-key',
-          algorithms: { cipher: expect.any(Array), kex: expect.any(Array) },
-          // Wait up to 60 seconds for the SSH handshake
-          readyTimeout: expect.any(Number),
-          // Retry 5 times over a minute
-          retries: expect.any(Number),
-          retry_minTimeout: expect.any(Number),
-          debug: expect.any(Function)
-        })
-      )
-    })
-    it('defaults the sftp port to 22 if the environment variable is not configured', async () => {
-      delete process.env.POCL_FTP_PORT
-      await config.initialise()
-      expect(config.ftp.port).toEqual('22')
-    })
-  })
-
   describe('s3', () => {
     it('provides properties relating the use of Amazon S3', async () => {
       expect(config.s3.bucket).toEqual('test-bucket')

package/src/__tests__/pocl-processor.spec.js

@@ -41,7 +41,7 @@ jest.mock('../config.js', () => ({
     bucket: 'testbucket'
   }
 }))
-jest.mock('../transport/ftp-to-s3.js')
+jest.mock('../transport/storeS3MetaData.js')
 jest.mock('../transport/s3-to-local.js')
 jest.mock('../io/db.js')
 jest.mock('../io/s3.js')

package/src/config.js

@@ -1,49 +1,5 @@
-import { AWS } from '@defra-fish/connectors-lib'
-import db from 'debug'
-const { secretsManager } = AWS()
-
-/**
- * Key exchange algorithms for public key authentication - in descending order of priority
- * @type {string[]}
- */
-export const SFTP_KEY_EXCHANGE_ALGORITHMS = [
-  'curve25519-sha256@libssh.org',
-  'curve25519-sha256',
-  'ecdh-sha2-nistp521',
-  'ecdh-sha2-nistp384',
-  'ecdh-sha2-nistp256',
-  'diffie-hellman-group-exchange-sha256',
-  'diffie-hellman-group14-sha256',
-  'diffie-hellman-group16-sha512',
-  'diffie-hellman-group18-sha512',
-  'diffie-hellman-group14-sha1',
-  'diffie-hellman-group-exchange-sha1',
-  'diffie-hellman-group1-sha1'
-]
-/**
- * Ciphers for SFTP support - in descending order of priority
- * @type {string[]}
- */
-export const SFTP_CIPHERS = [
-  // http://tools.ietf.org/html/rfc4344#section-4
-  'aes256-ctr',
-  'aes192-ctr',
-  'aes128-ctr',
-  'aes256-gcm',
-  'aes256-gcm@openssh.com',
-  'aes128-gcm',
-  'aes128-gcm@openssh.com',
-  'aes256-cbc',
-  'aes192-cbc',
-  'aes128-cbc',
-  'blowfish-cbc',
-  '3des-cbc',
-  'cast128-cbc'
-]
-
 class Config {
   _db
-  _ftp
   _s3
 
   async initialise () {
@@ -53,21 +9,6 @@ class Config {
       stagingTtlDelta: Number.parseInt(process.env.POCL_STAGING_TTL || 60 * 60 * 168)
     }
 
-    this.ftp = {
-      host: process.env.POCL_FTP_HOST,
-      port: process.env.POCL_FTP_PORT || '22',
-      path: process.env.POCL_FTP_PATH,
-      username: process.env.POCL_FTP_USERNAME,
-      privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.POCL_FTP_KEY_SECRET_ID }).promise()).SecretString,
-      algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS },
-      // Wait up to 60 seconds for the SSH handshake
-      readyTimeout: 60000,
-      // Retry 5 times over a minute
-      retries: 5,
-      retry_minTimeout: 12000,
-      debug: db('pocl:ftp')
-    }
-
     this.s3 = {
       bucket: process.env.POCL_S3_BUCKET
     }
@@ -85,18 +26,6 @@ class Config {
     this._db = cfg
   }
 
-  /**
-   * FTP configuration settings
-   * @type {object}
-   */
-  get ftp () {
-    return this._ftp
-  }
-
-  set ftp (cfg) {
-    this._ftp = cfg
-  }
-
   /**
    * S3 configuration settings
    * @type {object}

package/src/io/__tests__/s3.spec.js

@@ -5,7 +5,6 @@ import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../..
 import { salesApi } from '@defra-fish/connectors-lib'
 import fs from 'fs'
 import AwsMock from 'aws-sdk'
-import { mockedFtpMethods } from 'ssh2-sftp-client'
 
 jest.mock('fs')
 jest.mock('md5-file')
@@ -159,7 +158,6 @@ describe('s3 operations', () => {
     })
 
     it('skips file processing if a file has already been marked as processed in Dynamics', async () => {
-      mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }])
       fs.createReadStream.mockReturnValueOnce('teststream')
       fs.statSync.mockReturnValueOnce({ size: 1024 })
       salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } })

package/src/io/s3.js

@@ -2,7 +2,7 @@ import moment from 'moment'
 import filesize from 'filesize'
 import config from '../config.js'
 import { DYNAMICS_IMPORT_STAGE } from '../staging/constants.js'
-import { storeS3Metadata } from '../transport/ftp-to-s3.js'
+import { storeS3Metadata } from '../transport/storeS3MetaData.js'
 import { AWS, salesApi } from '@defra-fish/connectors-lib'
 const { s3 } = AWS()
 

package/src/transport/__tests__/storeS3MetaData.spec.js

@@ -0,0 +1,64 @@
+import moment from 'moment'
+import { salesApi } from '@defra-fish/connectors-lib'
+import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js'
+import { updateFileStagingTable } from '../../io/db.js'
+import { storeS3Metadata } from '../storeS3MetaData.js'
+
+jest.mock('../../io/db.js', () => ({
+  updateFileStagingTable: jest.fn()
+}))
+jest.mock('@defra-fish/connectors-lib', () => ({
+  salesApi: {
+    upsertTransactionFile: jest.fn()
+  }
+}))
+
+describe('storeS3Metadata', () => {
+  const md5 = 'mockMd5Hash'
+  const fileSize = 12345
+  const filename = 'testfile'
+  const s3Key = 'mock/s3/key/testfile'
+  const receiptMoment = new Date('2024-10-17T00:00:00Z')
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('console log should output "Storing metadata for s3Key"', async () => {
+    const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn())
+    await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment)
+    expect(consoleLogSpy).toHaveBeenCalledWith(`Storing metadata for ${s3Key}`)
+  })
+
+  it('should call updateFileStagingTable with correct arguments', async () => {
+    await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment)
+    expect(updateFileStagingTable).toHaveBeenCalledWith({
+      filename,
+      md5,
+      fileSize,
+      s3Key,
+      stage: FILE_STAGE.Pending
+    })
+  })
+
+  it('should call salesApi.upsertTransactionFile with correct arguments', async () => {
+    const expectedSalesDate = moment(receiptMoment).subtract(1, 'days').toISOString()
+    const expectedReceiptTimestamp = receiptMoment.toISOString()
+
+    await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment)
+    expect(salesApi.upsertTransactionFile).toHaveBeenCalledWith(filename, {
+      status: DYNAMICS_IMPORT_STAGE.Pending,
+      dataSource: POST_OFFICE_DATASOURCE,
+      fileSize: fileSize,
+      salesDate: expectedSalesDate,
+      receiptTimestamp: expectedReceiptTimestamp,
+      notes: 'Retrieved from the remote server and awaiting processing'
+    })
+  })
+
+  test('should log "Stored metadata for s3Key"', async () => {
+    const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn())
+    await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment)
+    expect(consoleLogSpy).toHaveBeenCalledWith(`Stored metadata for ${s3Key}`)
+  })
+})

package/src/transport/storeS3MetaData.js

@@ -0,0 +1,20 @@
+import moment from 'moment'
+import { salesApi } from '@defra-fish/connectors-lib'
+import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js'
+import { updateFileStagingTable } from '../io/db.js'
+
+export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) {
+  console.log(`Storing metadata for ${s3Key}`)
+  await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending })
+
+  await salesApi.upsertTransactionFile(filename, {
+    status: DYNAMICS_IMPORT_STAGE.Pending,
+    dataSource: POST_OFFICE_DATASOURCE,
+    fileSize: fileSize,
+    salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(),
+    receiptTimestamp: receiptMoment.toISOString(),
+    notes: 'Retrieved from the remote server and awaiting processing'
+  })
+
+  console.log(`Stored metadata for ${s3Key}`)
+}

package/src/__mocks__/ssh2-sftp-client.js

@@ -1,11 +0,0 @@
-const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client')
-
-export const mockedFtpMethods = {
-  connect: jest.fn(),
-  list: jest.fn(),
-  fastGet: jest.fn(),
-  delete: jest.fn(),
-  end: jest.fn()
-}
-ssh2sftpClient.mockImplementation(() => mockedFtpMethods)
-export default ssh2sftpClient

package/src/transport/__tests__/ftp-to-s3.spec.js

@@ -1,145 +0,0 @@
-import { ftpToS3 } from '../ftp-to-s3.js'
-import moment from 'moment'
-import { updateFileStagingTable } from '../../io/db.js'
-import { getTempDir } from '../../io/file.js'
-import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js'
-import { salesApi } from '@defra-fish/connectors-lib'
-import fs from 'fs'
-import md5File from 'md5-file'
-import AwsMock from 'aws-sdk'
-import { mockedFtpMethods } from 'ssh2-sftp-client'
-
-jest.mock('fs')
-jest.mock('md5-file')
-jest.mock('../../io/db.js')
-jest.mock('../../io/file.js')
-
-jest.mock('@defra-fish/connectors-lib', () => {
-  const actual = jest.requireActual('@defra-fish/connectors-lib')
-  return {
-    AWS: actual.AWS,
-    salesApi: {
-      ...Object.keys(actual.salesApi).reduce((acc, k) => ({ ...acc, [k]: jest.fn(async () => {}) }), {})
-    }
-  }
-})
-
-jest.mock('../../config.js', () => ({
-  ftp: {
-    path: '/ftpservershare/'
-  },
-  s3: {
-    bucket: 'testbucket'
-  }
-}))
-
-describe('ftp-to-s3', () => {
-  beforeAll(() => {
-    getTempDir.mockReturnValue('/local/tmp')
-    md5File.mockResolvedValue('example-md5')
-  })
-  beforeEach(() => {
-    jest.clearAllMocks()
-    AwsMock.__resetAll()
-  })
-
-  it('retrieves files from SFTP and stores in S3', async () => {
-    mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.xml' }, { name: 'test2.xml' }])
-    fs.createReadStream.mockReturnValueOnce('test1stream')
-    fs.createReadStream.mockReturnValueOnce('test2stream')
-    fs.statSync.mockReturnValueOnce({ size: 1024 })
-    fs.statSync.mockReturnValueOnce({ size: 2048 })
-    await ftpToS3()
-
-    const localPath1 = '/local/tmp/test1.xml'
-    const localPath2 = '/local/tmp/test2.xml'
-
-    const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml`
-    const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml`
-
-    expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(1, '/ftpservershare/test1.xml', localPath1, {})
-    expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(2, '/ftpservershare/test2.xml', localPath2, {})
-    expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(1, {
-      Bucket: 'testbucket',
-      Key: s3Key1,
-      Body: 'test1stream'
-    })
-    expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(2, {
-      Bucket: 'testbucket',
-      Key: s3Key2,
-      Body: 'test2stream'
-    })
-    expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, {
-      filename: 'test1.xml',
-      md5: 'example-md5',
-      fileSize: '1 KB',
-      stage: FILE_STAGE.Pending,
-      s3Key: s3Key1
-    })
-    expect(updateFileStagingTable).toHaveBeenNthCalledWith(2, {
-      filename: 'test2.xml',
-      md5: 'example-md5',
-      fileSize: '2 KB',
-      stage: FILE_STAGE.Pending,
-      s3Key: s3Key2
-    })
-    expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', {
-      status: DYNAMICS_IMPORT_STAGE.Pending,
-      dataSource: POST_OFFICE_DATASOURCE,
-      fileSize: '1 KB',
-      receiptTimestamp: expect.any(String),
-      salesDate: expect.any(String),
-      notes: 'Retrieved from the remote server and awaiting processing'
-    })
-    expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(2, 'test2.xml', {
-      status: DYNAMICS_IMPORT_STAGE.Pending,
-      dataSource: POST_OFFICE_DATASOURCE,
-      fileSize: '2 KB',
-      receiptTimestamp: expect.any(String),
-      salesDate: expect.any(String),
-      notes: 'Retrieved from the remote server and awaiting processing'
-    })
-    expect(fs.unlinkSync).toHaveBeenNthCalledWith(1, localPath1)
-    expect(fs.unlinkSync).toHaveBeenNthCalledWith(2, localPath2)
-    expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1)
-  })
-
-  it('moves the file to s3 but skips file processing if a file has already been marked as processed in Dynamics', async () => {
-    mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }])
-    fs.createReadStream.mockReturnValueOnce('teststream')
-    fs.statSync.mockReturnValueOnce({ size: 1024 })
-    salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } })
-    const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(jest.fn())
-    await ftpToS3()
-    const localPath = '/local/tmp/test-already-processed.xml'
-    const s3Key = `${moment().format('YYYY-MM-DD')}/test-already-processed.xml`
-    expect(mockedFtpMethods.fastGet).toHaveBeenCalledWith('/ftpservershare/test-already-processed.xml', localPath, {})
-    expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenCalledWith({
-      Bucket: 'testbucket',
-      Key: s3Key,
-      Body: 'teststream'
-    })
-    expect(updateFileStagingTable).not.toHaveBeenCalled()
-    expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled()
-    expect(consoleErrorSpy).toHaveBeenCalled()
-    expect(fs.unlinkSync).toHaveBeenCalledWith(localPath)
-    expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1)
-  })
-
-  it('logs and propogates errors back up the stack', async () => {
-    const testError = new Error('Test error')
-    mockedFtpMethods.list.mockRejectedValue(testError)
-    const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
-    await expect(ftpToS3).rejects.toThrow(testError)
-    expect(consoleErrorSpy).toHaveBeenCalled()
-  })
-
-  it('ignores non-xml files', async () => {
-    mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.pdf' }, { name: 'test2.md' }])
-    await ftpToS3()
-    expect(mockedFtpMethods.fastGet).not.toHaveBeenCalled()
-    expect(AwsMock.S3.mockedMethods.putObject).not.toHaveBeenCalled()
-    expect(fs.unlinkSync).not.toHaveBeenCalled()
-    expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1)
-  })
-})

package/src/transport/ftp-to-s3.js

@@ -1,91 +0,0 @@
-import FtpClient from 'ssh2-sftp-client'
-import moment from 'moment'
-import Path from 'path'
-import fs from 'fs'
-import db from 'debug'
-import md5File from 'md5-file'
-import filesize from 'filesize'
-import config from '../config.js'
-import { getTempDir } from '../io/file.js'
-import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js'
-import { AWS, salesApi } from '@defra-fish/connectors-lib'
-import { updateFileStagingTable } from '../io/db.js'
-const { s3 } = AWS()
-
-const debug = db('pocl:transport')
-const sftp = new FtpClient()
-
-export async function ftpToS3 () {
-  try {
-    debug('Connecting to SFTP endpoint at sftp://%s:%s%s', config.ftp.host, config.ftp.port, config.ftp.path)
-    await sftp.connect(config.ftp)
-    const fileList = await sftp.list(config.ftp.path)
-    debug('Discovered the following files on the SFTP server: %o', fileList)
-    const xmlFiles = fileList.filter(f => Path.extname(f.name).toLowerCase() === '.xml')
-
-    if (!xmlFiles.length) {
-      debug('No XML files were waiting to be processed on the SFTP server.')
-    } else {
-      await retrieveAllFiles(xmlFiles)
-    }
-  } catch (e) {
-    console.error('Error migrating files from the SFTP endpoint', e)
-    throw e
-  } finally {
-    debug('Closing SFTP connection.')
-    await sftp.end()
-  }
-}
-
-export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) {
-  console.log(`Storing metadata for ${s3Key}`)
-  await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending })
-
-  await salesApi.upsertTransactionFile(filename, {
-    status: DYNAMICS_IMPORT_STAGE.Pending,
-    dataSource: POST_OFFICE_DATASOURCE,
-    fileSize: fileSize,
-    salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(),
-    receiptTimestamp: receiptMoment.toISOString(),
-    notes: 'Retrieved from the remote server and awaiting processing'
-  })
-
-  console.log(`Stored metadata for ${s3Key}`)
-}
-
-const retrieveAllFiles = async xmlFiles => {
-  const tempDir = getTempDir('ftp')
-
-  for (const fileEntry of xmlFiles) {
-    const filename = fileEntry.name
-    const remoteFilePath = Path.join(config.ftp.path, filename)
-    const localFilePath = Path.resolve(tempDir, filename)
-
-    // Retrieve from FTP server to local temporary directory
-    debug('Transferring %s to %s', remoteFilePath, localFilePath)
-    await sftp.fastGet(remoteFilePath, localFilePath, {})
-
-    // Transfer to S3
-    const receiptMoment = moment()
-    const s3Key = Path.join(receiptMoment.format('YYYY-MM-DD'), filename)
-    debug('Transferring file to S3 bucket %s with key %s', config.s3.bucket, s3Key)
-    await s3.putObject({ Bucket: config.s3.bucket, Key: s3Key, Body: fs.createReadStream(localFilePath) }).promise()
-
-    const dynamicsRecord = await salesApi.getTransactionFile(filename)
-    if (dynamicsRecord && DYNAMICS_IMPORT_STAGE.isAlreadyProcessed(dynamicsRecord.status.description)) {
-      console.error(
-        'Retrieved file %s from SFTP and stored in S3, however an entry already exists in Dynamics with this filename.  Skipping import.',
-        filename
-      )
-    } else {
-      const md5 = await md5File(localFilePath)
-      const fileSize = filesize(fs.statSync(localFilePath).size)
-      await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment)
-    }
-
-    // Remove from FTP server and local tmp
-    debug('Removing remote file %s', remoteFilePath)
-    await sftp.delete(remoteFilePath)
-    fs.unlinkSync(localFilePath)
-  }
-}