@defra-fish/fulfilment-job 1.55.0 → 1.56.0

package/README.md

@@ -19,21 +19,16 @@ provider.
 
 # Environment variables
 
-| name                                | description                                                                               | required | default | valid                                                                   | notes |
-| ----------------------------------- | ----------------------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- |
-| NODE_ENV                            | Node environment                                                                          |    no    |         | development, test, production                                           |       |
-| FULFILMENT_FILE_SIZE                | The maximum number of records written to an aggregated fulfilment file                    |   yes    |         |                                                                         |       |
-| FULFILMENT_FTP_HOST                 | The hostname of the target FTP server                                                     |   yes    |         |                                                                         |       |
-| FULFILMENT_FTP_PORT                 | The port of the FTP service on the target server                                          |   yes    |         |                                                                         |       |
-| FULFILMENT_FTP_PATH                 | The base path under which files should be written to the FTP server                       |   yes    |         |                                                                         |       |
-| FULFILMENT_FTP_USERNAME             | The username used to authenticate with the FTP server                                     |   yes    |         |                                                                         |       |
-| FULFILMENT_FTP_KEY_SECRET_ID        | The ID of the secret in AWS secrets manager which contains the SSH key for authentication |   yes    |         |                                                                         |       |
-| FULFILMENT_S3_BUCKET                | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data             |   yes    |         |                                                                         |       |
-| FULFILMENT_SEND_UNENCRYPTED_FILE    | Flag for whether to send the unencrypted fulfilment file                                  |    no    | false   | true, false, 0, 1                                                       |       |
-| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key                                          |   yes    |         |                                                                         |       |
-| DEBUG                               | Use to enable output of debug information to the console                                  |   yes    |         | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp |       |
-| AIRBRAKE_HOST                       | URL of airbrake host                                                                      |    no    |         |                                                                         |       |
-| AIRBRAKE_PROJECT_KEY                | Project key for airbrake logging                                                          |    no    |         |                                                                         |       |
+| name                                | description                                                                   | required | default | valid                                                                   | notes |
+| ----------------------------------- | ----------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- |
+| NODE_ENV                            | Node environment                                                              |    no    |         | development, test, production                                           |       |
+| FULFILMENT_FILE_SIZE                | The maximum number of records written to an aggregated fulfilment file        |   yes    |         |                                                                         |       |
+| FULFILMENT_S3_BUCKET                | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data |   yes    |         |                                                                         |       |
+| FULFILMENT_SEND_UNENCRYPTED_FILE    | Flag for whether to send the unencrypted fulfilment file                      |    no    | false   | true, false, 0, 1                                                       |       |
+| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key                              |   yes    |         |                                                                         |       |
+| DEBUG                               | Use to enable output of debug information to the console                      |   yes    |         | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp |       |
+| AIRBRAKE_HOST                       | URL of airbrake host                                                          |    no    |         |                                                                         |       |
+| AIRBRAKE_PROJECT_KEY                | Project key for airbrake logging                                              |    no    |         |                                                                         |       |
 
 ### See also:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defra-fish/fulfilment-job",
-  "version": "1.55.0",
+  "version": "1.56.0",
   "description": "Rod Licensing Sales Fulfilment Job",
   "type": "module",
   "engines": {
@@ -35,15 +35,14 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@defra-fish/connectors-lib": "1.55.0",
-    "@defra-fish/dynamics-lib": "1.55.0",
+    "@defra-fish/connectors-lib": "1.56.0",
+    "@defra-fish/dynamics-lib": "1.56.0",
     "commander": "^7.2.0",
     "debug": "^4.3.3",
     "merge2": "^1.4.1",
     "moment": "^2.29.1",
     "openpgp": "^5.0.0-1",
-    "pluralize": "^8.0.0",
-    "ssh2-sftp-client": "^6.0.1"
+    "pluralize": "^8.0.0"
   },
-  "gitHead": "3405322b81a2016ba22e530de8d096a49be0bea4"
+  "gitHead": "4edcdd349b077ad0bdb3e7b4029df3aba7c04b49"
 }

package/src/__tests__/config.spec.js

@@ -17,11 +17,6 @@ const clearEnvVars = () => {
 
 const envVars = Object.freeze({
   FULFILMENT_FILE_SIZE: 1234,
-  FULFILMENT_FTP_HOST: 'test-host',
-  FULFILMENT_FTP_PORT: 2222,
-  FULFILMENT_FTP_PATH: '/remote/share',
-  FULFILMENT_FTP_USERNAME: 'test-user',
-  FULFILMENT_FTP_KEY_SECRET_ID: 'test-secret-id',
   FULFILMENT_S3_BUCKET: 'test-bucket',
   FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID: 'pgp-key-secret-id',
   FULFILMENT_SEND_UNENCRYPTED_FILE: 'false'
@@ -44,32 +39,6 @@ describe('config', () => {
     })
   })
 
-  describe('ftp', () => {
-    it('provides properties relating the use of SFTP', async () => {
-      expect(config.ftp).toEqual(
-        expect.objectContaining({
-          host: 'test-host',
-          port: '2222',
-          path: '/remote/share',
-          username: 'test-user',
-          privateKey: 'test-ssh-key',
-          algorithms: { cipher: expect.any(Array), kex: expect.any(Array) },
-          // Wait up to 60 seconds for the SSH handshake
-          readyTimeout: expect.any(Number),
-          // Retry 5 times over a minute
-          retries: expect.any(Number),
-          retry_minTimeout: expect.any(Number),
-          debug: expect.any(Function)
-        })
-      )
-    })
-    it('defaults the sftp port to 22 if the environment variable is not configured', async () => {
-      delete process.env.FULFILMENT_FTP_PORT
-      await config.initialise()
-      expect(config.ftp.port).toEqual('22')
-    })
-  })
-
   describe('s3', () => {
     it('provides properties relating the use of Amazon S3', async () => {
       expect(config.s3.bucket).toEqual('test-bucket')
@@ -79,7 +48,7 @@ describe('config', () => {
 
 describe('pgp config', () => {
   const init = async (samplePublicKey = 'sample-pgp-key') => {
-    AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: 'test-ssh-key' }, { SecretString: samplePublicKey })
+    AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: samplePublicKey })
     await config.initialise()
   }
   beforeAll(setEnvVars)

package/src/config.js

@@ -1,45 +1,6 @@
 import { AWS } from '@defra-fish/connectors-lib'
-import db from 'debug'
-const { secretsManager } = AWS()
 
-/**
- * Key exchange algorithms for public key authentication - in descending order of priority
- * @type {string[]}
- */
-export const SFTP_KEY_EXCHANGE_ALGORITHMS = [
-  'curve25519-sha256@libssh.org',
-  'curve25519-sha256',
-  'ecdh-sha2-nistp521',
-  'ecdh-sha2-nistp384',
-  'ecdh-sha2-nistp256',
-  'diffie-hellman-group-exchange-sha256',
-  'diffie-hellman-group14-sha256',
-  'diffie-hellman-group16-sha512',
-  'diffie-hellman-group18-sha512',
-  'diffie-hellman-group14-sha1',
-  'diffie-hellman-group-exchange-sha1',
-  'diffie-hellman-group1-sha1'
-]
-/**
- * Ciphers for SFTP support - in descending order of priority
- * @type {string[]}
- */
-export const SFTP_CIPHERS = [
-  // http://tools.ietf.org/html/rfc4344#section-4
-  'aes256-ctr',
-  'aes192-ctr',
-  'aes128-ctr',
-  'aes256-gcm',
-  'aes256-gcm@openssh.com',
-  'aes128-gcm',
-  'aes128-gcm@openssh.com',
-  'aes256-cbc',
-  'aes192-cbc',
-  'aes128-cbc',
-  'blowfish-cbc',
-  '3des-cbc',
-  'cast128-cbc'
-]
+const { secretsManager } = AWS()
 const falseRegEx = /(false|0)/i
 const trueRegEx = /(true|1)/i
 const toBoolean = val => {
@@ -54,7 +15,6 @@ const toBoolean = val => {
 
 class Config {
   _file
-  _ftp
   _s3
   _pgp
 
@@ -68,20 +28,6 @@ class Config {
        */
       partFileSize: Math.min(Number.parseInt(process.env.FULFILMENT_FILE_SIZE), 999)
     }
-    this.ftp = {
-      host: process.env.FULFILMENT_FTP_HOST,
-      port: process.env.FULFILMENT_FTP_PORT || '22',
-      path: process.env.FULFILMENT_FTP_PATH,
-      username: process.env.FULFILMENT_FTP_USERNAME,
-      privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.FULFILMENT_FTP_KEY_SECRET_ID }).promise()).SecretString,
-      algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS },
-      // Wait up to 60 seconds for the SSH handshake
-      readyTimeout: 60000,
-      // Retry 5 times over a minute
-      retries: 5,
-      retry_minTimeout: 12000,
-      debug: db('fulfilment:ftp')
-    }
     this.s3 = {
       bucket: process.env.FULFILMENT_S3_BUCKET
     }
@@ -104,18 +50,6 @@ class Config {
     this._file = cfg
   }
 
-  /**
-   * FTP configuration settings
-   * @type {object}
-   */
-  get ftp () {
-    return this._ftp
-  }
-
-  set ftp (cfg) {
-    this._ftp = cfg
-  }
-
   /**
    * S3 configuration settings
    * @type {object}

package/src/staging/__tests__/deliver-fulfilment-files.spec.js

@@ -1,7 +1,6 @@
 import { Readable, PassThrough, Writable } from 'stream'
 import { deliverFulfilmentFiles } from '../deliver-fulfilment-files.js'
 import { createS3WriteStream, readS3PartFiles } from '../../transport/s3.js'
-import { createFtpWriteStream } from '../../transport/ftp.js'
 import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from '../staging-common.js'
 import { FulfilmentRequestFile, executeQuery, persist } from '@defra-fish/dynamics-lib'
 import openpgp from 'openpgp'
@@ -10,7 +9,6 @@ import streamHelper from '../streamHelper.js'
 import merge2 from 'merge2'
 
 jest.mock('../../transport/s3.js')
-jest.mock('../../transport/ftp.js')
 jest.mock('openpgp', () => ({
   readKey: jest.fn(() => ({})),
   encrypt: jest.fn(({ message: readableStream }) => readableStream),
@@ -46,20 +44,10 @@ describe('deliverFulfilmentFiles', () => {
     executeQuery.mockResolvedValue([{ entity: mockFulfilmentRequestFile2 }, { entity: mockFulfilmentRequestFile1 }])
 
     // Streams for file1
-    const {
-      s3DataStreamFile: s3DataStreamFile1,
-      ftpDataStreamFile: ftpDataStreamFile1,
-      s3HashStreamFile: s3HashStreamFile1,
-      ftpHashStreamFile: ftpHashStreamFile1
-    } = createMockFileStreams()
+    const { s3DataStreamFile: s3DataStreamFile1, s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams()
 
     // Streams for file2
-    const {
-      s3DataStreamFile: s3DataStreamFile2,
-      ftpDataStreamFile: ftpDataStreamFile2,
-      s3HashStreamFile: s3HashStreamFile2,
-      ftpHashStreamFile: ftpHashStreamFile2
-    } = createMockFileStreams()
+    const { s3DataStreamFile: s3DataStreamFile2, s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams()
 
     // Run the delivery
     await expect(deliverFulfilmentFiles()).resolves.toBeUndefined()
@@ -69,22 +57,14 @@ describe('deliverFulfilmentFiles', () => {
     // File 1 expectations
     expect(createS3WriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json')
     expect(createS3WriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256')
     expect(JSON.parse(s3DataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
-    expect(JSON.parse(ftpDataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
     expect(s3HashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated
-    expect(ftpHashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated
 
     // File 2 expectations
     expect(createS3WriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json')
     expect(createS3WriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256')
     expect(JSON.parse(s3DataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
-    expect(JSON.parse(ftpDataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
     expect(s3HashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated
-    expect(ftpHashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated
 
     // Persist to dynamics for file 1
     expect(persist).toHaveBeenNthCalledWith(1, [
@@ -197,10 +177,7 @@ describe('deliverFulfilmentFiles', () => {
     const s3 = createTestableStream()
     streamHelper.pipelinePromise.mockResolvedValue()
     openpgp.encrypt.mockResolvedValue(s2)
-    merge2
-      .mockReturnValueOnce(s1)
-      .mockReturnValueOnce(s2)
-      .mockReturnValueOnce(s3)
+    merge2.mockReturnValueOnce(s1).mockReturnValueOnce(s2).mockReturnValueOnce(s3)
     await mockExecuteQuery()
     createMockFileStreams()
 
@@ -227,27 +204,18 @@ const createMockFulfilmentRequestFile = async (fileName, date) =>
 
 const createMockFileStreams = () => {
   const s3DataStreamFile = createTestableStream()
-  const ftpDataStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3DataStreamFile, managedUpload: Promise.resolve() })
-  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpDataStreamFile, managedUpload: Promise.resolve() })
 
   const s3EncryptedDataStreamFile = createTestableStream()
-  const ftpEncryptedDataStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3EncryptedDataStreamFile, managedUpload: Promise.resolve() })
-  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpEncryptedDataStreamFile, managedUpload: Promise.resolve() })
 
   const s3HashStreamFile = createTestableStream()
-  const ftpHashStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3HashStreamFile, managedUpload: Promise.resolve() })
-  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpHashStreamFile, managedUpload: Promise.resolve() })
 
   return {
     s3DataStreamFile,
-    ftpDataStreamFile,
     s3EncryptedDataStreamFile,
-    ftpEncryptedDataStreamFile,
-    s3HashStreamFile,
-    ftpHashStreamFile
+    s3HashStreamFile
   }
 }
 

package/src/staging/deliver-fulfilment-files.js

@@ -4,7 +4,6 @@ import merge2 from 'merge2'
 import moment from 'moment'
 import { executeQuery, persist, findFulfilmentFiles } from '@defra-fish/dynamics-lib'
 import { createS3WriteStream, readS3PartFiles } from '../transport/s3.js'
-import { createFtpWriteStream } from '../transport/ftp.js'
 import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from './staging-common.js'
 import db from 'debug'
 import openpgp from 'openpgp'
@@ -69,11 +68,6 @@ const createEncryptedDataReadStream = async file => {
  */
 const deliver = async (targetFileName, readableStream, ...transforms) => {
   const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName)
-  const { ftpWriteStream: ftpDataStream, managedUpload: ftpDataManagedUpload } = createFtpWriteStream(targetFileName)
 
-  await Promise.all([
-    streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream, ftpDataStream]),
-    s3DataManagedUpload,
-    ftpDataManagedUpload
-  ])
+  await Promise.all([streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), s3DataManagedUpload])
 }

package/src/__mocks__/ssh2-sftp-client.js

@@ -1,9 +0,0 @@
-const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client')
-
-export const mockedFtpMethods = {
-  connect: jest.fn(async () => {}),
-  put: jest.fn(async () => {}),
-  end: jest.fn()
-}
-ssh2sftpClient.mockImplementation(() => mockedFtpMethods)
-export default ssh2sftpClient

package/src/transport/__tests__/ftp.spec.js

@@ -1,63 +0,0 @@
-import { createFtpWriteStream } from '../ftp.js'
-import { mockedFtpMethods } from 'ssh2-sftp-client'
-
-jest.mock('stream')
-jest.mock('../../config.js', () => ({
-  ftp: {
-    host: 'testhost',
-    port: 2222,
-    path: 'testpath/',
-    username: 'testusername',
-    privateKey: 'testprivatekey'
-  }
-}))
-
-describe('ftp', () => {
-  beforeEach(() => {
-    jest.clearAllMocks()
-  })
-
-  describe('createFtpWriteStream', () => {
-    it('creates a stream to write to the configured FTP server', async () => {
-      const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json')
-      ftpWriteStream.write('Some data')
-      ftpWriteStream.end()
-      await managedUpload
-      expect(mockedFtpMethods.connect).toHaveBeenCalledWith(
-        expect.objectContaining({
-          host: 'testhost',
-          port: 2222,
-          username: 'testusername',
-          privateKey: 'testprivatekey'
-        })
-      )
-      expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', {
-        flags: 'w',
-        encoding: 'UTF-8',
-        autoClose: false
-      })
-      expect(mockedFtpMethods.end).toHaveBeenCalled()
-    })
-
-    it('rejects the managed upload promise if an FTP upload error occurs', async () => {
-      const testError = new Error('Test error')
-      mockedFtpMethods.put.mockImplementationOnce(() => Promise.reject(testError))
-      const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json')
-      await expect(managedUpload).rejects.toThrow('Test error')
-      expect(mockedFtpMethods.connect).toHaveBeenCalledWith(
-        expect.objectContaining({
-          host: 'testhost',
-          port: 2222,
-          username: 'testusername',
-          privateKey: 'testprivatekey'
-        })
-      )
-      expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', {
-        flags: 'w',
-        encoding: 'UTF-8',
-        autoClose: false
-      })
-      expect(mockedFtpMethods.end).toHaveBeenCalled()
-    })
-  })
-})

package/src/transport/ftp.js

@@ -1,28 +0,0 @@
-import FtpClient from 'ssh2-sftp-client'
-import Path from 'path'
-import { PassThrough } from 'stream'
-import config from '../config.js'
-import db from 'debug'
-const debug = db('fulfilment:transport')
-
-/**
- * Create a stream to write to the configured FTP server
- *
- * @param {string} filename The name of the file to be written to the remote server
- * @returns {{ftpWriteStream: module:stream.internal.PassThrough, managedUpload: Promise<*>}}
- */
-export const createFtpWriteStream = filename => {
-  const sftp = new FtpClient()
-  const passThrough = new PassThrough()
-  const remoteFilePath = Path.join(config.ftp.path, filename)
-  return {
-    ftpWriteStream: passThrough,
-    managedUpload: sftp
-      .connect(config.ftp)
-      .then(() => sftp.put(passThrough, remoteFilePath, { flags: 'w', encoding: 'UTF-8', autoClose: false }))
-      .then(() =>
-        debug('File successfully uploaded to fulfilment provider at sftp://%s:%s%s', config.ftp.host, config.ftp.port, remoteFilePath)
-      )
-      .finally(() => sftp.end())
-  }
-}