@defra-fish/fulfilment-job 1.55.0-rc.6 → 1.55.0

package/README.md

@@ -19,16 +19,21 @@ provider.
 
 # Environment variables
 
-| name                                | description                                                                   | required | default | valid                                                                   | notes |
-| ----------------------------------- | ----------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- |
-| NODE_ENV                            | Node environment                                                              |    no    |         | development, test, production                                           |       |
-| FULFILMENT_FILE_SIZE                | The maximum number of records written to an aggregated fulfilment file        |   yes    |         |                                                                         |       |
-| FULFILMENT_S3_BUCKET                | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data |   yes    |         |                                                                         |       |
-| FULFILMENT_SEND_UNENCRYPTED_FILE    | Flag for whether to send the unencrypted fulfilment file                      |    no    | false   | true, false, 0, 1                                                       |       |
-| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key                              |   yes    |         |                                                                         |       |
-| DEBUG                               | Use to enable output of debug information to the console                      |   yes    |         | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp |       |
-| AIRBRAKE_HOST                       | URL of airbrake host                                                          |    no    |         |                                                                         |       |
-| AIRBRAKE_PROJECT_KEY                | Project key for airbrake logging                                              |    no    |         |                                                                         |       |
+| name                                | description                                                                               | required | default | valid                                                                   | notes |
+| ----------------------------------- | ----------------------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- |
+| NODE_ENV                            | Node environment                                                                          |    no    |         | development, test, production                                           |       |
+| FULFILMENT_FILE_SIZE                | The maximum number of records written to an aggregated fulfilment file                    |   yes    |         |                                                                         |       |
+| FULFILMENT_FTP_HOST                 | The hostname of the target FTP server                                                     |   yes    |         |                                                                         |       |
+| FULFILMENT_FTP_PORT                 | The port of the FTP service on the target server                                          |   yes    |         |                                                                         |       |
+| FULFILMENT_FTP_PATH                 | The base path under which files should be written to the FTP server                       |   yes    |         |                                                                         |       |
+| FULFILMENT_FTP_USERNAME             | The username used to authenticate with the FTP server                                     |   yes    |         |                                                                         |       |
+| FULFILMENT_FTP_KEY_SECRET_ID        | The ID of the secret in AWS secrets manager which contains the SSH key for authentication |   yes    |         |                                                                         |       |
+| FULFILMENT_S3_BUCKET                | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data             |   yes    |         |                                                                         |       |
+| FULFILMENT_SEND_UNENCRYPTED_FILE    | Flag for whether to send the unencrypted fulfilment file                                  |    no    | false   | true, false, 0, 1                                                       |       |
+| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key                                          |   yes    |         |                                                                         |       |
+| DEBUG                               | Use to enable output of debug information to the console                                  |   yes    |         | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp |       |
+| AIRBRAKE_HOST                       | URL of airbrake host                                                                      |    no    |         |                                                                         |       |
+| AIRBRAKE_PROJECT_KEY                | Project key for airbrake logging                                                          |    no    |         |                                                                         |       |
 
 ### See also:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defra-fish/fulfilment-job",
-  "version": "1.55.0-rc.6",
+  "version": "1.55.0",
   "description": "Rod Licensing Sales Fulfilment Job",
   "type": "module",
   "engines": {
@@ -35,14 +35,15 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@defra-fish/connectors-lib": "1.55.0-rc.6",
-    "@defra-fish/dynamics-lib": "1.55.0-rc.6",
+    "@defra-fish/connectors-lib": "1.55.0",
+    "@defra-fish/dynamics-lib": "1.55.0",
     "commander": "^7.2.0",
     "debug": "^4.3.3",
     "merge2": "^1.4.1",
     "moment": "^2.29.1",
     "openpgp": "^5.0.0-1",
-    "pluralize": "^8.0.0"
+    "pluralize": "^8.0.0",
+    "ssh2-sftp-client": "^6.0.1"
   },
-  "gitHead": "2c55000bca5980f5de132040edd1b7d1166e5d95"
+  "gitHead": "3405322b81a2016ba22e530de8d096a49be0bea4"
 }

package/src/__mocks__/ssh2-sftp-client.js

@@ -0,0 +1,9 @@
+const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client')
+
+export const mockedFtpMethods = {
+  connect: jest.fn(async () => {}),
+  put: jest.fn(async () => {}),
+  end: jest.fn()
+}
+ssh2sftpClient.mockImplementation(() => mockedFtpMethods)
+export default ssh2sftpClient

package/src/__tests__/config.spec.js

@@ -17,6 +17,11 @@ const clearEnvVars = () => {
 
 const envVars = Object.freeze({
   FULFILMENT_FILE_SIZE: 1234,
+  FULFILMENT_FTP_HOST: 'test-host',
+  FULFILMENT_FTP_PORT: 2222,
+  FULFILMENT_FTP_PATH: '/remote/share',
+  FULFILMENT_FTP_USERNAME: 'test-user',
+  FULFILMENT_FTP_KEY_SECRET_ID: 'test-secret-id',
   FULFILMENT_S3_BUCKET: 'test-bucket',
   FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID: 'pgp-key-secret-id',
   FULFILMENT_SEND_UNENCRYPTED_FILE: 'false'
@@ -39,6 +44,32 @@ describe('config', () => {
     })
   })
 
+  describe('ftp', () => {
+    it('provides properties relating the use of SFTP', async () => {
+      expect(config.ftp).toEqual(
+        expect.objectContaining({
+          host: 'test-host',
+          port: '2222',
+          path: '/remote/share',
+          username: 'test-user',
+          privateKey: 'test-ssh-key',
+          algorithms: { cipher: expect.any(Array), kex: expect.any(Array) },
+          // Wait up to 60 seconds for the SSH handshake
+          readyTimeout: expect.any(Number),
+          // Retry 5 times over a minute
+          retries: expect.any(Number),
+          retry_minTimeout: expect.any(Number),
+          debug: expect.any(Function)
+        })
+      )
+    })
+    it('defaults the sftp port to 22 if the environment variable is not configured', async () => {
+      delete process.env.FULFILMENT_FTP_PORT
+      await config.initialise()
+      expect(config.ftp.port).toEqual('22')
+    })
+  })
+
   describe('s3', () => {
     it('provides properties relating the use of Amazon S3', async () => {
       expect(config.s3.bucket).toEqual('test-bucket')
@@ -48,7 +79,7 @@ describe('config', () => {
 
 describe('pgp config', () => {
   const init = async (samplePublicKey = 'sample-pgp-key') => {
-    AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: samplePublicKey })
+    AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: 'test-ssh-key' }, { SecretString: samplePublicKey })
     await config.initialise()
   }
   beforeAll(setEnvVars)

package/src/config.js

@@ -1,6 +1,45 @@
 import { AWS } from '@defra-fish/connectors-lib'
-
+import db from 'debug'
 const { secretsManager } = AWS()
+
+/**
+ * Key exchange algorithms for public key authentication - in descending order of priority
+ * @type {string[]}
+ */
+export const SFTP_KEY_EXCHANGE_ALGORITHMS = [
+  'curve25519-sha256@libssh.org',
+  'curve25519-sha256',
+  'ecdh-sha2-nistp521',
+  'ecdh-sha2-nistp384',
+  'ecdh-sha2-nistp256',
+  'diffie-hellman-group-exchange-sha256',
+  'diffie-hellman-group14-sha256',
+  'diffie-hellman-group16-sha512',
+  'diffie-hellman-group18-sha512',
+  'diffie-hellman-group14-sha1',
+  'diffie-hellman-group-exchange-sha1',
+  'diffie-hellman-group1-sha1'
+]
+/**
+ * Ciphers for SFTP support - in descending order of priority
+ * @type {string[]}
+ */
+export const SFTP_CIPHERS = [
+  // http://tools.ietf.org/html/rfc4344#section-4
+  'aes256-ctr',
+  'aes192-ctr',
+  'aes128-ctr',
+  'aes256-gcm',
+  'aes256-gcm@openssh.com',
+  'aes128-gcm',
+  'aes128-gcm@openssh.com',
+  'aes256-cbc',
+  'aes192-cbc',
+  'aes128-cbc',
+  'blowfish-cbc',
+  '3des-cbc',
+  'cast128-cbc'
+]
 const falseRegEx = /(false|0)/i
 const trueRegEx = /(true|1)/i
 const toBoolean = val => {
@@ -15,6 +54,7 @@ const toBoolean = val => {
 
 class Config {
   _file
+  _ftp
   _s3
   _pgp
 
@@ -28,6 +68,20 @@ class Config {
        */
       partFileSize: Math.min(Number.parseInt(process.env.FULFILMENT_FILE_SIZE), 999)
     }
+    this.ftp = {
+      host: process.env.FULFILMENT_FTP_HOST,
+      port: process.env.FULFILMENT_FTP_PORT || '22',
+      path: process.env.FULFILMENT_FTP_PATH,
+      username: process.env.FULFILMENT_FTP_USERNAME,
+      privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.FULFILMENT_FTP_KEY_SECRET_ID }).promise()).SecretString,
+      algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS },
+      // Wait up to 60 seconds for the SSH handshake
+      readyTimeout: 60000,
+      // Retry 5 times over a minute
+      retries: 5,
+      retry_minTimeout: 12000,
+      debug: db('fulfilment:ftp')
+    }
     this.s3 = {
       bucket: process.env.FULFILMENT_S3_BUCKET
     }
@@ -50,6 +104,18 @@ class Config {
     this._file = cfg
   }
 
+  /**
+   * FTP configuration settings
+   * @type {object}
+   */
+  get ftp () {
+    return this._ftp
+  }
+
+  set ftp (cfg) {
+    this._ftp = cfg
+  }
+
   /**
    * S3 configuration settings
    * @type {object}

package/src/staging/__tests__/deliver-fulfilment-files.spec.js

@@ -1,6 +1,7 @@
 import { Readable, PassThrough, Writable } from 'stream'
 import { deliverFulfilmentFiles } from '../deliver-fulfilment-files.js'
 import { createS3WriteStream, readS3PartFiles } from '../../transport/s3.js'
+import { createFtpWriteStream } from '../../transport/ftp.js'
 import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from '../staging-common.js'
 import { FulfilmentRequestFile, executeQuery, persist } from '@defra-fish/dynamics-lib'
 import openpgp from 'openpgp'
@@ -9,6 +10,7 @@ import streamHelper from '../streamHelper.js'
 import merge2 from 'merge2'
 
 jest.mock('../../transport/s3.js')
+jest.mock('../../transport/ftp.js')
 jest.mock('openpgp', () => ({
   readKey: jest.fn(() => ({})),
   encrypt: jest.fn(({ message: readableStream }) => readableStream),
@@ -44,10 +46,20 @@ describe('deliverFulfilmentFiles', () => {
     executeQuery.mockResolvedValue([{ entity: mockFulfilmentRequestFile2 }, { entity: mockFulfilmentRequestFile1 }])
 
     // Streams for file1
-    const { s3DataStreamFile: s3DataStreamFile1, s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams()
+    const {
+      s3DataStreamFile: s3DataStreamFile1,
+      ftpDataStreamFile: ftpDataStreamFile1,
+      s3HashStreamFile: s3HashStreamFile1,
+      ftpHashStreamFile: ftpHashStreamFile1
+    } = createMockFileStreams()
 
     // Streams for file2
-    const { s3DataStreamFile: s3DataStreamFile2, s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams()
+    const {
+      s3DataStreamFile: s3DataStreamFile2,
+      ftpDataStreamFile: ftpDataStreamFile2,
+      s3HashStreamFile: s3HashStreamFile2,
+      ftpHashStreamFile: ftpHashStreamFile2
+    } = createMockFileStreams()
 
     // Run the delivery
     await expect(deliverFulfilmentFiles()).resolves.toBeUndefined()
@@ -57,14 +69,22 @@ describe('deliverFulfilmentFiles', () => {
     // File 1 expectations
     expect(createS3WriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json')
     expect(createS3WriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256')
+    expect(createFtpWriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json')
+    expect(createFtpWriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256')
     expect(JSON.parse(s3DataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
+    expect(JSON.parse(ftpDataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
     expect(s3HashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated
+    expect(ftpHashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated
 
     // File 2 expectations
     expect(createS3WriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json')
     expect(createS3WriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256')
+    expect(createFtpWriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json')
+    expect(createFtpWriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256')
     expect(JSON.parse(s3DataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
+    expect(JSON.parse(ftpDataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
     expect(s3HashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated
+    expect(ftpHashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated
 
     // Persist to dynamics for file 1
     expect(persist).toHaveBeenNthCalledWith(1, [
@@ -177,7 +197,10 @@ describe('deliverFulfilmentFiles', () => {
     const s3 = createTestableStream()
     streamHelper.pipelinePromise.mockResolvedValue()
     openpgp.encrypt.mockResolvedValue(s2)
-    merge2.mockReturnValueOnce(s1).mockReturnValueOnce(s2).mockReturnValueOnce(s3)
+    merge2
+      .mockReturnValueOnce(s1)
+      .mockReturnValueOnce(s2)
+      .mockReturnValueOnce(s3)
     await mockExecuteQuery()
     createMockFileStreams()
 
@@ -204,18 +227,27 @@ const createMockFulfilmentRequestFile = async (fileName, date) =>
 
 const createMockFileStreams = () => {
   const s3DataStreamFile = createTestableStream()
+  const ftpDataStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3DataStreamFile, managedUpload: Promise.resolve() })
+  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpDataStreamFile, managedUpload: Promise.resolve() })
 
   const s3EncryptedDataStreamFile = createTestableStream()
+  const ftpEncryptedDataStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3EncryptedDataStreamFile, managedUpload: Promise.resolve() })
+  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpEncryptedDataStreamFile, managedUpload: Promise.resolve() })
 
   const s3HashStreamFile = createTestableStream()
+  const ftpHashStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3HashStreamFile, managedUpload: Promise.resolve() })
+  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpHashStreamFile, managedUpload: Promise.resolve() })
 
   return {
     s3DataStreamFile,
+    ftpDataStreamFile,
     s3EncryptedDataStreamFile,
-    s3HashStreamFile
+    ftpEncryptedDataStreamFile,
+    s3HashStreamFile,
+    ftpHashStreamFile
   }
 }
 

package/src/staging/deliver-fulfilment-files.js

@@ -4,6 +4,7 @@ import merge2 from 'merge2'
 import moment from 'moment'
 import { executeQuery, persist, findFulfilmentFiles } from '@defra-fish/dynamics-lib'
 import { createS3WriteStream, readS3PartFiles } from '../transport/s3.js'
+import { createFtpWriteStream } from '../transport/ftp.js'
 import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from './staging-common.js'
 import db from 'debug'
 import openpgp from 'openpgp'
@@ -68,6 +69,11 @@ const createEncryptedDataReadStream = async file => {
  */
 const deliver = async (targetFileName, readableStream, ...transforms) => {
   const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName)
+  const { ftpWriteStream: ftpDataStream, managedUpload: ftpDataManagedUpload } = createFtpWriteStream(targetFileName)
 
-  await Promise.all([streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), s3DataManagedUpload])
+  await Promise.all([
+    streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream, ftpDataStream]),
+    s3DataManagedUpload,
+    ftpDataManagedUpload
+  ])
 }

package/src/transport/__tests__/ftp.spec.js

@@ -0,0 +1,63 @@
+import { createFtpWriteStream } from '../ftp.js'
+import { mockedFtpMethods } from 'ssh2-sftp-client'
+
+jest.mock('stream')
+jest.mock('../../config.js', () => ({
+  ftp: {
+    host: 'testhost',
+    port: 2222,
+    path: 'testpath/',
+    username: 'testusername',
+    privateKey: 'testprivatekey'
+  }
+}))
+
+describe('ftp', () => {
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  describe('createFtpWriteStream', () => {
+    it('creates a stream to write to the configured FTP server', async () => {
+      const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json')
+      ftpWriteStream.write('Some data')
+      ftpWriteStream.end()
+      await managedUpload
+      expect(mockedFtpMethods.connect).toHaveBeenCalledWith(
+        expect.objectContaining({
+          host: 'testhost',
+          port: 2222,
+          username: 'testusername',
+          privateKey: 'testprivatekey'
+        })
+      )
+      expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', {
+        flags: 'w',
+        encoding: 'UTF-8',
+        autoClose: false
+      })
+      expect(mockedFtpMethods.end).toHaveBeenCalled()
+    })
+
+    it('rejects the managed upload promise if an FTP upload error occurs', async () => {
+      const testError = new Error('Test error')
+      mockedFtpMethods.put.mockImplementationOnce(() => Promise.reject(testError))
+      const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json')
+      await expect(managedUpload).rejects.toThrow('Test error')
+      expect(mockedFtpMethods.connect).toHaveBeenCalledWith(
+        expect.objectContaining({
+          host: 'testhost',
+          port: 2222,
+          username: 'testusername',
+          privateKey: 'testprivatekey'
+        })
+      )
+      expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', {
+        flags: 'w',
+        encoding: 'UTF-8',
+        autoClose: false
+      })
+      expect(mockedFtpMethods.end).toHaveBeenCalled()
+    })
+  })
+})

package/src/transport/ftp.js

@@ -0,0 +1,28 @@
+import FtpClient from 'ssh2-sftp-client'
+import Path from 'path'
+import { PassThrough } from 'stream'
+import config from '../config.js'
+import db from 'debug'
+const debug = db('fulfilment:transport')
+
+/**
+ * Create a stream to write to the configured FTP server
+ *
+ * @param {string} filename The name of the file to be written to the remote server
+ * @returns {{ftpWriteStream: module:stream.internal.PassThrough, managedUpload: Promise<*>}}
+ */
+export const createFtpWriteStream = filename => {
+  const sftp = new FtpClient()
+  const passThrough = new PassThrough()
+  const remoteFilePath = Path.join(config.ftp.path, filename)
+  return {
+    ftpWriteStream: passThrough,
+    managedUpload: sftp
+      .connect(config.ftp)
+      .then(() => sftp.put(passThrough, remoteFilePath, { flags: 'w', encoding: 'UTF-8', autoClose: false }))
+      .then(() =>
+        debug('File successfully uploaded to fulfilment provider at sftp://%s:%s%s', config.ftp.host, config.ftp.port, remoteFilePath)
+      )
+      .finally(() => sftp.end())
+  }
+}