@velocitycareerlabs/data-loader 1.20.0-dev-build.1cb592b8f → 1.20.0-dev-build.1e6832e17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@velocitycareerlabs/data-loader",
-  "version": "1.20.0-dev-build.1cb592b8f",
+  "version": "1.20.0-dev-build.1e6832e17",
   "description": "A tool for uploading data to the different target systems.",
   "repository": "https://github.com/velocitycareerlabs/packages",
   "main": "src/index.js",
@@ -26,6 +26,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-watch": "^7.0.0",
     "jest": "29.6.2",
+    "nock": "^13.2.9",
     "prettier": "^2.2.1"
   },
   "dependencies": {
@@ -39,5 +40,5 @@
     "lodash": "~4.17.20",
     "nanoid": "3.3.1"
   },
-  "gitHead": "1d9b55e87dc8ed0df48d736d72ca68d5841a10ff"
+  "gitHead": "f137575a659703ef89f3cd454989c46d0e0dc231"
 }

package/src/batch-issuing/constants.js

@@ -3,6 +3,31 @@ const DisclosureType = {
   EXISTING: 'existing',
 };
 
+const CREDENTIAL_TYPES = {
+  EMAIL: 'EmailV1.0',
+  PHONE: 'PhoneV1.0',
+  DRIVER_LICENSE: 'DriversLicenseV1.0',
+  ID_DOCUMENT: 'IdDocumentV1.0',
+  NATIONAL_ID_CARD: 'NationalIdCardV1.0',
+  PASSPORT: 'PassportV1.0',
+  PROOF_OF_AGE: 'ProofOfAgeV1.0',
+  RESIDENT_PERMIT: 'ResidentPermitV1.0',
+};
+const DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH =
+  '$.idDocumentCredentials[*].credentialSubject.identifier';
+const idCredentialTypeToPick = {
+  [CREDENTIAL_TYPES.EMAIL]: '$.emails',
+  [CREDENTIAL_TYPES.PHONE]: '$.phones',
+  [CREDENTIAL_TYPES.DRIVER_LICENSE]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.ID_DOCUMENT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.NATIONAL_ID_CARD]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.PASSPORT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.PROOF_OF_AGE]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.RESIDENT_PERMIT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+};
+
 module.exports = {
   DisclosureType,
+  CREDENTIAL_TYPES,
+  idCredentialTypeToPick,
 };

package/src/batch-issuing/file-writers.js

@@ -18,14 +18,11 @@ const writeJsonFile = async (obj, jsonName, { path }) => {
   return { fileName, filePath };
 };
 
-const writeOutputCsv = (
-  data,
-  { path, outputCsvName = 'output', vendorUserIdName = 'email' }
-) => {
+const writeOutputCsv = (data, { path, outputCsvName, vendorUserIdColName }) => {
   const fileName = `${outputCsvName}.csv`;
   const filePath = join(path, fileName);
   const buff = fs.createWriteStream(filePath, {});
-  buff.write([vendorUserIdName, ...OUTPUT_CSV_HEADERS].join(','));
+  buff.write([vendorUserIdColName, ...OUTPUT_CSV_HEADERS].join(','));
   for (const { vendorUserId, deeplink, qrcodeFilePath } of data) {
     buff.write(`\n${vendorUserId},${deeplink},${qrcodeFilePath}`);
   }

package/src/batch-issuing/index.js

@@ -1,79 +1,60 @@
 const { program } = require('commander');
 const { reduce } = require('lodash/fp');
-const { printInfo, printError } = require('../helpers/common');
+const { printInfo, printError, parseColumn } = require('../helpers');
 const { runBatchIssuing } = require('./orchestrators');
 
-const parseVar = reduce((acc, pair) => {
-  const [key, value] = pair.split('=');
-  acc[key] = value;
-  return acc;
-}, {});
-
-const validateOptions = (options) => {
-  if (options.dryrun == null && options.endpoint == null) {
-    throw new Error(
-      '"-e" or "--endpoint" is required unless executing a "dryrun"'
-    );
-  }
-  if (options.endpoint != null && options.authToken == null) {
-    throw new Error('"-a" or "--auth-token" is required');
-  }
-};
-
 program
   .name('data-loader vendorcreds')
   .description('Loads data into a db')
   .usage('[options]')
   .requiredOption(
     '-c, --csv-filename <filename>',
-    'File name containing variables'
+    'file name containing variables'
   )
   .requiredOption(
     '-o, --offer-template-filename <filename>',
-    'File name containing the credential template file'
-  )
-  .requiredOption(
-    '-d, --did <did>',
-    'DID of the organization that should do the issuing'
+    'file name containing the credential template file'
   )
+  .requiredOption('-d, --did <did>', "the issuer's DID")
   .requiredOption(
     '-p, --path <path>',
-    'path of directory for the qr-code images and output CSV'
+    'the output directory to use where QR codes and output state files are stored'
   )
   .requiredOption(
     '-t, --terms-url <termsUrl>',
-    'The terms and conditions url for the holder to accept'
+    'the url to the T&Cs that holder must consent to'
   )
   .option(
-    '--legacy',
-    'the target credential agent is running in the "LEGACY" offer type mode. Default is false'
+    '-m --identifier-match-column <identifierMatchColumn>',
+    `the column from the CSV for the user to be matched against the ID credential's "identifier" property
+    For example this should be the email column if matching against an Email credential type, or the phone number if 
+    matching against a Phone credential type. Accepts header name or index. Default is 0.`,
+    parseColumn,
+    '0'
   )
   .option(
-    '-x --outputcsv',
-    "if passed an output csv is generated including the vendor's user id as the first column and the generated qrcode filename and deeplink"
+    '-u  --vendor-userid-column <vendorUseridColumn>',
+    `the column from the CSV that is users id. Value is made available as "vendorUserId" in the offer template. Accepts 
+    header name or index. Default is 0.`,
+    parseColumn,
+    '0'
   )
   .option(
-    '--x-name <outputCsvName>',
-    'The file name for the output CSV. Default is "output"'
+    '-e, --endpoint <url>',
+    'Credential Agent Endpoint to call to execute the issuing'
   )
   .option(
-    '--x-vendor-userid-name <vendorUserIdName>',
-    'The column name to use for the vendor user id in the output csv. Default is "email"'
+    '-a, --auth-token <url>',
+    'Bearer Auth Token to be used on the Agent API'
   )
+  .option('-l, --label <label>', 'A label to attach to the documents inserted')
   .option(
-    '--dryrun',
-    'if passed in then a dry run executes showing how the data will be formatted'
+    '-v, --var <var...>',
+    'A variable that will be injected into the credential template renderer. use name=value'
   )
   .option(
-    '-e, --endpoint <url>',
-    'Credential Agent Endpoint to call to execute the issuing'
-  )
-  .option('-a, --auth-token <url>', 'Bearer Auth Token to use')
-  .option('-l, --label <label>', 'A label to attach to the documents inserted')
-  .option('-v, --var <var...>', 'A variable to add. use name=value')
-  .option(
-    '-y, --credential-type <credentialType>',
-    'The type of credentials. Default is EmailV1.0.',
+    '-y, --credential-type <idCredentialType>',
+    'the credential type used for identifying the user. Default is EmailV1.0.',
     'EmailV1.0'
   )
   .option(
@@ -82,17 +63,32 @@ program
   )
   .option(
     '--authTokenExpiresIn <authTokenExpiresIn>',
-    'The number of minutes that the offer will be available for after activation. Default is 90 days.',
-    '10080'
+    'The number of minutes that the offer will be available for after activation. Default is 365 days.',
+    '525600'
   )
   .option('--new', 'Use a new disclosure for batch issuing')
   .option(
     '-i, --disclosure [disclosure]',
     'An existing disclosure to use for the batch issuing'
   )
+  .option(
+    '--legacy',
+    'the target credential agent is running in the "LEGACY" offer type mode. Default is false'
+  )
+  .option(
+    '-x --outputcsv',
+    "if passed an output csv is generated including the vendor's user id as the first column and the generated qrcode filename and deeplink"
+  )
+  .option(
+    '--x-name <outputCsvName>',
+    'The file name for the output CSV. Default is "output"'
+  )
+  .option(
+    '--dryrun',
+    'if passed in then a dry run executes showing how the data will be formatted'
+  )
   .action(async () => {
     const options = program.opts();
-    validateOptions(options);
     // eslint-disable-next-line better-mutation/no-mutation
     options.vars = parseVar(options.var);
     printInfo(options);
@@ -113,3 +109,9 @@ program
     }
   })
   .parseAsync(process.argv);
+
+const parseVar = reduce((acc, pair) => {
+  const [key, value] = pair.split('=');
+  acc[key] = value;
+  return acc;
+}, {});

package/src/batch-issuing/orchestrators.js

@@ -1,93 +1,117 @@
-const { omitBy, isNil, find, isString, isEmpty } = require('lodash/fp');
+const { omitBy, isNil, find, isString, isEmpty, values } = require('lodash/fp');
 const { validateDirectoryExists } = require('./validate-directory-exists');
 const { initFetchers } = require('./fetchers');
-const { prepareData } = require('./prepare-data');
+const {
+  prepareNewDisclosureRequest,
+  prepareExchangeOffers,
+} = require('./prepare-data');
 const { printInfo } = require('../helpers/common');
 const {
   writeQrCodeFile,
   writeJsonFile,
   writeOutputCsv,
 } = require('./file-writers');
-const { DisclosureType } = require('./constants');
+const { CREDENTIAL_TYPES, DisclosureType } = require('./constants');
 const {
   askDisclosureType,
   askDisclosureList,
   askUseNewDisclosure,
 } = require('./prompts');
+const { loadCsv, getColName } = require('../helpers/load-csv');
 
-// eslint-disable-next-line complexity
-const getDisclosureType = async (hasIntegratedDisclosures, options) => {
-  const isIntegratedMode = !options.new && !options.disclosure;
-
-  switch (true) {
-    case isIntegratedMode && !hasIntegratedDisclosures: {
-      const useNewDisclosure = await askUseNewDisclosure();
-      return useNewDisclosure ? DisclosureType.NEW : null;
-    }
-    case isIntegratedMode && hasIntegratedDisclosures:
-      return askDisclosureType();
-    case Boolean(options.disclosure):
-      return DisclosureType.EXISTING;
-    default:
-      return DisclosureType.NEW;
-  }
+const defaultOptions = {
+  vendorUseridColumn: 0,
+  identifierMatchColumn: 0,
+  authTokenExpiresIn: 525600,
+  legacy: false,
+  outputCsvName: 'output',
+  idCredentialType: 'EmailV1.0',
 };
 
-const runBatchIssuing = async (options) => {
-  validateDirectoryExists(options);
-  const fetchers = initFetchers(options);
-  const disclosures = await getIntegratedDisclosures(fetchers);
-  const disclosureType = await getDisclosureType(
-    !isEmpty(disclosures),
-    options
-  );
+// eslint-disable-next-line consistent-return
+const runBatchIssuing = async (opts) => {
+  const options = { ...defaultOptions, ...opts };
+  validateOptions(options);
 
-  if (isEmpty(disclosureType)) {
-    return;
-  }
+  const context = { fetchers: initFetchers(options) };
 
-  const { newDisclosureRequest, newExchangeOffers } = await prepareData(
-    disclosureType,
+  const [csvHeaders, csvRows] = await loadCsv(options.csvFilename);
+
+  const disclosureRequest = await loadOrPrepareNewDisclosureRequest(
+    csvHeaders,
+    options,
+    context
+  );
+
+  const newExchangeOffers = await prepareExchangeOffers(
+    csvHeaders,
+    csvRows,
     options
   );
 
-  if (options.endpoint == null) {
+  if (options.dryrun) {
     printInfo('Dry Run would create:');
     printInfo(
       JSON.stringify(
-        omitBy(isNil, { newDisclosureRequest, newExchangeOffers }),
+        omitBy(isNil, { disclosureRequest, newExchangeOffers }),
         0,
         2
       )
     );
-    return;
+
+    return { disclosureRequest, newExchangeOffers };
   }
 
-  const disclosure = await createOrGotDisclosure({
-    disclosureType,
-    disclosures,
-    newDisclosureRequest,
-    fetchers,
-    options,
-  });
+  if (disclosureRequest.id == null) {
+    await createDisclosureRequest(disclosureRequest, context);
+  }
 
-  const disclosureRequest = await writeDisclosureToJson(disclosure, options);
+  await writeDisclosureToJson(disclosureRequest, options);
 
   const outputs = options.legacy
     ? await runLegacyBatchIssuing(
         disclosureRequest,
         newExchangeOffers,
-        fetchers,
-        options
+        options,
+        context
       )
     : await runSingleQrCodeBatchIssuing(
         disclosureRequest,
         newExchangeOffers,
-        fetchers,
-        options
+        options,
+        context
       );
 
-  writeOutput(outputs, options);
+  writeOutput(outputs, {
+    ...options,
+    vendorUserIdColName: getColName(csvHeaders, options.vendorUseridColumn),
+  });
+};
+
+const loadExistingDisclosuresIfRequired = async (options, context) => {
+  if (options.new) {
+    return [];
+  }
+
+  const disclosures = await loadIntegratedIdentificationDisclosures(context);
+  if (options.disclosure) {
+    return disclosures;
+  }
+
+  // interactive mode is handled below
+
+  if (isEmpty(disclosures)) {
+    const useNewDisclosure = await askUseNewDisclosure();
+    if (!useNewDisclosure)
+      throw new Error(
+        'no existing disclosures on the target agent. Use a new disclosure'
+      );
+
+    return [];
+  }
+
+  const disclosureType = await askDisclosureType();
+  return disclosureType === DisclosureType.NEW ? [] : disclosures;
 };
 
 const writeOutput = (outputs, options) => {
@@ -99,19 +123,21 @@ const writeOutput = (outputs, options) => {
 const runLegacyBatchIssuing = async (
   disclosureRequest,
   newExchangeOffers,
-  fetchers,
-  options
+  options,
+  context
 ) => {
   const outputs = [];
   for (const newExchangeOffer of newExchangeOffers) {
     outputs.push(
       // eslint-disable-next-line no-await-in-loop
-      await createOfferExchangeAndQrCode({
-        ...newExchangeOffer,
-        disclosureRequest,
-        fetchers,
+      await createOfferExchangeAndQrCode(
+        {
+          ...newExchangeOffer,
+          disclosureRequest,
+        },
         options,
-      })
+        context
+      )
     );
   }
 
@@ -121,22 +147,24 @@ const runLegacyBatchIssuing = async (
 const runSingleQrCodeBatchIssuing = async (
   disclosureRequest,
   newExchangeOffers,
-  fetchers,
-  options
+  options,
+  context
 ) => {
   const outputs = [];
   for (const newExchangeOffer of newExchangeOffers) {
     outputs.push(
       // eslint-disable-next-line no-await-in-loop
-      await createOfferExchange({
-        ...newExchangeOffer,
-        disclosureRequest,
-        fetchers,
-        options,
-      })
+      await createOfferExchange(
+        {
+          ...newExchangeOffer,
+          disclosureRequest,
+        },
+        context
+      )
     );
   }
 
+  const { fetchers } = context;
   printInfo('Generating qrcode & deep link');
   const deeplink = await fetchers.loadDisclosureDeeplink(disclosureRequest);
   printInfo(`Deep link: ${deeplink}`);
@@ -147,21 +175,32 @@ const runSingleQrCodeBatchIssuing = async (
   return outputs;
 };
 
-const getIntegratedDisclosures = async (fetchers) => {
-  const vendorEndpoints = ['integrated-issuing-identification'];
-  const disclosures = await fetchers.getDisclosureList(vendorEndpoints);
-  return disclosures;
-};
+const loadOrPrepareNewDisclosureRequest = async (
+  csvHeaders,
+  options,
+  context
+) => {
+  const disclosures = await loadExistingDisclosuresIfRequired(options, context);
 
-const getDisclosure = async (disclosures, fetchers, options) => {
-  const { disclosure } = options;
+  if (isEmpty(disclosures)) {
+    return prepareNewDisclosureRequest(csvHeaders, options);
+  }
 
-  if (isString(disclosure)) {
-    return fetchers.getDisclosure(disclosure);
+  const disclosureId = isString(options.disclosure)
+    ? options.disclosure
+    : await askDisclosureList(disclosures);
+  const disclosure = find({ id: disclosureId }, disclosures);
+  if (disclosure == null) {
+    throw new Error('existing disclosure not found');
   }
+  return disclosure;
+};
 
-  const disclosureId = await askDisclosureList(disclosures);
-  return find(({ id }) => id === disclosureId, disclosures);
+const loadIntegratedIdentificationDisclosures = async ({ fetchers }) =>
+  fetchers.getDisclosureList(['integrated-issuing-identification']);
+
+const createDisclosureRequest = async (newDisclosureRequest, { fetchers }) => {
+  return fetchers.createDisclosure(newDisclosureRequest);
 };
 
 const writeDisclosureToJson = async (disclosureRequest, options) => {
@@ -183,16 +222,13 @@ const writeDisclosureToJson = async (disclosureRequest, options) => {
     'lastrun',
     options
   );
-  return disclosureRequest;
 };
 
-const createOfferExchangeAndQrCode = async ({
-  newExchange,
-  newOffer,
-  disclosureRequest,
+const createOfferExchangeAndQrCode = async (
+  { newExchange, newOffer, disclosureRequest },
   options,
-  fetchers,
-}) => {
+  { fetchers }
+) => {
   const { exchange, offer, vendorUserId } = await createOfferExchange({
     newExchange,
     newOffer,
@@ -220,12 +256,10 @@ const createOfferExchangeAndQrCode = async ({
   };
 };
 
-const createOfferExchange = async ({
-  newExchange,
-  newOffer,
-  disclosureRequest,
-  fetchers,
-}) => {
+const createOfferExchange = async (
+  { newExchange, newOffer, disclosureRequest },
+  { fetchers }
+) => {
   const { vendorUserId } = newOffer.credentialSubject;
   printInfo(`Setting up vendorUserId:${vendorUserId}`);
   const exchange = await fetchers.createOfferExchange({
@@ -241,18 +275,31 @@ const createOfferExchange = async ({
   };
 };
 
-const createOrGotDisclosure = async ({
-  disclosureType,
-  disclosures,
-  newDisclosureRequest,
-  fetchers,
-  options,
-}) => {
-  const disclosure =
-    disclosureType === DisclosureType.NEW
-      ? await fetchers.createDisclosure(newDisclosureRequest)
-      : await getDisclosure(disclosures, fetchers, options);
-  return disclosure;
+const validateOptions = (options) => {
+  if (options.dryrun == null && options.endpoint == null) {
+    throw new Error(
+      '"-e" or "--endpoint" is required unless executing a "dryrun"'
+    );
+  }
+  if (options.endpoint != null && options.authToken == null) {
+    throw new Error('"-a" or "--auth-token" is required');
+  }
+
+  validateDirectoryExists(options);
+
+  validateCredentialType(options.idCredentialType);
+};
+
+const validateCredentialType = (idCredentialType) => {
+  const allowedIdCredentialTypes = values(CREDENTIAL_TYPES);
+  if (
+    !idCredentialType ||
+    !allowedIdCredentialTypes.includes(idCredentialType)
+  ) {
+    throw new Error(
+      `${idCredentialType} doesn't exist. Please use one of ${allowedIdCredentialTypes}`
+    );
+  }
 };
 
 module.exports = { runBatchIssuing };