@velocitycareerlabs/data-loader 1.20.0-dev-build.1cb592b8f → 1.20.0-dev-build.1e6832e17

package/src/batch-issuing/prepare-data.js

@@ -1,60 +1,21 @@
-const { get, includes, isNil, map, omitBy, values } = require('lodash/fp');
+const { get, includes, isNil, map, omitBy, isEmpty } = require('lodash/fp');
 const { formatISO } = require('date-fns/fp');
 const { nanoid } = require('nanoid');
-const {
-  loadHandlebarsTemplate,
-  loadCsvVariableSets,
-  computeActivationDate,
-} = require('../helpers');
-const { DisclosureType } = require('./constants');
+const { loadHandlebarsTemplate, computeActivationDate } = require('../helpers');
+const { idCredentialTypeToPick } = require('./constants');
+const { getColIndex, prepareVariableSets } = require('../helpers');
 
 const EMAIL_REGEX =
   // eslint-disable-next-line max-len
   /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
-
 const PHONE_REGEX = /\(?([0-9]{3})\)?([ .-]?)([0-9]{3})\2([0-9]{4})/;
-const VENDOR_USER_ID_PATH = 'credentialSubject.vendorUserId';
-
-const CREDENTIAL_TYPES = {
-  EMAIL: 'EmailV1.0',
-  PHONE: 'PhoneV1.0',
-  DRIVER_LICENSE: 'DriversLicenseV1.0',
-  ID_DOCUMENT: 'IdDocumentV1.0',
-  NATIONAL_ID_CARD: 'NationalIdCardV1.0',
-  PASSPORT: 'PassportV1.0',
-  PROOF_OF_AGE: 'ProofOfAgeV1.0',
-  RESIDENT_PERMIT: 'ResidentPermitV1.0',
-};
-
-const DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH =
-  '$.idDocumentCredentials[*].credentialSubject.identifier';
-const credentialTypeToPick = {
-  [CREDENTIAL_TYPES.EMAIL]: '$.emails',
-  [CREDENTIAL_TYPES.PHONE]: '$.phones',
-  [CREDENTIAL_TYPES.DRIVER_LICENSE]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
-  [CREDENTIAL_TYPES.ID_DOCUMENT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
-  [CREDENTIAL_TYPES.NATIONAL_ID_CARD]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
-  [CREDENTIAL_TYPES.PASSPORT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
-  [CREDENTIAL_TYPES.PROOF_OF_AGE]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
-  [CREDENTIAL_TYPES.RESIDENT_PERMIT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
-};
 
-const prepareData = async (
-  disclosureType,
-  { csvFilename, offerTemplateFilename, vars: defaultVars, did, ...options }
-) => {
-  validateCredentialType(options.credentialType);
-  const newDisclosureRequest = prepareNewDisclosureRequest(
-    disclosureType,
-    options
-  );
+const VENDOR_USER_ID_PATH = 'credentialSubject.vendorUserId';
 
-  const variableSets = await loadCsvVariableSets(csvFilename, {
-    ...defaultVars,
-    did,
-  });
-  const offerTemplate = loadHandlebarsTemplate(offerTemplateFilename);
-  const newExchangeOffers = map(
+const prepareExchangeOffers = async (csvHeaders, csvRows, options) => {
+  const variableSets = await prepareVariableSets(csvHeaders, csvRows, options);
+  const offerTemplate = loadHandlebarsTemplate(options.offerTemplateFilename);
+  return map(
     (variableSet) => ({
       newOffer: prepareNewOffer({
         template: offerTemplate,
@@ -65,15 +26,13 @@ const prepareData = async (
     }),
     variableSets
   );
-
-  return { newDisclosureRequest, newExchangeOffers };
 };
 
 const prepareNewOffer = ({ template, variableSet, label }) => {
   validateVariableSet(variableSet);
   const offerString = template(variableSet);
   const offer = JSON.parse(offerString);
-  validateVendorUserId(offer, variableSet);
+  validateVendorUserIdInOffer(offer, variableSet);
   return omitBy(isNil, {
     ...offer,
     offerId: variableSet.offerId ?? nanoid(),
@@ -81,15 +40,6 @@ const prepareNewOffer = ({ template, variableSet, label }) => {
   });
 };
 
-const validateCredentialType = (credentialType) => {
-  const allowedCredentialTypes = values(CREDENTIAL_TYPES);
-  if (!credentialType || !allowedCredentialTypes.includes(credentialType)) {
-    throw new Error(
-      `${credentialType} doesn't exist. Please use one of ${allowedCredentialTypes}`
-    );
-  }
-};
-
 const validateVariableSet = (variableSet) => {
   if (
     variableSet.email == null &&
@@ -99,13 +49,22 @@ const validateVariableSet = (variableSet) => {
     throw new Error('"email", "phone", or "identifier" column must be defined');
   }
 
-  const { email, phone } = variableSet;
+  const { email, phone, vendorUserId } = variableSet;
 
-  validateEmail(email);
-  validatePhone(phone);
+  validateVendorUserIdVariable(vendorUserId);
+  validateEmailVariable(email);
+  validatePhoneVariable(phone);
 };
 
-const validateEmail = (email) => {
+const validateVendorUserIdVariable = (vendorUserId) => {
+  if (isEmpty(vendorUserId)) {
+    throw new Error(
+      'vendorUserId variable must exist. Use the -u <column> option'
+    );
+  }
+};
+
+const validateEmailVariable = (email) => {
   if (!email) return;
 
   if (!EMAIL_REGEX.test(email)) {
@@ -113,7 +72,7 @@ const validateEmail = (email) => {
   }
 };
 
-const validatePhone = (phone) => {
+const validatePhoneVariable = (phone) => {
   if (!phone) return;
 
   if (!PHONE_REGEX.test(phone)) {
@@ -121,7 +80,7 @@ const validatePhone = (phone) => {
   }
 };
 
-const validateVendorUserId = (offer, variableSet) => {
+const validateVendorUserIdInOffer = (offer, variableSet) => {
   const value = get(VENDOR_USER_ID_PATH, offer);
   if (!includes(value, variableSet)) {
     throw new Error(
@@ -132,7 +91,7 @@ const validateVendorUserId = (offer, variableSet) => {
   }
 };
 
-const matcherToCredentialType = (credentialType, variableSet) => {
+const matcherToIdCredentialType = (idCredentialType, variableSet) => {
   const typeToVariable = {
     'EmailV1.0': variableSet.email,
     'PhoneV1.0': variableSet.phone,
@@ -143,32 +102,31 @@ const matcherToCredentialType = (credentialType, variableSet) => {
     'ProofOfAgeV1.0': variableSet.identifier,
     'ResidentPermitV1.0': variableSet.identifier,
   };
-  return typeToVariable[credentialType] || variableSet.email;
+  return typeToVariable[idCredentialType] || variableSet.email;
 };
 
-const prepareNewExchange = ({ variableSet, label, credentialType }) =>
+const prepareNewExchange = ({ variableSet, label, idCredentialType }) =>
   omitBy(isNil, {
     type: 'ISSUING',
     identityMatcherValues: [
-      matcherToCredentialType(credentialType, variableSet),
+      matcherToIdCredentialType(idCredentialType, variableSet),
     ],
     label,
   });
 
 const prepareNewDisclosureRequest = (
-  disclosureType,
+  csvHeaders,
   {
     label,
     termsUrl,
     purpose,
-    credentialType,
-    authTokenExpiresIn = '10080',
+    idCredentialType,
+    identifierMatchColumn,
+    vendorUseridColumn,
+    authTokenExpiresIn,
     ...options
   }
 ) => {
-  if (disclosureType !== DisclosureType.NEW) {
-    return undefined;
-  }
   const activationDate = computeActivationDate(options);
 
   const newDisclosureRequest = {
@@ -177,18 +135,18 @@ const prepareNewDisclosureRequest = (
     vendorEndpoint: 'integrated-issuing-identification',
     types: [
       {
-        type: credentialType,
+        type: idCredentialType,
       },
     ],
     identityMatchers: {
       rules: [
         {
-          valueIndex: 0,
-          path: [credentialTypeToPick[credentialType]],
+          valueIndex: getColIndex(csvHeaders, identifierMatchColumn),
+          path: [idCredentialTypeToPick[idCredentialType]],
           rule: 'pick',
         },
       ],
-      vendorUserIdIndex: 0,
+      vendorUserIdIndex: getColIndex(csvHeaders, vendorUseridColumn),
     },
     setIssuingDefault: true,
     duration: '1h', // 1 hour by default
@@ -204,5 +162,6 @@ const prepareNewDisclosureRequest = (
 };
 
 module.exports = {
-  prepareData,
+  prepareNewDisclosureRequest,
+  prepareExchangeOffers,
 };

package/src/batch-issuing/validate-directory-exists.js

@@ -1,8 +1,8 @@
 const { existsSync } = require('fs');
 
-const validateDirectoryExists = ({ path }) => {
-  if (!existsSync(path)) {
-    throw new Error(`${path} does not exist`);
+const validateDirectoryExists = (options) => {
+  if (options.dryrun == null && !existsSync(options.path)) {
+    throw new Error('Path does not exist. Check the -p var');
   }
 };
 

package/src/helpers/index.js

@@ -3,4 +3,6 @@ module.exports = {
   ...require('./common'),
   ...require('./load-csv'),
   ...require('./load-handlebars-template'),
+  ...require('./parse-column'),
+  ...require('./prepare-variable-sets'),
 };

package/src/helpers/load-csv.js

@@ -1,29 +1,32 @@
 const fs = require('fs');
 const csv = require('csv-parser');
-const { map } = require('lodash/fp');
+const { isString, indexOf } = require('lodash/fp');
 
 const loadCsv = (fileName) => {
   return new Promise((resolve, reject) => {
-    const results = [];
+    const csvRows = [];
+    let csvHeaders;
     fs.createReadStream(fileName)
       .pipe(csv())
+      .on('headers', (headers) => {
+        // eslint-disable-next-line better-mutation/no-mutation
+        csvHeaders = headers;
+      })
       .on('data', (data) => {
         // eslint-disable-next-line better-mutation/no-mutating-methods
-        results.push(data);
+        csvRows.push(data);
       })
       .on('err', (err) => reject(err))
       .on('end', () => {
-        resolve(results);
+        resolve([csvHeaders, csvRows]);
       });
   });
 };
 
-const mergeRowsWithDefaults = (defaultVars) =>
-  map((csvRow) => ({ ...csvRow, ...defaultVars }));
+const getColIndex = (csvHeaders, column) =>
+  isString(column) ? indexOf(column, csvHeaders) : column;
 
-const loadCsvVariableSets = async (fileName, defaultVars) => {
-  const csvRows = await loadCsv(fileName);
-  return mergeRowsWithDefaults(defaultVars)(csvRows);
-};
+const getColName = (csvHeaders, column) =>
+  isString(column) ? column : csvHeaders[column];
 
-module.exports = { loadCsv, loadCsvVariableSets, mergeRowsWithDefaults };
+module.exports = { loadCsv, getColIndex, getColName };

package/src/helpers/parse-column.js

@@ -0,0 +1,8 @@
+const parseColumn = (value) => {
+  const parsedValue = parseInt(value, 10);
+  if (Number.isNaN(parsedValue)) {
+    return value;
+  }
+  return parsedValue;
+};
+module.exports = { parseColumn };

package/src/helpers/prepare-variable-sets.js

@@ -0,0 +1,23 @@
+const { map } = require('lodash/fp');
+const { getColName } = require('./load-csv');
+
+const prepareVariableSets = async (
+  csvHeaders,
+  csvRows,
+  { vendorUseridColumn, vars, did }
+) => {
+  const overrideVars = { ...vars };
+  if (did != null) {
+    overrideVars.did = did;
+  }
+  return map(
+    (csvRow) => ({
+      ...csvRow,
+      ...overrideVars,
+      vendorUserId: csvRow[getColName(csvHeaders, vendorUseridColumn)],
+    }),
+    csvRows
+  );
+};
+
+module.exports = { prepareVariableSets };

package/src/vendor-credentials/README.md

@@ -11,22 +11,62 @@ Checkout the [test data](../test/data).
 
 ## How to Use
 `data-loader vendorcreds [options]`
-### Options
 
-`-c [CSV_FILENAME]` required parameter containing the csv file
+### Config
 
-`-o [OFFER_TEMPLATE_FILENAME]` offer handlebars template. Use moustaches around variables such as `{{did}}` 
+#### General config
 
-`-p [PERSON_TEMPLATE_FILENAME]` user handlebars template. Use moustaches around variables such as `{{email}}`. 
-  If -p is left out, no users will be created; only offers will be created. 
+`-c [CSV_FILENAME]` **required** parameter containing the csv file
+
+`-p [OUTPUT_PATH]` **required** the output directory to use where QR codes and output state files are stored
+
+`-l [LABEL]` A label to attach to the records added to the agent
+
+`--dry-run` Run a test that parses the CSV and creates offers, but does not attempt to write the data to an agent. great for testing!
+
+#### Issuing config
+
+`--new` to create a new issuing configuration ("disclosure")
+
+`-i [DISCLOSURE_ID]` an existing disclosure to use for the batch issuing
+
+`-d [DID]` **required** the issuer's DID
+
+`-t [URL]` **required** the url to the T&Cs that holder must consent to
+
+`--purpose` The purpose to display to the user. Use a maximum for 64 chars. Default is "Career Credential Issuing"
+
+`--authTokenExpiresIn` The number of minutes that the offer will be available for after activation. Default is 365 days.
+
+#### User authentication and matching config
+
+`-y [ID_CREDENTIAL_TYPE]`  the credential type used for identifying the user. Default is Email.
+
+`-u [COLUMN]` the column from the CSV for the user id. Value is made available as "vendorUserId" in the offer template. Default is the first column
+
+`-m [COLUMN]` the column from the CSV for the user to be matched against the ID credential's "identifier" property. Default is the first column
+
+#### Offer generation config
+
+`-o [OFFER_TEMPLATE_FILENAME]` **required** offer handlebars template. Use moustaches around variables such as `{{did}}` 
 
 `--var=[VAR_NAME]=[VAR_VALUE]` variables used in the templates can be specified on the command line. They override any csv values
 
-`-e [URL]` the endpoint of the mockvendor server
+#### Credential Agent config
+
+`-e [URL]` **required if not a dryrun** the endpoint of the mockvendor server
+
+`-t [AUTH_TOKEN]` **required if not a dryrun** the bearer token to use when calling the mockvendor server
+
+`--legacy` the target credential agent is running in the "LEGACY" offer type mode. Default is false
+
+#### Output config
+
+`-x` if passed an output csv is generated including the vendor's user id as the first column and the generated qrcode filename and deeplink
 
-`-t [AUTH_TOKEN]` the bearer token to use when calling the mockvendor server
+`--x-name [OUTPUT_CSV_NAME]` The file name for the output CSV. Default is "output"
 
 ### Dry Run Example
 Dry runs that print out what updates will be issued should omit the `-e` prop
 
-`./data-loader vendorcreds -c ./test/data/variables.csv -o ./test/data/offer.template.json -p ./test/data/person.template.json --var=did=did:ion:sap456`
+`./data-loader vendorcreds -c ./test/data/variables.csv -o ./test/data/offer.template.json -d did:ion:sap456`

package/src/vendor-credentials/index.js

@@ -1,10 +1,7 @@
-/* eslint-disable no-await-in-loop */
-
 const { program } = require('commander');
 const { reduce } = require('lodash/fp');
-const { prepareData } = require('./prepare-data');
-const { printInfo } = require('../helpers/common');
-const { initExecuteUpdate } = require('./execute-update');
+const { printInfo, parseColumn } = require('../helpers');
+const { executeVendorCredentials } = require('./orchestrator');
 
 const parseVar = reduce((acc, pair) => {
   const [key, value] = pair.split('=');
@@ -32,6 +29,13 @@ program
     '-e, --endpoint <url>',
     'Endpoint to call to upload the people and credentials to'
   )
+  .option(
+    '-u  --vendor-userid-column <vendorUseridColumn>',
+    `the column from the CSV that is users id. Value is made available as "vendorUserId" in the offer template. Accepts 
+    header name or index. Default is 0.`,
+    parseColumn,
+    '0'
+  )
   .option('-t, --auth-token <url>', 'Bearer Auth Token to use')
   .option('-l, --label <label>', 'A label to attach to the documents inserted')
   .option('-v, --var <var...>', 'A variable to add. use name=value')
@@ -40,14 +44,6 @@ program
     // eslint-disable-next-line better-mutation/no-mutation
     options.vars = parseVar(options.var);
     printInfo(options);
-    const executeUpdate = initExecuteUpdate(options);
-    const updates = await prepareData(options);
-    if (options.endpoint != null) {
-      for (const update of updates) {
-        await executeUpdate(update);
-      }
-    } else {
-      printInfo(JSON.stringify({ updates }));
-    }
+    await executeVendorCredentials(options);
   })
   .parseAsync(process.argv);

package/src/vendor-credentials/orchestrator.js

@@ -0,0 +1,22 @@
+const { initExecuteUpdate } = require('./execute-update');
+const { prepareData } = require('./prepare-data');
+const { loadCsv, printInfo } = require('../helpers');
+
+// eslint-disable-next-line consistent-return
+const executeVendorCredentials = async (options) => {
+  const [csvHeaders, csvRows] = await loadCsv(options.csvFilename);
+  const updates = await prepareData(csvHeaders, csvRows, options);
+
+  if (options.endpoint == null) {
+    printInfo(JSON.stringify({ updates }));
+    return updates;
+  }
+
+  const executeUpdate = initExecuteUpdate(options);
+  for (const update of updates) {
+    // eslint-disable-next-line no-await-in-loop
+    await executeUpdate(update);
+  }
+};
+
+module.exports = { executeVendorCredentials };

package/src/vendor-credentials/prepare-data.js

@@ -2,16 +2,12 @@ const { map } = require('lodash/fp');
 const {
   loadHandlebarsTemplate,
 } = require('../helpers/load-handlebars-template');
-const { loadCsvVariableSets } = require('../helpers/load-csv');
+const { prepareVariableSets } = require('../helpers');
 
-const prepareData = async ({
-  csvFilename,
-  offerTemplateFilename,
-  personTemplateFilename,
-  label,
-  vars: defaultVars,
-}) => {
-  const variableSets = await loadCsvVariableSets(csvFilename, defaultVars);
+const prepareData = async (csvHeaders, csvRows, options) => {
+  const variableSets = await prepareVariableSets(csvHeaders, csvRows, options);
+
+  const { offerTemplateFilename, personTemplateFilename, label } = options;
   const offerTemplate = loadHandlebarsTemplate(offerTemplateFilename);
   const personTemplate = personTemplateFilename
     ? loadHandlebarsTemplate(personTemplateFilename)