@velocitycareerlabs/data-loader 1.19.0-dev-build.11b7cc7b6

package/src/batch-issuing/index.js

@@ -0,0 +1,115 @@
+const { program } = require('commander');
+const { reduce } = require('lodash/fp');
+const { printInfo, printError } = require('../helpers/common');
+const { runBatchIssuing } = require('./orchestrators');
+
+const parseVar = reduce((acc, pair) => {
+  const [key, value] = pair.split('=');
+  acc[key] = value;
+  return acc;
+}, {});
+
+const validateOptions = (options) => {
+  if (options.dryrun == null && options.endpoint == null) {
+    throw new Error(
+      '"-e" or "--endpoint" is required unless executing a "dryrun"'
+    );
+  }
+  if (options.endpoint != null && options.authToken == null) {
+    throw new Error('"-a" or "--auth-token" is required');
+  }
+};
+
+program
+  .name('data-loader vendorcreds')
+  .description('Loads data into a db')
+  .usage('[options]')
+  .requiredOption(
+    '-c, --csv-filename <filename>',
+    'File name containing variables'
+  )
+  .requiredOption(
+    '-o, --offer-template-filename <filename>',
+    'File name containing the credential template file'
+  )
+  .requiredOption(
+    '-d, --did <did>',
+    'DID of the organization that should do the issuing'
+  )
+  .requiredOption(
+    '-p, --path <path>',
+    'path of directory for the qr-code images and output CSV'
+  )
+  .requiredOption(
+    '-t, --terms-url <termsUrl>',
+    'The terms and conditions url for the holder to accept'
+  )
+  .option(
+    '--legacy',
+    'the target credential agent is running in the "LEGACY" offer type mode. Default is false'
+  )
+  .option(
+    '-x --outputcsv',
+    "if passed an output csv is generated including the vendor's user id as the first column and the generated qrcode filename and deeplink"
+  )
+  .option(
+    '--x-name <outputCsvName>',
+    'The file name for the output CSV. Default is "output"'
+  )
+  .option(
+    '--x-vendor-userid-name <vendorUserIdName>',
+    'The column name to use for the vendor user id in the output csv. Default is "email"'
+  )
+  .option(
+    '--dryrun',
+    'if passed in then a dry run executes showing how the data will be formatted'
+  )
+  .option(
+    '-e, --endpoint <url>',
+    'Credential Agent Endpoint to call to execute the issuing'
+  )
+  .option('-a, --auth-token <url>', 'Bearer Auth Token to use')
+  .option('-l, --label <label>', 'A label to attach to the documents inserted')
+  .option('-v, --var <var...>', 'A variable to add. use name=value')
+  .option(
+    '-y, --credential-type <credentialType>',
+    'The type of credentials. Default is EmailV1.0.',
+    'EmailV1.0'
+  )
+  .option(
+    '--purpose <purpose>',
+    'The purpose to display to the user. Use a maximum for 64 chars. Default is "Career Credential Issuing"'
+  )
+  .option(
+    '--authTokenExpiresIn <authTokenExpiresIn>',
+    'The number of minutes that the offer will be available for after activation. Default is 90 days.',
+    '10080'
+  )
+  .option('--new', 'Use a new disclosure for batch issuing')
+  .option(
+    '-i, --disclosure [disclosure]',
+    'An existing disclosure to use for the batch issuing'
+  )
+  .action(async () => {
+    const options = program.opts();
+    validateOptions(options);
+    // eslint-disable-next-line better-mutation/no-mutation
+    options.vars = parseVar(options.var);
+    printInfo(options);
+    try {
+      await runBatchIssuing(options);
+    } catch (error) {
+      printError('Batch Issuing Script Failure');
+      if (error.response) {
+        printError({
+          error: error.message,
+          statusCode: error.response.statusCode,
+          errorCode: error.response.errorCode,
+          response: error.response.body,
+        });
+      } else {
+        printError(error);
+      }
+    }
+  })
+  .parseAsync(process.argv);

package/src/batch-issuing/orchestrators.js

@@ -0,0 +1,258 @@
+const { omitBy, isNil, find, isString, isEmpty } = require('lodash/fp');
+const { validateDirectoryExists } = require('./validate-directory-exists');
+const { initFetchers } = require('./fetchers');
+const { prepareData } = require('./prepare-data');
+const { printInfo } = require('../helpers/common');
+const {
+  writeQrCodeFile,
+  writeJsonFile,
+  writeOutputCsv,
+} = require('./file-writers');
+const { DisclosureType } = require('./constants');
+const {
+  askDisclosureType,
+  askDisclosureList,
+  askUseNewDisclosure,
+} = require('./prompts');
+
+// eslint-disable-next-line complexity
+const getDisclosureType = async (hasIntegratedDisclosures, options) => {
+  const isIntegratedMode = !options.new && !options.disclosure;
+
+  switch (true) {
+    case isIntegratedMode && !hasIntegratedDisclosures: {
+      const useNewDisclosure = await askUseNewDisclosure();
+      return useNewDisclosure ? DisclosureType.NEW : null;
+    }
+    case isIntegratedMode && hasIntegratedDisclosures:
+      return askDisclosureType();
+    case Boolean(options.disclosure):
+      return DisclosureType.EXISTING;
+    default:
+      return DisclosureType.NEW;
+  }
+};
+
+const runBatchIssuing = async (options) => {
+  validateDirectoryExists(options);
+  const fetchers = initFetchers(options);
+  const disclosures = await getIntegratedDisclosures(fetchers);
+  const disclosureType = await getDisclosureType(
+    !isEmpty(disclosures),
+    options
+  );
+
+  if (isEmpty(disclosureType)) {
+    return;
+  }
+
+  const { newDisclosureRequest, newExchangeOffers } = await prepareData(
+    disclosureType,
+    options
+  );
+
+  if (options.endpoint == null) {
+    printInfo('Dry Run would create:');
+    printInfo(
+      JSON.stringify(
+        omitBy(isNil, { newDisclosureRequest, newExchangeOffers }),
+        0,
+        2
+      )
+    );
+    return;
+  }
+
+  const disclosure = await createOrGotDisclosure({
+    disclosureType,
+    disclosures,
+    newDisclosureRequest,
+    fetchers,
+    options,
+  });
+
+  const disclosureRequest = await writeDisclosureToJson(disclosure, options);
+
+  const outputs = options.legacy
+    ? await runLegacyBatchIssuing(
+        disclosureRequest,
+        newExchangeOffers,
+        fetchers,
+        options
+      )
+    : await runSingleQrCodeBatchIssuing(
+        disclosureRequest,
+        newExchangeOffers,
+        fetchers,
+        options
+      );
+
+  writeOutput(outputs, options);
+};
+
+const writeOutput = (outputs, options) => {
+  if (options.outputcsv && options.legacy) {
+    writeOutputCsv(outputs, options);
+  }
+};
+
+const runLegacyBatchIssuing = async (
+  disclosureRequest,
+  newExchangeOffers,
+  fetchers,
+  options
+) => {
+  const outputs = [];
+  for (const newExchangeOffer of newExchangeOffers) {
+    outputs.push(
+      // eslint-disable-next-line no-await-in-loop
+      await createOfferExchangeAndQrCode({
+        ...newExchangeOffer,
+        disclosureRequest,
+        fetchers,
+        options,
+      })
+    );
+  }
+
+  return outputs;
+};
+
+const runSingleQrCodeBatchIssuing = async (
+  disclosureRequest,
+  newExchangeOffers,
+  fetchers,
+  options
+) => {
+  const outputs = [];
+  for (const newExchangeOffer of newExchangeOffers) {
+    outputs.push(
+      // eslint-disable-next-line no-await-in-loop
+      await createOfferExchange({
+        ...newExchangeOffer,
+        disclosureRequest,
+        fetchers,
+        options,
+      })
+    );
+  }
+
+  printInfo('Generating qrcode & deep link');
+  const deeplink = await fetchers.loadDisclosureDeeplink(disclosureRequest);
+  printInfo(`Deep link: ${deeplink}`);
+  const qrcode = await fetchers.loadDisclosureQrcode(disclosureRequest);
+  const { filePath } = await writeQrCodeFile('qrcode-generic', qrcode, options);
+  printInfo(`QRCode saved: ${filePath}`);
+
+  return outputs;
+};
+
+const getIntegratedDisclosures = async (fetchers) => {
+  const vendorEndpoints = ['integrated-issuing-identification'];
+  const disclosures = await fetchers.getDisclosureList(vendorEndpoints);
+  return disclosures;
+};
+
+const getDisclosure = async (disclosures, fetchers, options) => {
+  const { disclosure } = options;
+
+  if (isString(disclosure)) {
+    return fetchers.getDisclosure(disclosure);
+  }
+
+  const disclosureId = await askDisclosureList(disclosures);
+  return find(({ id }) => id === disclosureId, disclosures);
+};
+
+const writeDisclosureToJson = async (disclosureRequest, options) => {
+  printInfo(`Using disclosureId:${disclosureRequest.id}`);
+  printInfo('');
+
+  const { filePath: disclosureFilePath } = await writeJsonFile(
+    disclosureRequest,
+    `disclosure-${disclosureRequest.id}`,
+    options
+  );
+  await writeJsonFile(
+    {
+      disclosureId: disclosureRequest.id,
+      disclosureFile: disclosureFilePath,
+      timestamp: new Date().toISOString(),
+      ...options,
+    },
+    'lastrun',
+    options
+  );
+  return disclosureRequest;
+};
+
+const createOfferExchangeAndQrCode = async ({
+  newExchange,
+  newOffer,
+  disclosureRequest,
+  options,
+  fetchers,
+}) => {
+  const { exchange, offer, vendorUserId } = await createOfferExchange({
+    newExchange,
+    newOffer,
+    disclosureRequest,
+    fetchers,
+  });
+
+  const deeplink = await fetchers.loadExchangeDeeplink(exchange);
+  const qrcode = await fetchers.loadExchangeQrcode(exchange);
+  const { filePath } = await writeQrCodeFile(
+    `qrcode-${vendorUserId}`,
+    qrcode,
+    options
+  );
+  printInfo(`${vendorUserId} Done. Qrcode file:${filePath}`);
+  printInfo('');
+
+  return {
+    exchange,
+    offer,
+    qrcode,
+    qrcodeFilePath: filePath,
+    deeplink,
+    vendorUserId,
+  };
+};
+
+const createOfferExchange = async ({
+  newExchange,
+  newOffer,
+  disclosureRequest,
+  fetchers,
+}) => {
+  const { vendorUserId } = newOffer.credentialSubject;
+  printInfo(`Setting up vendorUserId:${vendorUserId}`);
+  const exchange = await fetchers.createOfferExchange({
+    ...newExchange,
+    disclosureId: disclosureRequest.id,
+  });
+  const offer = await fetchers.createOffer(exchange, newOffer);
+  await fetchers.submitCompleteOffer(exchange, [offer]);
+  return {
+    exchange,
+    offer,
+    vendorUserId,
+  };
+};
+
+const createOrGotDisclosure = async ({
+  disclosureType,
+  disclosures,
+  newDisclosureRequest,
+  fetchers,
+  options,
+}) => {
+  const disclosure =
+    disclosureType === DisclosureType.NEW
+      ? await fetchers.createDisclosure(newDisclosureRequest)
+      : await getDisclosure(disclosures, fetchers, options);
+  return disclosure;
+};
+
+module.exports = { runBatchIssuing };

package/src/batch-issuing/prepare-data.js

@@ -0,0 +1,208 @@
+const { get, includes, isNil, map, omitBy, values } = require('lodash/fp');
+const { formatISO } = require('date-fns/fp');
+const { nanoid } = require('nanoid');
+const {
+  loadHandlebarsTemplate,
+  loadCsvVariableSets,
+  computeActivationDate,
+} = require('../helpers');
+const { DisclosureType } = require('./constants');
+
+const EMAIL_REGEX =
+  // eslint-disable-next-line max-len
+  /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+
+const PHONE_REGEX = /\(?([0-9]{3})\)?([ .-]?)([0-9]{3})\2([0-9]{4})/;
+const VENDOR_USER_ID_PATH = 'credentialSubject.vendorUserId';
+
+const CREDENTIAL_TYPES = {
+  EMAIL: 'EmailV1.0',
+  PHONE: 'PhoneV1.0',
+  DRIVER_LICENSE: 'DriversLicenseV1.0',
+  ID_DOCUMENT: 'IdDocumentV1.0',
+  NATIONAL_ID_CARD: 'NationalIdCardV1.0',
+  PASSPORT: 'PassportV1.0',
+  PROOF_OF_AGE: 'ProofOfAgeV1.0',
+  RESIDENT_PERMIT: 'ResidentPermitV1.0',
+};
+
+const DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH =
+  '$.idDocumentCredentials[*].credentialSubject.identifier';
+const credentialTypeToPick = {
+  [CREDENTIAL_TYPES.EMAIL]: '$.emails',
+  [CREDENTIAL_TYPES.PHONE]: '$.phones',
+  [CREDENTIAL_TYPES.DRIVER_LICENSE]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.ID_DOCUMENT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.NATIONAL_ID_CARD]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.PASSPORT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.PROOF_OF_AGE]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+  [CREDENTIAL_TYPES.RESIDENT_PERMIT]: DEFAULT_ID_DOCUMENT_CREDENTIALS_PATH,
+};
+
+const prepareData = async (
+  disclosureType,
+  { csvFilename, offerTemplateFilename, vars: defaultVars, did, ...options }
+) => {
+  validateCredentialType(options.credentialType);
+  const newDisclosureRequest = prepareNewDisclosureRequest(
+    disclosureType,
+    options
+  );
+
+  const variableSets = await loadCsvVariableSets(csvFilename, {
+    ...defaultVars,
+    did,
+  });
+  const offerTemplate = loadHandlebarsTemplate(offerTemplateFilename);
+  const newExchangeOffers = map(
+    (variableSet) => ({
+      newOffer: prepareNewOffer({
+        template: offerTemplate,
+        variableSet,
+        ...options,
+      }),
+      newExchange: prepareNewExchange({ variableSet, ...options }),
+    }),
+    variableSets
+  );
+
+  return { newDisclosureRequest, newExchangeOffers };
+};
+
+const prepareNewOffer = ({ template, variableSet, label }) => {
+  validateVariableSet(variableSet);
+  const offerString = template(variableSet);
+  const offer = JSON.parse(offerString);
+  validateVendorUserId(offer, variableSet);
+  return omitBy(isNil, {
+    ...offer,
+    offerId: variableSet.offerId ?? nanoid(),
+    label,
+  });
+};
+
+const validateCredentialType = (credentialType) => {
+  const allowedCredentialTypes = values(CREDENTIAL_TYPES);
+  if (!credentialType || !allowedCredentialTypes.includes(credentialType)) {
+    throw new Error(
+      `${credentialType} doesn't exist. Please use one of ${allowedCredentialTypes}`
+    );
+  }
+};
+
+const validateVariableSet = (variableSet) => {
+  if (
+    variableSet.email == null &&
+    variableSet.phone == null &&
+    variableSet.identifier == null
+  ) {
+    throw new Error('"email", "phone", or "identifier" column must be defined');
+  }
+
+  const { email, phone } = variableSet;
+
+  validateEmail(email);
+  validatePhone(phone);
+};
+
+const validateEmail = (email) => {
+  if (!email) return;
+
+  if (!EMAIL_REGEX.test(email)) {
+    throw new Error(`${email} is not a valid email`);
+  }
+};
+
+const validatePhone = (phone) => {
+  if (!phone) return;
+
+  if (!PHONE_REGEX.test(phone)) {
+    throw new Error(`${phone} is not a valid phone`);
+  }
+};
+
+const validateVendorUserId = (offer, variableSet) => {
+  const value = get(VENDOR_USER_ID_PATH, offer);
+  if (!includes(value, variableSet)) {
+    throw new Error(
+      `${VENDOR_USER_ID_PATH}: ${value} cannot be hardcoded and must be defined in ${JSON.stringify(
+        variableSet
+      )})`
+    );
+  }
+};
+
+const matcherToCredentialType = (credentialType, variableSet) => {
+  const typeToVariable = {
+    'EmailV1.0': variableSet.email,
+    'PhoneV1.0': variableSet.phone,
+    'DriversLicenseV1.0': variableSet.identifier,
+    'IdDocumentV1.0': variableSet.identifier,
+    'NationalIdCardV1.0': variableSet.identifier,
+    'PassportV1.0': variableSet.identifier,
+    'ProofOfAgeV1.0': variableSet.identifier,
+    'ResidentPermitV1.0': variableSet.identifier,
+  };
+  return typeToVariable[credentialType] || variableSet.email;
+};
+
+const prepareNewExchange = ({ variableSet, label, credentialType }) =>
+  omitBy(isNil, {
+    type: 'ISSUING',
+    identityMatcherValues: [
+      matcherToCredentialType(credentialType, variableSet),
+    ],
+    label,
+  });
+
+const prepareNewDisclosureRequest = (
+  disclosureType,
+  {
+    label,
+    termsUrl,
+    purpose,
+    credentialType,
+    authTokenExpiresIn = '10080',
+    ...options
+  }
+) => {
+  if (disclosureType !== DisclosureType.NEW) {
+    return undefined;
+  }
+  const activationDate = computeActivationDate(options);
+
+  const newDisclosureRequest = {
+    offerMode: 'preloaded',
+    configurationType: 'issuing',
+    vendorEndpoint: 'integrated-issuing-identification',
+    types: [
+      {
+        type: credentialType,
+      },
+    ],
+    identityMatchers: {
+      rules: [
+        {
+          valueIndex: 0,
+          path: [credentialTypeToPick[credentialType]],
+          rule: 'pick',
+        },
+      ],
+      vendorUserIdIndex: 0,
+    },
+    setIssuingDefault: true,
+    duration: '1h', // 1 hour by default
+    vendorDisclosureId: Date.now(),
+    purpose: purpose ?? 'Issuing Career Credential', // by default have a generic message
+    activationDate: formatISO(activationDate),
+    authTokenExpiresIn: Number(authTokenExpiresIn),
+    termsUrl,
+    label,
+  };
+
+  return omitBy(isNil, newDisclosureRequest);
+};
+
+module.exports = {
+  prepareData,
+};

package/src/batch-issuing/prompts.js

@@ -0,0 +1,47 @@
+const inquirer = require('inquirer');
+const { format, parseISO } = require('date-fns');
+const { take, map, flow } = require('lodash/fp');
+
+const disclosureListQuestion = (disclosures) => ({
+  type: 'list',
+  name: 'disclosure',
+  message: 'Please select a disclosure',
+  choices: flow(
+    take(10),
+    map((disclosure) => ({
+      name: `${disclosure.purpose}, ${format(
+        parseISO(disclosure.createdAt),
+        'MMM d yyyy h:mma'
+      )}`,
+      value: disclosure.id,
+    }))
+  )(disclosures),
+});
+
+const askQuestion = async (question) => {
+  const result = await inquirer.prompt([question]);
+  return result[question.name];
+};
+
+const askDisclosureType = () =>
+  askQuestion({
+    type: 'list',
+    name: 'disclosureType',
+    message:
+      'Would you like to use an existing disclosure or create a new one?',
+    choices: ['existing', 'new'],
+  });
+const askUseNewDisclosure = () =>
+  askQuestion({
+    type: 'confirm',
+    name: 'useNewDisclosure',
+    message: 'The batch will create a new disclosure. Press enter to confirm.',
+  });
+const askDisclosureList = (disclosures) =>
+  askQuestion(disclosureListQuestion(disclosures));
+
+module.exports = {
+  askDisclosureType,
+  askDisclosureList,
+  askUseNewDisclosure,
+};

package/src/batch-issuing/validate-directory-exists.js

@@ -0,0 +1,11 @@
+const { existsSync } = require('fs');
+
+const validateDirectoryExists = ({ path }) => {
+  if (!existsSync(path)) {
+    throw new Error(`${path} does not exist`);
+  }
+};
+
+module.exports = {
+  validateDirectoryExists,
+};

package/src/data-loader.js

@@ -0,0 +1,14 @@
+const { program } = require('commander');
+const packageJson = require('../package.json');
+
+program
+  .version(packageJson.version, '-v, --vers', 'Current tool version')
+  .command('vendorcreds', 'load data from csv to the vendor', {
+    executableFile: `${__dirname}/vendor-credentials/index.js`,
+  })
+  .command('batchissuing', 'issue credentials from csv', {
+    executableFile: `${__dirname}/batch-issuing/index.js`,
+  })
+  .usage('[command] [options]')
+  .passThroughOptions()
+  .parse(process.argv);

package/src/helpers/common.js

@@ -0,0 +1,30 @@
+const console = require('console');
+const chalk = require('chalk');
+const fs = require('fs');
+const path = require('path');
+
+const writeFile = (filePath, fileContent) => {
+  const fileBasename = path.basename(filePath, '.*');
+
+  console.info(`${chalk.green('Writing:')} ${chalk.whiteBright(fileBasename)}`);
+
+  fs.writeFileSync(filePath, fileContent, 'utf8');
+};
+
+const readFile = (filePath, missingError) => {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(missingError);
+  }
+
+  return fs.readFileSync(filePath, 'utf8');
+};
+
+const printError = (ex) => console.error(ex);
+const printInfo = (data) => console.info(data);
+
+module.exports = {
+  printInfo,
+  writeFile,
+  readFile,
+  printError,
+};

package/src/helpers/compute-activation-date.js

@@ -0,0 +1,12 @@
+const { flow } = require('lodash/fp');
+const { addHours } = require('date-fns/fp');
+
+const computeActivationDate = ({
+  activatesInHours = 0 /* by default activate immediately */,
+}) => {
+  return flow(addHours(activatesInHours))(new Date());
+};
+
+module.exports = {
+  computeActivationDate,
+};

package/src/helpers/index.js

@@ -0,0 +1,6 @@
+module.exports = {
+  ...require('./compute-activation-date'),
+  ...require('./common'),
+  ...require('./load-csv'),
+  ...require('./load-handlebars-template'),
+};