@verii/data-loader 1.0.0-pre.1752076816

package/src/batch-issuing/index.js

@@ -0,0 +1,124 @@
+const { program } = require('commander');
+const { reduce } = require('lodash/fp');
+const { printInfo, printError, parseColumn } = require('../helpers');
+const { runBatchIssuing } = require('./orchestrators');
+
+const parseVar = reduce((acc, pair) => {
+  const [key, value] = pair.split('=');
+  acc[key] = value;
+  return acc;
+}, {});
+
+program
+  .name('data-loader vendorcreds')
+  .description('Loads data into a db')
+  .usage('[options]')
+  .requiredOption(
+    '-c, --csv-filename <filename>',
+    'file name containing variables'
+  )
+  .requiredOption(
+    '-o, --offer-template-filename <filename>',
+    'file name containing the credential template file'
+  )
+  .requiredOption(
+    '-p, --path <path>',
+    'the output directory to use where QR codes and output state files are stored'
+  )
+  .requiredOption(
+    '-t, --terms-url <termsUrl>',
+    'the url to the T&Cs that holder must consent to'
+  )
+  .option(
+    '-d, --did <did>',
+    'DID of the issuing organization. One of `tenant` or `did` must be specified.'
+  )
+  .option(
+    '-n, --tenant <tenantId>',
+    "Id of the issuing organization's tenant. One of `tenant` or `did` must be specified."
+  )
+  .option(
+    '-m, --identifier-match-column <identifierMatchColumn>',
+    `the column from the CSV for the user to be matched against the ID credential's "identifier" property
+    For example this should be the email column if matching against an Email credential type, or the phone number if 
+    matching against a Phone credential type. Accepts header name or index. Default is 0.`,
+    parseColumn,
+    0
+  )
+  .option(
+    '-u, --vendor-userid-column <vendorUseridColumn>',
+    `the column from the CSV that is users id. Value is made available as "vendorUserId" in the offer template. Accepts 
+    header name or index. Default is 0.`,
+    parseColumn,
+    0
+  )
+  .option(
+    '-e, --endpoint <url>',
+    'Credential Agent Endpoint to call to execute the issuing'
+  )
+  .option(
+    '-a, --auth-token <url>',
+    'Bearer Auth Token to be used on the Agent API'
+  )
+  .option('-l, --label <label>', 'A label to attach to the documents inserted')
+  .option(
+    '-v, --var <var...>',
+    'A variable that will be injected into the credential template renderer. use name=value'
+  )
+  .option(
+    '-y, --credential-type <idCredentialType>',
+    'the credential type used for identifying the user. Default is EmailV1.0.',
+    'EmailV1.0'
+  )
+  .option(
+    '--purpose <purpose>',
+    'The purpose to display to the user. Use a maximum for 64 chars. Default is "Career Credential Issuing"'
+  )
+  .option(
+    '--authTokenExpiresIn <authTokenExpiresIn>',
+    'The number of minutes that the offer will be available for after activation. Default is 365 days.',
+    '525600'
+  )
+  .option('--new', 'Use a new disclosure for batch issuing')
+  .option(
+    '-i, --disclosure [disclosure]',
+    'An existing disclosure to use for the batch issuing'
+  )
+  .option(
+    '--legacy',
+    'the target credential agent is running in the "LEGACY" offer type mode. Default is false'
+  )
+  .option(
+    '-x --outputcsv',
+    "if passed an output csv is generated including the vendor's user id as the first column and the generated qrcode filename and deeplink"
+  )
+  .option(
+    '--x-name <outputCsvName>',
+    'The file name for the output CSV. Default is "output"'
+  )
+  .option(
+    '--dryrun',
+    'if passed in then a dry run executes showing how the data will be formatted'
+  )
+  .action(async () => {
+    const options = program.opts();
+    // eslint-disable-next-line better-mutation/no-mutation
+    options.vars = parseVar(options.var);
+    printInfo(options);
+    try {
+      await runBatchIssuing(options);
+    } catch (error) {
+      printError('Batch Issuing Script Failure');
+      if (error.response) {
+        printError({
+          error: error.message,
+          statusCode: error.response.statusCode,
+          errorCode: error.response.errorCode,
+          response: error.response.body,
+        });
+      } else {
+        printError(error);
+      }
+    }
+  })
+  .parseAsync(process.argv);

package/src/batch-issuing/orchestrators.js

@@ -0,0 +1,332 @@
+const { omitBy, isNil, find, isString, isEmpty, values } = require('lodash/fp');
+const { validateDirectoryExists } = require('./validate-directory-exists');
+const { initFetchers } = require('./fetchers');
+const {
+  prepareNewDisclosureRequest,
+  prepareExchangeOffers,
+} = require('./prepare-data');
+const { printInfo } = require('../helpers/common');
+const {
+  writeQrCodeFile,
+  writeJsonFile,
+  writeOutputCsv,
+} = require('./file-writers');
+const { CREDENTIAL_TYPES, DisclosureType } = require('./constants');
+const {
+  askDisclosureType,
+  askDisclosureList,
+  askUseNewDisclosure,
+} = require('./prompts');
+const { loadCsv, getColName } = require('../helpers/load-csv');
+
+const defaultOptions = {
+  vendorUseridColumn: 0,
+  identifierMatchColumn: 0,
+  authTokenExpiresIn: 525600,
+  legacy: false,
+  outputCsvName: 'output',
+  idCredentialType: 'EmailV1.0',
+};
+
+// eslint-disable-next-line consistent-return
+const runBatchIssuing = async (opts) => {
+  const options = { ...defaultOptions, ...opts };
+  validateOptions(options);
+
+  const context = { fetchers: initFetchers(options) };
+  await setupDidOption(options, context);
+  const [csvHeaders, csvRows] = await loadCsv(options.csvFilename);
+
+  const disclosureRequest = await loadOrPrepareNewDisclosureRequest(
+    csvHeaders,
+    options,
+    context
+  );
+
+  const newExchangeOffers = await prepareExchangeOffers(
+    csvHeaders,
+    csvRows,
+    options
+  );
+
+  if (options.dryrun) {
+    printInfo('Dry Run would create:');
+    printInfo(
+      JSON.stringify(
+        omitBy(isNil, { disclosureRequest, newExchangeOffers }),
+        0,
+        2
+      )
+    );
+
+    return { disclosureRequest, newExchangeOffers };
+  }
+
+  const issuingDisclosure =
+    disclosureRequest.id != null
+      ? disclosureRequest
+      : await createDisclosureRequest(disclosureRequest, context);
+
+  await writeDisclosureToJson(issuingDisclosure, options);
+
+  const outputs = options.legacy
+    ? await runLegacyBatchIssuing(
+        issuingDisclosure,
+        newExchangeOffers,
+        options,
+        context
+      )
+    : await runSingleQrCodeBatchIssuing(
+        issuingDisclosure,
+        newExchangeOffers,
+        options,
+        context
+      );
+
+  writeOutput(outputs, {
+    ...options,
+    vendorUserIdColName: getColName(csvHeaders, options.vendorUseridColumn),
+  });
+};
+
+const loadExistingDisclosuresIfRequired = async (options, context) => {
+  if (options.new) {
+    return [];
+  }
+
+  const disclosures = await loadIntegratedIdentificationDisclosures(context);
+  if (options.disclosure) {
+    return disclosures;
+  }
+
+  // interactive mode is handled below
+
+  if (isEmpty(disclosures)) {
+    const useNewDisclosure = await askUseNewDisclosure();
+    if (!useNewDisclosure)
+      throw new Error(
+        'no existing disclosures on the target agent. Use a new disclosure'
+      );
+
+    return [];
+  }
+
+  const disclosureType = await askDisclosureType();
+  return disclosureType === DisclosureType.NEW ? [] : disclosures;
+};
+
+const writeOutput = (outputs, options) => {
+  if (options.outputcsv && options.legacy) {
+    writeOutputCsv(outputs, options);
+  }
+};
+
+const runLegacyBatchIssuing = async (
+  disclosureRequest,
+  newExchangeOffers,
+  options,
+  context
+) => {
+  const outputs = [];
+  for (const newExchangeOffer of newExchangeOffers) {
+    outputs.push(
+      // eslint-disable-next-line no-await-in-loop
+      await createOfferExchangeAndQrCode(
+        {
+          ...newExchangeOffer,
+          disclosureRequest,
+        },
+        options,
+        context
+      )
+    );
+  }
+
+  return outputs;
+};
+
+const runSingleQrCodeBatchIssuing = async (
+  disclosureRequest,
+  newExchangeOffers,
+  options,
+  context
+) => {
+  const outputs = [];
+  for (const newExchangeOffer of newExchangeOffers) {
+    outputs.push(
+      // eslint-disable-next-line no-await-in-loop
+      await createOfferExchange(
+        {
+          ...newExchangeOffer,
+          disclosureRequest,
+        },
+        context
+      )
+    );
+  }
+
+  const { fetchers } = context;
+  printInfo('Generating qrcode & deep link');
+  const deeplink = await fetchers.loadDisclosureDeeplink(disclosureRequest);
+  printInfo(`Deep link: ${deeplink}`);
+  const qrcode = await fetchers.loadDisclosureQrcode(disclosureRequest);
+  const { filePath } = await writeQrCodeFile('qrcode-generic', qrcode, options);
+  printInfo(`QRCode saved: ${filePath}`);
+
+  return outputs;
+};
+
+const loadOrPrepareNewDisclosureRequest = async (
+  csvHeaders,
+  options,
+  context
+) => {
+  const disclosures = await loadExistingDisclosuresIfRequired(options, context);
+
+  if (isEmpty(disclosures)) {
+    return prepareNewDisclosureRequest(csvHeaders, options);
+  }
+
+  const disclosureId = isString(options.disclosure)
+    ? options.disclosure
+    : await askDisclosureList(disclosures);
+  const disclosure = find({ id: disclosureId }, disclosures);
+  if (disclosure == null) {
+    throw new Error('existing disclosure not found');
+  }
+  return disclosure;
+};
+
+const loadIntegratedIdentificationDisclosures = async ({ fetchers }) =>
+  fetchers.getDisclosureList(['integrated-issuing-identification']);
+
+const createDisclosureRequest = async (newDisclosureRequest, { fetchers }) => {
+  return fetchers.createDisclosure(newDisclosureRequest);
+};
+
+const setupDidOption = async (options, { fetchers }) => {
+  if (options.did != null) {
+    return;
+  }
+  let did = 'did to be determined at runtime';
+  if (options.dryrun == null) {
+    const tenant = await fetchers.getTenant();
+    // eslint-disable-next-line better-mutation/no-mutation
+    ({ did } = tenant);
+  }
+  // eslint-disable-next-line better-mutation/no-mutation
+  options.did = did;
+};
+
+const writeDisclosureToJson = async (disclosureRequest, options) => {
+  printInfo(`Using disclosureId:${disclosureRequest.id}`);
+  printInfo('');
+
+  const { filePath: disclosureFilePath } = await writeJsonFile(
+    disclosureRequest,
+    `disclosure-${disclosureRequest.id}`,
+    options
+  );
+  await writeJsonFile(
+    {
+      disclosureId: disclosureRequest.id,
+      disclosureFile: disclosureFilePath,
+      timestamp: new Date().toISOString(),
+      ...options,
+    },
+    'lastrun',
+    options
+  );
+};
+
+const createOfferExchangeAndQrCode = async (
+  { newExchange, newOffer, disclosureRequest },
+  options,
+  context
+) => {
+  const { fetchers } = context;
+  const { exchange, offer, vendorUserId } = await createOfferExchange(
+    {
+      newExchange,
+      newOffer,
+      disclosureRequest,
+    },
+    context
+  );
+
+  const deeplink = await fetchers.loadExchangeDeeplink(exchange);
+  const qrcode = await fetchers.loadExchangeQrcode(exchange);
+  const { filePath } = await writeQrCodeFile(
+    `qrcode-${vendorUserId}`,
+    qrcode,
+    options
+  );
+  printInfo(`${vendorUserId} Done. Qrcode file:${filePath}`);
+  printInfo('');
+
+  return {
+    exchange,
+    offer,
+    qrcode,
+    qrcodeFilePath: filePath,
+    deeplink,
+    vendorUserId,
+  };
+};
+
+const createOfferExchange = async (
+  { newExchange, newOffer, disclosureRequest },
+  context
+) => {
+  const { fetchers } = context;
+  const { vendorUserId } = newOffer.credentialSubject;
+  printInfo(`Setting up vendorUserId:${vendorUserId}`);
+  const exchange = await fetchers.createOfferExchange({
+    ...newExchange,
+    disclosureId: disclosureRequest.id,
+  });
+  const offer = await fetchers.createOffer(exchange, newOffer);
+  await fetchers.submitCompleteOffer(exchange, [offer]);
+  return {
+    exchange,
+    offer,
+    vendorUserId,
+  };
+};
+
+const validateOptions = (options) => {
+  if (options.dryrun == null && options.endpoint == null) {
+    throw new Error(
+      '"-e" or "--endpoint" is required unless executing a "dryrun"'
+    );
+  }
+  if (options.endpoint != null && options.authToken == null) {
+    throw new Error('"-a" or "--auth-token" is required');
+  }
+
+  validateTenantAndDidArgs(options);
+
+  validateDirectoryExists(options);
+
+  validateCredentialType(options.idCredentialType);
+};
+
+const validateTenantAndDidArgs = (options) => {
+  if (options.tenant == null && options.did == null) {
+    throw new Error('one of "--tenant" or "--did" is required');
+  }
+};
+
+const validateCredentialType = (idCredentialType) => {
+  const allowedIdCredentialTypes = values(CREDENTIAL_TYPES);
+  if (
+    !idCredentialType ||
+    !allowedIdCredentialTypes.includes(idCredentialType)
+  ) {
+    throw new Error(
+      `${idCredentialType} doesn't exist. Please use one of ${allowedIdCredentialTypes}`
+    );
+  }
+};
+
+module.exports = { runBatchIssuing };

package/src/batch-issuing/prepare-data.js

@@ -0,0 +1,167 @@
+const { get, includes, isNil, map, omitBy, isEmpty } = require('lodash/fp');
+const { formatISO } = require('date-fns/fp');
+const { nanoid } = require('nanoid');
+const { loadHandlebarsTemplate, computeActivationDate } = require('../helpers');
+const { idCredentialTypeToPick } = require('./constants');
+const { getColIndex, prepareVariableSets } = require('../helpers');
+
+const EMAIL_REGEX =
+  // eslint-disable-next-line max-len
+  /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+const PHONE_REGEX = /\(?([0-9]{3})\)?([ .-]?)([0-9]{3})\2([0-9]{4})/;
+
+const VENDOR_USER_ID_PATH = 'credentialSubject.vendorUserId';
+
+const prepareExchangeOffers = async (csvHeaders, csvRows, options) => {
+  const variableSets = await prepareVariableSets(csvHeaders, csvRows, options);
+  const offerTemplate = loadHandlebarsTemplate(options.offerTemplateFilename);
+  return map(
+    (variableSet) => ({
+      newOffer: prepareNewOffer({
+        template: offerTemplate,
+        variableSet,
+        ...options,
+      }),
+      newExchange: prepareNewExchange({ variableSet, ...options }),
+    }),
+    variableSets
+  );
+};
+
+const prepareNewOffer = ({ template, variableSet, label }) => {
+  validateVariableSet(variableSet);
+  const offerString = template(variableSet);
+  const offer = JSON.parse(offerString);
+  validateVendorUserIdInOffer(offer, variableSet);
+  return omitBy(isNil, {
+    ...offer,
+    offerId: variableSet.offerId ?? nanoid(),
+    label,
+  });
+};
+
+const validateVariableSet = (variableSet) => {
+  if (
+    variableSet.email == null &&
+    variableSet.phone == null &&
+    variableSet.identifier == null
+  ) {
+    throw new Error('"email", "phone", or "identifier" column must be defined');
+  }
+
+  const { email, phone, vendorUserId } = variableSet;
+
+  validateVendorUserIdVariable(vendorUserId);
+  validateEmailVariable(email);
+  validatePhoneVariable(phone);
+};
+
+const validateVendorUserIdVariable = (vendorUserId) => {
+  if (isEmpty(vendorUserId)) {
+    throw new Error(
+      'vendorUserId variable must exist. Use the -u <column> option'
+    );
+  }
+};
+
+const validateEmailVariable = (email) => {
+  if (!email) return;
+
+  if (!EMAIL_REGEX.test(email)) {
+    throw new Error(`${email} is not a valid email`);
+  }
+};
+
+const validatePhoneVariable = (phone) => {
+  if (!phone) return;
+
+  if (!PHONE_REGEX.test(phone)) {
+    throw new Error(`${phone} is not a valid phone`);
+  }
+};
+
+const validateVendorUserIdInOffer = (offer, variableSet) => {
+  const value = get(VENDOR_USER_ID_PATH, offer);
+  if (!includes(value, variableSet)) {
+    throw new Error(
+      `${VENDOR_USER_ID_PATH}: ${value} cannot be hardcoded and must be defined in ${JSON.stringify(
+        variableSet
+      )})`
+    );
+  }
+};
+
+const matcherToIdCredentialType = (idCredentialType, variableSet) => {
+  const typeToVariable = {
+    'EmailV1.0': variableSet.email,
+    'PhoneV1.0': variableSet.phone,
+    'DriversLicenseV1.0': variableSet.identifier,
+    'IdDocumentV1.0': variableSet.identifier,
+    'NationalIdCardV1.0': variableSet.identifier,
+    'PassportV1.0': variableSet.identifier,
+    'ProofOfAgeV1.0': variableSet.identifier,
+    'ResidentPermitV1.0': variableSet.identifier,
+  };
+  return typeToVariable[idCredentialType] || variableSet.email;
+};
+
+const prepareNewExchange = ({ variableSet, label, idCredentialType }) =>
+  omitBy(isNil, {
+    type: 'ISSUING',
+    identityMatcherValues: [
+      matcherToIdCredentialType(idCredentialType, variableSet),
+    ],
+    label,
+  });
+
+const prepareNewDisclosureRequest = (
+  csvHeaders,
+  {
+    label,
+    termsUrl,
+    purpose,
+    idCredentialType,
+    identifierMatchColumn,
+    vendorUseridColumn,
+    authTokenExpiresIn,
+    ...options
+  }
+) => {
+  const activationDate = computeActivationDate(options);
+
+  const newDisclosureRequest = {
+    offerMode: 'preloaded',
+    configurationType: 'issuing',
+    vendorEndpoint: 'integrated-issuing-identification',
+    types: [
+      {
+        type: idCredentialType,
+      },
+    ],
+    identityMatchers: {
+      rules: [
+        {
+          valueIndex: getColIndex(csvHeaders, identifierMatchColumn),
+          path: [idCredentialTypeToPick[idCredentialType]],
+          rule: 'pick',
+        },
+      ],
+      vendorUserIdIndex: getColIndex(csvHeaders, vendorUseridColumn),
+    },
+    setIssuingDefault: true,
+    duration: '1h', // 1 hour by default
+    vendorDisclosureId: Date.now(),
+    purpose: purpose ?? 'Issuing Career Credential', // by default have a generic message
+    activationDate: formatISO(activationDate),
+    authTokenExpiresIn: Number(authTokenExpiresIn),
+    termsUrl,
+    label,
+  };
+
+  return omitBy(isNil, newDisclosureRequest);
+};
+
+module.exports = {
+  prepareNewDisclosureRequest,
+  prepareExchangeOffers,
+};

package/src/batch-issuing/prompts.js

@@ -0,0 +1,47 @@
+const { prompt } = require('inquirer');
+const { format, parseISO } = require('date-fns');
+const { take, map, flow } = require('lodash/fp');
+
+const disclosureListQuestion = (disclosures) => ({
+  type: 'list',
+  name: 'disclosure',
+  message: 'Please select a disclosure',
+  choices: flow(
+    take(10),
+    map((disclosure) => ({
+      name: `${disclosure.purpose}, ${format(
+        parseISO(disclosure.createdAt),
+        'MMM d yyyy h:mma'
+      )}`,
+      value: disclosure.id,
+    }))
+  )(disclosures),
+});
+
+const askQuestion = async (question) => {
+  const result = await prompt([question]);
+  return result[question.name];
+};
+
+const askDisclosureType = () =>
+  askQuestion({
+    type: 'list',
+    name: 'disclosureType',
+    message:
+      'Would you like to use an existing disclosure or create a new one?',
+    choices: ['existing', 'new'],
+  });
+const askUseNewDisclosure = () =>
+  askQuestion({
+    type: 'confirm',
+    name: 'useNewDisclosure',
+    message: 'The batch will create a new disclosure. Press enter to confirm.',
+  });
+const askDisclosureList = (disclosures) =>
+  askQuestion(disclosureListQuestion(disclosures));
+
+module.exports = {
+  askDisclosureType,
+  askDisclosureList,
+  askUseNewDisclosure,
+};

package/src/batch-issuing/validate-directory-exists.js

@@ -0,0 +1,11 @@
+const { existsSync } = require('fs');
+
+const validateDirectoryExists = (options) => {
+  if (options.dryrun == null && !existsSync(options.path)) {
+    throw new Error('Path does not exist. Check the -p var');
+  }
+};
+
+module.exports = {
+  validateDirectoryExists,
+};

package/src/data-loader.js

@@ -0,0 +1,14 @@
+const { program } = require('commander');
+const packageJson = require('../package.json');
+
+program
+  .version(packageJson.version, '-v, --vers', 'Current tool version')
+  .command('vendorcreds', 'load data from csv to the vendor', {
+    executableFile: `${__dirname}/vendor-credentials/index.js`,
+  })
+  .command('batchissuing', 'issue credentials from csv', {
+    executableFile: `${__dirname}/batch-issuing/index.js`,
+  })
+  .usage('[command] [options]')
+  .passThroughOptions()
+  .parse(process.argv);