@verii/data-loader 1.0.0-pre.1752076816

package/src/helpers/common.js

@@ -0,0 +1,30 @@
+const console = require('console');
+const chalk = require('chalk');
+const fs = require('fs');
+const path = require('path');
+
+const writeFile = (filePath, fileContent) => {
+  const fileBasename = path.basename(filePath, '.*');
+
+  console.info(`${chalk.green('Writing:')} ${chalk.whiteBright(fileBasename)}`);
+
+  fs.writeFileSync(filePath, fileContent, 'utf8');
+};
+
+const readFile = (filePath, missingError) => {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(missingError);
+  }
+
+  return fs.readFileSync(filePath, 'utf8');
+};
+
+const printError = (ex) => console.error(ex);
+const printInfo = (data) => console.info(data);
+
+module.exports = {
+  printInfo,
+  writeFile,
+  readFile,
+  printError,
+};

package/src/helpers/compute-activation-date.js

@@ -0,0 +1,12 @@
+const { flow } = require('lodash/fp');
+const { addHours } = require('date-fns/fp');
+
+const computeActivationDate = ({
+  activatesInHours = 0 /* by default activate immediately */,
+}) => {
+  return flow(addHours(activatesInHours))(new Date());
+};
+
+module.exports = {
+  computeActivationDate,
+};

package/src/helpers/index.js

@@ -0,0 +1,8 @@
+module.exports = {
+  ...require('./compute-activation-date'),
+  ...require('./common'),
+  ...require('./load-csv'),
+  ...require('./load-handlebars-template'),
+  ...require('./parse-column'),
+  ...require('./prepare-variable-sets'),
+};

package/src/helpers/load-csv.js

@@ -0,0 +1,34 @@
+const fs = require('fs');
+const csv = require('csv-parser');
+const stripBom = require('strip-bom-stream');
+const { isString, indexOf } = require('lodash/fp');
+
+const loadCsv = (fileName) => {
+  return new Promise((resolve, reject) => {
+    const csvRows = [];
+    let csvHeaders;
+    fs.createReadStream(fileName)
+      .pipe(stripBom())
+      .pipe(csv())
+      .on('headers', (headers) => {
+        // eslint-disable-next-line better-mutation/no-mutation
+        csvHeaders = headers;
+      })
+      .on('data', (data) => {
+        // eslint-disable-next-line better-mutation/no-mutating-methods
+        csvRows.push(data);
+      })
+      .on('err', (err) => reject(err))
+      .on('end', () => {
+        resolve([csvHeaders, csvRows]);
+      });
+  });
+};
+
+const getColIndex = (csvHeaders, column) =>
+  isString(column) ? indexOf(column, csvHeaders) : column;
+
+const getColName = (csvHeaders, column) =>
+  isString(column) ? column : csvHeaders[column];
+
+module.exports = { loadCsv, getColIndex, getColName };

package/src/helpers/load-handlebars-template.js

@@ -0,0 +1,9 @@
+const Handlebars = require('handlebars');
+const { readFile } = require('./common');
+
+const loadHandlebarsTemplate = (filename) => {
+  const templateString = readFile(filename);
+  return Handlebars.compile(templateString);
+};
+
+module.exports = { loadHandlebarsTemplate };

package/src/helpers/parse-column.js

@@ -0,0 +1,8 @@
+const parseColumn = (value) => {
+  const parsedValue = parseInt(value, 10);
+  if (Number.isNaN(parsedValue)) {
+    return value;
+  }
+  return parsedValue;
+};
+module.exports = { parseColumn };

package/src/helpers/prepare-variable-sets.js

@@ -0,0 +1,23 @@
+const { map } = require('lodash/fp');
+const { getColName } = require('./load-csv');
+
+const prepareVariableSets = async (
+  csvHeaders,
+  csvRows,
+  { vendorUseridColumn, vars, did }
+) => {
+  const overrideVars = { ...vars };
+  if (did != null) {
+    overrideVars.did = did;
+  }
+  return map(
+    (csvRow) => ({
+      ...csvRow,
+      ...overrideVars,
+      vendorUserId: csvRow[getColName(csvHeaders, vendorUseridColumn)],
+    }),
+    csvRows
+  );
+};
+
+module.exports = { prepareVariableSets };

package/src/index.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+require('./data-loader');

package/src/vendor-credentials/README.md

@@ -0,0 +1,72 @@
+# vendor-credentials command
+
+Used for creating users and then attaching them to specific credentials. 
+
+- Supports new users only, not matching to existing users
+- Credentials & People must be specified as a handlebars templates. The data is populated from two sources
+   - the csv specified with variable names the same as the headers
+   - any `-v` arg parsed in
+
+Checkout the [test data](../test/data).
+
+## How to Use
+`data-loader vendorcreds [options]`
+
+### Config
+
+#### General config
+
+`-c [CSV_FILENAME]` **required** parameter containing the csv file
+
+`-p [OUTPUT_PATH]` **required** the output directory to use where QR codes and output state files are stored
+
+`-l [LABEL]` A label to attach to the records added to the agent
+
+`--dry-run` Run a test that parses the CSV and creates offers, but does not attempt to write the data to an agent. great for testing!
+
+#### Issuing config
+
+`--new` to create a new issuing configuration ("disclosure")
+
+`-i [DISCLOSURE_ID]` an existing disclosure to use for the batch issuing
+
+`-d [DID]` **required** the issuer's DID
+
+`-t [URL]` **required** the url to the T&Cs that holder must consent to
+
+`--purpose` The purpose to display to the user. Use a maximum for 64 chars. Default is "Career Credential Issuing"
+
+`--authTokenExpiresIn` The number of minutes that the offer will be available for after activation. Default is 365 days.
+
+#### User authentication and matching config
+
+`-y [ID_CREDENTIAL_TYPE]`  the credential type used for identifying the user. Default is Email.
+
+`-u [COLUMN]` the column from the CSV for the user id. Value is made available as "vendorUserId" in the offer template. Default is the first column
+
+`-m [COLUMN]` the column from the CSV for the user to be matched against the ID credential's "identifier" property. Default is the first column
+
+#### Offer generation config
+
+`-o [OFFER_TEMPLATE_FILENAME]` **required** offer handlebars template. Use moustaches around variables such as `{{did}}` 
+
+`--var=[VAR_NAME]=[VAR_VALUE]` variables used in the templates can be specified on the command line. They override any csv values
+
+#### Credential Agent config
+
+`-e [URL]` **required if not a dryrun** the endpoint of the mockvendor server
+
+`-t [AUTH_TOKEN]` **required if not a dryrun** the bearer token to use when calling the mockvendor server
+
+`--legacy` the target credential agent is running in the "LEGACY" offer type mode. Default is false
+
+#### Output config
+
+`-x` if passed an output csv is generated including the vendor's user id as the first column and the generated qrcode filename and deeplink
+
+`--x-name [OUTPUT_CSV_NAME]` The file name for the output CSV. Default is "output"
+
+### Dry Run Example
+Dry runs that print out what updates will be issued should omit the `-e` prop
+
+`./data-loader vendorcreds -c ./test/data/variables.csv -o ./test/data/offer.template.json -d did:ion:sap456`

package/src/vendor-credentials/execute-update.js

@@ -0,0 +1,32 @@
+const got = require('got');
+const { printInfo } = require('../helpers/common');
+
+const setupGot = ({ endpoint, authToken }) => {
+  const options = {
+    prefixUrl: endpoint,
+  };
+  if (authToken != null) {
+    options.headers = { Authorization: `Bearer ${authToken}` };
+  }
+  return got.extend(options);
+};
+
+const initExecuteUpdate = (options) => {
+  const vendorGot = setupGot(options);
+  return async ({ person, offer }) => {
+    if (person) {
+      printInfo({
+        createdPerson: await vendorGot
+          .post('api/users', { json: person })
+          .json(),
+      });
+    }
+    printInfo({
+      createdOffer: await vendorGot.post('api/offers', { json: offer }).json(),
+    });
+  };
+};
+
+module.exports = {
+  initExecuteUpdate,
+};

package/src/vendor-credentials/index.js

@@ -0,0 +1,49 @@
+const { program } = require('commander');
+const { reduce } = require('lodash/fp');
+const { printInfo, parseColumn } = require('../helpers');
+const { executeVendorCredentials } = require('./orchestrator');
+
+const parseVar = reduce((acc, pair) => {
+  const [key, value] = pair.split('=');
+  acc[key] = value;
+  return acc;
+}, {});
+
+program
+  .name('data-loader vendorcreds')
+  .description('Loads data into a db')
+  .usage('[options]')
+  .requiredOption(
+    '-c, --csv-filename <filename>',
+    'File name containing variables'
+  )
+  .requiredOption(
+    '-o, --offer-template-filename <filename>',
+    'File name containing the credential template file'
+  )
+  .option(
+    '-p, --person-template-filename <filename>',
+    'File name containing the credential template file'
+  )
+  .option(
+    '-e, --endpoint <url>',
+    'Endpoint to call to upload the people and credentials to'
+  )
+  .option(
+    '-u  --vendor-userid-column <vendorUseridColumn>',
+    `the column from the CSV that is users id. Value is made available as "vendorUserId" in the offer template. Accepts 
+    header name or index. Default is 0.`,
+    parseColumn,
+    '0'
+  )
+  .option('-t, --auth-token <url>', 'Bearer Auth Token to use')
+  .option('-l, --label <label>', 'A label to attach to the documents inserted')
+  .option('-v, --var <var...>', 'A variable to add. use name=value')
+  .action(async () => {
+    const options = program.opts();
+    // eslint-disable-next-line better-mutation/no-mutation
+    options.vars = parseVar(options.var);
+    printInfo(options);
+    await executeVendorCredentials(options);
+  })
+  .parseAsync(process.argv);

package/src/vendor-credentials/orchestrator.js

@@ -0,0 +1,22 @@
+const { initExecuteUpdate } = require('./execute-update');
+const { prepareData } = require('./prepare-data');
+const { loadCsv, printInfo } = require('../helpers');
+
+// eslint-disable-next-line consistent-return
+const executeVendorCredentials = async (options) => {
+  const [csvHeaders, csvRows] = await loadCsv(options.csvFilename);
+  const updates = await prepareData(csvHeaders, csvRows, options);
+
+  if (options.endpoint == null) {
+    printInfo(JSON.stringify({ updates }));
+    return updates;
+  }
+
+  const executeUpdate = initExecuteUpdate(options);
+  for (const update of updates) {
+    // eslint-disable-next-line no-await-in-loop
+    await executeUpdate(update);
+  }
+};
+
+module.exports = { executeVendorCredentials };

package/src/vendor-credentials/prepare-data.js

@@ -0,0 +1,36 @@
+const { map } = require('lodash/fp');
+const {
+  loadHandlebarsTemplate,
+} = require('../helpers/load-handlebars-template');
+const { prepareVariableSets } = require('../helpers');
+
+const prepareData = async (csvHeaders, csvRows, options) => {
+  const variableSets = await prepareVariableSets(csvHeaders, csvRows, options);
+
+  const { offerTemplateFilename, personTemplateFilename, label } = options;
+  const offerTemplate = loadHandlebarsTemplate(offerTemplateFilename);
+  const personTemplate = personTemplateFilename
+    ? loadHandlebarsTemplate(personTemplateFilename)
+    : undefined;
+  return map((variableSet) => {
+    return {
+      offer: prepareDocument({ template: offerTemplate, variableSet, label }),
+      person: personTemplate
+        ? prepareDocument({ template: personTemplate, variableSet, label })
+        : undefined,
+    };
+  }, variableSets);
+};
+
+const prepareDocument = ({ template, variableSet, label }) => {
+  const offerString = template(variableSet);
+  const json = JSON.parse(offerString);
+  return {
+    ...json,
+    label,
+  };
+};
+
+module.exports = {
+  prepareData,
+};