jupyter-ijavascript-utils 1.56.0 → 1.58.0

package/DOCS.md

@@ -74,6 +74,15 @@ Give it a try here:
 [![Binder:what can I do with this](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/paulroth3d/jupyter-ijavascript-utils/main?labpath=example.ipynb)
 
 ## What's New
+* 1.58 - 
+    * #91 - add {@link module:object.splitIntoDatums|splitIntoDatums(collection, fieldsToSplitBy)} - to make working with datum libraries - like vega-lite - easier
+    * #92 - make using the cache easier for {@link module:file.useCache|file.useCache()} - as you can say to use the cache, and it will still work if the cache file does not exist yet.
+    * #93 / #94 - bug fixes
+* 1.57 - #86 - include examples on binning based on time series {@link https://github.com/paulroth3d/jupyter-ijavascript-utils/issues/86|see more here}
+    * #87 - add in {@link module:file.fileExists|file.fileExists} to make things easier for new people getting started
+    * #89 - allow {@link module:array.resize|array.resize} to work with defaults, if zipping arrays of different sizes
+    * #90 - correct issue with timezone offsets, so it is no longer possible to get a timezone offset +24:00 from any of the localizations - like {@link module:date.toLocalISO|date.toLocalISO}
+    * #92 - make it easier to handle big expensive calculations with a cache, see {@link module:ijs.useCache|ijs.useCache()} and {@link module:file.useCache|file.useCache()}
 * 1.56 - #84 (object.renamePropertiesFromList), #82 (date.getWeekday)
 * 1.55 - #76, #77, #74, #78
 * 1.54 - additional Date logic, and formatting. #70 #71 #72

package/Dockerfile

@@ -1,3 +1,3 @@
 # syntax=docker/dockerfile:1
 
-FROM darkbluestudios/jupyter-ijavascript-utils:binder_1.55.0
+FROM darkbluestudios/jupyter-ijavascript-utils:binder_1.58.0

package/README.md

@@ -54,6 +54,15 @@ This is not intended to be the only way to accomplish many of these tasks, and a
 ![Screenshot of example notebook](docResources/img/mainExampleNotebook.png)
 
 # What's New 
+* 1.58 - 
+    * #91 - add object.splitIntoDatums - to make working with datum libraries - like vega-lite - easier
+    * #92 - make using the cache easier - as you can say to use the cache, and it will still work if the cache file does not exist yet.
+    * #93 / #94 - bug fixes
+* 1.57 - #86 - include examples on binning based on time series
+    * #87 - add in fileExists to make things easier for new people getting started
+    * #89 - allow resizing arrays with defaults, if zipping arrays of different sizes
+    * #90 - correct issue with timezone offsets, so it is no longer possible to get a timezone offset +24:00
+    * #92 - make it easier to handle big expensive calculations with a cache
 * 1.56 - #84 (object.renamePropertiesFromList), #82 (date.getWeekday)
 * 1.55 - #76, #77, #74, #78
 * 1.54 - additional Date logic, and formatting. #70 #71 #72

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jupyter-ijavascript-utils",
-  "version": "1.56.0",
+  "version": "1.58.0",
   "description": "Utilities for working with iJavaScript - a Jupyter Kernel",
   "homepage": "https://jupyter-ijavascript-utils.onrender.com/",
   "license": "MIT",

package/src/array.js

@@ -1460,18 +1460,25 @@ module.exports.asyncWaitAndChain = (seconds, fn, rows) => {
  * 
  * utils.array.resize(categoryValues, 2); // ['rock', 'paper']
  * utils.array.resize(categoryValues, 7); // ['rock', 'paper', 'scissors',
- * 'rock', 'paper', 'scissors', 'rock];
+ * undefined, undefined, undefined, undefined];
  * ```
  * 
  * @param {Array} sourceList - array of values
- * @param {Number} length - new number of items in the list
+ * @param {Number} length - new length of the list
  */
-module.exports.resize = function resize(sourceList, length) {
+module.exports.resize = function resize(sourceList, length, defaultValue = undefined) {
   if (!sourceList || !Array.isArray(sourceList)) return [];
   if (length < 1 || sourceList.length < 1) return [];
-  return new Array(length)
-    .fill(0)
-    .map((_, index) => sourceList[index % sourceList.length]);
+
+  const result = new Array(length)
+    .fill(defaultValue);
+
+  sourceList.forEach((val, index) => {
+    if (index >= length) return;
+    result[index] = val;
+  });
+
+  return result;
 };
 
 /**
@@ -1532,9 +1539,9 @@ module.exports.zip = function zip(arrayLeft, arrayRight, ...rest) {
   if (cleanLeft.length === 0 && cleanRight.length === 0) {
     result = [[]];
   } else if (cleanLeft.length === 0) {
-    result = cleanRight.map((val) => Array.isArray(val) ? val : [val]);
+    result = cleanRight.map((val) => Array.isArray(val) ? val : (typeof val === 'string') ? [val] : [...val]);
   } else if (cleanRight.length === 0) {
-    result = cleanLeft.map((val) => Array.isArray(val) ? val : [val]);
+    result = cleanLeft.map((val) => Array.isArray(val) ? val : (typeof val === 'string') ? [val] : [...val]);
   } else {
     const cleanLeftLen = cleanLeft.length;
     const cleanRightLen = cleanRight.length;

package/src/date.js

@@ -258,7 +258,7 @@ module.exports.getTimezoneEntry = function getTimezoneEntry(timezoneStr) {
       }, {});
     //   const impactedDate = new Date(d.toLocaleString('en-US', { timezone: timezoneStr }));
     const dateStr = `${dm.year}-${DateUtils.padTime(dm.month)}-${DateUtils.padTime(dm.day)}T${
-      DateUtils.padTime(dm.hour)}:${DateUtils.padTime(dm.minute)}:${DateUtils.padTime(dm.second)}.${
+      DateUtils.padTime(dm.hour % 24)}:${DateUtils.padTime(dm.minute)}:${DateUtils.padTime(dm.second)}.${
       DateUtils.padTime(dm.fractionalSecond, 3)}`;
     
     return dateStr;
@@ -266,7 +266,8 @@ module.exports.getTimezoneEntry = function getTimezoneEntry(timezoneStr) {
   
   const impactedDate = getOffset(d);
 
-  const diff = d.getTime() - impactedDate.getTime();
+  //-- avoid +24:00 or -24:00 - the number of seconds within a day
+  const diff = (d.getTime() - impactedDate.getTime()) % 86400000;
 
   const diffSign = diff > 0 ? '-' : '+';
   let remainder = DateUtils.divideRemainder(Math.abs(diff), DateUtils.TIME.HOUR);

package/src/file.js

@@ -29,6 +29,9 @@ const logger = require('./logger');
  *   * {@link module:file.matchFiles|matchFiles(path, matchingFn)} - find files or directories based on type of file or name
  * * checking files exist
  *   * {@link module:file.checkFile|checkFile(...paths)} - check if a file at a path exists
+ *   * {@link module:file.fileExists|fileExists(filePath)} - check if a single file at a path exists
+ * * using a cache for long running executions
+ *   * {@link module:file.useCache|file.useCache()} - perform an expensive calculation and write to a cache, or read from the cache transparently
  * 
  * ---
  * 
@@ -96,6 +99,8 @@ const FileUtil = module.exports;
  * 
  * @param {string} filePath - path of the file to load
  * @param {Object} fsOptions - options to pass for fsRead (ex: { encoding: 'utf-8' })
+ * @param {Function} fsOptions.reviver - reviver to use when writing the JSON
+ * @param {String} fsOptions.encoding - the encoding to write the JSON out with
  * @example
  * const weather = [
  *   { id: 1, city: 'Seattle',  month: 'Aug', precip: 0.87 },
@@ -222,17 +227,20 @@ module.exports.readFile = function readFile(filePath, fsOptions = {}) {
  * @param {Boolean} fsOptions.append - if true, will append the text to the file
  * @param {Boolean} fsOptions.prefix - string to add before writing the json, like an opening bracket '[' or comma ','
  * @param {Boolean} fsOptions.prefix - string to add before writing the json, like a closing bracket ']'
+ * @param {Function} fsOptions.replacer - function to use when writing JSON passed to stringify
  * @param {String} fsOptions.encoding - encoding to use when writing the file.
  * @see {@link module:file.readJSON|readJSON(filePath, fsOptions)} - for reading
  */
 module.exports.writeJSON = function writeJSON(filePath, contents, fsOptions = {}) {
   //-- if it isn't desired, simply pass as a string.
-  const jsonContents = JSON.stringify(contents, null, 2);
   const optionsDefaults = { encoding: 'utf-8' };
   const cleanedOptions = { ...optionsDefaults, ...fsOptions };
   const isAppend = cleanedOptions.append === true;
   const prefix = cleanedOptions.prefix || '';
   const suffix = cleanedOptions.suffix || '';
+  const replacer = cleanedOptions.replacer || null;
+  const spacing = cleanedOptions.spacing || 2;
+  const jsonContents = JSON.stringify(contents, replacer, spacing);
 
   // const resolvedPath = path.resolve(filePath);
   try {
@@ -465,23 +473,104 @@ module.exports.checkFile = function checkFile(...files) {
   return notFoundFiles;
 };
 
-/*
- * Execute an async function if any of the files do not exist
- * @param {String[]} filePaths - list of paths of files to check that they exist
- * @param {*} fnIfFailed - async function tha will run - but only if any of the files are not found.
+/**
+ * Checks if a single file exists
+ * @param {String} filePath - path to check if the file exists.
+ * @returns {Boolean} - if the file exists (true) or not (false)
+ * @see {@link module:file.checkFile|file.checkFile} - if checking multiple files
  */
+module.exports.fileExists = function fileExists(filePath) {
+  const resolvedPath = path.resolve(filePath);
+  return fs.existsSync(resolvedPath);
+};
+
 /*
-module.exports.ifNotExists = async function ifNotExists(filePaths, fnIfFailed) {
-  const filesNotFound = FileUtil.checkFile(filePaths);
+//-- not needed - dates already serialize to iso Strings
+module.exports.cacheSerializer = (key, value) => {
+  if (key && (key === 'date' || key.endsWith('_date')) && (value instanceof Date)) {
+    return value.toISOString();
+  }
+  return value;
+};
+*/
 
-  let results;
+module.exports.cacheDeserializer = (key, value) => {
+  if (key && (key === 'date' || key.endsWith('_date') || key.endsWith('Date'))) {
+    return new Date(value);
+  }
+  return value;
+};
+
+/**
+ * For very long or time-intensive executions, sometimes it is better to cache the results
+ * than to execute them every single time.
+ * 
+ * Note that this works synchronously, and can be easier to use than if promises are involved.
+ * 
+ * As opposed to {@link module:ijs.useCache|ijs.useCache} - which works with promises.
+ * 
+ * ```
+ * shouldWrite = true; /// we will write to the cache with the results from the execution
+ * expensiveResults = utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *    const data = d3.csvParse(utils.file.readFile('./someFile.csv'))
+ *      .map(obj => ({ ...obj, date: Date.parse(obj.epoch) }));
+ *    
+ *    const earliestDate = utils.date.startOfDay( utils.agg.min(data, 'date') );
+ *    const lastDate = utils.date.endOfDay( utils.agg.max(data, 'date') );
+ * 
+ *    // binning or lots of other things.
+ * 
+ *    return finalResults;
+ * });
+ * 
+ * expensiveresults.length = 1023424;
+ * ```
+ * 
+ * but sometimes I would rather just skip to the end
+ * 
+ * ```
+ * shouldWrite = false; /// we will read from the cache instead,
+ * // everything else remains the same
+ * 
+ * expensiveResults = utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *    const data = d3.csvParse(utils.file.readFile('./someFile.csv'))
+ *      .map(obj => ({ ...obj, date: Date.parse(obj.epoch) }));
+ * 
+ *    //-- function can remain untouched,
+ *    //-- BUT nothing in here will be executed
+ *    //-- since we are reading from the cache
+ * });
+ * 
+ * //-- completely transparent to the runner
+ * expensiveresults.length = 1023424;
+ * ```
+ * 
+ * @param {Boolean} shouldWrite - whether we should write to the cache (true) or read from the cache (false)
+ * @param {String} cachePath - Path to the cache folder, ex: './cache'
+ * @param {String} cacheFile - Filename of the cache file to use for this execution, ex: 'ExecutionsPerMin.js'
+ * @param {Function} expensiveFn - function that returns the results to be stored in the cache
+ * @param {Object} fsOptions - options to use when writing or reading files
+ * @returns {any} - either the deserialized json from the cache or the results from the expensive function
+ * @see {@link module:file.readJSON|file.readJSON} - reads a local JSON file
+ * @see {@link module:file.writeJSON|file.writeJSON} - writes to a JSON file
+ * @see {@link module:ijs.useCache|ijs.useCache} - similar idea - but supports promises
+ */
+module.exports.useCache = function useCache(shouldWrite, cachePath, cacheFile, expensiveFn, fsOptions = null) {
+  const ensureEndsWithSlash = (str) => str.endsWith('/') ? str : `${str}/`;
+  const cacheFilePath = `${ensureEndsWithSlash(cachePath)}${cacheFile}`;
 
-  if (filesNotFound) {
-    results = await fnIfFailed(filesNotFound);
-  } else {
-    results = null;
+  const cacheExists = FileUtil.fileExists(cacheFilePath);
+  
+  if (cacheExists && !shouldWrite) {
+    const cleanOptions = { ...fsOptions, reviver: FileUtil.cacheDeserializer };
+    const results = FileUtil.readJSON(cacheFilePath, cleanOptions);
+    return results;
   }
 
+  const results = expensiveFn();
+
+  const cleanOptions = fsOptions; // FileUtil.cacheSerializer not needed
+  FileUtil.writeJSON(cacheFilePath, results, cleanOptions);
+
   return results;
 };
-*/

package/src/hashMap.js

@@ -83,12 +83,11 @@ module.exports.add = function add(map, key, value) {
  * @param {any} functor.value - the first argument is the current value
  * @param {any} functor.key - the second argument is the key passed
  * @param {any} functor.map - the third argument is the map being acted upon
- * @param {any} [setKey = null] - optional separate key to use to set the updated value
  * @returns {Map}
  */
-module.exports.getSet = function getSet(map, key, functor, setKey = null) {
-  const cleanSetKey = setKey || key;
-  map.set(cleanSetKey, functor(map.get(key), key, map));
+module.exports.getSet = function getSet(map, key, functor) {
+  const currentValue = map.has(key) ? map.get(key) : undefined;
+  map.set(key, functor(currentValue, key, map));
   return map;
 };
 module.exports.update = module.exports.getSet;

package/src/ijs.js

@@ -2,6 +2,8 @@
 
 const uuid = require('uuid').v4;
 
+const FileUtil = require('./file');
+
 require('./_types/global');
 
 /**
@@ -24,9 +26,11 @@ require('./_types/global');
  *   * {@link module:ijs.markdown|ijs.markdown} - Render output as markdown
  *   * {@link module:ijs.htmlScript|ijs.htmlScript} - Leverage external libraries like D3, Leaflet, etc.
  * * Printing
- *   * {@link module:ijs.noOutputNeeded|ijs.noOutputNeeded} - clears the output to declutter results (like importing libraries, or functions)
+ *   * {@link module:ijs.clearOutput|ijs.clearOutput} - clears the output to declutter results (like importing libraries, or functions)
  *   * {@link module:ijs.initializePageBreaks|ijs.initializePageBreaks} - call at least once to allow pageBreaks when rendering PDFs
  *   * {@link module:ijs.printPageBreak|ijs.printPageBreak} - call to print a page break when rendering PDFs
+ * * using a cache for long running executions
+ *   * {@link module:ijs.useCache|ijs.useCache()} - perform an expensive calculation and write to a cache, or read from the cache transparently
  * 
  * For example:
  * 
@@ -588,7 +592,7 @@ module.exports.htmlScript = function htmlScripts(
  * (This is useful to put after importing libraries,
  * or defining a list of functions)
  */
-module.exports.noOutputNeeded = function clearOutput(outputText = '') {
+module.exports.clearOutput = function clearOutput(outputText = '') {
   //-- you must be in iJavaScript container to rendeer
   const context = IJSUtils.detectContext();
   
@@ -598,7 +602,7 @@ module.exports.noOutputNeeded = function clearOutput(outputText = '') {
 
   context.$$.text(outputText);
 };
-module.exports.clearOutput = module.exports.noOutputNeeded;
+module.exports.noOutputNeeded = module.exports.clearOutput;
 
 /**
  * Required to be called first - in order to write page-breaks in the html results.
@@ -662,3 +666,102 @@ module.exports.printPageBreak = function printPageBreak() {
 
   context.$$.html('<div class="pagebreak"></div>');
 };
+
+/**
+ * For very long or time-intensive executions, sometimes it is better to cache the results
+ * than to execute them every single time.
+ * 
+ * Note that this supports promises, and can be a bit harder to understand.
+ * 
+ * As opposed to {@link module:file.useCache|file.useCache} - which works synchronously
+ * 
+ * ```
+ * shouldWrite = true; /// we will write to the cache with the results from the execution
+ * utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *   //-- all the items to cache 
+ *   return Promise.resolve()
+ *      .then(() => ajax.retrieve(...))
+ *      .then((results) => {
+ *          const data = results
+ *            .map(obj => ({ ...obj, date: Date.parse(obj.epoch) }));
+ *          .. other things to do
+ *          return data;
+ *       });
+ * })
+ *   //-- using the information AFTER the expensive function or retrieval from cache
+ *   //-- this can ALSO be run in a subsequence cell
+ *   .then((results) => {
+ *      expensiveresults = results;
+ *      conosole.log(`expensiveResults.length: ${results.length}`);
+ *    });
+ * ```
+ * 
+ * but sometimes I would rather just skip to the end
+ * 
+ * ```
+ * shouldWrite = false; /// we will read from the cache instead,
+ * // everything else remains the same
+ * 
+ * utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *   //-- all the items to cache 
+ *   return Promise.resolve()
+ *     ... blah blah - none of this will get executed
+ * })
+ *   //-- using the information AFTER the expensive function or retrieval from cache
+ *   //-- this can ALSO be run in a subsequence cell
+ *   .then((results) => {
+ *      expensiveresults = results;
+ *      conosole.log(`expensiveResults.length: ${results.length}`);
+ *    });
+ * 
+ * //-- completely transparent to the runner
+ * expensiveresults.length = 1023424;
+ * ```
+ * 
+ * @param {Boolean} shouldWrite - whether we should write to the cache (true) or read from the cache (false)
+ * @param {String} cachePath - Path to the cache folder, ex: './cache'
+ * @param {String} cacheFile - Filename of the cache file to use for this execution, ex: 'ExecutionsPerMin.js'
+ * @param {Function} expensiveFn - function that returns the results to be stored in the cache
+ * @param {Object} fsOptions - options to use when writing or reading files
+ * @returns {any} - either the deserialized json from the cache or the results from the expensive function
+ * @see {@link module:file.readJSON|file.readJSON} - reads a local JSON file
+ * @see {@link module:file.writeJSON|file.writeJSON} - writes to a JSON file
+ * @see {@link module:file.useCache|file.useCache} - can be much easier than using promises
+ */
+module.exports.useCache = async function useCache(shouldWrite, cachePath, cacheFile, expensiveFn, fsOptions = null) {
+  const ensureEndsWithSlash = (str) => str.endsWith('/') ? str : `${str}/`;
+  const cacheFilePath = `${ensureEndsWithSlash(cachePath)}${cacheFile}`;
+  const context = IJSUtils.detectContext();
+
+  if (!shouldWrite) {
+    const cleanOptions = { ...fsOptions, formatter: FileUtil.cacheDeserializer };
+    //-- we read the info and be done with it
+    //context.console.log(`before retrieving cache:${cacheFilePath}`);
+    const results = FileUtil.readJSON(cacheFilePath, cleanOptions);
+    //context.console.log(`after retrieving cache`);
+    return Promise.resolve(results);
+  }
+
+  if (!context) {
+    throw (Error('IJSUtils.async must be run within iJavaScript. Otherwise, use normal async methods'));
+  }
+
+  context.$$.async();
+
+  try {
+    context.console.log('before starting expensive fn');
+    const results = await expensiveFn(context.$$, context.console);
+    context.console.log('AFTER starting expensive fn');
+
+    FileUtil.writeJSON(cacheFilePath, results);
+    // context.$$.sendResult('success');
+    return results;
+  } catch (err) {
+    context.console.error('error occurred');
+    context.console.error(err);
+    context.$$.sendResult(err);
+
+    // return Promise.reject(err);
+    throw err;
+  }
+};

package/src/object.js

@@ -26,6 +26,7 @@ const FormatUtils = require('./format');
  *   * {@link module:object.extractObjectProperties|extractObjectProperties(list, propertyNameOrFnMap)} - extracts multiple propertie or fn across all objects in list.
  * * Apply deep values safely
  *   * {@link module:object.assign|objAssign(object, property, value)} - Applies properties to an object in functional programming style.
+ *   * {@link module:object.getSet|getSet(object, property, functor)} - calls a function with the current value on an object, allowing for decrementing/incrementing/etc.
  *   * {@link module:object.augment|augment(object, augmentFn)} - Applies properties to an object similar to Map
  *   * {@link module:object.assignEntities|objAssignEntities(object, [property, value])} - Applies properties to an object using Array values - [key,value]
  *   * {@link module:object.setPropertyDefaults|setPropertyDefaults()} - sets values for objects that don't currently have the property
@@ -50,13 +51,14 @@ const FormatUtils = require('./format');
  *   * {@link module:object.flatten|flatten()} - creates dot notation properties (similar to arrow notation) of all child objects.
  *   * {@link module:object.expand|expand()}  - expands dot notation properties onto sub children (inverse of flatten)
  * * Create Map of objects by key
- *   * {@link module:object.mapByProperty|mapByProperty()} -
- *   * {@link module:group.by|group(collection, accessor)}
+ *   * {@link module:object.mapByProperty|mapByProperty()}
+ *   * {@link module:group.by|group.by(collection, accessor)}
  * * Convert collections of objects
  *   * {@link module:object.objectCollectionFromArray|objectCollectionFromArray} - convert rows/columns 2d array to objects
- *   * {@link module:object.objectCollectionToArray} - convert objects to a rows/columns 2d array
- *   * {@link module:object.objectCollectionFromDataFrameObject} - convert tensor object with each field as 1d array of values
- *   * {@link module:object.objectCollectionToDataFrameObject} - convert objects from a tensor object
+ *   * {@link module:object.objectCollectionToArray|objectCollectionToArray} - convert objects to a rows/columns 2d array
+ *   * {@link module:object.objectCollectionFromDataFrameObject|objectCollectionFromDataFrameObject} - convert tensor object with each field as 1d array of values
+ *   * {@link module:object.objectCollectionToDataFrameObject|objectCollectionToDataFrameObject} - convert objects from a tensor object
+ *   * {@link module:object.splitIntoDatums|splitIntoDatums(object, fieldsToSplitBy)} - separate objects into series by fields
  * 
  * @module object
  * @exports object
@@ -169,6 +171,53 @@ module.exports.assignEntities = function objAssignEntities(obj, entities) {
 };
 module.exports.objAssignEntities = module.exports.assignEntities;
 
+/**
+ * Use this for times where you want to update a value
+ * 
+ * 
+ * ```
+ * key = 'somethingToIncrement';
+ * defaultValue = null;
+ * 
+ * const initialObject = {};
+ * initialObject[key] = defaultValue;
+ * 
+ * // { somethingToIncrement: null }
+ * 
+ * const functor = (value) => { //, key, map) => {
+ *   if (!value) return 1;
+ *   return value + 1;
+ * };
+ * 
+ * utils.object.getSet(initialObject, key, functor);
+ * 
+ * // equivalent to
+ * // intitialObject[key] = intitialObject[key] ? initialObject[key] + 1 : 1`
+ * // but in a way that is repeatable across many values
+ * 
+ * utils.object.getSet(initialObject, key, functor);
+ * utils.object.getSet(initialObject, key, functor);
+ * utils.object.getSet(initialObject, key, functor);
+ * utils.object.getSet(initialObject, key, functor);
+ * 
+ * initialObject.get(key); // 5
+ * ```
+ * 
+ * @param {Map} map - map to get and set values from
+ * @param {any} key - they key to GET and SET the value (unless setKey is provided)
+ * @param {Function} functor - the function called with the arguments below - returning the value to set
+ * @param {any} functor.value - the first argument is the current value
+ * @param {any} functor.key - the second argument is the key passed
+ * @param {any} functor.map - the third argument is the map being acted upon
+ * @returns {Map}
+ */
+module.exports.getSet = function getSet(obj, field, functor) {
+  const currentValue = Object.hasOwn(obj, field) ? obj[field] : undefined;
+  obj[field] = functor(currentValue, field, obj);
+  return obj;
+};
+module.exports.update = module.exports.getSet;
+
 /**
  * Runs a map over a collection, and adds properties the the objects.
  * 
@@ -283,6 +332,78 @@ module.exports.keys = function keys(objOrArray = {}, maxRows = -1) {
   return Array.from(result);
 };
 
+/**
+ * Identifies which keys provided are also in objOrArray.
+ * 
+ * ```
+ * dataSet = [
+ *  { first: 'john', last: 'McCartney' },
+ *  { first: 'ringo', last: 'Starr' }
+ * ];
+ * 
+ * utils.object.keysWithinList(dataSet, 'first', 'last', 'favouriteColor');
+ * // ['first', 'last'] // no favouriteColor defined
+ * ```
+ * 
+ * Note you can also pass the list of keys as an array in the first argument
+ * 
+ * ```
+ * fieldsToCheck = ['first', 'last', 'favouriteColor'];
+ * utils.object.keysWithinList(dataSet, fieldsToCheck);
+ * // ['first', 'last']
+ * ```
+ * 
+ * @param {Object|Object[]} objOrArray - object or list of objects to identify the keys they have
+ * @param  {...String} listOfKeys - a list of keys to check if they are defined within objOrArray
+ * @returns {String[]} - list of keys that are both in the objOrArray or within listOfKeys
+ */
+module.exports.keysWithinList = function keysWithinList(objOrArray, ...listOfKeys) {
+  const cleanListOfKeys = listOfKeys.length > 0 && Array.isArray(listOfKeys[0])
+    ? listOfKeys[0]
+    : listOfKeys;
+  
+  const keySet = new Set(ObjectUtils.keys(objOrArray));
+  const keyIntersection = cleanListOfKeys.filter((keyToTest) => keySet.has(keyToTest));
+  return keyIntersection;
+};
+
+/**
+ * Identifies which other keys are defined that are not in the list provided.
+ * 
+ * This is quite helpful for dynamic APIs.
+ * 
+ * ```
+ * dataSet = [
+ *  { first: 'john', last: 'McCartney', favouriteColor: 'blue' },
+ *  { first: 'ringo', last: 'Starr', favouriteColor: 'red' }
+ * ];
+ * 
+ * utils.object.keysNotInList(dataSet, 'first', 'last', 'favouriteColor');
+ * // ['favouriteColor']
+ * ```
+ * 
+ * Note you can also pass the list of keys as an array in the first argument
+ * 
+ * ```
+ * fieldsToCheck = ['first', 'last', 'favouriteColor'];
+ * utils.object.keysNotInList(dataSet, fieldsToCheck);
+ * // ['favouriteColor']
+ * ```
+ * 
+ * @param {Object|Object[]} objOrArray - object or list of objects to identify the keys they have
+ * @param  {...String} listOfKeys - a list of keys to check if they are defined within objOrArray
+ * @returns {String[]} - list of keys that are both in the objOrArray or within listOfKeys
+ */
+module.exports.keysNotInList = function keysNotInList(objOrArray, ...listOfKeys) {
+  const setOfKeysToCheck = listOfKeys.length > 0 && Array.isArray(listOfKeys[0])
+    ? new Set(listOfKeys[0])
+    : new Set(listOfKeys);
+  
+  const keys = ObjectUtils.keys(objOrArray);
+  const keyIntersection = keys.filter((keyToTest) => !setOfKeysToCheck.has(keyToTest));
+  return keyIntersection;
+};
+
 /**
  * Cleans all the properties of the array of objects in place (does not make Copies)
  * 
@@ -2122,3 +2243,92 @@ module.exports.objectCollectionToDataFrameObject = function objectCollectionToDa
   });
   return dataFrameObject;
 };
+
+/**
+ * Some charting software (such as vega-lite) does not allow a single object to be used
+ * for multiple line series.
+ * 
+ * This is intended to help with that.
+ * 
+ * ```
+ * [
+ *   { category: 'A', source: 'chicago', x: 0.1, y: 0.6, z: 0.9 },
+ *   { category: 'B', source: 'springfield', x: 0.7, y: 0.2, z: 1.1 },
+ *   { category: 'C', source: 'winnetka', x: 0.6, y: 0.1, z: 0.2 }
+ * ]
+ * ```
+ * 
+ * must have a separate object for each x, y and z field for the A category.
+ * 
+ * ```
+ * utils.object.splitIntoDatums(category, ['x', 'y', 'z']);
+ * [
+ *   { category: 'A', source: 'chicago', series: 'x', value: 0.1 },
+ *   { category: 'A', source: 'chicago', series: 'y', value: 0.6 },
+ *   { category: 'A', source: 'chicago', series: 'z', value: 0.9 },
+ *   { category: 'B', source: 'springfield', series: 'x', value: 0.7 },
+ *   { category: 'B', source: 'springfield', series: 'y', value: 0.2 },
+ *   { category: 'B', source: 'springfield', series: 'z', value: 1.1 },
+ *   { category: 'C', source: 'winnetka', series: 'x', value: 0.6 },
+ *   { category: 'C', source: 'winnetka', series: 'y', value: 0.1 },
+ *   { category: 'C', source: 'winnetka', series: 'z', value: 0.2 }
+ * ]
+ * ```
+ * 
+ * note that the fields NOT within the list of fields specified, are preserved
+ * in the denormalized objects...
+ * 
+ * while the fields listed are put into separate objects.
+ * 
+ * You can specify which fields that are generated in those new objects
+ * 
+ * ```
+ * utils.object.splitIntoDatums(category, ['x', 'y', 'z'], 'group', 'val');
+ * [
+ *   { category: 'A', source: 'chicago', group: 'x', val: 0.1 },
+ *   { category: 'A', source: 'chicago', group: 'y', val: 0.6 },
+ *   { category: 'A', source: 'chicago', group: 'z', val: 0.9 },
+ *   { category: 'B', source: 'springfield', group: 'x', val: 0.7 },
+ *   { category: 'B', source: 'springfield', group: 'y', val: 0.2 },
+ *   { category: 'B', source: 'springfield', group: 'z', val: 1.1 },
+ *   { category: 'C', source: 'winnetka', group: 'x', val: 0.6 },
+ *   { category: 'C', source: 'winnetka', group: 'y', val: 0.1 },
+ *   { category: 'C', source: 'winnetka', group: 'z', val: 0.2 }
+ * ]
+ * ```
+ * 
+ * @param {Object[]} objectCollection - collection
+ * @param {String[]} keysForEachSeries - fields that will each be a series spread across datum records
+ * @param {String} [seriesFieldName='series'] - the name of the field to indicate which series this datum is in
+ * @param {String} [valueFieldName='value'] - the name of the value for that datum
+ * @returns {Object} - list of objects x keysForEachSeries.length, where field within keysForEachSeries
+ *    are then individually assigned to a value, and series - to indicate which field is used.
+ */
+module.exports.splitIntoDatums = function splitIntoDatums(
+  collection,
+  keysToSplit,
+  seriesFieldName = 'series',
+  valueFieldName = 'value'
+) {
+  const keysToKeep = ObjectUtils.keysNotInList(collection, keysToSplit);
+  const keysToSeparate = ObjectUtils.keysWithinList(collection, keysToSplit);
+  const rowLength = keysToSeparate.length;
+
+  const results = new Array(collection.length * rowLength).fill(null);
+  collection.forEach((obj, objIndex) => {
+    const basis = {};
+    keysToKeep.forEach((key) => {
+      basis[key] = obj[key];
+    });
+
+    keysToSeparate.forEach((key, keyIndex) => {
+      const newRecord = { ...basis };
+      newRecord[seriesFieldName] = key;
+      newRecord[valueFieldName] = obj[key];
+      results[objIndex * rowLength + keyIndex] = newRecord;
+    });
+  });
+  
+  // return ({keysForSplitting, keysForKeeping});
+  return results;
+};