jupyter-ijavascript-utils 1.57.0 → 1.58.0

package/DOCS.md

@@ -74,6 +74,10 @@ Give it a try here:
 [![Binder:what can I do with this](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/paulroth3d/jupyter-ijavascript-utils/main?labpath=example.ipynb)
 
 ## What's New
+* 1.58 - 
+    * #91 - add {@link module:object.splitIntoDatums|splitIntoDatums(collection, fieldsToSplitBy)} - to make working with datum libraries - like vega-lite - easier
+    * #92 - make using the cache easier for {@link module:file.useCache|file.useCache()} - as you can say to use the cache, and it will still work if the cache file does not exist yet.
+    * #93 / #94 - bug fixes
 * 1.57 - #86 - include examples on binning based on time series {@link https://github.com/paulroth3d/jupyter-ijavascript-utils/issues/86|see more here}
     * #87 - add in {@link module:file.fileExists|file.fileExists} to make things easier for new people getting started
     * #89 - allow {@link module:array.resize|array.resize} to work with defaults, if zipping arrays of different sizes

package/Dockerfile

@@ -1,3 +1,3 @@
 # syntax=docker/dockerfile:1
 
-FROM darkbluestudios/jupyter-ijavascript-utils:binder_1.57.0
+FROM darkbluestudios/jupyter-ijavascript-utils:binder_1.58.0

package/README.md

@@ -54,6 +54,10 @@ This is not intended to be the only way to accomplish many of these tasks, and a
 ![Screenshot of example notebook](docResources/img/mainExampleNotebook.png)
 
 # What's New 
+* 1.58 - 
+    * #91 - add object.splitIntoDatums - to make working with datum libraries - like vega-lite - easier
+    * #92 - make using the cache easier - as you can say to use the cache, and it will still work if the cache file does not exist yet.
+    * #93 / #94 - bug fixes
 * 1.57 - #86 - include examples on binning based on time series
     * #87 - add in fileExists to make things easier for new people getting started
     * #89 - allow resizing arrays with defaults, if zipping arrays of different sizes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jupyter-ijavascript-utils",
-  "version": "1.57.0",
+  "version": "1.58.0",
   "description": "Utilities for working with iJavaScript - a Jupyter Kernel",
   "homepage": "https://jupyter-ijavascript-utils.onrender.com/",
   "license": "MIT",

package/src/date.js

@@ -258,7 +258,7 @@ module.exports.getTimezoneEntry = function getTimezoneEntry(timezoneStr) {
       }, {});
     //   const impactedDate = new Date(d.toLocaleString('en-US', { timezone: timezoneStr }));
     const dateStr = `${dm.year}-${DateUtils.padTime(dm.month)}-${DateUtils.padTime(dm.day)}T${
-      DateUtils.padTime(dm.hour)}:${DateUtils.padTime(dm.minute)}:${DateUtils.padTime(dm.second)}.${
+      DateUtils.padTime(dm.hour % 24)}:${DateUtils.padTime(dm.minute)}:${DateUtils.padTime(dm.second)}.${
       DateUtils.padTime(dm.fractionalSecond, 3)}`;
     
     return dateStr;

package/src/file.js

@@ -99,7 +99,7 @@ const FileUtil = module.exports;
  * 
  * @param {string} filePath - path of the file to load
  * @param {Object} fsOptions - options to pass for fsRead (ex: { encoding: 'utf-8' })
- * @param {Function} fsOptions.formatter - formatter to use when writing the JSON
+ * @param {Function} fsOptions.reviver - reviver to use when writing the JSON
  * @param {String} fsOptions.encoding - the encoding to write the JSON out with
  * @example
  * const weather = [
@@ -122,11 +122,7 @@ const FileUtil = module.exports;
 module.exports.readJSON = function readJSON(filePath, fsOptions = {}) {
   const resolvedPath = path.resolve(filePath);
   const optionsDefaults = { encoding: 'utf-8' };
-  let cleanedOptions = { ...optionsDefaults, ...fsOptions };
-
-  //-- unfortunately we cannot pass the formatter in addition, it must replace
-  //-- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse
-  if (cleanedOptions.formatter) cleanedOptions = cleanedOptions.formatter;
+  const cleanedOptions = { ...optionsDefaults, ...fsOptions };
 
   /** @type {string} */
   let result;
@@ -231,6 +227,7 @@ module.exports.readFile = function readFile(filePath, fsOptions = {}) {
  * @param {Boolean} fsOptions.append - if true, will append the text to the file
  * @param {Boolean} fsOptions.prefix - string to add before writing the json, like an opening bracket '[' or comma ','
  * @param {Boolean} fsOptions.prefix - string to add before writing the json, like a closing bracket ']'
+ * @param {Function} fsOptions.replacer - function to use when writing JSON passed to stringify
  * @param {String} fsOptions.encoding - encoding to use when writing the file.
  * @see {@link module:file.readJSON|readJSON(filePath, fsOptions)} - for reading
  */
@@ -241,9 +238,9 @@ module.exports.writeJSON = function writeJSON(filePath, contents, fsOptions = {}
   const isAppend = cleanedOptions.append === true;
   const prefix = cleanedOptions.prefix || '';
   const suffix = cleanedOptions.suffix || '';
-  const formatter = cleanedOptions.formatter || null;
+  const replacer = cleanedOptions.replacer || null;
   const spacing = cleanedOptions.spacing || 2;
-  const jsonContents = JSON.stringify(contents, formatter, spacing);
+  const jsonContents = JSON.stringify(contents, replacer, spacing);
 
   // const resolvedPath = path.resolve(filePath);
   try {
@@ -498,7 +495,7 @@ module.exports.cacheSerializer = (key, value) => {
 */
 
 module.exports.cacheDeserializer = (key, value) => {
-  if (key && (key === 'date' || key.endsWith('_date'))) {
+  if (key && (key === 'date' || key.endsWith('_date') || key.endsWith('Date'))) {
     return new Date(value);
   }
   return value;
@@ -561,17 +558,18 @@ module.exports.cacheDeserializer = (key, value) => {
 module.exports.useCache = function useCache(shouldWrite, cachePath, cacheFile, expensiveFn, fsOptions = null) {
   const ensureEndsWithSlash = (str) => str.endsWith('/') ? str : `${str}/`;
   const cacheFilePath = `${ensureEndsWithSlash(cachePath)}${cacheFile}`;
+
+  const cacheExists = FileUtil.fileExists(cacheFilePath);
   
-  if (!shouldWrite) {
-    const cleanOptions = { ...fsOptions, formatter: FileUtil.cacheDeserializer };
+  if (cacheExists && !shouldWrite) {
+    const cleanOptions = { ...fsOptions, reviver: FileUtil.cacheDeserializer };
     const results = FileUtil.readJSON(cacheFilePath, cleanOptions);
     return results;
   }
 
   const results = expensiveFn();
 
-  const cleanOptions = { ...fsOptions, formatter: null }; // FileUtil.cacheSerializer not needed
-
+  const cleanOptions = fsOptions; // FileUtil.cacheSerializer not needed
   FileUtil.writeJSON(cacheFilePath, results, cleanOptions);
 
   return results;

package/src/ijs.js

@@ -26,7 +26,7 @@ require('./_types/global');
  *   * {@link module:ijs.markdown|ijs.markdown} - Render output as markdown
  *   * {@link module:ijs.htmlScript|ijs.htmlScript} - Leverage external libraries like D3, Leaflet, etc.
  * * Printing
- *   * {@link module:ijs.noOutputNeeded|ijs.noOutputNeeded} - clears the output to declutter results (like importing libraries, or functions)
+ *   * {@link module:ijs.clearOutput|ijs.clearOutput} - clears the output to declutter results (like importing libraries, or functions)
  *   * {@link module:ijs.initializePageBreaks|ijs.initializePageBreaks} - call at least once to allow pageBreaks when rendering PDFs
  *   * {@link module:ijs.printPageBreak|ijs.printPageBreak} - call to print a page break when rendering PDFs
  * * using a cache for long running executions
@@ -592,7 +592,7 @@ module.exports.htmlScript = function htmlScripts(
  * (This is useful to put after importing libraries,
  * or defining a list of functions)
  */
-module.exports.noOutputNeeded = function clearOutput(outputText = '') {
+module.exports.clearOutput = function clearOutput(outputText = '') {
   //-- you must be in iJavaScript container to rendeer
   const context = IJSUtils.detectContext();
   
@@ -602,7 +602,7 @@ module.exports.noOutputNeeded = function clearOutput(outputText = '') {
 
   context.$$.text(outputText);
 };
-module.exports.clearOutput = module.exports.noOutputNeeded;
+module.exports.noOutputNeeded = module.exports.clearOutput;
 
 /**
  * Required to be called first - in order to write page-breaks in the html results.

package/src/object.js

@@ -51,13 +51,14 @@ const FormatUtils = require('./format');
  *   * {@link module:object.flatten|flatten()} - creates dot notation properties (similar to arrow notation) of all child objects.
  *   * {@link module:object.expand|expand()}  - expands dot notation properties onto sub children (inverse of flatten)
  * * Create Map of objects by key
- *   * {@link module:object.mapByProperty|mapByProperty()} -
- *   * {@link module:group.by|group(collection, accessor)}
+ *   * {@link module:object.mapByProperty|mapByProperty()}
+ *   * {@link module:group.by|group.by(collection, accessor)}
  * * Convert collections of objects
  *   * {@link module:object.objectCollectionFromArray|objectCollectionFromArray} - convert rows/columns 2d array to objects
- *   * {@link module:object.objectCollectionToArray} - convert objects to a rows/columns 2d array
- *   * {@link module:object.objectCollectionFromDataFrameObject} - convert tensor object with each field as 1d array of values
- *   * {@link module:object.objectCollectionToDataFrameObject} - convert objects from a tensor object
+ *   * {@link module:object.objectCollectionToArray|objectCollectionToArray} - convert objects to a rows/columns 2d array
+ *   * {@link module:object.objectCollectionFromDataFrameObject|objectCollectionFromDataFrameObject} - convert tensor object with each field as 1d array of values
+ *   * {@link module:object.objectCollectionToDataFrameObject|objectCollectionToDataFrameObject} - convert objects from a tensor object
+ *   * {@link module:object.splitIntoDatums|splitIntoDatums(object, fieldsToSplitBy)} - separate objects into series by fields
  * 
  * @module object
  * @exports object
@@ -331,6 +332,78 @@ module.exports.keys = function keys(objOrArray = {}, maxRows = -1) {
   return Array.from(result);
 };
 
+/**
+ * Identifies which keys provided are also in objOrArray.
+ * 
+ * ```
+ * dataSet = [
+ *  { first: 'john', last: 'McCartney' },
+ *  { first: 'ringo', last: 'Starr' }
+ * ];
+ * 
+ * utils.object.keysWithinList(dataSet, 'first', 'last', 'favouriteColor');
+ * // ['first', 'last'] // no favouriteColor defined
+ * ```
+ * 
+ * Note you can also pass the list of keys as an array in the first argument
+ * 
+ * ```
+ * fieldsToCheck = ['first', 'last', 'favouriteColor'];
+ * utils.object.keysWithinList(dataSet, fieldsToCheck);
+ * // ['first', 'last']
+ * ```
+ * 
+ * @param {Object|Object[]} objOrArray - object or list of objects to identify the keys they have
+ * @param  {...String} listOfKeys - a list of keys to check if they are defined within objOrArray
+ * @returns {String[]} - list of keys that are both in the objOrArray or within listOfKeys
+ */
+module.exports.keysWithinList = function keysWithinList(objOrArray, ...listOfKeys) {
+  const cleanListOfKeys = listOfKeys.length > 0 && Array.isArray(listOfKeys[0])
+    ? listOfKeys[0]
+    : listOfKeys;
+  
+  const keySet = new Set(ObjectUtils.keys(objOrArray));
+  const keyIntersection = cleanListOfKeys.filter((keyToTest) => keySet.has(keyToTest));
+  return keyIntersection;
+};
+
+/**
+ * Identifies which other keys are defined that are not in the list provided.
+ * 
+ * This is quite helpful for dynamic APIs.
+ * 
+ * ```
+ * dataSet = [
+ *  { first: 'john', last: 'McCartney', favouriteColor: 'blue' },
+ *  { first: 'ringo', last: 'Starr', favouriteColor: 'red' }
+ * ];
+ * 
+ * utils.object.keysNotInList(dataSet, 'first', 'last', 'favouriteColor');
+ * // ['favouriteColor']
+ * ```
+ * 
+ * Note you can also pass the list of keys as an array in the first argument
+ * 
+ * ```
+ * fieldsToCheck = ['first', 'last', 'favouriteColor'];
+ * utils.object.keysNotInList(dataSet, fieldsToCheck);
+ * // ['favouriteColor']
+ * ```
+ * 
+ * @param {Object|Object[]} objOrArray - object or list of objects to identify the keys they have
+ * @param  {...String} listOfKeys - a list of keys to check if they are defined within objOrArray
+ * @returns {String[]} - list of keys that are both in the objOrArray or within listOfKeys
+ */
+module.exports.keysNotInList = function keysNotInList(objOrArray, ...listOfKeys) {
+  const setOfKeysToCheck = listOfKeys.length > 0 && Array.isArray(listOfKeys[0])
+    ? new Set(listOfKeys[0])
+    : new Set(listOfKeys);
+  
+  const keys = ObjectUtils.keys(objOrArray);
+  const keyIntersection = keys.filter((keyToTest) => !setOfKeysToCheck.has(keyToTest));
+  return keyIntersection;
+};
+
 /**
  * Cleans all the properties of the array of objects in place (does not make Copies)
  * 
@@ -2170,3 +2243,92 @@ module.exports.objectCollectionToDataFrameObject = function objectCollectionToDa
   });
   return dataFrameObject;
 };
+
+/**
+ * Some charting software (such as vega-lite) does not allow a single object to be used
+ * for multiple line series.
+ * 
+ * This is intended to help with that.
+ * 
+ * ```
+ * [
+ *   { category: 'A', source: 'chicago', x: 0.1, y: 0.6, z: 0.9 },
+ *   { category: 'B', source: 'springfield', x: 0.7, y: 0.2, z: 1.1 },
+ *   { category: 'C', source: 'winnetka', x: 0.6, y: 0.1, z: 0.2 }
+ * ]
+ * ```
+ * 
+ * must have a separate object for each x, y and z field for the A category.
+ * 
+ * ```
+ * utils.object.splitIntoDatums(category, ['x', 'y', 'z']);
+ * [
+ *   { category: 'A', source: 'chicago', series: 'x', value: 0.1 },
+ *   { category: 'A', source: 'chicago', series: 'y', value: 0.6 },
+ *   { category: 'A', source: 'chicago', series: 'z', value: 0.9 },
+ *   { category: 'B', source: 'springfield', series: 'x', value: 0.7 },
+ *   { category: 'B', source: 'springfield', series: 'y', value: 0.2 },
+ *   { category: 'B', source: 'springfield', series: 'z', value: 1.1 },
+ *   { category: 'C', source: 'winnetka', series: 'x', value: 0.6 },
+ *   { category: 'C', source: 'winnetka', series: 'y', value: 0.1 },
+ *   { category: 'C', source: 'winnetka', series: 'z', value: 0.2 }
+ * ]
+ * ```
+ * 
+ * note that the fields NOT within the list of fields specified, are preserved
+ * in the denormalized objects...
+ * 
+ * while the fields listed are put into separate objects.
+ * 
+ * You can specify which fields that are generated in those new objects
+ * 
+ * ```
+ * utils.object.splitIntoDatums(category, ['x', 'y', 'z'], 'group', 'val');
+ * [
+ *   { category: 'A', source: 'chicago', group: 'x', val: 0.1 },
+ *   { category: 'A', source: 'chicago', group: 'y', val: 0.6 },
+ *   { category: 'A', source: 'chicago', group: 'z', val: 0.9 },
+ *   { category: 'B', source: 'springfield', group: 'x', val: 0.7 },
+ *   { category: 'B', source: 'springfield', group: 'y', val: 0.2 },
+ *   { category: 'B', source: 'springfield', group: 'z', val: 1.1 },
+ *   { category: 'C', source: 'winnetka', group: 'x', val: 0.6 },
+ *   { category: 'C', source: 'winnetka', group: 'y', val: 0.1 },
+ *   { category: 'C', source: 'winnetka', group: 'z', val: 0.2 }
+ * ]
+ * ```
+ * 
+ * @param {Object[]} objectCollection - collection
+ * @param {String[]} keysForEachSeries - fields that will each be a series spread across datum records
+ * @param {String} [seriesFieldName='series'] - the name of the field to indicate which series this datum is in
+ * @param {String} [valueFieldName='value'] - the name of the value for that datum
+ * @returns {Object} - list of objects x keysForEachSeries.length, where field within keysForEachSeries
+ *    are then individually assigned to a value, and series - to indicate which field is used.
+ */
+module.exports.splitIntoDatums = function splitIntoDatums(
+  collection,
+  keysToSplit,
+  seriesFieldName = 'series',
+  valueFieldName = 'value'
+) {
+  const keysToKeep = ObjectUtils.keysNotInList(collection, keysToSplit);
+  const keysToSeparate = ObjectUtils.keysWithinList(collection, keysToSplit);
+  const rowLength = keysToSeparate.length;
+
+  const results = new Array(collection.length * rowLength).fill(null);
+  collection.forEach((obj, objIndex) => {
+    const basis = {};
+    keysToKeep.forEach((key) => {
+      basis[key] = obj[key];
+    });
+
+    keysToSeparate.forEach((key, keyIndex) => {
+      const newRecord = { ...basis };
+      newRecord[seriesFieldName] = key;
+      newRecord[valueFieldName] = obj[key];
+      results[objIndex * rowLength + keyIndex] = newRecord;
+    });
+  });
+  
+  // return ({keysForSplitting, keysForKeeping});
+  return results;
+};