jupyter-ijavascript-utils 1.56.0 → 1.57.0

package/DOCS.md

@@ -74,6 +74,11 @@ Give it a try here:
 [![Binder:what can I do with this](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/paulroth3d/jupyter-ijavascript-utils/main?labpath=example.ipynb)
 
 ## What's New
+* 1.57 - #86 - include examples on binning based on time series {@link https://github.com/paulroth3d/jupyter-ijavascript-utils/issues/86|see more here}
+    * #87 - add in {@link module:file.fileExists|file.fileExists} to make things easier for new people getting started
+    * #89 - allow {@link module:array.resize|array.resize} to work with defaults, if zipping arrays of different sizes
+    * #90 - correct issue with timezone offsets, so it is no longer possible to get a timezone offset +24:00 from any of the localizations - like {@link module:date.toLocalISO|date.toLocalISO}
+    * #92 - make it easier to handle big expensive calculations with a cache, see {@link module:ijs.useCache|ijs.useCache()} and {@link module:file.useCache|file.useCache()}
 * 1.56 - #84 (object.renamePropertiesFromList), #82 (date.getWeekday)
 * 1.55 - #76, #77, #74, #78
 * 1.54 - additional Date logic, and formatting. #70 #71 #72

package/Dockerfile

@@ -1,3 +1,3 @@
 # syntax=docker/dockerfile:1
 
-FROM darkbluestudios/jupyter-ijavascript-utils:binder_1.55.0
+FROM darkbluestudios/jupyter-ijavascript-utils:binder_1.57.0

package/README.md

@@ -54,6 +54,11 @@ This is not intended to be the only way to accomplish many of these tasks, and a
 ![Screenshot of example notebook](docResources/img/mainExampleNotebook.png)
 
 # What's New 
+* 1.57 - #86 - include examples on binning based on time series
+    * #87 - add in fileExists to make things easier for new people getting started
+    * #89 - allow resizing arrays with defaults, if zipping arrays of different sizes
+    * #90 - correct issue with timezone offsets, so it is no longer possible to get a timezone offset +24:00
+    * #92 - make it easier to handle big expensive calculations with a cache
 * 1.56 - #84 (object.renamePropertiesFromList), #82 (date.getWeekday)
 * 1.55 - #76, #77, #74, #78
 * 1.54 - additional Date logic, and formatting. #70 #71 #72

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jupyter-ijavascript-utils",
-  "version": "1.56.0",
+  "version": "1.57.0",
   "description": "Utilities for working with iJavaScript - a Jupyter Kernel",
   "homepage": "https://jupyter-ijavascript-utils.onrender.com/",
   "license": "MIT",

package/src/array.js

@@ -1460,18 +1460,25 @@ module.exports.asyncWaitAndChain = (seconds, fn, rows) => {
  * 
  * utils.array.resize(categoryValues, 2); // ['rock', 'paper']
  * utils.array.resize(categoryValues, 7); // ['rock', 'paper', 'scissors',
- * 'rock', 'paper', 'scissors', 'rock];
+ * undefined, undefined, undefined, undefined];
  * ```
  * 
  * @param {Array} sourceList - array of values
- * @param {Number} length - new number of items in the list
+ * @param {Number} length - new length of the list
  */
-module.exports.resize = function resize(sourceList, length) {
+module.exports.resize = function resize(sourceList, length, defaultValue = undefined) {
   if (!sourceList || !Array.isArray(sourceList)) return [];
   if (length < 1 || sourceList.length < 1) return [];
-  return new Array(length)
-    .fill(0)
-    .map((_, index) => sourceList[index % sourceList.length]);
+
+  const result = new Array(length)
+    .fill(defaultValue);
+
+  sourceList.forEach((val, index) => {
+    if (index >= length) return;
+    result[index] = val;
+  });
+
+  return result;
 };
 
 /**
@@ -1532,9 +1539,9 @@ module.exports.zip = function zip(arrayLeft, arrayRight, ...rest) {
   if (cleanLeft.length === 0 && cleanRight.length === 0) {
     result = [[]];
   } else if (cleanLeft.length === 0) {
-    result = cleanRight.map((val) => Array.isArray(val) ? val : [val]);
+    result = cleanRight.map((val) => Array.isArray(val) ? val : (typeof val === 'string') ? [val] : [...val]);
   } else if (cleanRight.length === 0) {
-    result = cleanLeft.map((val) => Array.isArray(val) ? val : [val]);
+    result = cleanLeft.map((val) => Array.isArray(val) ? val : (typeof val === 'string') ? [val] : [...val]);
   } else {
     const cleanLeftLen = cleanLeft.length;
     const cleanRightLen = cleanRight.length;

package/src/date.js

@@ -266,7 +266,8 @@ module.exports.getTimezoneEntry = function getTimezoneEntry(timezoneStr) {
   
   const impactedDate = getOffset(d);
 
-  const diff = d.getTime() - impactedDate.getTime();
+  //-- avoid +24:00 or -24:00 - the number of seconds within a day
+  const diff = (d.getTime() - impactedDate.getTime()) % 86400000;
 
   const diffSign = diff > 0 ? '-' : '+';
   let remainder = DateUtils.divideRemainder(Math.abs(diff), DateUtils.TIME.HOUR);

package/src/file.js

@@ -29,6 +29,9 @@ const logger = require('./logger');
  *   * {@link module:file.matchFiles|matchFiles(path, matchingFn)} - find files or directories based on type of file or name
  * * checking files exist
  *   * {@link module:file.checkFile|checkFile(...paths)} - check if a file at a path exists
+ *   * {@link module:file.fileExists|fileExists(filePath)} - check if a single file at a path exists
+ * * using a cache for long running executions
+ *   * {@link module:file.useCache|file.useCache()} - perform an expensive calculation and write to a cache, or read from the cache transparently
  * 
  * ---
  * 
@@ -96,6 +99,8 @@ const FileUtil = module.exports;
  * 
  * @param {string} filePath - path of the file to load
  * @param {Object} fsOptions - options to pass for fsRead (ex: { encoding: 'utf-8' })
+ * @param {Function} fsOptions.formatter - formatter to use when writing the JSON
+ * @param {String} fsOptions.encoding - the encoding to write the JSON out with
  * @example
  * const weather = [
  *   { id: 1, city: 'Seattle',  month: 'Aug', precip: 0.87 },
@@ -117,7 +122,11 @@ const FileUtil = module.exports;
 module.exports.readJSON = function readJSON(filePath, fsOptions = {}) {
   const resolvedPath = path.resolve(filePath);
   const optionsDefaults = { encoding: 'utf-8' };
-  const cleanedOptions = { ...optionsDefaults, ...fsOptions };
+  let cleanedOptions = { ...optionsDefaults, ...fsOptions };
+
+  //-- unfortunately we cannot pass the formatter in addition, it must replace
+  //-- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse
+  if (cleanedOptions.formatter) cleanedOptions = cleanedOptions.formatter;
 
   /** @type {string} */
   let result;
@@ -227,12 +236,14 @@ module.exports.readFile = function readFile(filePath, fsOptions = {}) {
  */
 module.exports.writeJSON = function writeJSON(filePath, contents, fsOptions = {}) {
   //-- if it isn't desired, simply pass as a string.
-  const jsonContents = JSON.stringify(contents, null, 2);
   const optionsDefaults = { encoding: 'utf-8' };
   const cleanedOptions = { ...optionsDefaults, ...fsOptions };
   const isAppend = cleanedOptions.append === true;
   const prefix = cleanedOptions.prefix || '';
   const suffix = cleanedOptions.suffix || '';
+  const formatter = cleanedOptions.formatter || null;
+  const spacing = cleanedOptions.spacing || 2;
+  const jsonContents = JSON.stringify(contents, formatter, spacing);
 
   // const resolvedPath = path.resolve(filePath);
   try {
@@ -465,23 +476,103 @@ module.exports.checkFile = function checkFile(...files) {
   return notFoundFiles;
 };
 
-/*
- * Execute an async function if any of the files do not exist
- * @param {String[]} filePaths - list of paths of files to check that they exist
- * @param {*} fnIfFailed - async function tha will run - but only if any of the files are not found.
+/**
+ * Checks if a single file exists
+ * @param {String} filePath - path to check if the file exists.
+ * @returns {Boolean} - if the file exists (true) or not (false)
+ * @see {@link module:file.checkFile|file.checkFile} - if checking multiple files
  */
+module.exports.fileExists = function fileExists(filePath) {
+  const resolvedPath = path.resolve(filePath);
+  return fs.existsSync(resolvedPath);
+};
+
 /*
-module.exports.ifNotExists = async function ifNotExists(filePaths, fnIfFailed) {
-  const filesNotFound = FileUtil.checkFile(filePaths);
+//-- not needed - dates already serialize to iso Strings
+module.exports.cacheSerializer = (key, value) => {
+  if (key && (key === 'date' || key.endsWith('_date')) && (value instanceof Date)) {
+    return value.toISOString();
+  }
+  return value;
+};
+*/
 
-  let results;
+module.exports.cacheDeserializer = (key, value) => {
+  if (key && (key === 'date' || key.endsWith('_date'))) {
+    return new Date(value);
+  }
+  return value;
+};
 
-  if (filesNotFound) {
-    results = await fnIfFailed(filesNotFound);
-  } else {
-    results = null;
+/**
+ * For very long or time-intensive executions, sometimes it is better to cache the results
+ * than to execute them every single time.
+ * 
+ * Note that this works synchronously, and can be easier to use than if promises are involved.
+ * 
+ * As opposed to {@link module:ijs.useCache|ijs.useCache} - which works with promises.
+ * 
+ * ```
+ * shouldWrite = true; /// we will write to the cache with the results from the execution
+ * expensiveResults = utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *    const data = d3.csvParse(utils.file.readFile('./someFile.csv'))
+ *      .map(obj => ({ ...obj, date: Date.parse(obj.epoch) }));
+ *    
+ *    const earliestDate = utils.date.startOfDay( utils.agg.min(data, 'date') );
+ *    const lastDate = utils.date.endOfDay( utils.agg.max(data, 'date') );
+ * 
+ *    // binning or lots of other things.
+ * 
+ *    return finalResults;
+ * });
+ * 
+ * expensiveresults.length = 1023424;
+ * ```
+ * 
+ * but sometimes I would rather just skip to the end
+ * 
+ * ```
+ * shouldWrite = false; /// we will read from the cache instead,
+ * // everything else remains the same
+ * 
+ * expensiveResults = utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *    const data = d3.csvParse(utils.file.readFile('./someFile.csv'))
+ *      .map(obj => ({ ...obj, date: Date.parse(obj.epoch) }));
+ * 
+ *    //-- function can remain untouched,
+ *    //-- BUT nothing in here will be executed
+ *    //-- since we are reading from the cache
+ * });
+ * 
+ * //-- completely transparent to the runner
+ * expensiveresults.length = 1023424;
+ * ```
+ * 
+ * @param {Boolean} shouldWrite - whether we should write to the cache (true) or read from the cache (false)
+ * @param {String} cachePath - Path to the cache folder, ex: './cache'
+ * @param {String} cacheFile - Filename of the cache file to use for this execution, ex: 'ExecutionsPerMin.js'
+ * @param {Function} expensiveFn - function that returns the results to be stored in the cache
+ * @param {Object} fsOptions - options to use when writing or reading files
+ * @returns {any} - either the deserialized json from the cache or the results from the expensive function
+ * @see {@link module:file.readJSON|file.readJSON} - reads a local JSON file
+ * @see {@link module:file.writeJSON|file.writeJSON} - writes to a JSON file
+ * @see {@link module:ijs.useCache|ijs.useCache} - similar idea - but supports promises
+ */
+module.exports.useCache = function useCache(shouldWrite, cachePath, cacheFile, expensiveFn, fsOptions = null) {
+  const ensureEndsWithSlash = (str) => str.endsWith('/') ? str : `${str}/`;
+  const cacheFilePath = `${ensureEndsWithSlash(cachePath)}${cacheFile}`;
+  
+  if (!shouldWrite) {
+    const cleanOptions = { ...fsOptions, formatter: FileUtil.cacheDeserializer };
+    const results = FileUtil.readJSON(cacheFilePath, cleanOptions);
+    return results;
   }
 
+  const results = expensiveFn();
+
+  const cleanOptions = { ...fsOptions, formatter: null }; // FileUtil.cacheSerializer not needed
+
+  FileUtil.writeJSON(cacheFilePath, results, cleanOptions);
+
   return results;
 };
-*/

package/src/hashMap.js

@@ -83,12 +83,11 @@ module.exports.add = function add(map, key, value) {
  * @param {any} functor.value - the first argument is the current value
  * @param {any} functor.key - the second argument is the key passed
  * @param {any} functor.map - the third argument is the map being acted upon
- * @param {any} [setKey = null] - optional separate key to use to set the updated value
  * @returns {Map}
  */
-module.exports.getSet = function getSet(map, key, functor, setKey = null) {
-  const cleanSetKey = setKey || key;
-  map.set(cleanSetKey, functor(map.get(key), key, map));
+module.exports.getSet = function getSet(map, key, functor) {
+  const currentValue = map.has(key) ? map.get(key) : undefined;
+  map.set(key, functor(currentValue, key, map));
   return map;
 };
 module.exports.update = module.exports.getSet;

package/src/ijs.js

@@ -2,6 +2,8 @@
 
 const uuid = require('uuid').v4;
 
+const FileUtil = require('./file');
+
 require('./_types/global');
 
 /**
@@ -27,6 +29,8 @@ require('./_types/global');
  *   * {@link module:ijs.noOutputNeeded|ijs.noOutputNeeded} - clears the output to declutter results (like importing libraries, or functions)
  *   * {@link module:ijs.initializePageBreaks|ijs.initializePageBreaks} - call at least once to allow pageBreaks when rendering PDFs
  *   * {@link module:ijs.printPageBreak|ijs.printPageBreak} - call to print a page break when rendering PDFs
+ * * using a cache for long running executions
+ *   * {@link module:ijs.useCache|ijs.useCache()} - perform an expensive calculation and write to a cache, or read from the cache transparently
  * 
  * For example:
  * 
@@ -662,3 +666,102 @@ module.exports.printPageBreak = function printPageBreak() {
 
   context.$$.html('<div class="pagebreak"></div>');
 };
+
+/**
+ * For very long or time-intensive executions, sometimes it is better to cache the results
+ * than to execute them every single time.
+ * 
+ * Note that this supports promises, and can be a bit harder to understand.
+ * 
+ * As opposed to {@link module:file.useCache|file.useCache} - which works synchronously
+ * 
+ * ```
+ * shouldWrite = true; /// we will write to the cache with the results from the execution
+ * utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *   //-- all the items to cache 
+ *   return Promise.resolve()
+ *      .then(() => ajax.retrieve(...))
+ *      .then((results) => {
+ *          const data = results
+ *            .map(obj => ({ ...obj, date: Date.parse(obj.epoch) }));
+ *          .. other things to do
+ *          return data;
+ *       });
+ * })
+ *   //-- using the information AFTER the expensive function or retrieval from cache
+ *   //-- this can ALSO be run in a subsequence cell
+ *   .then((results) => {
+ *      expensiveresults = results;
+ *      conosole.log(`expensiveResults.length: ${results.length}`);
+ *    });
+ * ```
+ * 
+ * but sometimes I would rather just skip to the end
+ * 
+ * ```
+ * shouldWrite = false; /// we will read from the cache instead,
+ * // everything else remains the same
+ * 
+ * utils.file.useCache(shouldWrite, './cache', 'expensive.json', () => {
+ *   //-- all the items to cache 
+ *   return Promise.resolve()
+ *     ... blah blah - none of this will get executed
+ * })
+ *   //-- using the information AFTER the expensive function or retrieval from cache
+ *   //-- this can ALSO be run in a subsequence cell
+ *   .then((results) => {
+ *      expensiveresults = results;
+ *      conosole.log(`expensiveResults.length: ${results.length}`);
+ *    });
+ * 
+ * //-- completely transparent to the runner
+ * expensiveresults.length = 1023424;
+ * ```
+ * 
+ * @param {Boolean} shouldWrite - whether we should write to the cache (true) or read from the cache (false)
+ * @param {String} cachePath - Path to the cache folder, ex: './cache'
+ * @param {String} cacheFile - Filename of the cache file to use for this execution, ex: 'ExecutionsPerMin.js'
+ * @param {Function} expensiveFn - function that returns the results to be stored in the cache
+ * @param {Object} fsOptions - options to use when writing or reading files
+ * @returns {any} - either the deserialized json from the cache or the results from the expensive function
+ * @see {@link module:file.readJSON|file.readJSON} - reads a local JSON file
+ * @see {@link module:file.writeJSON|file.writeJSON} - writes to a JSON file
+ * @see {@link module:file.useCache|file.useCache} - can be much easier than using promises
+ */
+module.exports.useCache = async function useCache(shouldWrite, cachePath, cacheFile, expensiveFn, fsOptions = null) {
+  const ensureEndsWithSlash = (str) => str.endsWith('/') ? str : `${str}/`;
+  const cacheFilePath = `${ensureEndsWithSlash(cachePath)}${cacheFile}`;
+  const context = IJSUtils.detectContext();
+
+  if (!shouldWrite) {
+    const cleanOptions = { ...fsOptions, formatter: FileUtil.cacheDeserializer };
+    //-- we read the info and be done with it
+    //context.console.log(`before retrieving cache:${cacheFilePath}`);
+    const results = FileUtil.readJSON(cacheFilePath, cleanOptions);
+    //context.console.log(`after retrieving cache`);
+    return Promise.resolve(results);
+  }
+
+  if (!context) {
+    throw (Error('IJSUtils.async must be run within iJavaScript. Otherwise, use normal async methods'));
+  }
+
+  context.$$.async();
+
+  try {
+    context.console.log('before starting expensive fn');
+    const results = await expensiveFn(context.$$, context.console);
+    context.console.log('AFTER starting expensive fn');
+
+    FileUtil.writeJSON(cacheFilePath, results);
+    // context.$$.sendResult('success');
+    return results;
+  } catch (err) {
+    context.console.error('error occurred');
+    context.console.error(err);
+    context.$$.sendResult(err);
+
+    // return Promise.reject(err);
+    throw err;
+  }
+};

package/src/object.js

@@ -26,6 +26,7 @@ const FormatUtils = require('./format');
  *   * {@link module:object.extractObjectProperties|extractObjectProperties(list, propertyNameOrFnMap)} - extracts multiple propertie or fn across all objects in list.
  * * Apply deep values safely
  *   * {@link module:object.assign|objAssign(object, property, value)} - Applies properties to an object in functional programming style.
+ *   * {@link module:object.getSet|getSet(object, property, functor)} - calls a function with the current value on an object, allowing for decrementing/incrementing/etc.
  *   * {@link module:object.augment|augment(object, augmentFn)} - Applies properties to an object similar to Map
  *   * {@link module:object.assignEntities|objAssignEntities(object, [property, value])} - Applies properties to an object using Array values - [key,value]
  *   * {@link module:object.setPropertyDefaults|setPropertyDefaults()} - sets values for objects that don't currently have the property
@@ -169,6 +170,53 @@ module.exports.assignEntities = function objAssignEntities(obj, entities) {
 };
 module.exports.objAssignEntities = module.exports.assignEntities;
 
+/**
+ * Use this for times where you want to update a value
+ * 
+ * 
+ * ```
+ * key = 'somethingToIncrement';
+ * defaultValue = null;
+ * 
+ * const initialObject = {};
+ * initialObject[key] = defaultValue;
+ * 
+ * // { somethingToIncrement: null }
+ * 
+ * const functor = (value) => { //, key, map) => {
+ *   if (!value) return 1;
+ *   return value + 1;
+ * };
+ * 
+ * utils.object.getSet(initialObject, key, functor);
+ * 
+ * // equivalent to
+ * // intitialObject[key] = intitialObject[key] ? initialObject[key] + 1 : 1`
+ * // but in a way that is repeatable across many values
+ * 
+ * utils.object.getSet(initialObject, key, functor);
+ * utils.object.getSet(initialObject, key, functor);
+ * utils.object.getSet(initialObject, key, functor);
+ * utils.object.getSet(initialObject, key, functor);
+ * 
+ * initialObject.get(key); // 5
+ * ```
+ * 
+ * @param {Map} map - map to get and set values from
+ * @param {any} key - they key to GET and SET the value (unless setKey is provided)
+ * @param {Function} functor - the function called with the arguments below - returning the value to set
+ * @param {any} functor.value - the first argument is the current value
+ * @param {any} functor.key - the second argument is the key passed
+ * @param {any} functor.map - the third argument is the map being acted upon
+ * @returns {Map}
+ */
+module.exports.getSet = function getSet(obj, field, functor) {
+  const currentValue = Object.hasOwn(obj, field) ? obj[field] : undefined;
+  obj[field] = functor(currentValue, field, obj);
+  return obj;
+};
+module.exports.update = module.exports.getSet;
+
 /**
  * Runs a map over a collection, and adds properties the the objects.
  *