manyfest 1.0.7 → 1.0.9

package/.config/code-server/config.yaml

@@ -1,4 +1,4 @@
 bind-addr: 127.0.0.1:8080
 auth: password
-password: luxury
+password: retold
 cert: false

package/.config/configstore/update-notifier-npm.json

@@ -1,4 +1,4 @@
 {
 	"optOut": false,
-	"lastUpdateCheck": 1666021392557
+	"lastUpdateCheck": 1666107934926
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "manyfest",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "description": "JSON Object Manifest for Data Description and Parsing",
   "main": "source/Manyfest.js",
   "scripts": {
@@ -58,7 +58,10 @@
     "vinyl-buffer": "^1.0.1",
     "vinyl-source-stream": "^2.0.0"
   },
-  "dependencies": {},
+  "dependencies": {
+    "elucidator": "^1.0.2",
+    "precedent": "^1.0.6"
+  },
   "author": "steven velozo <steven@velozo.com>",
   "license": "MIT",
   "bugs": {

package/source/Manyfest-ObjectAddressGeneration.js

@@ -0,0 +1,123 @@
+/**
+* @license MIT
+* @author <steven@velozo.com>
+*/
+let libSimpleLog = require('./Manyfest-LogToConsole.js');
+
+/**
+* Object Address Generation
+*
+* Automagically generate addresses and properties based on a passed-in object, 
+* to be used for easy creation of schemas.  Meant to simplify the lives of
+* developers wanting to create schemas without typing a bunch of stuff.
+* 
+* IMPORTANT NOTE: This code is intentionally more verbose than necessary, to
+*                 be extremely clear what is going on in the recursion for
+*                 each of the three address resolution functions.
+* 
+*                 Although there is some opportunity to repeat ourselves a
+*                 bit less in this codebase (e.g. with detection of arrays
+*                 versus objects versus direct properties), it can make
+*                 debugging.. challenging.  The minified version of the code
+*                 optimizes out almost anything repeated in here.  So please
+*                 be kind and rewind... meaning please keep the codebase less
+*                 terse and more verbose so humans can comprehend it.
+*                 
+*
+* @class ManyfestObjectAddressGeneration
+*/
+class ManyfestObjectAddressGeneration
+{
+	constructor(pInfoLog, pErrorLog)
+	{
+		// Wire in logging
+		this.logInfo = (typeof(pInfoLog) == 'function') ? pInfoLog : libSimpleLog;
+		this.logError = (typeof(pErrorLog) == 'function') ? pErrorLog : libSimpleLog;
+	}
+
+	// generateAddressses
+	//
+	// This flattens an object into a set of key:value pairs for *EVERY SINGLE
+	// POSSIBLE ADDRESS* in the object.  It can get ... really insane really
+	// quickly.  This is not meant to be used directly to generate schemas, but
+	// instead as a starting point for scripts or UIs.
+	//
+	// This will return a mega set of key:value pairs with all possible schema 
+	// permutations and default values (when not an object) and everything else.
+	generateAddressses (pObject, pBaseAddress, pSchema)
+	{
+		let tmpBaseAddress = (typeof(pBaseAddress) == 'string') ? pBaseAddress : '';
+		let tmpSchema = (typeof(pSchema) == 'object') ? pSchema : {};
+
+		let tmpObjectType = typeof(pObject);
+
+		let tmpSchemaObjectEntry = (
+			{
+				Address: tmpBaseAddress,
+				Hash: tmpBaseAddress,
+				Name: tmpBaseAddress,
+				// This is so scripts and UI controls can force a developer to opt-in.
+				InSchema: false
+			}
+		)
+
+		switch(tmpObjectType)
+		{
+			case 'string':
+				tmpSchemaObjectEntry.DataType = 'String';
+				tmpSchemaObjectEntry.Default = pObject;
+				tmpSchema[tmpBaseAddress] = tmpSchemaObjectEntry;
+				break;
+			case 'number':
+			case 'bigint':
+				tmpSchemaObjectEntry.DataType = 'Number';
+				tmpSchemaObjectEntry.Default = pObject;
+				tmpSchema[tmpBaseAddress] = tmpSchemaObjectEntry;
+				break;
+			case 'undefined':
+				tmpSchemaObjectEntry.DataType = 'Any';
+				tmpSchemaObjectEntry.Default = pObject;
+				tmpSchema[tmpBaseAddress] = tmpSchemaObjectEntry;
+				break;
+			case 'object':
+				if (Array.isArray(pObject))
+				{
+					tmpSchemaObjectEntry.DataType = 'Array';
+					if (tmpBaseAddress != '')
+					{
+						tmpSchema[tmpBaseAddress] = tmpSchemaObjectEntry;
+					}
+	
+					for (let i = 0; i < pObject.length; i++)
+					{
+						this.generateAddressses(pObject[i], `${tmpBaseAddress}[${i}]`, tmpSchema);
+					}
+				}
+				else
+				{
+					tmpSchemaObjectEntry.DataType = 'Object';
+					if (tmpBaseAddress != '')
+					{
+						tmpSchema[tmpBaseAddress] = tmpSchemaObjectEntry;
+						tmpBaseAddress += '.';
+					}
+	
+					let tmpObjectProperties = Object.keys(pObject);
+
+					for (let i = 0; i < tmpObjectProperties.length; i++)
+					{
+						this.generateAddressses(pObject[tmpObjectProperties[i]], `${tmpBaseAddress}${tmpObjectProperties[i]}`, tmpSchema);
+					}			
+				}
+				break;
+			case 'symbol':
+			case 'function':
+				// Symbols and functions neither recurse nor get added to the schema
+				break;
+		}
+
+		return tmpSchema;
+	}
+};
+
+module.exports = ManyfestObjectAddressGeneration;

package/source/Manyfest-ObjectAddressResolver.js

@@ -3,6 +3,7 @@
 * @author <steven@velozo.com>
 */
 let libSimpleLog = require('./Manyfest-LogToConsole.js');
+let libPrecedent = require('precedent');
 
 /**
 * Object Address Resolver
@@ -19,6 +20,8 @@ let libSimpleLog = require('./Manyfest-LogToConsole.js');
 *                 be kind and rewind... meaning please keep the codebase less
 *                 terse and more verbose so humans can comprehend it.
 *                 
+* TODO: Once we validate this pattern is good to go, break these out into 
+*       three separate modules.
 *
 * @class ManyfestObjectAddressResolver
 */
@@ -29,6 +32,9 @@ class ManyfestObjectAddressResolver
 		// Wire in logging
 		this.logInfo = (typeof(pInfoLog) == 'function') ? pInfoLog : libSimpleLog;
 		this.logError = (typeof(pErrorLog) == 'function') ? pErrorLog : libSimpleLog;
+
+		this.elucidatorSolver = false;
+		this.elucidatorSolverState = {};
 	}
 
 	// When a boxed property is passed in, it should have quotes of some
@@ -236,13 +242,57 @@ class ManyfestObjectAddressResolver
 		}
 	}
 
+	checkFilters(pAddress, pRecord)
+	{
+		let tmpPrecedent = new libPrecedent();
+		// If we don't copy the string, precedent takes it out for good.
+		// TODO: Consider adding a "don't replace" option for precedent
+		let tmpAddress = pAddress;
+
+		if (this.elucidatorSolver)
+		{
+			// This allows the magic filtration with elucidator configuration
+			// TODO: We could pass more state in (e.g. parent address, object, etc.)
+			// TODO: Discuss this metaprogramming AT LENGTH
+			let tmpFilterState = (
+				{
+					Record: pRecord,
+					keepRecord: true 
+				});
+
+			// This is about as complex as it gets.
+			// TODO: Optimize this so it is only initialized once.
+			// TODO: That means figuring out a healthy pattern for passing in state to this
+			tmpPrecedent.addPattern('{<~~', '~~>}', 
+				(pInstructionHash) => 
+				{
+					// This is for internal config on the solution steps.  Right now config is not shared across steps.
+					if (this.elucidatorSolverState.hasOwnProperty(pInstructionHash))
+					{
+						tmpFilterState.SolutionState = this.elucidatorSolverState[pInstructionHash];
+					}
+					this.elucidatorSolver.solveInternalOperation('Custom', pInstructionHash, tmpFilterState);
+				});
+			
+			tmpPrecedent.parseString(tmpAddress);
+
+			// It is expected that the operation will mutate this to some truthy value
+			return tmpFilterState.keepRecord;
+		}
+		else
+		{
+			return true;
+		}
+	}
+
 	// Get the value of an element at an address
 	getValueAtAddress (pObject, pAddress, pParentAddress)
 	{
-		// Make sure pObject is an object
+		// Make sure pObject (the object we are meant to be recursing) is an object (which could be an array or object)
 		if (typeof(pObject) != 'object') return undefined;
-		// Make sure pAddress is a string
+		// Make sure pAddress (the address we are resolving) is a string
 		if (typeof(pAddress) != 'string') return undefined;
+		// Stash the parent address for later resolution
 		let tmpParentAddress = "";
 		if (typeof(pParentAddress) == 'string')
 		{
@@ -337,7 +387,19 @@ class ManyfestObjectAddressResolver
 					return false;
 				}
 
-				return pObject[tmpBoxedPropertyName];
+				let tmpInputArray = pObject[tmpBoxedPropertyName];
+				let tmpOutputArray = [];
+				for (let i = 0; i < tmpInputArray.length; i++)
+				{
+					// The filtering is complex but allows config-based metaprogramming directly from schema
+					let tmpKeepRecord = this.checkFilters(pAddress, tmpInputArray[i]);
+					if (tmpKeepRecord)
+					{
+						tmpOutputArray.push(tmpInputArray[i]);
+					}
+				}
+
+				return tmpOutputArray;
 			}
 			// The object has been flagged as an object set, so treat it as such
 			else if (tmpObjectTypeMarkerIndex > 0)
@@ -456,7 +518,8 @@ class ManyfestObjectAddressResolver
 				for (let i = 0; i < tmpArrayProperty.length; i++)
 				{
 					let tmpPropertyParentAddress = `${tmpParentAddress}[${i}]`;
-					let tmpValue = this.getValueAtAddress(pObject[tmpBoxedPropertyName][i], tmpNewAddress, tmpPropertyParentAddress);;
+					let tmpValue = this.getValueAtAddress(pObject[tmpBoxedPropertyName][i], tmpNewAddress, tmpPropertyParentAddress);
+					
 					tmpContainerObject[`${tmpPropertyParentAddress}.${tmpNewAddress}`] = tmpValue;
 				}
 
@@ -486,8 +549,14 @@ class ManyfestObjectAddressResolver
 				for (let i = 0; i < tmpObjectPropertyKeys.length; i++)
 				{
 					let tmpPropertyParentAddress = `${tmpParentAddress}.${tmpObjectPropertyKeys[i]}`;
-					let tmpValue = this.getValueAtAddress(pObject[tmpObjectPropertyName][tmpObjectPropertyKeys[i]], tmpNewAddress, tmpPropertyParentAddress);;
-					tmpContainerObject[`${tmpPropertyParentAddress}.${tmpNewAddress}`] = tmpValue;
+					let tmpValue = this.getValueAtAddress(pObject[tmpObjectPropertyName][tmpObjectPropertyKeys[i]], tmpNewAddress, tmpPropertyParentAddress);
+	
+					// The filtering is complex but allows config-based metaprogramming directly from schema
+					let tmpKeepRecord = this.checkFilters(pAddress, tmpValue);
+					if (tmpKeepRecord)
+					{
+						tmpContainerObject[`${tmpPropertyParentAddress}.${tmpNewAddress}`] = tmpValue;
+					}
 				}
 
 				return tmpContainerObject;

package/source/Manyfest.js

@@ -3,10 +3,15 @@
 * @author <steven@velozo.com>
 */
 let libSimpleLog = require('./Manyfest-LogToConsole.js');
-let libObjectAddressResolver = require('./Manyfest-ObjectAddressResolver.js');
+
+let libPrecedent = require('precedent');
+
 let libHashTranslation = require('./Manyfest-HashTranslation.js');
+let libObjectAddressResolver = require('./Manyfest-ObjectAddressResolver.js');
+let libObjectAddressGeneration = require('./Manyfest-ObjectAddressGeneration.js');
 let libSchemaManipulation = require('./Manyfest-SchemaManipulation.js');
 
+
 /**
 * Manyfest object address-based descriptions and manipulations.
 *
@@ -45,6 +50,10 @@ class Manyfest
 		this.elementAddresses = undefined;
 		this.elementHashes = undefined;
 		this.elementDescriptors = undefined;
+		// This can cause a circular dependency chain, so it only gets initialized if the schema specifically calls for it.
+		this.dataSolvers = undefined;
+		// So solvers can use their own state
+		this.dataSolverState = undefined;
 
 		this.reset();
 
@@ -54,6 +63,7 @@ class Manyfest
 		}
 
 		this.schemaManipulations = new libSchemaManipulation(this.logInfo, this.logError);
+		this.objectAddressGeneration = new libObjectAddressGeneration(this.logInfo, this.logError);
 
 		this.hashTranslations = new libHashTranslation(this.logInfo, this.logError);
 	}
@@ -69,6 +79,11 @@ class Manyfest
 		this.elementAddresses = [];
 		this.elementHashes = {};
 		this.elementDescriptors = {};
+		this.dataSolvers = undefined;
+		this.dataSolverState = {};
+
+		this.libElucidator = undefined;
+		this.objectAddressResolver.elucidatorSolver = false;
 	}
 
 	clone()
@@ -135,6 +150,35 @@ class Manyfest
 		{
 			this.logError(`(${this.scope}) Error loading object description from manifest object.  Property "Descriptors" does not exist in the root of the Manifest object.`, pManifest);
 		}
+
+		// This seems like it would create a circular dependency issue but it only goes as deep as the schema defines Solvers
+		if ((pManifest.hasOwnProperty('Solvers')) && (typeof(pManifest.Solvers) == 'object'))
+		{
+			// There are elucidator solvers passed-in, so we will create one to filter data.
+			let libElucidator = require('elucidator');
+			// WARNING THESE CAN MUTATE THE DATA
+				// The pattern for the solver is: {<~~SolverName~~>} anywhere in a property.
+				//   Yes, this means your Javascript elements can't have my self-styled jellyfish brackets in them.
+				//   This does, though, mean we can filter at multiple layers safely.
+				//   Because these can be put at any address
+			// The solver themselves:
+				//   They are passed-in an object, and the current record is in the Record subobject.
+				//   Basic operations can just write to the root object but...
+				//   IF YOU PERMUTE THE Record SUBOBJECT YOU CAN AFFECT RECURSION
+			// This is mostly meant for if statements to filter.
+				//   Basically on aggregation, if a filter is set it will set "keep record" to true and let the solver decide differently.
+			this.dataSolvers = new libElucidator(pManifest.Solvers, this.logInfo, this.logError);
+			this.objectAddressResolver.elucidatorSolver = this.dataSolvers;
+
+			// Load the solver state in so each instruction can have internal config
+			// TODO: Should this just be a part of the lower layer pattern?
+			let tmpSolverKeys = Object.keys(pManifest.Solvers)
+			for (let i = 0; i < tmpSolverKeys.length; i++)
+			{
+				this.dataSolverState[tmpSolverKeys] = pManifest.Solvers[tmpSolverKeys[i]];
+			}
+			this.objectAddressResolver.elucidatorSolverState = this.dataSolverState;
+		}
 	}
 
 	// Serialize the Manifest to a string
@@ -256,13 +300,29 @@ class Manyfest
 	// Get the value of an element by its hash
 	getValueByHash (pObject, pHash)
 	{
-		return this.getValueAtAddress(pObject, this.resolveHashAddress(pHash));
+		let tmpValue = this.getValueAtAddress(pObject, this.resolveHashAddress(pHash));
+
+		if (typeof(tmpValue) == 'undefined')
+		{
+			// Try to get a default if it exists
+			tmpValue = this.getDefaultValue(this.getDescriptorByHash(pHash));
+		}
+
+		return tmpValue;
 	}
 
 	// Get the value of an element at an address
 	getValueAtAddress (pObject, pAddress)
 	{
-		return this.objectAddressResolver.getValueAtAddress(pObject, pAddress);
+		let tmpValue = this.objectAddressResolver.getValueAtAddress(pObject, pAddress);
+
+		if (typeof(tmpValue) == 'undefined')
+		{
+			// Try to get a default if it exists
+			tmpValue = this.getDefaultValue(this.getDescriptor(pAddress));
+		}
+
+		return tmpValue;
 	}
 
 	// Set the value of an element by its hash
@@ -304,9 +364,10 @@ class Manyfest
 		for (let i = 0; i < this.elementAddresses.length; i++)
 		{
 			let tmpDescriptor = this.getDescriptor(this.elementAddresses[i]);
+			let tmpValueExists = this.checkAddressExists(pObject, tmpDescriptor.Address);
 			let tmpValue = this.getValueAtAddress(pObject, tmpDescriptor.Address);
 
-			if (typeof(tmpValue) == 'undefined')
+			if ((typeof(tmpValue) == 'undefined') || !tmpValueExists)
 			{
 				// This will technically mean that `Object.Some.Value = undefined` will end up showing as "missing"
 				// TODO: Do we want to do a different message based on if the property exists but is undefined?
@@ -385,6 +446,11 @@ class Manyfest
 	// Returns a default value, or, the default value for the data type (which is overridable with configuration)
 	getDefaultValue(pDescriptor)
 	{
+		if (typeof(pDescriptor) != 'object')
+		{
+			return undefined;
+		}
+
 		if (pDescriptor.hasOwnProperty('Default'))
 		{
 			return pDescriptor.Default;

package/test/Manyfest_Embedded_Solvers_tests.js

@@ -0,0 +1,86 @@
+/**
+* Unit tests for Manyfest
+*
+* @license     MIT
+*
+* @author      Steven Velozo <steven@velozo.com>
+*/
+
+var Chai = require("chai");
+var Expect = Chai.expect;
+
+let libManyfest = require('../source/Manyfest.js');
+
+let _SampleDataArchiveOrgFrankenberry = require('./Data-Archive-org-Frankenberry.json');
+
+suite
+(
+	'Embedded Solvers',
+	function()
+	{
+		setup (()=> {} );
+
+		suite
+		(
+			'Set Filtration',
+			()=>
+			{
+				test
+				(
+					'We should be able to access sets of properties from objects with schema.',
+					(fTestComplete)=>
+					{
+						let _Manyfest = new libManyfest(
+							{
+								Scope:'Archive.org',
+								Descriptors:
+									{
+										'files[]': {Name:'Files', Hash:'FileSet'},
+										'files[].size': {Name:'FileSizes', Hash:'FileSizes'},
+                                        'files[]{<~~ThumbnailFilesOnly~~>}': {Name:'Thumbnail Files', Hash:'ThumbnailFiles'},
+										'metadata.creator': {Name:'Creator', Hash:'Creator'},
+										'metadata{}': {Name:'Metadata', Hash:'Metadata'}
+									},
+                                Solvers:
+                                    {
+                                        'ThumbnailFilesOnly': 
+                                            {
+                                                "Description":
+                                                {
+                                                    "Operation": "ThumbnailFilesOnly",
+                                                    "Synopsis": "Filter files down to just thumbnails"
+                                                },
+												"Config": { "SearchTerm": "Thumbnail" },
+                                                "Steps":
+                                                [
+                                                    {
+                                                        "Namespace": "Logic",
+                                                        "Instruction": "if",
+
+														"InputHashAddressMap": 
+                                                            {
+                                                                "leftValue": "Record.format",
+                                                                "rightValue": "SolutionState.Config.SearchTerm",
+                                                                "comparator": "=="
+                                                            },
+                                                        "OutputHashAddressMap": { "truthinessResult":"keepRecord" }
+                                                    }
+                                                ]
+                                            }
+                                    }
+							});
+						let tmpThumbnailFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'ThumbnailFiles');
+						Expect(tmpThumbnailFiles).to.be.an('array');
+						// There are 7 thumbnail files in the set.
+						Expect(tmpThumbnailFiles.length).to.equal(7);
+						let tmpFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'FileSet');
+						Expect(tmpFiles).to.be.an('array');
+						// There are 7 thumbnail files in the set.
+						Expect(tmpFiles.length).to.equal(17);
+						fTestComplete();
+					}
+				);
+			}
+		);
+	}
+);

package/test/Manyfest_Object_Read_tests.js

@@ -57,7 +57,7 @@ suite
 				);
 				test
 				(
-					'Exercise more hash accesss scenarios..',
+					'Return default values when none are supplied.',
 					(fTestComplete)=>
 					{
 						let animalManyfest = new libManyfest(
@@ -79,13 +79,19 @@ suite
 												"Name":"Comfortable Environmental Temperature",
 												"NameShort":"Comf Env Temp",
 												"Hash":"ComfET",
-												"Description":"The most comfortable temperature for this animal to survive in."
+												"Description":"The most comfortable temperature for this animal to survive in.",
+												"Default": "96.8"
 											}
 									}
 							});
 
-						Expect(animalManyfest.getValueByHash({MedicalStats: { Temps: { CET:200 }},Name:'Froggy'}, 'ComfET'))
-							.to.equal(200);
+							Expect(animalManyfest.getValueByHash({MedicalStats: { Temps: { CET:200 }},Name:'Froggy'}, 'ComfET')).to.equal(200);
+							Expect(animalManyfest.getValueByHash({MedicalStats: { Temps: { MinET:200 }},Name:'Froggy'}, 'ComfET')).to.equal('96.8');
+							Expect(animalManyfest.getValueByHash({MedicalStats: { Temps: { MinET:200 }},Name:'Froggy'}, 'CurrentTemperature')).to.equal(undefined);
+
+							Expect(animalManyfest.getValueAtAddress({MedicalStats: { Temps: { CET:200 }},Name:'Froggy'}, 'MedicalStats.Temps.CET')).to.equal(200);
+							Expect(animalManyfest.getValueAtAddress({MedicalStats: { Temps: { MinET:200 }},Name:'Froggy'}, 'MedicalStats.Temps.CET')).to.equal('96.8');
+							Expect(animalManyfest.getValueAtAddress({MedicalStats: { Temps: { MinET:200 }},Name:'Froggy'}, 'MedicalStats.Temps.HighET')).to.equal(undefined);
 
 						fTestComplete();
 					}

package/test/Manyfest_Object_SchemaManipulation_tests.js

@@ -194,6 +194,52 @@ suite
 						Expect(_ClonedManyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'Director')).to.equal('General Mills');
 						Expect(_Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'Director')).to.equal(undefined);
 	
+						fTestComplete();
+					}
+				);
+				test
+				(
+					'Schema definition prototypes should be able to be generated from any JSON object shape.',
+					(fTestComplete)=>
+					{
+						let tmpSchemaDescriptors = (
+							{
+								"a": { "Hash": "a", "Type": "Number" },
+								"b": { "Hash": "b", "Type": "Number" },
+								"TranslationTable":
+								{
+									"a": "CarrotCost",
+									"b": "AppleCost"
+								}
+							});
+						
+						let _Manyfest = new libManyfest();
+						// Now remap the schema (in-place)
+						let tmpSchemaPrototype = _Manyfest.objectAddressGeneration.generateAddressses(tmpSchemaDescriptors);
+
+						// The schema should be fundamentally altered to point these addresses to the old hashes
+						Expect(tmpSchemaPrototype).to.be.an('object');
+
+						Expect(tmpSchemaPrototype['TranslationTable.a'].DataType).to.equal('String');
+
+						fTestComplete();
+					}
+				);
+				test
+				(
+					'Make a much bigger schema prototype.',
+					(fTestComplete)=>
+					{						
+						let _Manyfest = new libManyfest();
+						// Now remap the schema (in-place)
+						let tmpSchemaPrototype = _Manyfest.objectAddressGeneration.generateAddressses(_SampleDataArchiveOrgFrankenberry);
+
+						// The schema should be fundamentally altered to point these addresses to the old hashes
+						Expect(tmpSchemaPrototype).to.be.an('object');
+
+						Expect(tmpSchemaPrototype['files_count'].Default).to.equal(17);
+						Expect(tmpSchemaPrototype['files_count'].DataType).to.equal('Number');
+
 						fTestComplete();
 					}
 				);