manyfest 1.0.8 → 1.0.10

package/.config/configstore/update-notifier-npm.json

@@ -1,4 +1,4 @@
 {
 	"optOut": false,
-	"lastUpdateCheck": 1666021392557
+	"lastUpdateCheck": 1666107934926
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "manyfest",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "JSON Object Manifest for Data Description and Parsing",
   "main": "source/Manyfest.js",
   "scripts": {
@@ -58,7 +58,10 @@
     "vinyl-buffer": "^1.0.1",
     "vinyl-source-stream": "^2.0.0"
   },
-  "dependencies": {},
+  "dependencies": {
+    "elucidator": "^1.0.2",
+    "precedent": "^1.0.6"
+  },
   "author": "steven velozo <steven@velozo.com>",
   "license": "MIT",
   "bugs": {

package/source/Manyfest-ObjectAddressResolver.js

@@ -3,6 +3,7 @@
 * @author <steven@velozo.com>
 */
 let libSimpleLog = require('./Manyfest-LogToConsole.js');
+let libPrecedent = require('precedent');
 
 /**
 * Object Address Resolver
@@ -19,6 +20,8 @@ let libSimpleLog = require('./Manyfest-LogToConsole.js');
 *                 be kind and rewind... meaning please keep the codebase less
 *                 terse and more verbose so humans can comprehend it.
 *                 
+* TODO: Once we validate this pattern is good to go, break these out into 
+*       three separate modules.
 *
 * @class ManyfestObjectAddressResolver
 */
@@ -29,6 +32,9 @@ class ManyfestObjectAddressResolver
 		// Wire in logging
 		this.logInfo = (typeof(pInfoLog) == 'function') ? pInfoLog : libSimpleLog;
 		this.logError = (typeof(pErrorLog) == 'function') ? pErrorLog : libSimpleLog;
+
+		this.elucidatorSolver = false;
+		this.elucidatorSolverState = {};
 	}
 
 	// When a boxed property is passed in, it should have quotes of some
@@ -236,13 +242,111 @@ class ManyfestObjectAddressResolver
 		}
 	}
 
+	checkFilters(pAddress, pRecord)
+	{
+		let tmpPrecedent = new libPrecedent();
+		// If we don't copy the string, precedent takes it out for good.
+		// TODO: Consider adding a "don't replace" option for precedent
+		let tmpAddress = pAddress;
+
+		if (!this.elucidatorSolver)
+		{
+			// Again, manage against circular dependencies
+			let libElucidator = require('elucidator');
+			this.elucidatorSolver = new libElucidator({}, this.logInfo, this.logError);
+		}
+
+		if (this.elucidatorSolver)
+		{
+			// This allows the magic filtration with elucidator configuration
+			// TODO: We could pass more state in (e.g. parent address, object, etc.)
+			// TODO: Discuss this metaprogramming AT LENGTH
+			let tmpFilterState = (
+				{
+					Record: pRecord,
+					keepRecord: true 
+				});
+
+			// This is about as complex as it gets.
+			// TODO: Optimize this so it is only initialized once.
+			// TODO: That means figuring out a healthy pattern for passing in state to this
+			tmpPrecedent.addPattern('<<~~', '~~>>',
+				(pInstructionHash) => 
+				{
+					// This is for internal config on the solution steps.  Right now config is not shared across steps.
+					if (this.elucidatorSolverState.hasOwnProperty(pInstructionHash))
+					{
+						tmpFilterState.SolutionState = this.elucidatorSolverState[pInstructionHash];
+					}
+					this.elucidatorSolver.solveInternalOperation('Custom', pInstructionHash, tmpFilterState);
+				});
+			tmpPrecedent.addPattern('<<~?', '?~>>',
+				(pMagicSearchExpression) => 
+				{
+					if (typeof(pMagicSearchExpression) !== 'string')
+					{
+						return false;
+					}
+					// This expects a comma separated expression:
+					//     Some.Address.In.The.Object,==,Search Term to Match
+					let tmpMagicComparisonPatternSet = pMagicSearchExpression.split(',');
+
+					let tmpSearchAddress = tmpMagicComparisonPatternSet[0];
+					let tmpSearchComparator = tmpMagicComparisonPatternSet[1];
+					let tmpSearchValue = tmpMagicComparisonPatternSet[2];
+
+					tmpFilterState.ComparisonState = (
+						{
+							SearchAddress: tmpSearchAddress,
+							Comparator: tmpSearchComparator,
+							SearchTerm: tmpSearchValue
+						});
+
+					this.elucidatorSolver.solveOperation(
+						{
+							"Description":
+							{
+								"Operation": "Simple_If",
+								"Synopsis": "Test for "
+							},
+							"Steps":
+							[
+								{
+									"Namespace": "Logic",
+									"Instruction": "if",
+		
+									"InputHashAddressMap": 
+										{
+											// This is ... dynamically assigning the address in the instruction
+											// The complexity is astounding.
+											"leftValue": `Record.${tmpSearchAddress}`,
+											"rightValue": "ComparisonState.SearchTerm",
+											"comparator": "ComparisonState.Comparator"
+										},
+									"OutputHashAddressMap": { "truthinessResult":"keepRecord" }
+								}
+							]
+						}, tmpFilterState);
+				});
+			tmpPrecedent.parseString(tmpAddress);
+
+			// It is expected that the operation will mutate this to some truthy value
+			return tmpFilterState.keepRecord;
+		}
+		else
+		{
+			return true;
+		}
+	}
+
 	// Get the value of an element at an address
 	getValueAtAddress (pObject, pAddress, pParentAddress)
 	{
-		// Make sure pObject is an object
+		// Make sure pObject (the object we are meant to be recursing) is an object (which could be an array or object)
 		if (typeof(pObject) != 'object') return undefined;
-		// Make sure pAddress is a string
+		// Make sure pAddress (the address we are resolving) is a string
 		if (typeof(pAddress) != 'string') return undefined;
+		// Stash the parent address for later resolution
 		let tmpParentAddress = "";
 		if (typeof(pParentAddress) == 'string')
 		{
@@ -337,7 +441,19 @@ class ManyfestObjectAddressResolver
 					return false;
 				}
 
-				return pObject[tmpBoxedPropertyName];
+				let tmpInputArray = pObject[tmpBoxedPropertyName];
+				let tmpOutputArray = [];
+				for (let i = 0; i < tmpInputArray.length; i++)
+				{
+					// The filtering is complex but allows config-based metaprogramming directly from schema
+					let tmpKeepRecord = this.checkFilters(pAddress, tmpInputArray[i]);
+					if (tmpKeepRecord)
+					{
+						tmpOutputArray.push(tmpInputArray[i]);
+					}
+				}
+
+				return tmpOutputArray;
 			}
 			// The object has been flagged as an object set, so treat it as such
 			else if (tmpObjectTypeMarkerIndex > 0)
@@ -456,7 +572,8 @@ class ManyfestObjectAddressResolver
 				for (let i = 0; i < tmpArrayProperty.length; i++)
 				{
 					let tmpPropertyParentAddress = `${tmpParentAddress}[${i}]`;
-					let tmpValue = this.getValueAtAddress(pObject[tmpBoxedPropertyName][i], tmpNewAddress, tmpPropertyParentAddress);;
+					let tmpValue = this.getValueAtAddress(pObject[tmpBoxedPropertyName][i], tmpNewAddress, tmpPropertyParentAddress);
+					
 					tmpContainerObject[`${tmpPropertyParentAddress}.${tmpNewAddress}`] = tmpValue;
 				}
 
@@ -486,8 +603,14 @@ class ManyfestObjectAddressResolver
 				for (let i = 0; i < tmpObjectPropertyKeys.length; i++)
 				{
 					let tmpPropertyParentAddress = `${tmpParentAddress}.${tmpObjectPropertyKeys[i]}`;
-					let tmpValue = this.getValueAtAddress(pObject[tmpObjectPropertyName][tmpObjectPropertyKeys[i]], tmpNewAddress, tmpPropertyParentAddress);;
-					tmpContainerObject[`${tmpPropertyParentAddress}.${tmpNewAddress}`] = tmpValue;
+					let tmpValue = this.getValueAtAddress(pObject[tmpObjectPropertyName][tmpObjectPropertyKeys[i]], tmpNewAddress, tmpPropertyParentAddress);
+	
+					// The filtering is complex but allows config-based metaprogramming directly from schema
+					let tmpKeepRecord = this.checkFilters(pAddress, tmpValue);
+					if (tmpKeepRecord)
+					{
+						tmpContainerObject[`${tmpPropertyParentAddress}.${tmpNewAddress}`] = tmpValue;
+					}
 				}
 
 				return tmpContainerObject;

package/source/Manyfest.js

@@ -4,6 +4,8 @@
 */
 let libSimpleLog = require('./Manyfest-LogToConsole.js');
 
+let libPrecedent = require('precedent');
+
 let libHashTranslation = require('./Manyfest-HashTranslation.js');
 let libObjectAddressResolver = require('./Manyfest-ObjectAddressResolver.js');
 let libObjectAddressGeneration = require('./Manyfest-ObjectAddressGeneration.js');
@@ -48,6 +50,10 @@ class Manyfest
 		this.elementAddresses = undefined;
 		this.elementHashes = undefined;
 		this.elementDescriptors = undefined;
+		// This can cause a circular dependency chain, so it only gets initialized if the schema specifically calls for it.
+		this.dataSolvers = undefined;
+		// So solvers can use their own state
+		this.dataSolverState = undefined;
 
 		this.reset();
 
@@ -73,6 +79,11 @@ class Manyfest
 		this.elementAddresses = [];
 		this.elementHashes = {};
 		this.elementDescriptors = {};
+		this.dataSolvers = undefined;
+		this.dataSolverState = {};
+
+		this.libElucidator = undefined;
+		this.objectAddressResolver.elucidatorSolver = false;
 	}
 
 	clone()
@@ -139,6 +150,35 @@ class Manyfest
 		{
 			this.logError(`(${this.scope}) Error loading object description from manifest object.  Property "Descriptors" does not exist in the root of the Manifest object.`, pManifest);
 		}
+
+		// This seems like it would create a circular dependency issue but it only goes as deep as the schema defines Solvers
+		if ((pManifest.hasOwnProperty('Solvers')) && (typeof(pManifest.Solvers) == 'object'))
+		{
+			// There are elucidator solvers passed-in, so we will create one to filter data.
+			let libElucidator = require('elucidator');
+			// WARNING THESE CAN MUTATE THE DATA
+				// The pattern for the solver is: {<~~SolverName~~>} anywhere in a property.
+				//   Yes, this means your Javascript elements can't have my self-styled jellyfish brackets in them.
+				//   This does, though, mean we can filter at multiple layers safely.
+				//   Because these can be put at any address
+			// The solver themselves:
+				//   They are passed-in an object, and the current record is in the Record subobject.
+				//   Basic operations can just write to the root object but...
+				//   IF YOU PERMUTE THE Record SUBOBJECT YOU CAN AFFECT RECURSION
+			// This is mostly meant for if statements to filter.
+				//   Basically on aggregation, if a filter is set it will set "keep record" to true and let the solver decide differently.
+			this.dataSolvers = new libElucidator(pManifest.Solvers, this.logInfo, this.logError);
+			this.objectAddressResolver.elucidatorSolver = this.dataSolvers;
+
+			// Load the solver state in so each instruction can have internal config
+			// TODO: Should this just be a part of the lower layer pattern?
+			let tmpSolverKeys = Object.keys(pManifest.Solvers)
+			for (let i = 0; i < tmpSolverKeys.length; i++)
+			{
+				this.dataSolverState[tmpSolverKeys] = pManifest.Solvers[tmpSolverKeys[i]];
+			}
+			this.objectAddressResolver.elucidatorSolverState = this.dataSolverState;
+		}
 	}
 
 	// Serialize the Manifest to a string

package/test/Manyfest_Embedded_Solvers_tests.js

@@ -0,0 +1,123 @@
+/**
+* Unit tests for Manyfest
+*
+* @license     MIT
+*
+* @author      Steven Velozo <steven@velozo.com>
+*/
+
+var Chai = require("chai");
+var Expect = Chai.expect;
+
+let libManyfest = require('../source/Manyfest.js');
+
+let _SampleDataArchiveOrgFrankenberry = require('./Data-Archive-org-Frankenberry.json');
+
+suite
+(
+	'Embedded Solvers',
+	function()
+	{
+		setup (()=> {} );
+
+		suite
+		(
+			'Set Filtration',
+			()=>
+			{
+				test
+				(
+					'Magic filters should be magic.',
+					(fTestComplete)=>
+					{
+						let _Manyfest = new libManyfest(
+							{
+								Scope:'Archive.org',
+								Descriptors:
+									{
+										'files[]': {Name:'Files', Hash:'FileSet'},
+                                        'files[]<<~?format,==,Thumbnail?~>>': {Name:'Thumbnail Files', Hash:'ThumbnailFiles'},
+                                        'files[]<<~?format,==,Metadata?~>>': {Name:'Metadata Files', Hash:'MetadataFiles'},
+										'metadata.creator': {Name:'Creator', Hash:'Creator'}
+									}
+							});
+
+						// Grab magic filtered thumbnails
+						let tmpThumbnailFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'ThumbnailFiles');
+						Expect(tmpThumbnailFiles).to.be.an('array');
+						// There are 7 thumbnail files in the set.
+						Expect(tmpThumbnailFiles.length).to.equal(7);
+
+						// Grab magic filtered etadataFiles
+						let tmpMetadataFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'MetadataFiles');
+						Expect(tmpMetadataFiles).to.be.an('array');
+						// There are 3 metadata files in the set.
+						Expect(tmpMetadataFiles.length).to.equal(3);
+
+						let tmpFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'FileSet');
+						Expect(tmpFiles).to.be.an('array');
+						// There are 17 total files in the set.
+						Expect(tmpFiles.length).to.equal(17);
+
+						fTestComplete();
+					}
+				);
+				test
+				(
+					'We should be able to access sets of properties from objects with schema.',
+					(fTestComplete)=>
+					{
+						let _Manyfest = new libManyfest(
+							{
+								Scope:'Archive.org',
+								Descriptors:
+									{
+										'files[]': {Name:'Files', Hash:'FileSet'},
+										'files[].size': {Name:'FileSizes', Hash:'FileSizes'},
+                                        'files[]<<~~ThumbnailFilesOnly~~>>': {Name:'Thumbnail Files', Hash:'ThumbnailFiles'},
+										'metadata.creator': {Name:'Creator', Hash:'Creator'},
+										'metadata{}': {Name:'Metadata', Hash:'Metadata'}
+									},
+                                Solvers:
+                                    {
+                                        'ThumbnailFilesOnly': 
+                                            {
+                                                "Description":
+                                                {
+                                                    "Operation": "ThumbnailFilesOnly",
+                                                    "Synopsis": "Filter files down to just thumbnails"
+                                                },
+												"Config": { "SearchTerm": "Thumbnail" },
+                                                "Steps":
+                                                [
+                                                    {
+                                                        "Namespace": "Logic",
+                                                        "Instruction": "if",
+
+														"InputHashAddressMap": 
+                                                            {
+                                                                "leftValue": "Record.format",
+                                                                "rightValue": "SolutionState.Config.SearchTerm",
+                                                                "comparator": "=="
+                                                            },
+                                                        "OutputHashAddressMap": { "truthinessResult":"keepRecord" }
+                                                    }
+                                                ]
+                                            }
+                                    }
+							});
+						let tmpThumbnailFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'ThumbnailFiles');
+						Expect(tmpThumbnailFiles).to.be.an('array');
+						// There are 7 thumbnail files in the set.
+						Expect(tmpThumbnailFiles.length).to.equal(7);
+						let tmpFiles = _Manyfest.getValueByHash(_SampleDataArchiveOrgFrankenberry, 'FileSet');
+						Expect(tmpFiles).to.be.an('array');
+						// There are 7 thumbnail files in the set.
+						Expect(tmpFiles.length).to.equal(17);
+						fTestComplete();
+					}
+				);
+			}
+		);
+	}
+);