@cepharum/concrete-db 0.1.1-alpha.1 → 0.2.0

package/lib/shaper.mjs

@@ -1,38 +1,12 @@
-/**
- * (c) 2021 cepharum GmbH, Berlin, http://cepharum.de
- *
- * The MIT License (MIT)
- *
- * Copyright (c) 2021 cepharum GmbH
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * @author: cepharum
- */
+// noinspection ExceptionCaughtLocallyJS
 
 import EventEmitter from "events";
 
 import { Compiler, Functions } from "simple-terms";
 import merge from "lodash.merge";
 
-import { DefaultShaperOptions } from "./defaults.mjs";
-import { augmentedDataSpace, deepFreeze } from "./helper.mjs";
+import { DefaultShape, DefaultShaperOptions } from "./defaults.mjs";
+import { augmentDataSpace } from "./helper.mjs";
 import * as TermFunctions from "./term-functions.mjs";
 
 
@@ -55,16 +29,17 @@ export class Shaper extends EventEmitter {
 	constructor( options ) {
 		super();
 
+		const compiledOptions = { ...DefaultShaperOptions, ...options };
+
 		Object.defineProperties( this, {
-			options: { value: { ...DefaultShaperOptions, ...options }, enumerable: true },
+			options: { value: compiledOptions, enumerable: true },
 
 			termsCache: { value: new Map() },
 
-			termFunctions: { value: { ...Functions, ...TermFunctions } },
-		} );
+			// SECURITY: prevent custom library functions from replacing regular ones by intention
+			termFunctions: { value: { ...compiledOptions.library, ...Functions, ...TermFunctions } },
 
-		Object.defineProperties( this, {
-			prefix: { value: normalizePathname( this.options.prefix, true ) },
+			prefix: { value: normalizePathname( compiledOptions.prefix, true ) },
 		} );
 	}
 
@@ -72,13 +47,19 @@ export class Shaper extends EventEmitter {
 	 * Creates database from provided collection of records.
 	 *
 	 * @param {Map<string,CollectedRecord[]>} recordsCollection collection of records to transform
+	 * @param {Shape[]} localShapes local shapes of folders records have been collected from
 	 * @returns {Database} resulting database records
 	 */
-	transform( recordsCollection ) {
-		const models = this.groupByModel( recordsCollection );
+	transform( recordsCollection, localShapes ) {
+		if ( recordsCollection.size > 0 && !localShapes?.length ) {
+			throw new Error( "invalid use of Shaper: missing list of local shapes per folder records have been collected from" );
+		}
+
+		const mergedLocalShapes = merge( {}, DefaultShape, ...localShapes );
+		const models = this.handleContributions( this.groupByModel( recordsCollection ), mergedLocalShapes );
 		const endpoints = {};
 
-		this.populateModels( endpoints, models );
+		this.populateModels( endpoints, models, mergedLocalShapes );
 
 		return { prefix: this.prefix, endpoints };
 	}
@@ -86,62 +67,587 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Reorganizes records to by grouped by model.
 	 *
-	 * @param {Map<string,CollectedRecord[]>} collection collection of records to transform
-	 * @returns {object<string,object<string, CollectedRecord[]>>} provided records grouped by model and unique item ID
+	 * @param {Map<string,CollectedRecord[]>} collectedRecordsByName collection of records to transform
+	 * @returns {Object<string,Object<string, ModelItem>>} provided records grouped by model and unique item ID
 	 */
-	groupByModel( collection ) {
-		const models = {};
-		const modelShapes = {};
-
-		for ( const records of collection.values() ) {
-			for ( const record of records ) {
-				const { segments, shape, data } = record;
-				const metaProperties = {
-					$segments: segments,
-				};
-				const wrapped = augmentedDataSpace( data, metaProperties );
-
-				const modelName = this.detectModel( wrapped, shape );
-				metaProperties.$model = modelName;
-
-				if ( !modelShapes.hasOwnProperty( modelName ) ) {
-					modelShapes[modelName] = deepFreeze( merge( {}, shape?.common, shape?.models?.[modelName] ) );
+	groupByModel( collectedRecordsByName ) {
+		const globalItemsPerModel = {};
+		let index = 0;
+
+		if ( this.options.verbose ) {
+			console.error( `shaping records ...` );
+		}
+
+		for ( const records of collectedRecordsByName.values() ) {
+			if ( this.options.verbose ) {
+				process.stderr.write( `\r[${++index}/${collectedRecordsByName.size}]` );
+			}
+
+			this.compileRecords( globalItemsPerModel, records );
+		}
+
+		return globalItemsPerModel;
+	}
+
+	/**
+	 * Implements separate stage for computing contributions per model item
+	 * based on either model's compiled shape.
+	 *
+	 * @param {Object<string,Object<string, CollectedRecord[]>>} globalItemsPerModel collections of items per model
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
+	 * @returns {Object<string,Object<string, CollectedRecord[]>>} collections of items per model as provided
+	 */
+	handleContributions( globalItemsPerModel, mergedLocalShapes ) {
+		const postProcessingQueue = new Set();
+
+		if ( this.options.verbose ) {
+			console.error( `\nprocessing contributions ...` );
+		}
+
+		this.processContributions( globalItemsPerModel, postProcessingQueue, mergedLocalShapes );
+
+		if ( this.options.verbose ) {
+			console.error( `post-processing ${postProcessingQueue.size} contributions` );
+		}
+
+		this.recompileContributionTargets( globalItemsPerModel, postProcessingQueue );
+
+		return globalItemsPerModel;
+	}
+
+	/**
+	 * Compiles all items according to collected records as provided and collect
+	 * them grouped by described items' models.
+	 *
+	 * @param {Object<string,Object<string, ModelItem>>} itemsPerModel collections of items per model to be populated
+	 * @param {CollectedRecord[]} records list of records collected from a source
+	 * @returns {Object<string,Object<string,ModelItem>>} compiled items grouped by model, addressable by either item's ID
+	 */
+	compileRecords( itemsPerModel, records ) {
+		const shapesPerModelCache = {};
+
+		for ( let read = 0, numRecords = records.length; read < numRecords; read++ ) {
+			const item = this.compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, records[read] );
+			const modelPool = itemsPerModel[item.$model];
+
+			if ( item.$variantId == null ) {
+				modelPool[item.$id] = this.mergeModelItem( modelPool[item.$id], item );
+			} else {
+				const target = modelPool[item.$id];
+
+				if ( !target.$variants ) {
+					target.$variants = {};
 				}
 
-				const modelShape = modelShapes[modelName];
-				metaProperties.$shape = modelShape;
+				target.$variants[item.$variantId] = this.mergeModelItem( target.$variants[item.$variantId], item );
+			}
+		}
 
-				const id = this.identifyRecord( wrapped, modelShape );
-				metaProperties.$id = id;
+		return itemsPerModel;
+	}
 
-				let model = models[modelName];
-				if ( !model ) {
-					model = models[modelName] = {};
+	/**
+	 * Compiles item of model from a provided source record. This includes
+	 * detecting model and selected ID of resulting item from provided record
+	 * based on record's shape.
+	 *
+	 * @param {Object<string,ShapeModel>} shapesPerModelCache caches per-model shapes compiled before
+	 * @param {Object<string,Object<string,Object>>} itemsPerModel collection of previously compiled items per model
+	 * @param {CollectedRecord} record augmented data collected from a source to be compiled
+	 * @returns {object} instance of model with hidden meta information injected
+	 */
+	compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, record ) {
+		const { segments, shape, data } = record;
 
-					Object.defineProperties( model, {
-						$shape: { value: modelShape },
-						$model: { value: modelName },
-					} );
+		const metaProperties = {
+			$segments: segments
+		};
+
+		const augmentedData = augmentDataSpace( data, metaProperties );
+
+		metaProperties.$model = this.detectModel( augmentedData, shape );
+		metaProperties.$shape = this.getShapeOfModel( {}, shape, metaProperties.$model );
+		metaProperties.$id = this.identifyRecord( augmentedData, metaProperties.$shape );
+
+		if ( !itemsPerModel.hasOwnProperty( metaProperties.$model ) ) {
+			// eslint-disable-next-line no-param-reassign
+			itemsPerModel[metaProperties.$model] = {};
+		}
+
+		const model = itemsPerModel[metaProperties.$model];
+
+		const { properties, defaults } = metaProperties.$shape;
+		const item = this.compileItem( augmentedData, properties, model[metaProperties.$id] ? {} : defaults );
+
+		Object.defineProperties( item, {
+			$segments: { value: segments },
+			$original: { value: data, configurable: true },
+			$shape: { value: metaProperties.$shape, configurable: true },
+			$model: { value: metaProperties.$model },
+			$id: { value: metaProperties.$id },
+			$variantId: { value: this.computeVariantIdOfItem( metaProperties.$shape.properties.$variant, augmentedData ) },
+		} );
+
+		return item;
+	}
+
+	/**
+	 * Re-compiles provided item this time based on item's properties'
+	 * themselves instead of some record originally collected from a source.
+	 *
+	 * Re-compiling is used to compute data contributed by other models.
+	 *
+	 * @param {ModelItem} item previously compiled item to re-compile
+	 * @returns {ModelItem} re-compiled item
+	 */
+	recompileItem( item ) {
+		const data = augmentDataSpace( merge( {}, item.$original, item ), { ...item, $post: true } );
+		const updatedItem = this.compileItem( data, item.$shape.properties, {} );
+
+		Object.defineProperties( updatedItem, {
+			$segments: { value: item.$segments },
+			$original: { value: item.$original, configurable: true },
+			$shape: { value: item.$shape, configurable: true },
+			$model: { value: item.$model },
+			$id: { value: item.$id },
+		} );
+
+		return updatedItem;
+	}
+
+	/**
+	 * Computes ID of variant provided item is describing. If nullish, the item
+	 * isn't describing any variant but the base version of the item.
+	 *
+	 * @param {string?} term term expressing computation of variant ID
+	 * @param {ModelItem} item an item to get variant ID for
+	 * @returns {null|any} computed variant ID of provided item, nullish if item is not a variant
+	 */
+	computeVariantIdOfItem( term, item ) {
+		if ( term == null ) {
+			return null;
+		}
+
+		// compile data space based on item's properties overlaying the item's originating record
+		const data = augmentDataSpace( merge( {}, item.$original, item ), item );
+
+		try {
+			const value = Compiler.compile( String( term ), this.termFunctions, this.termsCache )( data );
+
+			// support processed term requesting to fail computation of variant ID
+			if ( value instanceof Error ) {
+				throw value;
+			}
+
+			return value;
+		} catch ( cause ) {
+			throw new Error( `computing variant ID of ${item.$model}#${item.$id} failed: ${cause.message} in ${term}` );
+		}
+	}
+
+	/**
+	 * Fetches compiled shape of named model using provided cache.
+	 *
+	 * @param {Object<string,ShapeModel>} shapesPerModelCache cache to use on fetching shape for same model multiple times
+	 * @param {Shape} localShape shape including desired definition of selected model's shape
+	 * @param {string} modelName name of model
+	 * @returns {ShapeModel} compiled shape of model incorporating common definitions for all models
+	 */
+	getShapeOfModel( shapesPerModelCache, localShape, modelName ) {
+		if ( !shapesPerModelCache.hasOwnProperty( modelName ) ) {
+			// eslint-disable-next-line no-param-reassign
+			shapesPerModelCache[modelName] = merge( { defaults: DefaultShape.common.defaults }, localShape?.common, localShape?.models?.[modelName] );
+		}
+
+		return shapesPerModelCache[modelName];
+	}
+
+	/**
+	 * Computes dependencies of models on contributions by provided items
+	 * grouped by model.
+	 *
+	 * @param {Object<string,Object<string,Object>>} itemsPerModel compiled items grouped by model
+	 * @returns {string[]} sorted list of names of models in provided pool starting with model depending on other models' contributions the least
+	 */
+	listContributingModels( itemsPerModel ) {
+		const contributorsPerModel = {};
+
+		// create map of immediate relations between every two models involved
+		// in a contribution
+		for ( const model of Object.keys( itemsPerModel ) ) {
+			for ( const item of Object.values( itemsPerModel[model] ) ) {
+				for ( const contributedModel of Object.keys( item.$shape.contributions || {} ) ) {
+					if ( !contributorsPerModel.hasOwnProperty( contributedModel ) ) {
+						contributorsPerModel[contributedModel] = {};
+					}
+
+					contributorsPerModel[contributedModel][item.$model] = 1;
 				}
+			}
+		}
+
+		// surface mediate dependencies and collect all contributing models' names
+		const contributingModelNames = {};
+
+		for ( const surfaceModel of Object.keys( contributorsPerModel ) ) {
+			descend( surfaceModel, surfaceModel, [] );
+		}
 
-				const isVariant = model.hasOwnProperty( id );
-				const item = this.compileItem( wrapped, modelShape, !isVariant );
+		// sort names of contributing models from models depending on other
+		// contributing models the least to those doing it the most
+		const names = Object.keys( contributingModelNames );
 
-				Object.defineProperties( item, {
-					$shape: { value: modelShape },
-					$model: { value: modelName },
-					$id: { value: id },
-				} );
+		names.sort( ( left, right ) => {
+			if ( contributorsPerModel.hasOwnProperty( left ) && contributorsPerModel[left].hasOwnProperty( right ) ) {
+				// left model depends on contributions by right model
+				return 1;
+			}
+
+			if ( contributorsPerModel.hasOwnProperty( right ) && contributorsPerModel[right].hasOwnProperty( left ) ) {
+				// right model depends on contributions by left model
+				return -1;
+			}
 
-				if ( isVariant ) {
-					// FIXME implement variants or overlays or whatever there is to do with multiple records for the same item
+			return 0;
+		} );
+
+		return names;
+
+		/**
+		 * Surfaces dependencies of a named model.
+		 *
+		 * @param {string} surfaceModel name of model triggering current processing
+		 * @param {string} currentModel name of currently processed model
+		 * @param {string[]} breadcrumb names of models processed while descending into dependencies
+		 * @returns {void}
+		 */
+		function descend( surfaceModel, currentModel, breadcrumb ) {
+			if ( breadcrumb.indexOf( currentModel ) > -1 ) {
+				throw new Error( `invalid circular dependency on contributing to model ${surfaceModel}: [${breadcrumb}]` );
+			}
+
+			if ( contributorsPerModel.hasOwnProperty( currentModel ) ) {
+				const current = contributorsPerModel[currentModel];
+				const surface = contributorsPerModel[surfaceModel];
+
+				for ( const contributingModel of Object.keys( current ) ) {
+					contributingModelNames[contributingModel] = true;
+
+					if ( !surface.hasOwnProperty( contributingModel ) ) {
+						surface[contributingModel] = -1;
+					} else if ( current[contributingModel] > 0 ) {
+						breadcrumb.push( currentModel );
+						descend( surfaceModel, contributingModel, breadcrumb );
+						breadcrumb.pop();
+					}
+				}
+			}
+		}
+	}
+
+	/**
+	 * Processes contributions of models in provided collection of items
+	 * augmenting that collection.
+	 *
+	 * @param {Object<string,Object<string,Object>>} itemsPerModel map of model names into either model's map of item IDs into related item
+	 * @param {Set<string>} contributedItems tracks IDs of items or their variants that have been contributed to
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
+	 * @returns {void}
+	 */
+	processContributions( itemsPerModel, contributedItems, mergedLocalShapes ) {
+		const contributingModels = this.listContributingModels( itemsPerModel );
+		const shapePerModelCache = {};
+
+		for ( const modelName of contributingModels ) {
+			const items = itemsPerModel[modelName];
+
+			for ( const item of Object.values( items ) ) {
+				const { contributions } = item.$shape;
+				if ( !contributions ) {
+					continue;
+				}
+
+				for ( const contributedModelName of Object.keys( contributions ) ) {
+					const isOptionalContribution = contributedModelName.trim().endsWith( "?" );
+					const targetModelName = contributedModelName.replace( /\s*\?\s*$/, "" );
+
+					const targetModelShape = this.getShapeOfModel( shapePerModelCache, mergedLocalShapes, targetModelName );
+
+
+					// extract contributions to actually declared properties of target model
+					const contributionTerms = {};
+					const contributionModes = {};
+					let warn = false;
+					let numActualProperties = 0;
+
+					for ( const termTargetName of Object.keys( contributions[contributedModelName] ) ) {
+						const match = /^\s*(\S.+?)\s*(\[\])?\s*(\.\.\.|\u2026)?\s*$/u.exec( termTargetName );
+						const propertyName = match ? match[1] : termTargetName;
+
+						if ( match && ( match[2] || match[3] ) && propertyName.startsWith( "$" ) ) {
+							throw new Error( `invalid collecting or distributed contribution of record #${item.$id} in model ${item.$model} to property ${propertyName} of model ${targetModelName}` ); // eslint-disable-line max-len
+						}
+
+						if ( propertyName === "$variant" || propertyName === "$id" || Object.prototype.hasOwnProperty.call( targetModelShape.properties, propertyName ) ) {
+							if ( contributionTerms.hasOwnProperty( propertyName ) ) {
+								throw new Error( `double contribution of record #${item.$id} in model ${item.$model} to property ${propertyName} of model ${targetModelName}` ); // eslint-disable-line max-len
+							}
+
+							if ( !propertyName.startsWith( "$" ) ) {
+								numActualProperties++;
+							}
+
+							contributionTerms[propertyName] = contributions[contributedModelName][termTargetName];
+							contributionModes[propertyName] = {
+								$isCollecting: Boolean( match && match[2] ),
+								$isDistributed: Boolean( match && match[3] ),
+							};
+						} else {
+							warn = true;
+						}
+					}
+
+					if ( numActualProperties < 1 && !isOptionalContribution ) {
+						console.warn( `ignoring eventually empty contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName}` );
+						continue;
+					}
+
+					if ( warn ) {
+						console.warn( `ignoring undeclared properties in contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName}` ); // eslint-disable-line max-len
+					}
+
+
+					// pre-compute ID(s) of item(s) to contribute to
+					const contributionMeta = {
+						$segments: item.$segments,
+						$original: item.$original,
+						$shape: item.$shape,
+						$model: item.$model,
+						$id: item.$id,
+					};
+
+					const augmentedData = augmentDataSpace( item, contributionMeta );
+					const targetId = this.computeId( contributionTerms.$id, augmentedData );
+
+					if ( !isOptionalContribution && ( !targetId || ( Array.isArray( targetId ) && !targetId.length ) ) ) {
+						console.warn( `ignoring contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName} due to lack of target ID` ); // eslint-disable-line max-len
+						continue;
+					}
+
+					const targetIds = Array.isArray( targetId ) ? targetId : [targetId];
+
+
+					for ( let index = 0, count = targetIds.length; index < count; index++ ) {
+						const id = targetIds[index];
+
+						// compile contribution per target item
+						contributionMeta.$targetid = id;
+						contributionMeta.$targetindex = index;
+
+
+						// try computing variant ID based on contributing item
+						const variantId = this.computeVariantIdOfItem( contributionTerms.$variant, {
+							...item,
+							$segments: item.$segments,
+							$original: item.$original,
+							$shape: targetModelShape,
+							$model: targetModelName,
+							$id: targetId,
+						} );
+
+						// track contributions to selected target item or its variant
+						contributedItems.add( JSON.stringify( [ targetModelName, id, variantId ] ) );
+
+
+						// gain access on selected target item, create its containers as necessary
+						let target;
+
+						if ( !itemsPerModel.hasOwnProperty( targetModelName ) ) {
+							// eslint-disable-next-line no-param-reassign
+							itemsPerModel[targetModelName] = {};
+						}
+
+						const model = itemsPerModel[targetModelName];
+						const targetItemExists = model.hasOwnProperty( id );
+
+						if ( !targetItemExists ) {
+							model[id] = Object.defineProperties( {}, {
+								$segments: { value: item.$segments },
+								$original: { value: {}, configurable: true },
+								$shape: { value: targetModelShape, configurable: true },
+								$model: { value: targetModelName },
+								$id: { value: id },
+							} );
+						}
+
+						const targetItem = model[id];
+
+						if ( variantId == null ) {
+							target = targetItem;
+						} else {
+							if ( !targetItem.$variants ) {
+								targetItem.$variants = {};
+							}
+
+							if ( !targetItem.$variants.hasOwnProperty( variantId ) ) {
+								targetItem.$variants[variantId] = {};
+							}
+
+							target = targetItem.$variants[variantId];
+						}
+
+
+						// compile actual properties of contribution
+						const contributionDefaults = targetItemExists || variantId != null ? {} : targetModelShape.defaults;
+						const contribution = this.compileItem( augmentedData, contributionTerms, contributionDefaults );
+
+
+						// merge this item's contribution with existing
+						// properties of selected target using reduced version
+						// of merge strategy found in mergeModelItem()
+						for ( const name of Object.keys( contribution ) ) {
+							if ( String( name ).startsWith( "$" ) ) {
+								continue;
+							}
+
+							const { $isCollecting, $isDistributed } = contributionModes[name] || {};
+							let value = contribution[name];
+
+							if ( $isDistributed ) {
+								// contribution is providing another value for every target item
+								if ( Array.isArray( value ) && value.length === targetIds.length ) { // eslint-disable-line max-depth
+									value = value[index];
+								}
+							}
+
+							if ( $isCollecting ) {
+								// contribution has been declared with one or more suffices, e.g. []
+								// -> collect value in target
+								if ( Array.isArray( target[name] ) ) { // eslint-disable-line no-lonely-if,max-depth
+									target[name].push( value );
+								} else if ( target[name] == null ) {
+									target[name] = [value];
+								} else {
+									target[name] = [ target[name], value ];
+								}
+							} else {
+								// contribution has been declared without any suffix
+								// -> do not collect, but replace value in target
+								target[name] = value;
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	/**
+	 * Deeply merges provided source item into given target item including their
+	 * attached model shape and original data.
+	 *
+	 * @param {ModelItem} target model item to adjust by merging
+	 * @param {ModelItem} source model item to merge into provided target
+	 * @returns {ModelItem} provided target
+	 */
+	mergeModelItem( target, source ) {
+		const _to = target || {};
+		const oldNames = target ? Object.keys( _to.$shape.properties ) : [];
+		const newNames = Object.keys( source.$shape.properties );
+
+		// merge properties of items
+		for ( const name of Object.keys( source ) ) {
+			if ( !String( name ).startsWith( "$" ) ) {
+				const wasCollecting = oldNames.some( n => n.replace( /\s*\[]$/, "" ).trim() === name );
+				const isCollecting = newNames.some( n => n.replace( /\s*\[]$/, "" ).trim() === name );
+
+				if ( _to.hasOwnProperty( name ) ) {
+					if ( isCollecting ) {
+						if ( wasCollecting && Array.isArray( _to[name] ) ) {
+							_to[name].splice( _to[name].length, 0, ...source[name] );
+						} else if ( _to[name] == null ) {
+							_to[name] = source[name];
+						} else {
+							_to[name] = [ _to[name], ...source[name] ];
+						}
+					} else {
+						_to[name] = source[name];
+					}
 				} else {
-					model[id] = item;
+					_to[name] = source[name];
 				}
 			}
 		}
 
-		return models;
+		// assure some freshly created target has all common meta properties
+		if ( !target ) {
+			Object.defineProperties( _to, {
+				$segments: { value: source.$segments },
+				$model: { value: source.$model },
+				$id: { value: source.$id },
+			} );
+		}
+
+		// merge some complex meta properties
+		Object.defineProperties( _to, {
+			$shape: {
+				value: merge( _to.$shape || {}, source.$shape ),
+				configurable: true,
+			},
+			$original: {
+				value: merge( _to.$original || {}, source.$original ),
+				configurable: true,
+			},
+		} );
+
+		return _to;
+	}
+
+	/**
+	 * Re-compiles all items in provided pool that have been contributed to.
+	 *
+	 * @param {Object<string,Object<string,ModelItem>>} itemsPerModel pool of items per model to update
+	 * @param {Set<string>} contributions lists IDs of items or their variants that have been contributed to
+	 * @returns {Object<string,Object<string,ModelItem>>} pool of items per model as provided
+	 */
+	recompileContributionTargets( itemsPerModel, contributions ) {
+		for ( const track of contributions.values() ) {
+			const [ model, id, variantId ] = JSON.parse( track );
+
+			if ( !itemsPerModel.hasOwnProperty( model ) ) {
+				console.warn( `inconsistency: missing items pool of model ${model} though contributions have been tracked` );
+				continue;
+			}
+
+			const modelPool = itemsPerModel[model];
+
+			if ( !modelPool.hasOwnProperty( id ) ) {
+				console.warn( `inconsistency: missing item ${id} of model ${model} though contributions have been tracked` );
+				continue;
+			}
+
+			const item = modelPool[id];
+
+			if ( variantId == null ) {
+				modelPool[id] = this.recompileItem( item );
+			} else {
+				if ( !item.$variants ) {
+					console.warn( `inconsistency: missing variants of item ${id} of model ${model} though contributions have been tracked` );
+					continue;
+				}
+
+				const variants = item.$variants;
+
+				if ( !variants.hasOwnProperty( variantId ) ) {
+					console.warn( `inconsistency: missing variant #${variantId} of item #${id} of model ${model} though contributions have been tracked` );
+					continue;
+				}
+
+				variants[variantId] = this.recompileItem( variants[variantId] );
+			}
+		}
+
+		return itemsPerModel;
 	}
 
 	/**
@@ -160,6 +666,11 @@ export class Shaper extends EventEmitter {
 
 		const name = Compiler.compile( selector, this.termFunctions, this.termsCache )( data );
 
+		// support processed term requesting to fail model detection
+		if ( name instanceof Error ) {
+			throw new Error( `detecting model of record failed: ${name.message} in context of ${JSON.stringify( data )}` );
+		}
+
 		if ( !name ) {
 			throw new Error( `detecting model of record failed: ${JSON.stringify( data )}` );
 		}
@@ -175,13 +686,7 @@ export class Shaper extends EventEmitter {
 	 * @returns {string} ID of record
 	 */
 	identifyRecord( data, modelShape ) {
-		const selector = modelShape?.properties?.$id;
-
-		if ( !selector || typeof selector !== "string" ) {
-			throw new Error( `missing or invalid definition of $id in shape of ${data.$segments.join( "/" )}` );
-		}
-
-		const id = Compiler.compile( selector )( data );
+		const id = this.computeId( modelShape?.properties?.$id, data );
 
 		if ( !id ) {
 			throw new Error( `detecting ID of item related to record ${data.$segments.join( "/" )} failed` );
@@ -193,66 +698,133 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Compiles resulting item by evaluating properties declared in model shape.
 	 *
-	 * @param {object} data source of record
-	 * @param {ShapeModel} modelShape shape of particular model
-	 * @param {boolean} useDefaults set true to include defaults configured in shape
+	 * @param {object} data (augmented) source of record
+	 * @param {ShapeProperties} properties definition of resulting item's properties
+	 * @param {ShapePropertyDefaults} defaults definition of default values for resulting item's properties
+	 * @param {string[]} accept list names of properties to accept even when starting with a dollar sign
 	 * @returns {object} compiled item
 	 */
-	compileItem( data, modelShape, useDefaults = true ) {
-		const properties = modelShape?.properties || {};
-		const defaults = modelShape?.defaults || {};
+	compileItem( data, properties = {}, defaults = {}, accept = [] ) {
 		const item = {};
 
 		for ( const property of Object.keys( properties ) ) {
-			if ( !String( property ).startsWith( "$" ) ) {
-				const selector = properties[property];
-				if ( typeof selector !== "string" || selector.trim().length === 0 ) {
-					throw new Error( `invalid or missing term for computing value of property ${data.$model}#${property}` );
+			if ( accept.indexOf( property ) > -1 || !String( property ).startsWith( "$" ) ) {
+				const propertyName = property.replace( /\s*\[]$/, "" );
+				const isCollecting = property !== propertyName;
+
+				const term = properties[property];
+				if ( typeof term !== "string" || term.trim().length === 0 ) {
+					throw new Error( `invalid or missing term for computing value of property ${propertyName} of ${data.$model}` );
+				}
+
+				if ( isCollecting && properties.hasOwnProperty( propertyName ) ) {
+					throw new Error( `double definition: property ${propertyName} of ${data.$model} must be either regular or collecting property` );
 				}
 
 				let value;
 
 				try {
-					value = Compiler.compile( selector, this.termFunctions, this.termsCache )( data );
+					value = Compiler.compile( term, this.termFunctions, this.termsCache )( data );
+
+					// support processed term requesting to fail computation of property's value
+					if ( value instanceof Error ) {
+						throw value;
+					}
 				} catch ( cause ) {
-					throw new Error( `fetching value of property ${data.$model}#${property} failed: ${cause.message} in ${selector}` );
+					throw new Error( `fetching value of property ${propertyName} of ${data.$model} failed: ${cause.message} in ${term}` );
 				}
 
 				if ( value == null ) {
-					if ( useDefaults && defaults.hasOwnProperty( property ) ) {
-						const defaultValue = defaults[property];
+					if ( defaults.hasOwnProperty( propertyName ) || defaults.hasOwnProperty( property ) ) {
+						const defaultValue = defaults.hasOwnProperty( propertyName ) ? defaults[propertyName] : defaults[property];
 						const trimmed = typeof defaultValue === "string" ? defaultValue.trim() : "";
 
 						try {
 							const code = trimmed.startsWith( "=" ) ? trimmed.slice( 1 ) : undefined;
 							value = code ? Compiler.compile( code, this.termFunctions, this.termsCache )( data ) : defaultValue;
+
+							// support processed term requesting to fail computation of default value
+							if ( value instanceof Error ) {
+								throw value;
+							}
 						} catch ( cause ) {
-							throw new Error( `fetching default of property ${data.$model}#${property} failed: "${cause.message}" in "${defaultValue}"` );
+							throw new Error( `fetching default of property ${propertyName} of ${data.$model} failed: "${cause.message}" in "${defaultValue}"` );
 						}
 					}
 				}
 
 				if ( value != null ) {
-					item[property] = value;
+					if ( isCollecting ) {
+						if ( Array.isArray( item[propertyName] ) ) { // eslint-disable-line no-lonely-if
+							item[propertyName].push( value );
+						} else if ( item[propertyName] == null ) {
+							item[propertyName] = [value];
+						} else {
+							item[propertyName] = [ item[propertyName], value ];
+						}
+					} else {
+						item[propertyName] = value;
+					}
 				}
 			}
 		}
 
-		return item;
+		// deeply merge fresh object with compiled item to get rid of deep data
+		// exposed to terms as Proxy
+		return merge( {}, item );
+	}
+
+	/**
+	 * Computes ID using provided term in context of given data space.
+	 *
+	 * @param {string} term term to process
+	 * @param {object} data data space available to term processing
+	 * @returns {any} processed term's result
+	 */
+	computeId( term, data ) {
+		if ( !term || typeof term !== "string" ) {
+			throw new Error( `missing or invalid term for computing ID of ${data.$segments.join( "/" )}` );
+		}
+
+		try {
+			const value = Compiler.compile( term, this.termFunctions, this.termsCache )( data );
+
+			// support processed term requesting to fail computation of ID
+			if ( value instanceof Error ) {
+				throw value;
+			}
+
+			return value;
+		} catch ( cause ) {
+			throw new Error( `computing ID failed: ${cause.message} in ${term}` );
+		}
 	}
 
 	/**
 	 * Populates pool of endpoints with collected records of discovered models.
 	 *
 	 * @param {DatabaseEndpoints} endpoints map of routes into data sets to expose eventually
-	 * @param {object<string,DatabaseKeyedCollection>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {DatabaseEndpoints} map of routes into data sets to expose eventually
 	 */
-	populateModels( endpoints, models ) {
-		for ( const modelName of Object.keys( models ) ) {
+	populateModels( endpoints, models, mergedLocalShapes ) {
+		const modelNames = Object.keys( models );
+		let index = 0;
+
+		for ( const modelName of modelNames ) {
 			const model = models[modelName];
+			const itemIds = Object.keys( model );
 
-			for ( const itemId of Object.keys( model ) ) {
+			if ( this.options.verbose ) {
+				console.error( `creating endpoints for model ${modelName} [${++index}/${modelNames.length}] ...` );
+			}
+
+			if ( this.options.verbose ) {
+				console.error( `  - ${itemIds.length} items ...` );
+			}
+
+			for ( const itemId of itemIds ) {
 				const normalized = normalizePathname( `${this.prefix}${modelName}/item/${itemId}` );
 				if ( normalized.endsWith( "/" ) ) {
 					throw new Error( `item ID "${itemId}" of model "${modelName}" results in invalid route for endpoint exposing it` );
@@ -261,7 +833,7 @@ export class Shaper extends EventEmitter {
 				endpoints[normalized] = model[itemId]; // eslint-disable-line no-param-reassign
 			}
 
-			this.populateModelCollections( endpoints, models, modelName );
+			this.populateModelCollections( endpoints, models, modelName, mergedLocalShapes );
 		}
 
 		return endpoints;
@@ -272,13 +844,14 @@ export class Shaper extends EventEmitter {
 	 * a model.
 	 *
 	 * @param {DatabaseEndpoints} endpoints map of routes into data sets to expose eventually
-	 * @param {object<string,DatabaseKeyedCollection>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {DatabaseEndpoints} map of routes into data sets to expose eventually
 	 */
-	populateModelCollections( endpoints, models, modelName ) {
-		const model = models[modelName];
-		const collections = model.$shape?.collections || {};
+	populateModelCollections( endpoints, models, modelName, mergedLocalShapes ) {
+		const finalModelShape = merge( {}, mergedLocalShapes.common, mergedLocalShapes.models[modelName] );
+		const collections = finalModelShape.collections || {};
 		const cache = {};
 
 		// prepare to process collection instances prior to collection references
@@ -297,9 +870,15 @@ export class Shaper extends EventEmitter {
 		} );
 
 		// process collection by collection
+		let index = 0;
+
 		for ( const route of keys ) {
 			const normalized = normalizePathname( route );
 
+			if ( this.options.verbose ) {
+				console.error( `  - collection /${normalized} [${++index}/${keys.length}] ...` );
+			}
+
 			if ( String( normalized ).startsWith( "item/" ) || /(^|\/)\.\.(\/|$)|[\\%]/.test( normalized ) ) {
 				throw new Error( `invalid custom route "${normalized}" for collection of model "${modelName}"` );
 			}
@@ -339,14 +918,14 @@ export class Shaper extends EventEmitter {
 
 						for ( const key in collection ) {
 							if ( collection.hasOwnProperty( key ) ) {
-								this.populateSlicedEndpoint( endpoints, normalized.replace( ptnKey, key ), collection[key], false, modelName );
+								this.populateSlicedEndpoint( endpoints, normalized.replace( ptnKey, key ), collection[key], false, modelName, definition );
 							}
 						}
 					} else {
 						throw new Error( `invalid use of {key} in route "${normalized}" of regular collection of model "${modelName}"` );
 					}
 				} else {
-					this.populateSlicedEndpoint( endpoints, normalized, collection, isKeyed, modelName );
+					this.populateSlicedEndpoint( endpoints, normalized, collection, isKeyed, modelName, definition );
 				}
 			}
 		}
@@ -361,12 +940,13 @@ export class Shaper extends EventEmitter {
 	 *
 	 * @param {DatabaseEndpoints} endpoints map of routes into data sets to expose eventually
 	 * @param {string} route route of current collection
-	 * @param {{items: Array<object>}|object<string, Array<object>>} collection collection of items to expose
+	 * @param {{items: Array<object>}|Object<string, Array<object>>} collection collection of items to expose
 	 * @param {boolean} isKeyed indicates if collection lists item per key or all items in a single list
 	 * @param {string} modelName name of model this collection is defined for
+	 * @param {object} definition provided collection's definition in shape
 	 * @returns {void}
 	 */
-	populateSlicedEndpoint( endpoints, route, collection, isKeyed, modelName ) {
+	populateSlicedEndpoint( endpoints, route, collection, isKeyed, modelName, definition ) {
 		ptnOffset.lastIndex = 0;
 		const match = ptnOffset.exec( route );
 
@@ -387,22 +967,30 @@ export class Shaper extends EventEmitter {
 				const normalized = normalizePathname( prefix + route.replace( ptnOffset, offset ) );
 				endpoints[normalized] = { count: items.length, items: items.slice( offset, offset + limit ) }; // eslint-disable-line no-param-reassign
 			}
-		} else {
+		} else if ( isKeyed ) {
 			const normalized = normalizePathname( this.prefix + modelName + "/" + route );
+			const base = endpoints[normalized] = {}; // eslint-disable-line no-param-reassign
 
-			if ( isKeyed ) {
-				const qualified = endpoints[normalized] = {}; // eslint-disable-line no-param-reassign
+			for ( const key of Object.keys( collection ) ) {
+				const items = collection[key].items;
 
-				for ( const key of Object.keys( collection ) ) {
-					const items = collection[key].items;
+				base[key] = { count: items.length };
 
-					qualified[key] = { count: items.length, items };
-				}
-			} else {
-				const items = collection.items;
+				if ( definition.reduce ) {
+					// generate single endpoint listing all items of collection grouped by key
+					base[key].items = items;
+				} else {
+					// expose separate endpoint per group of items
+					const subNormalized = normalizePathname( this.prefix + modelName + "/" + route + "/" + key );
 
-				endpoints[normalized] = { count: items.length, items }; // eslint-disable-line no-param-reassign
+					endpoints[subNormalized] = { count: items.length, items }; // eslint-disable-line no-param-reassign
+				}
 			}
+		} else {
+			const normalized = normalizePathname( this.prefix + modelName + "/" + route );
+			const items = collection.items;
+
+			endpoints[normalized] = { count: items.length, items }; // eslint-disable-line no-param-reassign
 		}
 	}
 
@@ -410,15 +998,15 @@ export class Shaper extends EventEmitter {
 	 * Creates managed data space for evaluating terms in context of a set of
 	 * items.
 	 *
-	 * @param {object<string,DatabaseKeyedCollection>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
 	 * @param {ShapeCollection} collectionDefinition rules controlling compilation of collection
-	 * @returns {{handler: {get: ((function(*, *=): (*))|*)}, context: {$models, $model, $collection}}} handler and extensible context for term evaluation
+	 * @returns {{handler: {get: ((function(*, *=): (*))|*)}, context: {$database, $model, $collection}}} handler and extensible context for term evaluation
 	 */
 	createTermContextCollection( models, modelName, collectionDefinition ) {
 		const context = {
 			$collection: collectionDefinition,
-			$models: models,
+			$database: models,
 			$model: modelName,
 		};
 
@@ -438,7 +1026,7 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Compiles collection according to its definition in provided shape.
 	 *
-	 * @param {object<string,DatabaseKeyedCollection>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
 	 * @param {ShapeCollection} definition rules controlling compilation of collection
 	 * @param {string} route route for resulting collection
@@ -451,7 +1039,7 @@ export class Shaper extends EventEmitter {
 		const collection = isKeyed ? {} : { items: [] };
 
 
-		const { handler, context } = this.createTermContextCollection();
+		const { handler, context } = this.createTermContextCollection( models, modelName, definition );
 
 
 		// prepare terms used in compiling collection
@@ -461,12 +1049,22 @@ export class Shaper extends EventEmitter {
 
 		try {
 			keyFn = isKeyed ? Compiler.compile( $key, this.termFunctions, this.termsCache ) : undefined;
+
+			// support processed term requesting to fail
+			if ( keyFn instanceof Error ) {
+				throw keyFn;
+			}
 		} catch ( cause ) {
 			throw new Error( `invalid rule for defining key of collection at "${route}" in ${modelName}: ${cause.message}` );
 		}
 
 		try {
 			filterFn = $filter && typeof $filter === "string" ? Compiler.compile( $filter, this.termFunctions, this.termsCache ) : undefined;
+
+			// support processed term requesting to fail
+			if ( filterFn instanceof Error ) {
+				throw filterFn;
+			}
 		} catch ( cause ) {
 			throw new Error( `invalid rule for defining filter of collection at "${route}" in ${modelName}: ${cause.message}` );
 		}
@@ -478,6 +1076,11 @@ export class Shaper extends EventEmitter {
 				if ( code && typeof code === "string" ) {
 					const fn = Compiler.compile( code, this.termFunctions, this.termsCache );
 
+					// support processed term requesting to fail computed definition of collection
+					if ( fn instanceof Error ) {
+						throw fn;
+					}
+
 					fn.target = target;
 
 					return fn;
@@ -492,7 +1095,7 @@ export class Shaper extends EventEmitter {
 
 		// process all items of model
 		for ( const id in model ) {
-			if ( model.hasOwnProperty( id ) && typeof id === "string" && id[0] !== "$" ) {
+			if ( model[id] && typeof id === "string" && id[0] !== "$" ) {
 				const item = model[id];
 				const data = new Proxy( item, handler );
 				let extracted;
@@ -548,7 +1151,7 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Sorts provided collection either in-place or as a copy.
 	 *
-	 * @param {object<string,DatabaseKeyedCollection>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
 	 * @param {ShapeCollection} collectionDefinition rules controlling compilation of collection
 	 * @param {string} route route for resulting collection
@@ -562,6 +1165,11 @@ export class Shaper extends EventEmitter {
 
 		try {
 			$sort = $sort && typeof $sort === "string" ? Compiler.compile( $sort, this.termFunctions, this.termsCache ) : $sort?.property ? $sort : undefined;
+
+			// support processed term requesting to fail
+			if ( $sort instanceof Error ) {
+				throw $sort;
+			}
 		} catch ( cause ) {
 			throw new Error( `invalid rule for defining sorting of collection at "${route}" in ${modelName}: ${cause.message}` );
 		}