@cepharum/concrete-db 0.1.2-alpha.1 → 0.2.0

package/lib/shaper.mjs

@@ -1,9 +1,11 @@
+// noinspection ExceptionCaughtLocallyJS
+
 import EventEmitter from "events";
 
 import { Compiler, Functions } from "simple-terms";
 import merge from "lodash.merge";
 
-import { DefaultShaperOptions } from "./defaults.mjs";
+import { DefaultShape, DefaultShaperOptions } from "./defaults.mjs";
 import { augmentDataSpace } from "./helper.mjs";
 import * as TermFunctions from "./term-functions.mjs";
 
@@ -27,16 +29,17 @@ export class Shaper extends EventEmitter {
 	constructor( options ) {
 		super();
 
+		const compiledOptions = { ...DefaultShaperOptions, ...options };
+
 		Object.defineProperties( this, {
-			options: { value: { ...DefaultShaperOptions, ...options }, enumerable: true },
+			options: { value: compiledOptions, enumerable: true },
 
 			termsCache: { value: new Map() },
 
-			termFunctions: { value: { ...Functions, ...TermFunctions } },
-		} );
+			// SECURITY: prevent custom library functions from replacing regular ones by intention
+			termFunctions: { value: { ...compiledOptions.library, ...Functions, ...TermFunctions } },
 
-		Object.defineProperties( this, {
-			prefix: { value: normalizePathname( this.options.prefix, true ) },
+			prefix: { value: normalizePathname( compiledOptions.prefix, true ) },
 		} );
 	}
 
@@ -44,13 +47,19 @@ export class Shaper extends EventEmitter {
 	 * Creates database from provided collection of records.
 	 *
 	 * @param {Map<string,CollectedRecord[]>} recordsCollection collection of records to transform
+	 * @param {Shape[]} localShapes local shapes of folders records have been collected from
 	 * @returns {Database} resulting database records
 	 */
-	transform( recordsCollection ) {
-		const models = this.groupByModel( recordsCollection );
+	transform( recordsCollection, localShapes ) {
+		if ( recordsCollection.size > 0 && !localShapes?.length ) {
+			throw new Error( "invalid use of Shaper: missing list of local shapes per folder records have been collected from" );
+		}
+
+		const mergedLocalShapes = merge( {}, DefaultShape, ...localShapes );
+		const models = this.handleContributions( this.groupByModel( recordsCollection ), mergedLocalShapes );
 		const endpoints = {};
 
-		this.populateModels( endpoints, models );
+		this.populateModels( endpoints, models, mergedLocalShapes );
 
 		return { prefix: this.prefix, endpoints };
 	}
@@ -58,44 +67,71 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Reorganizes records to by grouped by model.
 	 *
-	 * @param {Map<string,CollectedRecord[]>} collection collection of records to transform
-	 * @returns {Object<string,Object<string, CollectedRecord[]>>} provided records grouped by model and unique item ID
+	 * @param {Map<string,CollectedRecord[]>} collectedRecordsByName collection of records to transform
+	 * @returns {Object<string,Object<string, ModelItem>>} provided records grouped by model and unique item ID
 	 */
-	groupByModel( collection ) {
+	groupByModel( collectedRecordsByName ) {
 		const globalItemsPerModel = {};
+		let index = 0;
+
+		if ( this.options.verbose ) {
+			console.error( `shaping records ...` );
+		}
+
+		for ( const records of collectedRecordsByName.values() ) {
+			if ( this.options.verbose ) {
+				process.stderr.write( `\r[${++index}/${collectedRecordsByName.size}]` );
+			}
+
+			this.compileRecords( globalItemsPerModel, records );
+		}
+
+		return globalItemsPerModel;
+	}
+
+	/**
+	 * Implements separate stage for computing contributions per model item
+	 * based on either model's compiled shape.
+	 *
+	 * @param {Object<string,Object<string, CollectedRecord[]>>} globalItemsPerModel collections of items per model
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
+	 * @returns {Object<string,Object<string, CollectedRecord[]>>} collections of items per model as provided
+	 */
+	handleContributions( globalItemsPerModel, mergedLocalShapes ) {
 		const postProcessingQueue = new Set();
 
-		for ( const source of collection.values() ) {
-			const localItemsPerModel = this.compileModelsOfSource( source );
+		if ( this.options.verbose ) {
+			console.error( `\nprocessing contributions ...` );
+		}
 
-			this.processContributions( localItemsPerModel, postProcessingQueue );
+		this.processContributions( globalItemsPerModel, postProcessingQueue, mergedLocalShapes );
 
-			this.mergeItemsPerModel( globalItemsPerModel, localItemsPerModel );
+		if ( this.options.verbose ) {
+			console.error( `post-processing ${postProcessingQueue.size} contributions` );
 		}
 
-		this.postProcessContributions( globalItemsPerModel, postProcessingQueue );
+		this.recompileContributionTargets( globalItemsPerModel, postProcessingQueue );
 
 		return globalItemsPerModel;
 	}
 
 	/**
-	 * Compiles all items according to collected records of provided source and
-	 * collect them grouped by items' model.
+	 * Compiles all items according to collected records as provided and collect
+	 * them grouped by described items' models.
 	 *
-	 * @param {CollectedRecord[]} source list of records collected from a source
-	 * @returns {Object<string,Object<string,Object>>} compiled items grouped by model, addressable by either item's ID
+	 * @param {Object<string,Object<string, ModelItem>>} itemsPerModel collections of items per model to be populated
+	 * @param {CollectedRecord[]} records list of records collected from a source
+	 * @returns {Object<string,Object<string,ModelItem>>} compiled items grouped by model, addressable by either item's ID
 	 */
-	compileModelsOfSource( source ) {
+	compileRecords( itemsPerModel, records ) {
 		const shapesPerModelCache = {};
-		const itemsPerModel = {};
 
-		for ( let read = 0, numRecords = source.length; read < numRecords; read++ ) {
-			const item = this.compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, source[read] );
+		for ( let read = 0, numRecords = records.length; read < numRecords; read++ ) {
+			const item = this.compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, records[read] );
 			const modelPool = itemsPerModel[item.$model];
 
-			const variantId = this.computeVariantIdOfItem( item );
-			if ( variantId == null ) {
-				modelPool[item.$id] = modelPool[item.$id] ? merge( modelPool[item.$id], item ) : item;
+			if ( item.$variantId == null ) {
+				modelPool[item.$id] = this.mergeModelItem( modelPool[item.$id], item );
 			} else {
 				const target = modelPool[item.$id];
 
@@ -103,7 +139,7 @@ export class Shaper extends EventEmitter {
 					target.$variants = {};
 				}
 
-				target.$variants[variantId] = target.$variants[variantId] ? merge( target.$variants[variantId], item ) : item;
+				target.$variants[item.$variantId] = this.mergeModelItem( target.$variants[item.$variantId], item );
 			}
 		}
 
@@ -130,15 +166,12 @@ export class Shaper extends EventEmitter {
 		const augmentedData = augmentDataSpace( data, metaProperties );
 
 		metaProperties.$model = this.detectModel( augmentedData, shape );
-		metaProperties.$shape = this.getShapeOfModel( shapesPerModelCache, shape, metaProperties.$model );
+		metaProperties.$shape = this.getShapeOfModel( {}, shape, metaProperties.$model );
 		metaProperties.$id = this.identifyRecord( augmentedData, metaProperties.$shape );
 
 		if ( !itemsPerModel.hasOwnProperty( metaProperties.$model ) ) {
 			// eslint-disable-next-line no-param-reassign
-			itemsPerModel[metaProperties.$model] = Object.defineProperties( {}, {
-				$model: { value: metaProperties.$model },
-				$shape: { value: metaProperties.$shape },
-			} );
+			itemsPerModel[metaProperties.$model] = {};
 		}
 
 		const model = itemsPerModel[metaProperties.$model];
@@ -147,37 +180,36 @@ export class Shaper extends EventEmitter {
 		const item = this.compileItem( augmentedData, properties, model[metaProperties.$id] ? {} : defaults );
 
 		Object.defineProperties( item, {
-			$original: { value: data },
-			$shape: { value: metaProperties.$shape },
+			$segments: { value: segments },
+			$original: { value: data, configurable: true },
+			$shape: { value: metaProperties.$shape, configurable: true },
 			$model: { value: metaProperties.$model },
 			$id: { value: metaProperties.$id },
-			$segments: { value: segments },
-			$localShape: { value: shape },
+			$variantId: { value: this.computeVariantIdOfItem( metaProperties.$shape.properties.$variant, augmentedData ) },
 		} );
 
 		return item;
 	}
 
 	/**
-	 * Re-compiles provided item this time based item's properties' themselves
-	 * instead of some record originally collected from a source.
+	 * Re-compiles provided item this time based on item's properties'
+	 * themselves instead of some record originally collected from a source.
 	 *
 	 * Re-compiling is used to compute data contributed by other models.
 	 *
-	 * @param {CollectedItem} item previously compiled item to re-compile
-	 * @returns {CollectedItem} re-compiled item
+	 * @param {ModelItem} item previously compiled item to re-compile
+	 * @returns {ModelItem} re-compiled item
 	 */
 	recompileItem( item ) {
-		const data = augmentDataSpace( merge( {}, item.$original, item ), { ...item, $postcontribution: true } );
+		const data = augmentDataSpace( merge( {}, item.$original, item ), { ...item, $post: true } );
 		const updatedItem = this.compileItem( data, item.$shape.properties, {} );
 
 		Object.defineProperties( updatedItem, {
-			$original: { value: item.$original },
-			$shape: { value: item.$shape },
+			$segments: { value: item.$segments },
+			$original: { value: item.$original, configurable: true },
+			$shape: { value: item.$shape, configurable: true },
 			$model: { value: item.$model },
 			$id: { value: item.$id },
-			$segments: { value: item.$segments },
-			$localShape: { value: item.$localShape },
 		} );
 
 		return updatedItem;
@@ -187,12 +219,11 @@ export class Shaper extends EventEmitter {
 	 * Computes ID of variant provided item is describing. If nullish, the item
 	 * isn't describing any variant but the base version of the item.
 	 *
-	 * @param {CollectedItem} item an item to get variant ID for
+	 * @param {string?} term term expressing computation of variant ID
+	 * @param {ModelItem} item an item to get variant ID for
 	 * @returns {null|any} computed variant ID of provided item, nullish if item is not a variant
 	 */
-	computeVariantIdOfItem( item ) {
-		const term = item.$shape.variant;
-
+	computeVariantIdOfItem( term, item ) {
 		if ( term == null ) {
 			return null;
 		}
@@ -201,7 +232,14 @@ export class Shaper extends EventEmitter {
 		const data = augmentDataSpace( merge( {}, item.$original, item ), item );
 
 		try {
-			return Compiler.compile( String( term ), this.termFunctions, this.termsCache )( data );
+			const value = Compiler.compile( String( term ), this.termFunctions, this.termsCache )( data );
+
+			// support processed term requesting to fail computation of variant ID
+			if ( value instanceof Error ) {
+				throw value;
+			}
+
+			return value;
 		} catch ( cause ) {
 			throw new Error( `computing variant ID of ${item.$model}#${item.$id} failed: ${cause.message} in ${term}` );
 		}
@@ -218,7 +256,7 @@ export class Shaper extends EventEmitter {
 	getShapeOfModel( shapesPerModelCache, localShape, modelName ) {
 		if ( !shapesPerModelCache.hasOwnProperty( modelName ) ) {
 			// eslint-disable-next-line no-param-reassign
-			shapesPerModelCache[modelName] = merge( {}, localShape?.common, localShape?.models?.[modelName] );
+			shapesPerModelCache[modelName] = merge( { defaults: DefaultShape.common.defaults }, localShape?.common, localShape?.models?.[modelName] );
 		}
 
 		return shapesPerModelCache[modelName];
@@ -313,9 +351,10 @@ export class Shaper extends EventEmitter {
 	 *
 	 * @param {Object<string,Object<string,Object>>} itemsPerModel map of model names into either model's map of item IDs into related item
 	 * @param {Set<string>} contributedItems tracks IDs of items or their variants that have been contributed to
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {void}
 	 */
-	processContributions( itemsPerModel, contributedItems ) {
+	processContributions( itemsPerModel, contributedItems, mergedLocalShapes ) {
 		const contributingModels = this.listContributingModels( itemsPerModel );
 		const shapePerModelCache = {};
 
@@ -328,33 +367,47 @@ export class Shaper extends EventEmitter {
 					continue;
 				}
 
-				for ( const targetModelName of Object.keys( contributions ) ) {
-					const targetModelShape = this.getShapeOfModel( shapePerModelCache, item.$localShape, targetModelName );
+				for ( const contributedModelName of Object.keys( contributions ) ) {
+					const isOptionalContribution = contributedModelName.trim().endsWith( "?" );
+					const targetModelName = contributedModelName.replace( /\s*\?\s*$/, "" );
+
+					const targetModelShape = this.getShapeOfModel( shapePerModelCache, mergedLocalShapes, targetModelName );
 
 
 					// extract contributions to actually declared properties of target model
 					const contributionTerms = {};
+					const contributionModes = {};
 					let warn = false;
+					let numActualProperties = 0;
 
-					for ( const termTargetName of Object.keys( contributions[targetModelName] ) ) {
-						const propertyName = termTargetName.replace( /\s*\[]$/, "" );
+					for ( const termTargetName of Object.keys( contributions[contributedModelName] ) ) {
+						const match = /^\s*(\S.+?)\s*(\[\])?\s*(\.\.\.|\u2026)?\s*$/u.exec( termTargetName );
+						const propertyName = match ? match[1] : termTargetName;
 
-						if ( propertyName !== termTargetName && propertyName.startsWith( "$" ) ) {
-							throw new Error( `invalid collecting contribution of record #${item.$id} in model ${item.$model} to property ${propertyName} of model ${targetModelName}` ); // eslint-disable-line max-len
+						if ( match && ( match[2] || match[3] ) && propertyName.startsWith( "$" ) ) {
+							throw new Error( `invalid collecting or distributed contribution of record #${item.$id} in model ${item.$model} to property ${propertyName} of model ${targetModelName}` ); // eslint-disable-line max-len
 						}
 
-						if ( Object.prototype.hasOwnProperty.call( targetModelShape.properties, propertyName ) ) {
+						if ( propertyName === "$variant" || propertyName === "$id" || Object.prototype.hasOwnProperty.call( targetModelShape.properties, propertyName ) ) {
 							if ( contributionTerms.hasOwnProperty( propertyName ) ) {
 								throw new Error( `double contribution of record #${item.$id} in model ${item.$model} to property ${propertyName} of model ${targetModelName}` ); // eslint-disable-line max-len
 							}
 
-							contributionTerms[propertyName] = contributions[targetModelName][termTargetName];
+							if ( !propertyName.startsWith( "$" ) ) {
+								numActualProperties++;
+							}
+
+							contributionTerms[propertyName] = contributions[contributedModelName][termTargetName];
+							contributionModes[propertyName] = {
+								$isCollecting: Boolean( match && match[2] ),
+								$isDistributed: Boolean( match && match[3] ),
+							};
 						} else {
 							warn = true;
 						}
 					}
 
-					if ( Object.keys( contributionTerms ).length < 1 ) {
+					if ( numActualProperties < 1 && !isOptionalContribution ) {
 						console.warn( `ignoring eventually empty contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName}` );
 						continue;
 					}
@@ -364,46 +417,44 @@ export class Shaper extends EventEmitter {
 					}
 
 
-					// compile values contributed to target model in context of current item
-					const augmentedData = augmentDataSpace( item, item );
-					const contribution = this.compileItem( augmentedData, contributionTerms, {}, ["$id"] );
+					// pre-compute ID(s) of item(s) to contribute to
+					const contributionMeta = {
+						$segments: item.$segments,
+						$original: item.$original,
+						$shape: item.$shape,
+						$model: item.$model,
+						$id: item.$id,
+					};
 
-					if ( Object.keys( contribution ).length < 1 ) {
-						console.warn( `ignoring empty contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName}` );
+					const augmentedData = augmentDataSpace( item, contributionMeta );
+					const targetId = this.computeId( contributionTerms.$id, augmentedData );
+
+					if ( !isOptionalContribution && ( !targetId || ( Array.isArray( targetId ) && !targetId.length ) ) ) {
+						console.warn( `ignoring contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName} due to lack of target ID` ); // eslint-disable-line max-len
 						continue;
 					}
 
+					const targetIds = Array.isArray( targetId ) ? targetId : [targetId];
 
-					// use contribution data's $id property to pick item of target model to contribute to
-					const targetId = contribution.$id;
-					if ( !targetId ) {
-						throw new Error( `contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName} failed due to lack of ID` );
-					}
 
-					const targetIds = Array.isArray( targetId ) ? targetId : [targetId];
+					for ( let index = 0, count = targetIds.length; index < count; index++ ) {
+						const id = targetIds[index];
+
+						// compile contribution per target item
+						contributionMeta.$targetid = id;
+						contributionMeta.$targetindex = index;
 
-					// check if this contribution is regarding a variant of selected target item
-					let variantId;
 
-					if ( targetModelShape.variant ) {
-						// try computing variant ID based on this item's contribution to target item
-						variantId = this.computeVariantIdOfItem( {
-							...contribution,
+						// try computing variant ID based on contributing item
+						const variantId = this.computeVariantIdOfItem( contributionTerms.$variant, {
+							...item,
+							$segments: item.$segments,
+							$original: item.$original,
 							$shape: targetModelShape,
-							$id: targetId,
 							$model: targetModelName,
-							$original: { ...item.$original, data: {} },
-							$segments: item.$segments,
-							$localShape: item.$localShape,
+							$id: targetId,
 						} );
-					}
 
-					if ( variantId == null ) {
-						variantId = null;
-					}
-
-
-					for ( const id of targetIds ) {
 						// track contributions to selected target item or its variant
 						contributedItems.add( JSON.stringify( [ targetModelName, id, variantId ] ) );
 
@@ -413,22 +464,19 @@ export class Shaper extends EventEmitter {
 
 						if ( !itemsPerModel.hasOwnProperty( targetModelName ) ) {
 							// eslint-disable-next-line no-param-reassign
-							itemsPerModel[targetModelName] = Object.defineProperties( {}, {
-								$model: { value: targetModelName },
-								$shape: { value: targetModelShape },
-							} );
+							itemsPerModel[targetModelName] = {};
 						}
 
 						const model = itemsPerModel[targetModelName];
+						const targetItemExists = model.hasOwnProperty( id );
 
-						if ( !model.hasOwnProperty( id ) ) {
+						if ( !targetItemExists ) {
 							model[id] = Object.defineProperties( {}, {
-								$original: { value: {} },
-								$shape: { value: targetModelShape },
+								$segments: { value: item.$segments },
+								$original: { value: {}, configurable: true },
+								$shape: { value: targetModelShape, configurable: true },
 								$model: { value: targetModelName },
 								$id: { value: id },
-								$segments: { value: item.$segments },
-								$localShape: { value: item.$localShape, configurable: true },
 							} );
 						}
 
@@ -449,20 +497,31 @@ export class Shaper extends EventEmitter {
 						}
 
 
-						// merge this item's contribution with any previous contribution to the selected target
+						// compile actual properties of contribution
+						const contributionDefaults = targetItemExists || variantId != null ? {} : targetModelShape.defaults;
+						const contribution = this.compileItem( augmentedData, contributionTerms, contributionDefaults );
+
+
+						// merge this item's contribution with existing
+						// properties of selected target using reduced version
+						// of merge strategy found in mergeModelItem()
 						for ( const name of Object.keys( contribution ) ) {
 							if ( String( name ).startsWith( "$" ) ) {
 								continue;
 							}
 
-							const value = contribution[name];
+							const { $isCollecting, $isDistributed } = contributionModes[name] || {};
+							let value = contribution[name];
 
-							if ( contributions[targetModelName][name] ) {
-								// contribution has been declared without suffix "[]"
-								// -> do not collect, but replace value in target
-								target[name] = value;
-							} else {
-								// contribution has been declared with suffix "[]"
+							if ( $isDistributed ) {
+								// contribution is providing another value for every target item
+								if ( Array.isArray( value ) && value.length === targetIds.length ) { // eslint-disable-line max-depth
+									value = value[index];
+								}
+							}
+
+							if ( $isCollecting ) {
+								// contribution has been declared with one or more suffices, e.g. []
 								// -> collect value in target
 								if ( Array.isArray( target[name] ) ) { // eslint-disable-line no-lonely-if,max-depth
 									target[name].push( value );
@@ -471,6 +530,10 @@ export class Shaper extends EventEmitter {
 								} else {
 									target[name] = [ target[name], value ];
 								}
+							} else {
+								// contribution has been declared without any suffix
+								// -> do not collect, but replace value in target
+								target[name] = value;
 							}
 						}
 					}
@@ -479,14 +542,75 @@ export class Shaper extends EventEmitter {
 		}
 	}
 
+	/**
+	 * Deeply merges provided source item into given target item including their
+	 * attached model shape and original data.
+	 *
+	 * @param {ModelItem} target model item to adjust by merging
+	 * @param {ModelItem} source model item to merge into provided target
+	 * @returns {ModelItem} provided target
+	 */
+	mergeModelItem( target, source ) {
+		const _to = target || {};
+		const oldNames = target ? Object.keys( _to.$shape.properties ) : [];
+		const newNames = Object.keys( source.$shape.properties );
+
+		// merge properties of items
+		for ( const name of Object.keys( source ) ) {
+			if ( !String( name ).startsWith( "$" ) ) {
+				const wasCollecting = oldNames.some( n => n.replace( /\s*\[]$/, "" ).trim() === name );
+				const isCollecting = newNames.some( n => n.replace( /\s*\[]$/, "" ).trim() === name );
+
+				if ( _to.hasOwnProperty( name ) ) {
+					if ( isCollecting ) {
+						if ( wasCollecting && Array.isArray( _to[name] ) ) {
+							_to[name].splice( _to[name].length, 0, ...source[name] );
+						} else if ( _to[name] == null ) {
+							_to[name] = source[name];
+						} else {
+							_to[name] = [ _to[name], ...source[name] ];
+						}
+					} else {
+						_to[name] = source[name];
+					}
+				} else {
+					_to[name] = source[name];
+				}
+			}
+		}
+
+		// assure some freshly created target has all common meta properties
+		if ( !target ) {
+			Object.defineProperties( _to, {
+				$segments: { value: source.$segments },
+				$model: { value: source.$model },
+				$id: { value: source.$id },
+			} );
+		}
+
+		// merge some complex meta properties
+		Object.defineProperties( _to, {
+			$shape: {
+				value: merge( _to.$shape || {}, source.$shape ),
+				configurable: true,
+			},
+			$original: {
+				value: merge( _to.$original || {}, source.$original ),
+				configurable: true,
+			},
+		} );
+
+		return _to;
+	}
+
 	/**
 	 * Re-compiles all items in provided pool that have been contributed to.
 	 *
-	 * @param {Object<string,Object<string,CollectedItem>>} itemsPerModel pool of items per model to update
+	 * @param {Object<string,Object<string,ModelItem>>} itemsPerModel pool of items per model to update
 	 * @param {Set<string>} contributions lists IDs of items or their variants that have been contributed to
-	 * @returns {Object<string,Object<string,CollectedItem>>} pool of items per model as provided
+	 * @returns {Object<string,Object<string,ModelItem>>} pool of items per model as provided
 	 */
-	postProcessContributions( itemsPerModel, contributions ) {
+	recompileContributionTargets( itemsPerModel, contributions ) {
 		for ( const track of contributions.values() ) {
 			const [ model, id, variantId ] = JSON.parse( track );
 
@@ -515,7 +639,7 @@ export class Shaper extends EventEmitter {
 				const variants = item.$variants;
 
 				if ( !variants.hasOwnProperty( variantId ) ) {
-					console.warn( `inconsistency: missing variant ${variantId} of item ${id} of model ${model} though contributions have been tracked` );
+					console.warn( `inconsistency: missing variant #${variantId} of item #${id} of model ${model} though contributions have been tracked` );
 					continue;
 				}
 
@@ -526,96 +650,6 @@ export class Shaper extends EventEmitter {
 		return itemsPerModel;
 	}
 
-	/**
-	 * Merges items collected per model from a single source into provided
-	 * target collection which is covering multiple possible sources.
-	 *
-	 * @param {Object<string,Object<string,Object>>} target collection of items per model to be augmented by provided source collection
-	 * @param {Object<string,Object<string,Object>>} source collection items per model to merge into target
-	 * @returns {void}
-	 */
-	mergeItemsPerModel( target, source ) {
-		for ( const modelName of Object.keys( source ) ) {
-			const sourcePool = source[modelName];
-
-			if ( target.hasOwnProperty( modelName ) ) {
-				// blend shapes of pools claiming to refer to same model
-				const mergedShape = merge( {}, target[modelName].$shape, sourcePool.$shape );
-
-				Object.defineProperties( target[modelName], {
-					$shape: { value: mergedShape, configurable: true },
-				} );
-			} else {
-				// create pool of items for target model as it hasn't been populated before
-				// eslint-disable-next-line no-param-reassign
-				target[modelName] = Object.defineProperties( {}, {
-					$model: { value: sourcePool.$model },
-					$shape: { value: sourcePool.$shape, configurable: true },
-				} );
-			}
-
-			const targetPool = target[modelName];
-
-			for ( const itemId of Object.keys( sourcePool ) ) {
-				if ( targetPool.hasOwnProperty( itemId ) ) {
-					const from = sourcePool[itemId];
-					const to = targetPool[itemId];
-
-					targetPool[itemId] = this.mergeItem( to, from );
-
-					if ( from.$variants ) {
-						if ( !to.$variants ) {
-							to.$variants = {};
-						}
-
-						for ( const variantId of Object.keys( from.$variants ) ) {
-							to.$variants[variantId] = this.mergeItem( to.$variants[variantId] || {}, from.$variants[variantId] );
-						}
-					}
-				} else {
-					targetPool[itemId] = sourcePool[itemId];
-				}
-			}
-		}
-	}
-
-	/**
-	 * Updates a given set of properties blending it with another set of item
-	 * properties.
-	 *
-	 * @param {object} to target object to merge properties of source into
-	 * @param {object} from properties to merge into target
-	 * @returns {object} provided target object, or new object if provided target was falsy
-	 */
-	mergeItem( to, from ) {
-		const _to = to || {};
-
-		for ( const name of Object.keys( from ) ) {
-			if ( !String( name ).startsWith( "$" ) ) {
-				// TODO reconsider this merging strategy for being beneficial
-				//      they way it is by now (e.g. for it keeps nullish data on
-				//      either end)
-				if ( Array.isArray( from[name] ) ) {
-					if ( Array.isArray( _to[name] ) ) {
-						_to[name].splice( _to[name].length, 0, ...from[name] );
-					} else if ( _to.hasOwnProperty( name ) ) {
-						_to[name] = [ _to[name], ...from[name] ];
-					} else {
-						_to[name] = [...from[name]];
-					}
-				} else if ( Array.isArray( _to[name] ) ) {
-					if ( from.hasOwnProperty( name ) ) {
-						_to[name].push( from[name] );
-					}
-				} else {
-					_to[name] = from[name];
-				}
-			}
-		}
-
-		return _to;
-	}
-
 	/**
 	 * Detects model of a record.
 	 *
@@ -632,6 +666,11 @@ export class Shaper extends EventEmitter {
 
 		const name = Compiler.compile( selector, this.termFunctions, this.termsCache )( data );
 
+		// support processed term requesting to fail model detection
+		if ( name instanceof Error ) {
+			throw new Error( `detecting model of record failed: ${name.message} in context of ${JSON.stringify( data )}` );
+		}
+
 		if ( !name ) {
 			throw new Error( `detecting model of record failed: ${JSON.stringify( data )}` );
 		}
@@ -647,13 +686,7 @@ export class Shaper extends EventEmitter {
 	 * @returns {string} ID of record
 	 */
 	identifyRecord( data, modelShape ) {
-		const selector = modelShape?.properties?.$id;
-
-		if ( !selector || typeof selector !== "string" ) {
-			throw new Error( `missing or invalid definition of $id in shape of ${data.$segments.join( "/" )}` );
-		}
-
-		const id = Compiler.compile( selector )( data );
+		const id = this.computeId( modelShape?.properties?.$id, data );
 
 		if ( !id ) {
 			throw new Error( `detecting ID of item related to record ${data.$segments.join( "/" )} failed` );
@@ -676,35 +709,62 @@ export class Shaper extends EventEmitter {
 
 		for ( const property of Object.keys( properties ) ) {
 			if ( accept.indexOf( property ) > -1 || !String( property ).startsWith( "$" ) ) {
+				const propertyName = property.replace( /\s*\[]$/, "" );
+				const isCollecting = property !== propertyName;
+
 				const term = properties[property];
 				if ( typeof term !== "string" || term.trim().length === 0 ) {
-					throw new Error( `invalid or missing term for computing value of property ${data.$model}#${property}` );
+					throw new Error( `invalid or missing term for computing value of property ${propertyName} of ${data.$model}` );
+				}
+
+				if ( isCollecting && properties.hasOwnProperty( propertyName ) ) {
+					throw new Error( `double definition: property ${propertyName} of ${data.$model} must be either regular or collecting property` );
 				}
 
 				let value;
 
 				try {
 					value = Compiler.compile( term, this.termFunctions, this.termsCache )( data );
+
+					// support processed term requesting to fail computation of property's value
+					if ( value instanceof Error ) {
+						throw value;
+					}
 				} catch ( cause ) {
-					throw new Error( `fetching value of property ${data.$model}#${property} failed: ${cause.message} in ${term}` );
+					throw new Error( `fetching value of property ${propertyName} of ${data.$model} failed: ${cause.message} in ${term}` );
 				}
 
 				if ( value == null ) {
-					if ( defaults.hasOwnProperty( property ) ) {
-						const defaultValue = defaults[property];
+					if ( defaults.hasOwnProperty( propertyName ) || defaults.hasOwnProperty( property ) ) {
+						const defaultValue = defaults.hasOwnProperty( propertyName ) ? defaults[propertyName] : defaults[property];
 						const trimmed = typeof defaultValue === "string" ? defaultValue.trim() : "";
 
 						try {
 							const code = trimmed.startsWith( "=" ) ? trimmed.slice( 1 ) : undefined;
 							value = code ? Compiler.compile( code, this.termFunctions, this.termsCache )( data ) : defaultValue;
+
+							// support processed term requesting to fail computation of default value
+							if ( value instanceof Error ) {
+								throw value;
+							}
 						} catch ( cause ) {
-							throw new Error( `fetching default of property ${data.$model}#${property} failed: "${cause.message}" in "${defaultValue}"` );
+							throw new Error( `fetching default of property ${propertyName} of ${data.$model} failed: "${cause.message}" in "${defaultValue}"` );
 						}
 					}
 				}
 
 				if ( value != null ) {
-					item[property] = value;
+					if ( isCollecting ) {
+						if ( Array.isArray( item[propertyName] ) ) { // eslint-disable-line no-lonely-if
+							item[propertyName].push( value );
+						} else if ( item[propertyName] == null ) {
+							item[propertyName] = [value];
+						} else {
+							item[propertyName] = [ item[propertyName], value ];
+						}
+					} else {
+						item[propertyName] = value;
+					}
 				}
 			}
 		}
@@ -714,18 +774,57 @@ export class Shaper extends EventEmitter {
 		return merge( {}, item );
 	}
 
+	/**
+	 * Computes ID using provided term in context of given data space.
+	 *
+	 * @param {string} term term to process
+	 * @param {object} data data space available to term processing
+	 * @returns {any} processed term's result
+	 */
+	computeId( term, data ) {
+		if ( !term || typeof term !== "string" ) {
+			throw new Error( `missing or invalid term for computing ID of ${data.$segments.join( "/" )}` );
+		}
+
+		try {
+			const value = Compiler.compile( term, this.termFunctions, this.termsCache )( data );
+
+			// support processed term requesting to fail computation of ID
+			if ( value instanceof Error ) {
+				throw value;
+			}
+
+			return value;
+		} catch ( cause ) {
+			throw new Error( `computing ID failed: ${cause.message} in ${term}` );
+		}
+	}
+
 	/**
 	 * Populates pool of endpoints with collected records of discovered models.
 	 *
 	 * @param {DatabaseEndpoints} endpoints map of routes into data sets to expose eventually
-	 * @param {Object<string,Object<string,CollectedItem>>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {DatabaseEndpoints} map of routes into data sets to expose eventually
 	 */
-	populateModels( endpoints, models ) {
-		for ( const modelName of Object.keys( models ) ) {
+	populateModels( endpoints, models, mergedLocalShapes ) {
+		const modelNames = Object.keys( models );
+		let index = 0;
+
+		for ( const modelName of modelNames ) {
 			const model = models[modelName];
+			const itemIds = Object.keys( model );
 
-			for ( const itemId of Object.keys( model ) ) {
+			if ( this.options.verbose ) {
+				console.error( `creating endpoints for model ${modelName} [${++index}/${modelNames.length}] ...` );
+			}
+
+			if ( this.options.verbose ) {
+				console.error( `  - ${itemIds.length} items ...` );
+			}
+
+			for ( const itemId of itemIds ) {
 				const normalized = normalizePathname( `${this.prefix}${modelName}/item/${itemId}` );
 				if ( normalized.endsWith( "/" ) ) {
 					throw new Error( `item ID "${itemId}" of model "${modelName}" results in invalid route for endpoint exposing it` );
@@ -734,7 +833,7 @@ export class Shaper extends EventEmitter {
 				endpoints[normalized] = model[itemId]; // eslint-disable-line no-param-reassign
 			}
 
-			this.populateModelCollections( endpoints, models, modelName );
+			this.populateModelCollections( endpoints, models, modelName, mergedLocalShapes );
 		}
 
 		return endpoints;
@@ -745,13 +844,14 @@ export class Shaper extends EventEmitter {
 	 * a model.
 	 *
 	 * @param {DatabaseEndpoints} endpoints map of routes into data sets to expose eventually
-	 * @param {Object<string,Object<string,CollectedItem>>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
+	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {DatabaseEndpoints} map of routes into data sets to expose eventually
 	 */
-	populateModelCollections( endpoints, models, modelName ) {
-		const model = models[modelName];
-		const collections = model.$shape.collections || {};
+	populateModelCollections( endpoints, models, modelName, mergedLocalShapes ) {
+		const finalModelShape = merge( {}, mergedLocalShapes.common, mergedLocalShapes.models[modelName] );
+		const collections = finalModelShape.collections || {};
 		const cache = {};
 
 		// prepare to process collection instances prior to collection references
@@ -770,9 +870,15 @@ export class Shaper extends EventEmitter {
 		} );
 
 		// process collection by collection
+		let index = 0;
+
 		for ( const route of keys ) {
 			const normalized = normalizePathname( route );
 
+			if ( this.options.verbose ) {
+				console.error( `  - collection /${normalized} [${++index}/${keys.length}] ...` );
+			}
+
 			if ( String( normalized ).startsWith( "item/" ) || /(^|\/)\.\.(\/|$)|[\\%]/.test( normalized ) ) {
 				throw new Error( `invalid custom route "${normalized}" for collection of model "${modelName}"` );
 			}
@@ -892,15 +998,15 @@ export class Shaper extends EventEmitter {
 	 * Creates managed data space for evaluating terms in context of a set of
 	 * items.
 	 *
-	 * @param {Object<string,Object<string,CollectedItem>>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
 	 * @param {ShapeCollection} collectionDefinition rules controlling compilation of collection
-	 * @returns {{handler: {get: ((function(*, *=): (*))|*)}, context: {$models, $model, $collection}}} handler and extensible context for term evaluation
+	 * @returns {{handler: {get: ((function(*, *=): (*))|*)}, context: {$database, $model, $collection}}} handler and extensible context for term evaluation
 	 */
 	createTermContextCollection( models, modelName, collectionDefinition ) {
 		const context = {
 			$collection: collectionDefinition,
-			$models: models,
+			$database: models,
 			$model: modelName,
 		};
 
@@ -920,7 +1026,7 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Compiles collection according to its definition in provided shape.
 	 *
-	 * @param {Object<string,Object<string,CollectedItem>>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
 	 * @param {ShapeCollection} definition rules controlling compilation of collection
 	 * @param {string} route route for resulting collection
@@ -943,12 +1049,22 @@ export class Shaper extends EventEmitter {
 
 		try {
 			keyFn = isKeyed ? Compiler.compile( $key, this.termFunctions, this.termsCache ) : undefined;
+
+			// support processed term requesting to fail
+			if ( keyFn instanceof Error ) {
+				throw keyFn;
+			}
 		} catch ( cause ) {
 			throw new Error( `invalid rule for defining key of collection at "${route}" in ${modelName}: ${cause.message}` );
 		}
 
 		try {
 			filterFn = $filter && typeof $filter === "string" ? Compiler.compile( $filter, this.termFunctions, this.termsCache ) : undefined;
+
+			// support processed term requesting to fail
+			if ( filterFn instanceof Error ) {
+				throw filterFn;
+			}
 		} catch ( cause ) {
 			throw new Error( `invalid rule for defining filter of collection at "${route}" in ${modelName}: ${cause.message}` );
 		}
@@ -960,6 +1076,11 @@ export class Shaper extends EventEmitter {
 				if ( code && typeof code === "string" ) {
 					const fn = Compiler.compile( code, this.termFunctions, this.termsCache );
 
+					// support processed term requesting to fail computed definition of collection
+					if ( fn instanceof Error ) {
+						throw fn;
+					}
+
 					fn.target = target;
 
 					return fn;
@@ -1030,7 +1151,7 @@ export class Shaper extends EventEmitter {
 	/**
 	 * Sorts provided collection either in-place or as a copy.
 	 *
-	 * @param {Object<string,Object<string,CollectedItem>>} models pool of items per discovered model
+	 * @param {Object<string,Object<string,ModelItem>>} models pool of items per discovered model
 	 * @param {string} modelName name of model to process
 	 * @param {ShapeCollection} collectionDefinition rules controlling compilation of collection
 	 * @param {string} route route for resulting collection
@@ -1044,6 +1165,11 @@ export class Shaper extends EventEmitter {
 
 		try {
 			$sort = $sort && typeof $sort === "string" ? Compiler.compile( $sort, this.termFunctions, this.termsCache ) : $sort?.property ? $sort : undefined;
+
+			// support processed term requesting to fail
+			if ( $sort instanceof Error ) {
+				throw $sort;
+			}
 		} catch ( cause ) {
 			throw new Error( `invalid rule for defining sorting of collection at "${route}" in ${modelName}: ${cause.message}` );
 		}