@cepharum/concrete-db 0.2.1 → 0.3.0

package/concrete-db.d.ts

@@ -1,32 +1,4 @@
-/**
- * (c) 2022 cepharum GmbH, Berlin, http://cepharum.de
- *
- * The MIT License (MIT)
- *
- * Copyright (c) 2021 cepharum GmbH
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * @author: cepharum
- */
-
-declare module "concrete-db" {
+declare namespace ConcreteDB {
 	export interface CuringOptions extends CollectorOptions, ShaperOptions, GeneratorOptions {
 		/**
 		 * Selects base folder for writing files of resulting concrete-db.
@@ -260,18 +232,34 @@ declare module "concrete-db" {
 	 */
 	export type ShapeCollections = ShapeCollection[];
 
+	/**
+	 * Defines optional constraints to apply.
+	 */
+	export interface ShapeConstraints {
+		/**
+		 * If set, multiple source records addressing same target record aren't
+		 * merged, but some exception is thrown.
+		 *
+		 * This doesn't affect definition of variants. However, merging single
+		 * variant from multiple sources is rejected, too.
+		 */
+		singleSource?: boolean;
+	}
+
 	/**
 	 * Describes rules controlling shaping of a model.
 	 */
 	export interface ShapeModel {
 		defaults: ShapePropertyDefaults;
-		properties: ShapeProperties;
-		collections: ShapeCollections;
-		contributions: {
+		properties?: ShapeProperties;
+		collections?: ShapeCollections;
+		contributions?: {
 			[contributedModelName: string]: {
 				[targetPropertyName: string]: string;
 			};
 		};
+
+		constraints?: ShapeConstraints;
 	}
 
 	export interface Shape {

package/lib/shaper.mjs

@@ -1,4 +1,5 @@
 // noinspection ExceptionCaughtLocallyJS
+/* eslint-disable no-await-in-loop */
 
 import EventEmitter from "events";
 
@@ -50,16 +51,17 @@ export class Shaper extends EventEmitter {
 	 * @param {Shape[]} localShapes local shapes of folders records have been collected from
 	 * @returns {Database} resulting database records
 	 */
-	transform( recordsCollection, localShapes ) {
+	async transform( recordsCollection, localShapes ) {
 		if ( recordsCollection.size > 0 && !localShapes?.length ) {
 			throw new Error( "invalid use of Shaper: missing list of local shapes per folder records have been collected from" );
 		}
 
 		const mergedLocalShapes = merge( {}, DefaultShape, ...localShapes );
-		const models = this.handleContributions( this.groupByModel( recordsCollection ), mergedLocalShapes );
+		const grouped = await this.groupByModel( recordsCollection );
+		const models = await this.handleContributions( grouped, mergedLocalShapes );
 		const endpoints = {};
 
-		this.populateModels( endpoints, models, mergedLocalShapes );
+		await this.populateModels( endpoints, models, mergedLocalShapes );
 
 		return { prefix: this.prefix, endpoints };
 	}
@@ -70,7 +72,7 @@ export class Shaper extends EventEmitter {
 	 * @param {Map<string,CollectedRecord[]>} collectedRecordsByName collection of records to transform
 	 * @returns {Object<string,Object<string, ModelItem>>} provided records grouped by model and unique item ID
 	 */
-	groupByModel( collectedRecordsByName ) {
+	async groupByModel( collectedRecordsByName ) {
 		const globalItemsPerModel = {};
 		let index = 0;
 
@@ -83,7 +85,7 @@ export class Shaper extends EventEmitter {
 				process.stderr.write( `\r[${++index}/${collectedRecordsByName.size}]` );
 			}
 
-			this.compileRecords( globalItemsPerModel, records );
+			await this.compileRecords( globalItemsPerModel, records );
 		}
 
 		return globalItemsPerModel;
@@ -97,20 +99,20 @@ export class Shaper extends EventEmitter {
 	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {Object<string,Object<string, CollectedRecord[]>>} collections of items per model as provided
 	 */
-	handleContributions( globalItemsPerModel, mergedLocalShapes ) {
+	async handleContributions( globalItemsPerModel, mergedLocalShapes ) {
 		const postProcessingQueue = new Set();
 
 		if ( this.options.verbose ) {
 			console.error( `\nprocessing contributions ...` );
 		}
 
-		this.processContributions( globalItemsPerModel, postProcessingQueue, mergedLocalShapes );
+		await this.processContributions( globalItemsPerModel, postProcessingQueue, mergedLocalShapes );
 
 		if ( this.options.verbose ) {
 			console.error( `post-processing ${postProcessingQueue.size} contributions` );
 		}
 
-		this.recompileContributionTargets( globalItemsPerModel, postProcessingQueue );
+		await this.recompileContributionTargets( globalItemsPerModel, postProcessingQueue );
 
 		return globalItemsPerModel;
 	}
@@ -123,11 +125,11 @@ export class Shaper extends EventEmitter {
 	 * @param {CollectedRecord[]} records list of records collected from a source
 	 * @returns {Object<string,Object<string,ModelItem>>} compiled items grouped by model, addressable by either item's ID
 	 */
-	compileRecords( itemsPerModel, records ) {
+	async compileRecords( itemsPerModel, records ) {
 		const shapesPerModelCache = {};
 
 		for ( let read = 0, numRecords = records.length; read < numRecords; read++ ) {
-			const item = this.compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, records[read] );
+			const item = await this.compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, records[read] );
 			const modelPool = itemsPerModel[item.$model];
 
 			if ( item.$variantId == null ) {
@@ -154,9 +156,9 @@ export class Shaper extends EventEmitter {
 	 * @param {Object<string,ShapeModel>} shapesPerModelCache caches per-model shapes compiled before
 	 * @param {Object<string,Object<string,Object>>} itemsPerModel collection of previously compiled items per model
 	 * @param {CollectedRecord} record augmented data collected from a source to be compiled
-	 * @returns {object} instance of model with hidden meta information injected
+	 * @returns {Promise<ModelItem>} instance of model with hidden meta information injected
 	 */
-	compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, record ) {
+	async compileModelItemFromRecord( shapesPerModelCache, itemsPerModel, record ) {
 		const { segments, shape, data } = record;
 
 		const metaProperties = {
@@ -165,9 +167,9 @@ export class Shaper extends EventEmitter {
 
 		const augmentedData = augmentDataSpace( data, metaProperties );
 
-		metaProperties.$model = this.detectModel( augmentedData, shape );
-		metaProperties.$shape = this.getShapeOfModel( {}, shape, metaProperties.$model );
-		metaProperties.$id = this.identifyRecord( augmentedData, metaProperties.$shape );
+		metaProperties.$model = await this.detectModel( augmentedData, shape );
+		metaProperties.$shape = this.getShapeOfModel( shapesPerModelCache, shape, metaProperties.$model );
+		metaProperties.$id = await this.identifyRecord( augmentedData, metaProperties.$shape );
 
 		if ( !itemsPerModel.hasOwnProperty( metaProperties.$model ) ) {
 			// eslint-disable-next-line no-param-reassign
@@ -177,7 +179,7 @@ export class Shaper extends EventEmitter {
 		const model = itemsPerModel[metaProperties.$model];
 
 		const { properties, defaults } = metaProperties.$shape;
-		const item = this.compileItem( augmentedData, properties, model[metaProperties.$id] ? {} : defaults );
+		const item = await this.compileItem( augmentedData, properties, model[metaProperties.$id] ? {} : defaults );
 
 		Object.defineProperties( item, {
 			$segments: { value: segments },
@@ -185,7 +187,7 @@ export class Shaper extends EventEmitter {
 			$shape: { value: metaProperties.$shape, configurable: true },
 			$model: { value: metaProperties.$model },
 			$id: { value: metaProperties.$id },
-			$variantId: { value: this.computeVariantIdOfItem( metaProperties.$shape.properties.$variant, augmentedData ) },
+			$variantId: { value: await this.computeVariantIdOfItem( metaProperties.$shape.properties.$variant, augmentedData ) },
 		} );
 
 		return item;
@@ -200,9 +202,9 @@ export class Shaper extends EventEmitter {
 	 * @param {ModelItem} item previously compiled item to re-compile
 	 * @returns {ModelItem} re-compiled item
 	 */
-	recompileItem( item ) {
+	async recompileItem( item ) {
 		const data = augmentDataSpace( merge( {}, item.$original, item ), { ...item, $post: true } );
-		const updatedItem = this.compileItem( data, item.$shape.properties, {} );
+		const updatedItem = await this.compileItem( data, item.$shape.properties, {} );
 
 		Object.defineProperties( updatedItem, {
 			$segments: { value: item.$segments },
@@ -221,9 +223,9 @@ export class Shaper extends EventEmitter {
 	 *
 	 * @param {string?} term term expressing computation of variant ID
 	 * @param {ModelItem} item an item to get variant ID for
-	 * @returns {null|any} computed variant ID of provided item, nullish if item is not a variant
+	 * @returns {Promise<null|any>} computed variant ID of provided item, nullish if item is not a variant
 	 */
-	computeVariantIdOfItem( term, item ) {
+	async computeVariantIdOfItem( term, item ) {
 		if ( term == null ) {
 			return null;
 		}
@@ -232,7 +234,7 @@ export class Shaper extends EventEmitter {
 		const data = augmentDataSpace( merge( {}, item.$original, item ), item );
 
 		try {
-			const value = Compiler.compile( String( term ), this.termFunctions, this.termsCache )( data );
+			const value = await Compiler.compile( String( term ), this.termFunctions, this.termsCache )( data );
 
 			// support processed term requesting to fail computation of variant ID
 			if ( value instanceof Error ) {
@@ -354,7 +356,7 @@ export class Shaper extends EventEmitter {
 	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {void}
 	 */
-	processContributions( itemsPerModel, contributedItems, mergedLocalShapes ) {
+	async processContributions( itemsPerModel, contributedItems, mergedLocalShapes ) {
 		const contributingModels = this.listContributingModels( itemsPerModel );
 		const shapePerModelCache = {};
 
@@ -427,7 +429,7 @@ export class Shaper extends EventEmitter {
 					};
 
 					const augmentedData = augmentDataSpace( item, contributionMeta );
-					const targetId = this.computeId( contributionTerms.$id, augmentedData );
+					const targetId = await this.computeId( contributionTerms.$id, augmentedData );
 
 					if ( !isOptionalContribution && ( !targetId || ( Array.isArray( targetId ) && !targetId.length ) ) ) {
 						console.warn( `ignoring contribution of record #${item.$id} in model ${item.$model} to model ${targetModelName} due to lack of target ID` ); // eslint-disable-line max-len
@@ -446,7 +448,7 @@ export class Shaper extends EventEmitter {
 
 
 						// try computing variant ID based on contributing item
-						const variantId = this.computeVariantIdOfItem( contributionTerms.$variant, {
+						const variantId = await this.computeVariantIdOfItem( contributionTerms.$variant, {
 							...item,
 							$segments: item.$segments,
 							$original: item.$original,
@@ -499,7 +501,7 @@ export class Shaper extends EventEmitter {
 
 						// compile actual properties of contribution
 						const contributionDefaults = targetItemExists || variantId != null ? {} : targetModelShape.defaults;
-						const contribution = this.compileItem( augmentedData, contributionTerms, contributionDefaults );
+						const contribution = await this.compileItem( augmentedData, contributionTerms, contributionDefaults );
 
 
 						// merge this item's contribution with existing
@@ -551,6 +553,10 @@ export class Shaper extends EventEmitter {
 	 * @returns {ModelItem} provided target
 	 */
 	mergeModelItem( target, source ) {
+		if ( target?.$shape?.constraints?.singleSource ) {
+			throw new Error( `merging ${target.$model} with ID ${target.$id} from multiple source records rejected by shape` );
+		}
+
 		const _to = target || {};
 		const oldNames = target ? Object.keys( _to.$shape.properties ) : [];
 		const newNames = Object.keys( source.$shape.properties );
@@ -558,8 +564,8 @@ export class Shaper extends EventEmitter {
 		// merge properties of items
 		for ( const name of Object.keys( source ) ) {
 			if ( !String( name ).startsWith( "$" ) ) {
-				const wasCollecting = oldNames.some( n => n.replace( /\s*\[]$/, "" ).trim() === name );
-				const isCollecting = newNames.some( n => n.replace( /\s*\[]$/, "" ).trim() === name );
+				const wasCollecting = oldNames.some( n => n.replace( /\s+/g, "" ) === name + "[]" );
+				const isCollecting = newNames.some( n => n.replace( /\s+/g, "" ) === name + "[]" );
 
 				if ( _to.hasOwnProperty( name ) ) {
 					if ( isCollecting ) {
@@ -610,7 +616,7 @@ export class Shaper extends EventEmitter {
 	 * @param {Set<string>} contributions lists IDs of items or their variants that have been contributed to
 	 * @returns {Object<string,Object<string,ModelItem>>} pool of items per model as provided
 	 */
-	recompileContributionTargets( itemsPerModel, contributions ) {
+	async recompileContributionTargets( itemsPerModel, contributions ) {
 		for ( const track of contributions.values() ) {
 			const [ model, id, variantId ] = JSON.parse( track );
 
@@ -629,7 +635,7 @@ export class Shaper extends EventEmitter {
 			const item = modelPool[id];
 
 			if ( variantId == null ) {
-				modelPool[id] = this.recompileItem( item );
+				modelPool[id] = await this.recompileItem( item );
 			} else {
 				if ( !item.$variants ) {
 					console.warn( `inconsistency: missing variants of item ${id} of model ${model} though contributions have been tracked` );
@@ -643,7 +649,7 @@ export class Shaper extends EventEmitter {
 					continue;
 				}
 
-				variants[variantId] = this.recompileItem( variants[variantId] );
+				variants[variantId] = await this.recompileItem( variants[variantId] );
 			}
 		}
 
@@ -655,16 +661,16 @@ export class Shaper extends EventEmitter {
 	 *
 	 * @param {object} data source of record
 	 * @param {Shape} shape shape of database affecting record
-	 * @returns {string} name of model to use
+	 * @returns {Promise<string>} name of model to use
 	 */
-	detectModel( data, shape ) {
+	async detectModel( data, shape ) {
 		const selector = shape?.common?.properties?.$model;
 
 		if ( !selector || typeof selector !== "string" ) {
 			throw new Error( "missing or invalid definition of common property $model in shape" );
 		}
 
-		const name = Compiler.compile( selector, this.termFunctions, this.termsCache )( data );
+		const name = await Compiler.compile( selector, this.termFunctions, this.termsCache )( data );
 
 		// support processed term requesting to fail model detection
 		if ( name instanceof Error ) {
@@ -683,10 +689,10 @@ export class Shaper extends EventEmitter {
 	 *
 	 * @param {object} data source of record
 	 * @param {ShapeModel} modelShape shape of particular model
-	 * @returns {string} ID of record
+	 * @returns {Promise<string>} ID of record
 	 */
-	identifyRecord( data, modelShape ) {
-		const id = this.computeId( modelShape?.properties?.$id, data );
+	async identifyRecord( data, modelShape ) {
+		const id = await this.computeId( modelShape?.properties?.$id, data );
 
 		if ( !id ) {
 			throw new Error( `detecting ID of item related to record ${data.$segments.join( "/" )} failed` );
@@ -702,14 +708,14 @@ export class Shaper extends EventEmitter {
 	 * @param {ShapeProperties} properties definition of resulting item's properties
 	 * @param {ShapePropertyDefaults} defaults definition of default values for resulting item's properties
 	 * @param {string[]} accept list names of properties to accept even when starting with a dollar sign
-	 * @returns {object} compiled item
+	 * @returns {Promise<Partial<ModelItem>>} compiled item
 	 */
-	compileItem( data, properties = {}, defaults = {}, accept = [] ) {
+	async compileItem( data, properties = {}, defaults = {}, accept = [] ) {
 		const item = {};
 
 		for ( const property of Object.keys( properties ) ) {
 			if ( accept.indexOf( property ) > -1 || !String( property ).startsWith( "$" ) ) {
-				const propertyName = property.replace( /\s*\[]$/, "" );
+				const propertyName = property.replace( /\s*\[]\s*$/, "" );
 				const isCollecting = property !== propertyName;
 
 				const term = properties[property];
@@ -724,7 +730,7 @@ export class Shaper extends EventEmitter {
 				let value;
 
 				try {
-					value = Compiler.compile( term, this.termFunctions, this.termsCache )( data );
+					value = await Compiler.compile( term, this.termFunctions, this.termsCache )( data );
 
 					// support processed term requesting to fail computation of property's value
 					if ( value instanceof Error ) {
@@ -741,7 +747,7 @@ export class Shaper extends EventEmitter {
 
 						try {
 							const code = trimmed.startsWith( "=" ) ? trimmed.slice( 1 ) : undefined;
-							value = code ? Compiler.compile( code, this.termFunctions, this.termsCache )( data ) : defaultValue;
+							value = code ? await Compiler.compile( code, this.termFunctions, this.termsCache )( data ) : defaultValue;
 
 							// support processed term requesting to fail computation of default value
 							if ( value instanceof Error ) {
@@ -779,15 +785,15 @@ export class Shaper extends EventEmitter {
 	 *
 	 * @param {string} term term to process
 	 * @param {object} data data space available to term processing
-	 * @returns {any} processed term's result
+	 * @returns {Promise<any>} processed term's result
 	 */
-	computeId( term, data ) {
+	async computeId( term, data ) {
 		if ( !term || typeof term !== "string" ) {
 			throw new Error( `missing or invalid term for computing ID of ${data.$segments.join( "/" )}` );
 		}
 
 		try {
-			const value = Compiler.compile( term, this.termFunctions, this.termsCache )( data );
+			const value = await Compiler.compile( term, this.termFunctions, this.termsCache )( data );
 
 			// support processed term requesting to fail computation of ID
 			if ( value instanceof Error ) {
@@ -808,7 +814,7 @@ export class Shaper extends EventEmitter {
 	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {DatabaseEndpoints} map of routes into data sets to expose eventually
 	 */
-	populateModels( endpoints, models, mergedLocalShapes ) {
+	async populateModels( endpoints, models, mergedLocalShapes ) {
 		const modelNames = Object.keys( models );
 		let index = 0;
 
@@ -833,7 +839,7 @@ export class Shaper extends EventEmitter {
 				endpoints[normalized] = model[itemId]; // eslint-disable-line no-param-reassign
 			}
 
-			this.populateModelCollections( endpoints, models, modelName, mergedLocalShapes );
+			await this.populateModelCollections( endpoints, models, modelName, mergedLocalShapes );
 		}
 
 		return endpoints;
@@ -849,7 +855,7 @@ export class Shaper extends EventEmitter {
 	 * @param {Shape} mergedLocalShapes local shapes of source folders merged in order of collecting records
 	 * @returns {DatabaseEndpoints} map of routes into data sets to expose eventually
 	 */
-	populateModelCollections( endpoints, models, modelName, mergedLocalShapes ) {
+	async populateModelCollections( endpoints, models, modelName, mergedLocalShapes ) {
 		const finalModelShape = merge( {}, mergedLocalShapes.common, mergedLocalShapes.models[modelName] );
 		const collections = finalModelShape.collections || {};
 		const cache = {};
@@ -893,7 +899,7 @@ export class Shaper extends EventEmitter {
 					collection = cache[definition.reference];
 					isKeyed = collection.$isKeyed;
 				} else {
-					collection = this.compileCollection( models, modelName, definition, normalized );
+					collection = await this.compileCollection( models, modelName, definition, normalized );
 					isKeyed = definition.key && typeof definition.key === "string";
 
 					Object.defineProperties( collection, {
@@ -964,7 +970,7 @@ export class Shaper extends EventEmitter {
 			const items = collection.items;
 
 			for ( let offset = 0; offset < items.length; offset += limit ) {
-				const normalized = normalizePathname( prefix + route.replace( ptnOffset, offset ) );
+				const normalized = normalizePathname( prefix + route.replace( ptnOffset, String( offset ) ) );
 				endpoints[normalized] = { count: items.length, items: items.slice( offset, offset + limit ) }; // eslint-disable-line no-param-reassign
 			}
 		} else if ( isKeyed ) {
@@ -1032,7 +1038,7 @@ export class Shaper extends EventEmitter {
 	 * @param {string} route route for resulting collection
 	 * @returns {DatabaseCollection} compiled collection
 	 */
-	compileCollection( models, modelName, definition, route ) {
+	async compileCollection( models, modelName, definition, route ) {
 		const model = models[modelName];
 		const { key: $key, filter: $filter, properties: $properties } = definition;
 		const isKeyed = $key && typeof $key === "string";
@@ -1103,9 +1109,9 @@ export class Shaper extends EventEmitter {
 				context.$id = id;
 
 				// ignore this item or prepare its spot in resulting collection?
-				if ( !filterFn || filterFn( data ) ) {
+				if ( !filterFn || await filterFn( data ) ) {
 					if ( isKeyed ) {
-						const key = keyFn( data );
+						const key = await keyFn( data );
 						if ( key != null ) {
 							const _key = String( key );
 
@@ -1131,7 +1137,7 @@ export class Shaper extends EventEmitter {
 					if ( propertiesFn ) {
 						// compile custom representation of item in collection
 						for ( const fn of propertiesFn ) {
-							extracted[fn.target] = fn( data );
+							extracted[fn.target] = await fn( data );
 						}
 					} else {
 						// expose all properties of item in collection, too

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@cepharum/concrete-db",
-	"version": "0.2.1",
+	"version": "0.3.0",
 	"description": "a read-only web database generator",
 	"main": "lib/collector.mjs",
 	"types": "concrete-db.d.ts",
@@ -20,22 +20,22 @@
 	"author": "cepharum GmbH",
 	"license": "MIT",
 	"dependencies": {
-		"ajv": "^8.11.0",
+		"ajv": "^8.12.0",
 		"file-essentials": "^0.1.2",
 		"lodash.merge": "^4.6.2",
-		"minimist": "^1.2.6",
+		"minimist": "^1.2.8",
 		"promise-essentials": "^0.2.0",
-		"simple-terms": "^0.4.0",
-		"yaml": "^2.1.0"
+		"simple-terms": "^0.4.1",
+		"yaml": "^2.2.2"
 	},
 	"devDependencies": {
-		"c8": "^7.11.3",
-		"eslint": "^8.16.0",
-		"eslint-config-cepharum": "^1.0.12",
-		"eslint-plugin-promise": "^6.0.0",
-		"mocha": "^10.0.0",
+		"c8": "^7.13.0",
+		"eslint": "^8.40.0",
+		"eslint-config-cepharum": "^1.0.13",
+		"eslint-plugin-promise": "^6.1.1",
+		"mocha": "^10.2.0",
 		"should": "^13.2.3",
-		"vuepress": "^1.9.7",
+		"vuepress": "^1.9.9",
 		"vuepress-plugin-mermaidjs": "^1.9.1"
 	},
 	"engines": {