@cepharum/concrete-db 0.1.2-alpha.1 → 0.2.0

package/concrete-db.d.ts

@@ -1,5 +1,5 @@
 /**
- * (c) 2021 cepharum GmbH, Berlin, http://cepharum.de
+ * (c) 2022 cepharum GmbH, Berlin, http://cepharum.de
  *
  * The MIT License (MIT)
  *
@@ -32,6 +32,13 @@ declare module "concrete-db" {
 		 * Selects base folder for writing files of resulting concrete-db.
 		 */
 		outputFolder?: string;
+
+		/**
+		 * Lists custom functions to expose in simple terms used in shape
+		 * processing. For security reasons, you cannot replace regular term
+		 * functions this way by intention.
+		 */
+		library?: { [functionName: string]: () => any };
 	}
 
 	export interface CollectorOptions {
@@ -65,6 +72,11 @@ declare module "concrete-db" {
 		 */
 		records: Map<string, CollectedRecord[]>
 
+		/**
+		 * Exposes local shapes of processed folders in order of processing.
+		 */
+		localShapes: Shape[];
+
 		/**
 		 * Collects sources from provided folders.
 		 *
@@ -83,8 +95,9 @@ declare module "concrete-db" {
 		 * with associated data to expose.
 		 *
 		 * @param records
+		 * @param localShapes local shapes of folders records have been collected from in same order as folders processed
 		 */
-		transform( records: CollectedSources ): Promise<Database>;
+		transform( records: CollectedSources, localShapes: Shape[] ): Promise<Database>;
 	}
 
 	/**
@@ -147,17 +160,17 @@ declare module "concrete-db" {
 	/**
 	 * Describes a model's item processed during shaping stage.
 	 */
-	export interface CollectedItem {
+	export interface ModelItem {
 		/**
-		 * Refers to record this item is based on.
+		 * Lists segments of relative path of file this item's originating
+		 * record has been read from.
 		 */
-		$original: CollectedRecord;
+		$segments: string[];
 
 		/**
-		 * Exposes local shape declared in context of record item was collected
-		 * from.
+		 * Refers to record this item is based on.
 		 */
-		$localShape: ShapeModel;
+		$original: CollectedRecord;
 
 		/**
 		 * Exposes shape of item's model in context of its originating record.
@@ -174,12 +187,6 @@ declare module "concrete-db" {
 		 */
 		$id: string;
 
-		/**
-		 * Lists segments of relative path of file this item's originating
-		 * record has been read from.
-		 */
-		$segments: string[];
-
 		/**
 		 * item's properties
 		 */
@@ -198,6 +205,22 @@ declare module "concrete-db" {
 	 * of model or an such an item's representation in a collection).
 	 */
 	export interface ShapeProperties {
+		/**
+		 * Provides term computed to get the name of a record's model.
+		 */
+		$model?: string;
+
+		/**
+		 * Provides term computed to get ID of item described by a record.
+		 */
+		$id?: string;
+
+		/**
+		 * Provides term computed to get ID of selected item's variant described
+		 * by record.
+		 */
+		$variant?: string;
+
 		[propertyName: string]: any;
 	}
 
@@ -249,13 +272,6 @@ declare module "concrete-db" {
 				[targetPropertyName: string]: string;
 			};
 		};
-
-		/**
-		 * Declares term to compute an item's variant. If falsy or omitted, the
-		 * model does not have variants and multiple records describing an item
-		 * are merged property by property.
-		 */
-		variant?: string;
 	}
 
 	export interface Shape {

package/cure.mjs

@@ -22,10 +22,15 @@ Processes current folder by default.
 
 Supported options are:
 
- -h  /  --help                showing this information
- -V  /  --version             showing version of this package
- -s  /  --shape=file          selects global shape controlling transformation
- -o  /  --output=folder       selects folder resulting files are written to
+ -h  /  --help              showing this information
+ -V  /  --version           showing version of this package
+ -v  /  --verbose           be verbose about the progress of curing database
+        --debug             be extra verbose
+ -s  /  --shape=file        selects global shape controlling transformation
+ -o  /  --output=folder     selects folder resulting files are written to
+ -c  /  --clean             remove all existing files from output folder
+ -l  /  --library=file      selects file exposing custom term functions to use
+                            in addition to regular ones
 ` );
 	process.exit( 0 );
 }
@@ -40,10 +45,38 @@ if ( args.V || args.version ) {
 ( async() => {
 	const options = {
 		outputFolder: args.o || args.output || process.cwd(),
+		verbose: args.debug ? 2 : args.v || args.verbose ? 1 : 0,
+		clean: Boolean( args.c || args.clean ),
 	};
 
 	const globalShapeFile = args.s || args.shape || Path.resolve( process.cwd(), ShapeFileName );
 
+	const libraries = Array.isArray( args.l ) ? args.l : args.l ? [args.l] : [];
+	libraries.splice( libraries.length, 0, ...Array.isArray( args.library ) ? args.library : args.library ? [args.library] : [] );
+
+	options.library = {};
+	for ( const library of libraries ) {
+		let pathname = Path.resolve( ".", library );
+
+		if ( options.verbose ) {
+			console.error( `including library ${pathname}` );
+		}
+
+		if ( /^[a-z]:/i.test( pathname ) ) {
+			pathname = "file://" + pathname.replace( /\//g, "%2F" );
+		}
+
+		const functions = await import( pathname ); // eslint-disable-line no-await-in-loop
+
+		for ( const name of Object.keys( functions ) ) {
+			if ( options.verbose > 1 ) {
+				console.error( ` - ${name}` );
+			}
+
+			options.library[name] = functions[name];
+		}
+	}
+
 	try {
 		options.shape = Object.freeze( YAML.parse( await File.promises.readFile( globalShapeFile, { encoding: "utf8" } ), YAMLParserOptions ) );
 	} catch ( cause ) {

package/lib/collector.mjs

@@ -35,6 +35,7 @@ export class Collector extends EventEmitter {
 				enumerable: true
 			},
 			records: { value: new Map(), enumerable: true },
+			localShapes: { value: [], enumerable: true },
 		} );
 	}
 
@@ -72,10 +73,19 @@ export class Collector extends EventEmitter {
 	 * @returns {Promise<Collector>} promises collector having collected records from folders
 	 */
 	async fromFolders( folders ) {
+		const spinner = "⠁⠂⠄⡀⢀⠠⠐⠈";
 		const { pattern, nameToKey } = this.options;
 
-		await PromiseEssentials.each( folders || [], async folder => {
+		await PromiseEssentials.each( folders || [], async( folder, index ) => {
+			if ( this.options.verbose ) {
+				process.stderr.write( `collecting records from folder ${folder} [${index + 1}/${folders.length}] ... ` );
+			}
+
 			const localShape = await this.getLocalShape( folder );
+			let total = 0;
+			let additional = 0;
+
+			this.localShapes.push( localShape );
 
 			await FileEssentials.find( folder, {
 				filter: ( localPath, fullPath, stat ) => {
@@ -105,15 +115,26 @@ export class Collector extends EventEmitter {
 
 						if ( this.records.has( name ) ) {
 							this.records.get( name ).push( record );
+							additional++;
 						} else {
 							this.records.set( name, [record] );
 						}
+
+						total++;
+
+						if ( this.options.verbose ) {
+							process.stderr.write( `${spinner[Math.floor( total / 10 ) % spinner.length]}\x08` );
+						}
 					}
 
 					return undefined;
 				},
 				waitForConverter: true,
 			} );
+
+			if ( this.options.verbose ) {
+				process.stderr.write( `${total} records (${additional} overlays)\n` );
+			}
 		} );
 
 		return this;

package/lib/cure.mjs

@@ -31,6 +31,6 @@ export async function cure( folders, options = {}, stages = {} ) {
 	}
 
 	const collector = await _stages.collector.fromFolders( sources );
-	const database = await _stages.shaper.transform( collector.records );
+	const database = await _stages.shaper.transform( collector.records, collector.localShapes );
 	await _stages.generator.writeDatabase( database );
 }

package/lib/default.shape.yaml

@@ -1,6 +1,8 @@
 root: true
 
 common:
+  defaults: {}
+
   properties:
     $model: model
     $id: id

package/lib/generator.mjs

@@ -40,6 +40,10 @@ export class Generator extends EventEmitter {
 		await FileEssentials.mkdir( "/", subfolder );
 
 		if ( this.options.clean ) {
+			if ( this.options.verbose ) {
+				console.error( `cleaning output folder ${subfolder} ...` );
+			}
+
 			await FileEssentials.find( subfolder, {
 				depthFirst: true,
 				filter: ( localPath, fullPath, stat ) => {
@@ -49,11 +53,11 @@ export class Generator extends EventEmitter {
 
 					return stat.isFile();
 				},
-				converter: ( localPath, fullPath ) => {
+				converter: ( localPath, fullPath, stat ) => {
 					const name = fullPath.replace( /\.json$/, "" );
 
 					if ( !database.hasOwnProperty( name ) ) {
-						return File.promises.unlink( fullPath );
+						return stat.isDirectory() ? File.promises.rmdir( fullPath ) : File.promises.unlink( fullPath );
 					}
 
 					return undefined;
@@ -62,14 +66,29 @@ export class Generator extends EventEmitter {
 			} );
 		}
 
-		await PromiseEssentials.each( Object.keys( endpoints ), key => {
+		const keys = Object.keys( endpoints );
+		let index = 0;
+
+		if ( this.options.verbose ) {
+			console.error( `writing ${keys.length} files` );
+		}
+
+		await PromiseEssentials.each( keys, key => {
 			const name = key.endsWith( ".json" ) ? key : key + ".json";
 			const path = name.indexOf( "/" ) > -1 ? name.replace( /\/[^/]*?$/, "" ) : undefined;
 
+			if ( this.options.verbose ) {
+				process.stderr.write( `\r[${++index}/${keys.length}]` );
+			}
+
 			return FileEssentials.mkdir( folder, path )
 				.then( () => File.promises.writeFile( Path.resolve( folder, name ), JSON.stringify( endpoints[key] ), { encoding: "utf8" } ) );
 		} );
 
+		if ( this.options.verbose ) {
+			process.stderr.write( "\n" );
+		}
+
 		return this;
 	}
 }

package/lib/helper.mjs

@@ -1,3 +1,5 @@
+export * from "simple-terms/lib/helper.js";
+
 /**
  * Wraps provided object in a proxy offering case-insensitive access on
  * object's properties. Any accessed property's value is wrapped before