@routier/core 0.0.1-alpha.1

package/src/plugins/replication/OptimisticReplicationDbPlugin.ts

@@ -0,0 +1,202 @@
+import { PluginEventCallbackPartialResult, PluginEventCallbackResult, PluginEventResult, Result } from '../../results';
+import { DbPluginBulkPersistEvent, DbPluginEvent, DbPluginQueryEvent, IDbPlugin, OptimisticReplicationPluginOptions } from '../types';
+import { WorkPipeline } from '../../pipeline'; 7
+import { CompiledSchema } from '../../schema';
+import { Query } from '../query';
+import { resolveBulkPersistChanges, uuid } from '../../utilities';
+import { BulkPersistChanges, BulkPersistResult } from '../../collections';
+
+const getMemoryPluginCollectionSize = <T extends {}>(plugin: IDbPlugin, schema: CompiledSchema<T>): number => {
+
+    if ("getCollectionSize" in plugin && typeof plugin.getCollectionSize === "function") {
+        return plugin.getCollectionSize(schema.collectionName) as number;
+    }
+
+    throw new Error("Cannot get size of collection for MemoryPlugin, not an instance of MemoryPlugin")
+}
+
+export class OptimisticReplicationDbPlugin implements IDbPlugin {
+
+    protected plugins: OptimisticReplicationPluginOptions;
+
+    protected constructor(plugins: OptimisticReplicationPluginOptions) {
+        this.plugins = plugins;
+    }
+
+    /**
+     * Creates a new OptimisticDbPluginReplicator that coordinates operations between a source database and its replicas.
+     * 
+     * @param source The primary database plugin that will receive all operations first
+     * @param replicas Additional database plugins that will replicate operations from the source
+     * @returns A new DbPluginReplicator instance that manages the source-replica relationship
+     */
+    static create(plugins: OptimisticReplicationPluginOptions) {
+        return new OptimisticReplicationDbPlugin(plugins);
+    }
+
+    /**
+     * Will query the read plugin if there is one, otherwise the source plugin will be queried
+    */
+    query<TEntity extends {}, TShape extends any = TEntity>(event: DbPluginQueryEvent<TEntity, TShape>, done: PluginEventCallbackResult<TShape>): void {
+        try {
+
+            const readPlugin = this.plugins.read;
+            const sourcePlugin = this.plugins.source;
+            const collectionSize = getMemoryPluginCollectionSize(this.plugins.read, event.operation.schema);
+
+            if (collectionSize === 0) {
+                // nothing is hydrated, let's try and hydrate before querying
+                // Memory plugin might not be hydrated, lets hydrate it for the targeted schema only,
+                // Other queries will do the same and hydrate if needed
+                // We want to select all data here
+                sourcePlugin.query<TEntity, TShape>({
+                    id: uuid(8),
+                    schemas: event.schemas,
+
+                    // Select All Data
+                    operation: Query.EMPTY<TEntity, TShape>(event.operation.schema)
+                }, (sourceResult) => {
+
+                    if (sourceResult.ok === Result.ERROR) {
+                        done(sourceResult);
+                        return;
+                    }
+
+                    if (sourceResult == null || (Array.isArray(sourceResult.data) && sourceResult.data.length === 0)) {
+                        done(sourceResult);
+                        return;
+                    }
+
+                    if (Array.isArray(sourceResult.data) === false) {
+                        done(PluginEventResult.error(event.id, "Query result is not an array"));
+                        return;
+                    }
+
+                    const changesCollection = new BulkPersistChanges();
+                    const schemaChanges = changesCollection.resolve(event.operation.schema.id);
+
+                    // Add the existing items into the persist payload as adds
+                    schemaChanges.adds = sourceResult.data;
+
+                    readPlugin.bulkPersist({
+                        id: uuid(8),
+                        schemas: event.schemas,
+                        operation: changesCollection
+                    }, (readPersistResult) => {
+
+                        if (readPersistResult.ok === Result.ERROR) {
+                            done(readPersistResult);
+                            return;
+                        }
+
+                        // requery the read plugin
+                        readPlugin.query(event, done);
+                    });
+                });
+                return;
+            }
+
+            // Collection is hydrated for the targeted collection and should be in sync
+            readPlugin.query(event, (readResult) => {
+
+                if (readResult.ok === Result.ERROR) {
+                    done(readResult);
+                    return;
+                }
+
+                done(readResult);
+                return;
+
+            });
+        } catch (e: any) {
+            done(PluginEventResult.error(event.id, e));
+        }
+    }
+
+    destroy(event: DbPluginEvent, done: PluginEventCallbackResult<never>): void {
+        try {
+
+            const workPipeline = new WorkPipeline();
+            const plugins = [this.plugins.source, ...this.plugins.replicas];
+
+            for (let i = 0, length = plugins.length; i < length; i++) {
+                workPipeline.pipe((done) => plugins[i].destroy(event, done));
+            }
+
+            workPipeline.filter((result) => {
+
+                if (result.ok === Result.ERROR) {
+                    done(PluginEventResult.error(event.id, result.error));
+                    return;
+                }
+
+                done(PluginEventResult.success(event.id));
+            });
+
+        } catch (e: any) {
+            done(PluginEventResult.error(event.id, e));
+        }
+    }
+
+    bulkPersist(event: DbPluginBulkPersistEvent, done: PluginEventCallbackPartialResult<BulkPersistResult>): void {
+        try {
+
+            const workPipeline = new WorkPipeline();
+            const deferredPlugins = [this.plugins.source, ... this.plugins.replicas];
+
+            // Since we are doing optimistic, we insert into the read plugin first and assume later plugins will succeed
+            // This means the read plugin will generate ids for the source plugin
+            this.plugins.read.bulkPersist({
+                id: uuid(8),
+                operation: event.operation,
+                schemas: event.schemas
+            }, (r) => {
+
+                if (r.ok !== Result.SUCCESS) {
+                    done(r);
+                    return;
+                }
+
+                // optimistically call done and still continue saving the rest of the data.  We read from the memory
+                // db anyways
+                done(r);
+
+                const optimisticBulkPersistChanges = new BulkPersistChanges();
+
+                // make sure we swap the adds here, that way we can make sure other persist events
+                // don't take their additions and try to change subsequent calls
+                resolveBulkPersistChanges(event, r.data, optimisticBulkPersistChanges);
+
+                for (let i = 0, length = deferredPlugins.length; i < length; i++) {
+                    workPipeline.pipe((d) => {
+
+                        const plugin = deferredPlugins[i];
+
+                        plugin.bulkPersist({
+                            id: uuid(8),
+                            operation: optimisticBulkPersistChanges,
+                            schemas: event.schemas
+                        }, (r) => {
+
+                            if (r.ok === Result.ERROR) {
+                                d(r);
+                                return;
+                            }
+
+                            d(Result.success());
+                        });
+                    });
+                }
+
+                setTimeout(() => {
+                    workPipeline.filter((_) => {
+                        // no-op for now, maybe implement retries?
+                    });
+                }, 5);
+            });
+
+        } catch (e: any) {
+            done(PluginEventResult.error(event.id, e));
+        }
+    }
+}

package/src/plugins/replication/ReplicationDbPlugin.ts

@@ -0,0 +1,137 @@
+import { PluginEventCallbackPartialResult, PluginEventCallbackResult, PluginEventResult, Result } from '../../results';
+import { DbPluginBulkPersistEvent, DbPluginEvent, DbPluginQueryEvent, IDbPlugin, ReplicationPluginOptions } from '../types';
+import { AsyncPipeline, WorkPipeline } from '../../pipeline';
+import { BulkPersistResult } from '../../collections';
+import { resolveBulkPersistChanges } from '../../utilities';
+
+export class ReplicationDbPlugin implements IDbPlugin {
+
+    plugins: ReplicationPluginOptions;
+
+    protected constructor(plugins: ReplicationPluginOptions) {
+        this.plugins = plugins;
+    }
+
+    /**
+     * Creates a new DbPluginReplicator that coordinates operations between a source database and its replicas.
+     * 
+     * @param source The primary database plugin that will receive all operations first
+     * @param replicas Additional database plugins that will replicate operations from the source
+     * @returns A new DbPluginReplicator instance that manages the source-replica relationship
+     */
+    static create(plugins: ReplicationPluginOptions) {
+        return new ReplicationDbPlugin(plugins);
+    }
+
+    /**
+     * Will query the read plugin if there is one, otherwise the source plugin will be queried
+    */
+    query<TEntity extends {}, TShape extends any = TEntity>(event: DbPluginQueryEvent<TEntity, TShape>, done: PluginEventCallbackResult<TShape>): void {
+        try {
+
+            const plugin = this.plugins.read != null ? this.plugins.read : this.plugins.source;
+
+            plugin.query(event, done);
+        } catch (e: any) {
+            done(PluginEventResult.error(event.id, e));
+        }
+    }
+
+    destroy(event: DbPluginEvent, done: PluginEventCallbackResult<never>): void {
+        try {
+
+            const pipeline = new AsyncPipeline<IDbPlugin, never>();
+            const plugins = [this.plugins.source, ...this.plugins.replicas];
+
+            for (let i = 0, length = plugins.length; i < length; i++) {
+                pipeline.pipe(plugins[i], (plugin, done) => plugin.destroy(event, done));
+            }
+
+            pipeline.filter((result) => {
+
+                if (result.ok === Result.ERROR) {
+                    done(PluginEventResult.error(event.id, result.error));
+                    return;
+                }
+
+                done(PluginEventResult.success(event.id));
+            });
+
+        } catch (e: any) {
+            done(PluginEventResult.error(event.id, e));
+        }
+    }
+
+
+    bulkPersist(event: DbPluginBulkPersistEvent, done: PluginEventCallbackPartialResult<BulkPersistResult>): void {
+        try {
+            // insert into the source first to generate any ids, then take the result and persist that into the replicas
+            const pipeline = new WorkPipeline();
+            const plugins = [this.plugins.source, ...this.plugins.replicas];
+
+            if (this.plugins.read != null) {
+                plugins.push(this.plugins.read);
+            }
+
+            const result = new BulkPersistResult();
+
+            for (let i = 0, length = plugins.length; i < length; i++) {
+                pipeline.pipe((d) => {
+
+                    const plugin = plugins[i];
+
+                    // source is first
+                    if (i === 0) {
+                        plugin.bulkPersist(event, (r) => {
+
+                            if (r.ok === Result.ERROR) {
+                                d(r)
+                                return;
+                            }
+
+                            // make sure we swap the adds here, that way we can make sure other persist events
+                            // don't take their additions and try to change subsequent calls
+                            resolveBulkPersistChanges(event, r.data, event.operation);
+
+                            d(Result.success());
+                        });
+                        return;
+                    }
+
+                    plugin.bulkPersist(event, (r) => {
+
+                        if (r.ok === Result.ERROR) {
+                            d(r)
+                            return;
+                        }
+
+                        d(Result.success());
+                    });
+                })
+            }
+
+            let successCount = 0;
+
+            pipeline.filter((r) => {
+
+                if (r.ok === Result.ERROR) {
+
+                    if (successCount > 0) {
+                        done(PluginEventResult.partial(event.id, result, r.error));
+                        return;
+                    }
+
+                    done(PluginEventResult.error(event.id, r.error));
+                    return;
+                }
+
+                successCount++;
+
+                done(PluginEventResult.success(event.id, result));
+            });
+
+        } catch (e: any) {
+            done(PluginEventResult.error(event.id, e));
+        }
+    }
+}

package/src/plugins/replication/index.ts

@@ -0,0 +1,3 @@
+export * from './ReplicationDbPlugin';
+export * from './OptimisticReplicationDbPlugin';
+export * from './types';

package/src/plugins/replication/types.ts

@@ -0,0 +1,5 @@
+import { IDbPlugin, ReplicationPluginOptions, OptimisticReplicationPluginOptions } from '../types';
+
+export type IDbPluginReplicator = IDbPlugin & {
+    plugins: ReplicationPluginOptions | OptimisticReplicationPluginOptions;
+}

package/src/plugins/translators/DataTranslator.ts

@@ -0,0 +1,46 @@
+import { QueryOption, QueryOptionName } from "../query/types";
+import { IQuery } from "../types";
+
+export abstract class DataTranslator<TRoot extends {}, TShape> {
+
+    protected query: IQuery<TRoot, TShape>;
+    private functionMap: Record<QueryOptionName, (data: unknown, option: QueryOption<any, any>) => TShape> = {
+        count: (data: unknown, option: QueryOption<TShape, "count">) => this.count<any>(data, option),
+        distinct: (data: unknown, option: QueryOption<TShape, "distinct">) => this.distinct(data, option),
+        filter: (data: unknown, option: QueryOption<TShape, "filter">) => this.filter(data, option),
+        map: (data: unknown, option: QueryOption<TShape, "map">) => this.map(data, option),
+        max: (data: unknown, option: QueryOption<TShape, "max">) => this.max<any>(data, option),
+        min: (data: unknown, option: QueryOption<TShape, "min">) => this.min<any>(data, option),
+        skip: (data: unknown, option: QueryOption<TShape, "skip">) => this.skip(data, option),
+        sort: (data: unknown, option: QueryOption<TShape, "sort">) => this.sort(data, option),
+        sum: (data: unknown, option: QueryOption<TShape, "sum">) => this.sum<any>(data, option),
+        take: (data: unknown, option: QueryOption<TShape, "take">) => this.take(data, option)
+    };
+
+    constructor(query: IQuery<TRoot, TShape>) {
+        this.query = query;
+    }
+
+    // Terminators
+    abstract count<TResult extends number>(data: unknown, option: QueryOption<TShape, "count">): TResult;
+    abstract min<TResult extends string | number | Date>(data: unknown, option: QueryOption<TShape, "min">): TResult;
+    abstract max<TResult extends string | number | Date>(data: unknown, option: QueryOption<TShape, "max">): TResult;
+    abstract sum<TResult extends number>(data: unknown, option: QueryOption<TShape, "sum">): TResult;
+    abstract distinct<TResult>(data: unknown, option: QueryOption<TShape, "distinct">): TResult;
+
+    // Shapers
+    abstract filter<TResult>(data: unknown, option: QueryOption<TShape, "filter">): TResult;
+    abstract skip(data: unknown, option: QueryOption<TShape, "skip">): TShape;
+    abstract take(data: unknown, option: QueryOption<TShape, "take">): TShape;
+    abstract sort(data: unknown, option: QueryOption<TShape, "sort">): TShape;
+    abstract map(data: unknown, option: QueryOption<TShape, "map">): TShape;
+
+    translate(data: unknown): TShape {
+
+        this.query.options.forEach(item => {
+            data = this.functionMap[item.name](data, item);
+        });
+
+        return data as TShape;
+    }
+}