@sschepis/magazine 0.1.0

package/src/components/DataTransformer.js

@@ -0,0 +1,1172 @@
+/**
+ * DataTransformer - Transforms blockchain events into structured data models using GunDB
+ * Inspired by @nomyx/gundb but integrated with Magazine's existing architecture
+ */
+import Logger from './Logger.js';
+
+/**
+ * Custom error classes for data transformation
+ */
+export class ValidationError extends Error {
+    constructor(message, field = null) {
+        super(message);
+        this.name = 'ValidationError';
+        this.field = field;
+    }
+}
+
+export class ModelError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ModelError';
+    }
+}
+
+/**
+ * SchemaMigration - Tracks schema versions and migrates documents between versions.
+ */
+export class SchemaMigration {
+    constructor(modelName, gunInstance) {
+        this.modelName = modelName;
+        this.gun = gunInstance;
+        this.migrations = new Map();
+        this.currentVersion = 1;
+    }
+
+    version(v) {
+        this.currentVersion = v;
+        return this;
+    }
+
+    addMigration(fromVersion, toVersion, migrateFn) {
+        this.migrations.set(`${fromVersion}:${toVersion}`, migrateFn);
+        return this;
+    }
+
+    async migrate(document) {
+        let docVersion = document._schemaVersion || 1;
+        let migrated = { ...document };
+        while (docVersion < this.currentVersion) {
+            const nextVersion = docVersion + 1;
+            const key = `${docVersion}:${nextVersion}`;
+            const migrateFn = this.migrations.get(key);
+            if (!migrateFn) {
+                throw new ModelError(
+                    `No migration path from version ${docVersion} to ${nextVersion} for model '${this.modelName}'`
+                );
+            }
+            migrated = await migrateFn(migrated);
+            migrated._schemaVersion = nextVersion;
+            docVersion = nextVersion;
+        }
+        return migrated;
+    }
+
+    needsMigration(document) {
+        return (document._schemaVersion || 1) < this.currentVersion;
+    }
+}
+
+/**
+ * Simple schema validator (can be extended to use Joi or other validation libraries)
+ */
+class SchemaValidator {
+    static validate(data, schema) {
+        const errors = [];
+        
+        // Check required fields
+        for (const field of schema.required || []) {
+            if (!(field in data) || data[field] === null || data[field] === undefined) {
+                errors.push(`Field '${field}' is required`);
+            }
+        }
+        
+        // Check field types
+        for (const [field, definition] of Object.entries(schema.properties || {})) {
+            if (field in data && data[field] !== null && data[field] !== undefined) {
+                const value = data[field];
+                const { type, validate: customValidator } = definition;
+                
+                // Type validation
+                if (type && !this.validateType(value, type)) {
+                    errors.push(`Field '${field}' must be of type ${type}`);
+                }
+                
+                // Custom validation
+                if (customValidator && typeof customValidator === 'function') {
+                    const customResult = customValidator(value);
+                    if (customResult !== true) {
+                        errors.push(`Field '${field}': ${customResult || 'validation failed'}`);
+                    }
+                }
+            }
+        }
+        
+        return {
+            valid: errors.length === 0,
+            errors
+        };
+    }
+    
+    static validateType(value, type) {
+        switch (type) {
+            case 'string':
+                return typeof value === 'string';
+            case 'number':
+                return typeof value === 'number' && !isNaN(value);
+            case 'boolean':
+                return typeof value === 'boolean';
+            case 'array':
+                return Array.isArray(value);
+            case 'object':
+                return typeof value === 'object' && value !== null && !Array.isArray(value);
+            case 'address':
+                return typeof value === 'string' && /^0x[a-fA-F0-9]{40}$/.test(value);
+            case 'hash':
+                return typeof value === 'string' && /^0x[a-fA-F0-9]{64}$/.test(value);
+            default:
+                return true;
+        }
+    }
+}
+
+/**
+ * DataModel - Represents a structured data model with CRUD operations
+ */
+export class DataModel {
+    constructor(name, schema, gunInstance, transformer, options = {}) {
+        this.name = name;
+        this.schema = schema;
+        this.gun = gunInstance;
+        this.transformer = transformer;
+        this.options = {
+            autoId: true,
+            timestampFields: ['createdAt', 'updatedAt'],
+            ...options
+        };
+        
+        this.logger = new Logger({
+            context: `DataModel:${name}`,
+            level: options.logLevel || 'info'
+        });
+        
+        // Get model's Gun node
+        this.modelNode = this.gun.get('models').get(this.name);
+
+        // Lifecycle hooks: { 'pre:save': [fn], 'post:save': [fn], ... }
+        this._hooks = {};
+
+        // Schema migration instance (set via setMigration)
+        this.migration = null;
+
+        this.logger.info('DataModel initialized', {
+            name: this.name,
+            options: this.options
+        });
+    }
+
+    /**
+     * Register a lifecycle hook
+     * @param {'pre:save'|'post:save'|'pre:update'|'post:update'|'pre:delete'|'post:delete'} event
+     * @param {Function} fn - Hook function. Pre-hooks receive (data) and can return modified data. Post-hooks receive (result).
+     */
+    hook(event, fn) {
+        if (!this._hooks[event]) this._hooks[event] = [];
+        this._hooks[event].push(fn);
+        return this;
+    }
+
+    /**
+     * Attach a SchemaMigration instance so documents are auto-migrated on read.
+     * @param {SchemaMigration} schemaMigration
+     * @returns {DataModel} this (for chaining)
+     */
+    setMigration(schemaMigration) {
+        this.migration = schemaMigration;
+        return this;
+    }
+
+    async _runHooks(event, data) {
+        const hooks = this._hooks[event] || [];
+        let result = data;
+        for (const fn of hooks) {
+            const out = await fn(result);
+            if (out !== undefined) result = out;
+        }
+        return result;
+    }
+
+    /**
+     * Generate a unique ID for a new document
+     * @returns {string} Unique ID
+     */
+    generateId() {
+        return `${this.name}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+    }
+
+    /**
+     * Validate data against schema
+     * @param {Object} data - Data to validate
+     * @throws {ValidationError} If validation fails
+     */
+    validate(data) {
+        const result = SchemaValidator.validate(data, this.schema);
+        if (!result.valid) {
+            throw new ValidationError(`Validation failed: ${result.errors.join(', ')}`);
+        }
+    }
+
+    /**
+     * Prepare data for storage (add timestamps, ID, etc.)
+     * @param {Object} data - Raw data
+     * @param {boolean} isUpdate - Whether this is an update operation
+     * @returns {Object} Prepared data
+     */
+    prepareData(data, isUpdate = false) {
+        const prepared = { ...data };
+        
+        // Add auto-generated ID if needed
+        if (this.options.autoId && !prepared.id && !isUpdate) {
+            prepared.id = this.generateId();
+        }
+        
+        // Add timestamps
+        if (this.options.timestampFields.includes('updatedAt')) {
+            prepared.updatedAt = new Date().toISOString();
+        }
+        
+        if (this.options.timestampFields.includes('createdAt') && !isUpdate) {
+            prepared.createdAt = new Date().toISOString();
+        }
+
+        if (this.migration) {
+            prepared._schemaVersion = this.migration.currentVersion;
+        }
+
+        return prepared;
+    }
+
+    /**
+     * Save a document (create or update)
+     * @param {Object} data - Document data
+     * @returns {Promise<Object>} Saved document
+     */
+    async save(data) {
+        this.logger.debug('Saving document', { data });
+
+        try {
+            let prepared = this.prepareData(data);
+            prepared = await this._runHooks('pre:save', prepared);
+            this.validate(prepared);
+
+            const id = prepared.id;
+            if (!id) {
+                throw new ModelError('Document must have an ID');
+            }
+
+            await new Promise((resolve, reject) => {
+                this.modelNode.get(id).put(prepared, (ack) => {
+                    if (ack.err) {
+                        reject(new ModelError(`Failed to save document: ${ack.err}`));
+                    } else {
+                        resolve();
+                    }
+                });
+            });
+
+            await this._runHooks('post:save', prepared);
+            this.logger.info('Document saved', { id, model: this.name });
+            return prepared;
+
+        } catch (error) {
+            this.logger.error('Failed to save document', { error: error.message, data });
+            throw error;
+        }
+    }
+
+    /**
+     * Find documents by query
+     * @param {Object} query - Query object
+     * @param {Object} options - Query options
+     * @returns {Promise<Array>} Found documents
+     */
+    async find(query = {}, options = {}) {
+        this.logger.debug('Finding documents', { query, options });
+        
+        try {
+            const { limit = 100, skip = 0 } = options;
+            const results = [];
+            
+            await new Promise((resolve) => {
+                let count = 0;
+                let skipped = 0;
+                
+                this.modelNode.map().on((data, key) => {
+                    if (data && typeof data === 'object') {
+                        // Simple query matching (can be enhanced with more complex queries)
+                        if (this.matchesQuery(data, query)) {
+                            if (skipped < skip) {
+                                skipped++;
+                                return;
+                            }
+                            
+                            if (count < limit) {
+                                results.push({ ...data, _key: key });
+                                count++;
+                            }
+                            
+                            if (count >= limit) {
+                                resolve();
+                            }
+                        }
+                    }
+                });
+                
+                // Resolve after a timeout to handle cases with fewer results
+                setTimeout(resolve, 1000);
+            });
+            
+            if (this.migration) {
+                for (let i = 0; i < results.length; i++) {
+                    if (this.migration.needsMigration(results[i])) {
+                        results[i] = await this.migration.migrate(results[i]);
+                        await this.save(results[i]);
+                    }
+                }
+            }
+
+            this.logger.debug('Documents found', { count: results.length, query });
+            return results;
+            
+        } catch (error) {
+            this.logger.error('Failed to find documents', { error: error.message, query });
+            throw new ModelError(`Failed to find documents: ${error.message}`);
+        }
+    }
+
+    /**
+     * Find a single document by query
+     * @param {Object} query - Query object
+     * @returns {Promise<Object|null>} Found document or null
+     */
+    async findOne(query) {
+        const results = await this.find(query, { limit: 1 });
+        return results.length > 0 ? results[0] : null;
+    }
+
+    /**
+     * Find a document by ID
+     * @param {string} id - Document ID
+     * @returns {Promise<Object|null>} Found document or null
+     */
+    async findById(id) {
+        this.logger.debug('Finding document by ID', { id });
+
+        try {
+            let doc = await new Promise((resolve) => {
+                this.modelNode.get(id).once((data) => {
+                    if (data && typeof data === 'object') {
+                        resolve({ ...data, _key: id });
+                    } else {
+                        resolve(null);
+                    }
+                });
+            });
+
+            if (doc && this.migration && this.migration.needsMigration(doc)) {
+                doc = await this.migration.migrate(doc);
+                await this.save(doc);
+            }
+
+            return doc;
+        } catch (error) {
+            this.logger.error('Failed to find document by ID', { error: error.message, id });
+            throw new ModelError(`Failed to find document: ${error.message}`);
+        }
+    }
+
+    /**
+     * Update a document by ID
+     * @param {string} id - Document ID
+     * @param {Object} updateData - Data to update
+     * @returns {Promise<Object>} Updated document
+     */
+    async update(id, updateData) {
+        this.logger.debug('Updating document', { id, updateData });
+
+        try {
+            const existing = await this.findById(id);
+            if (!existing) {
+                throw new ModelError(`Document with ID ${id} not found`);
+            }
+
+            const updated = { ...existing, ...updateData, id };
+            let prepared = this.prepareData(updated, true);
+            prepared = await this._runHooks('pre:update', prepared);
+            this.validate(prepared);
+
+            await new Promise((resolve, reject) => {
+                this.modelNode.get(id).put(prepared, (ack) => {
+                    if (ack.err) {
+                        reject(new ModelError(`Failed to update document: ${ack.err}`));
+                    } else {
+                        resolve();
+                    }
+                });
+            });
+
+            await this._runHooks('post:update', prepared);
+            this.logger.info('Document updated', { id, model: this.name });
+            return prepared;
+
+        } catch (error) {
+            this.logger.error('Failed to update document', { error: error.message, id, updateData });
+            throw error;
+        }
+    }
+
+    /**
+     * Delete a document by ID
+     * @param {string} id - Document ID
+     * @returns {Promise<boolean>} Success status
+     */
+    async delete(id) {
+        this.logger.debug('Deleting document', { id });
+
+        try {
+            await this._runHooks('pre:delete', { id });
+
+            await new Promise((resolve, reject) => {
+                this.modelNode.get(id).put(null, (ack) => {
+                    if (ack.err) {
+                        reject(new ModelError(`Failed to delete document: ${ack.err}`));
+                    } else {
+                        resolve();
+                    }
+                });
+            });
+
+            await this._runHooks('post:delete', { id });
+            this.logger.info('Document deleted', { id, model: this.name });
+            return true;
+
+        } catch (error) {
+            this.logger.error('Failed to delete document', { error: error.message, id });
+            throw error;
+        }
+    }
+
+    /**
+     * Count documents matching query
+     * @param {Object} query - Query object
+     * @returns {Promise<number>} Document count
+     */
+    async count(query = {}) {
+        const results = await this.find(query, { limit: 10000 });
+        return results.length;
+    }
+
+    /**
+     * Insert multiple documents
+     * @param {Array<Object>} documents - Array of documents to insert
+     * @returns {Promise<Array<Object>>} Saved documents
+     */
+    async insertMany(documents) {
+        if (!Array.isArray(documents)) {
+            throw new ModelError('insertMany requires an array of documents');
+        }
+        const results = [];
+        const errors = [];
+        for (const doc of documents) {
+            try {
+                results.push(await this.save(doc));
+            } catch (error) {
+                errors.push({ document: doc, error: error.message });
+            }
+        }
+        if (errors.length > 0) {
+            this.logger.warn('Some documents failed to insert', {
+                total: documents.length, succeeded: results.length, failed: errors.length
+            });
+        }
+        return { inserted: results, errors };
+    }
+
+    /**
+     * Update multiple documents matching a query
+     * @param {Object} query - Query to match documents
+     * @param {Object} updateData - Data to update on each matched document
+     * @returns {Promise<Object>} Update results
+     */
+    async updateMany(query, updateData) {
+        const docs = await this.find(query, { limit: 100000 });
+        const results = [];
+        const errors = [];
+        for (const doc of docs) {
+            const id = doc.id || doc._key;
+            if (!id) continue;
+            try {
+                results.push(await this.update(id, updateData));
+            } catch (error) {
+                errors.push({ id, error: error.message });
+            }
+        }
+        this.logger.info('updateMany completed', {
+            matched: docs.length, updated: results.length, failed: errors.length
+        });
+        return { matched: docs.length, updated: results, errors };
+    }
+
+    /**
+     * Delete multiple documents matching a query
+     * @param {Object} query - Query to match documents for deletion
+     * @returns {Promise<Object>} Deletion results
+     */
+    async deleteMany(query) {
+        const docs = await this.find(query, { limit: 100000 });
+        let deleted = 0;
+        const errors = [];
+        for (const doc of docs) {
+            const id = doc.id || doc._key;
+            if (!id) continue;
+            try {
+                await this.delete(id);
+                deleted++;
+            } catch (error) {
+                errors.push({ id, error: error.message });
+            }
+        }
+        this.logger.info('deleteMany completed', {
+            matched: docs.length, deleted, failed: errors.length
+        });
+        return { matched: docs.length, deleted, errors };
+    }
+
+    /**
+     * Migrate all documents in this model to the current schema version.
+     * @returns {Promise<Object>} Migration results { migrated, errors }
+     */
+    async migrateAll() {
+        if (!this.migration) {
+            throw new ModelError('No migration configured for this model. Call setMigration() first.');
+        }
+        const docs = await this.find({}, { limit: 100000 });
+        let migrated = 0;
+        const errors = [];
+        for (const doc of docs) {
+            if (this.migration.needsMigration(doc)) {
+                try {
+                    const updated = await this.migration.migrate(doc);
+                    await this.save(updated);
+                    migrated++;
+                } catch (error) {
+                    errors.push({ id: doc.id || doc._key, error: error.message });
+                }
+            }
+        }
+        this.logger.info('migrateAll completed', {
+            total: docs.length, migrated, failed: errors.length
+        });
+        return { total: docs.length, migrated, errors };
+    }
+
+    /**
+     * Simple query matching (can be enhanced with MongoDB-like operators)
+     * @private
+     * @param {Object} document - Document to test
+     * @param {Object} query - Query object
+     * @returns {boolean} Whether document matches query
+     */
+    matchesQuery(document, query) {
+        for (const [field, value] of Object.entries(query)) {
+            if (typeof value === 'object' && value !== null) {
+                // Handle operators like { age: { $gte: 18 } }
+                for (const [operator, operand] of Object.entries(value)) {
+                    switch (operator) {
+                        case '$gte':
+                            if (!(document[field] >= operand)) return false;
+                            break;
+                        case '$lte':
+                            if (!(document[field] <= operand)) return false;
+                            break;
+                        case '$gt':
+                            if (!(document[field] > operand)) return false;
+                            break;
+                        case '$lt':
+                            if (!(document[field] < operand)) return false;
+                            break;
+                        case '$ne':
+                            if (document[field] === operand) return false;
+                            break;
+                        case '$in':
+                            if (!Array.isArray(operand) || !operand.includes(document[field])) return false;
+                            break;
+                        case '$nin':
+                            if (Array.isArray(operand) && operand.includes(document[field])) return false;
+                            break;
+                        case '$regex':
+                            if (!new RegExp(operand).test(document[field])) return false;
+                            break;
+                        default:
+                            // Unknown operator, treat as exact match
+                            if (document[field] !== value) return false;
+                    }
+                }
+            } else {
+                // Exact match
+                if (document[field] !== value) return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Create an aggregation pipeline from query results.
+     * @param {Object} [query={}] - Optional query to filter documents before aggregation
+     * @returns {Promise<AggregationPipeline>} A new AggregationPipeline instance
+     */
+    async aggregate(query = {}) {
+        const results = await this.find(query, { limit: 100000 });
+        return new AggregationPipeline(results);
+    }
+}
+
+/**
+ * AggregationPipeline - Provides aggregate functions, grouping, time-series
+ * bucketing, and a fluent API for chaining operations on in-memory datasets.
+ */
+export class AggregationPipeline {
+    constructor(data) {
+        this.data = Array.isArray(data) ? data : [];
+        this.stages = [];
+    }
+
+    /**
+     * Filter/match stage - filter data before aggregation.
+     * Uses the same query operators as DataModel.matchesQuery.
+     * @param {Object} query - Query object with optional operators ($gte, $lte, etc.)
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    match(query) {
+        this.stages.push({ type: 'match', query });
+        return this;
+    }
+
+    /**
+     * Group results by a field.
+     * @param {string} field - Field name to group by
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    group(field) {
+        this.stages.push({ type: 'group', field });
+        return this;
+    }
+
+    /**
+     * Sum aggregate on a numeric field.
+     * @param {string} field - Field to sum
+     * @param {string} [alias] - Optional alias for the result key (default: `sum_<field>`)
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    sum(field, alias) {
+        this.stages.push({ type: 'aggregate', op: 'sum', field, alias: alias || `sum_${field}` });
+        return this;
+    }
+
+    /**
+     * Average aggregate on a numeric field.
+     * @param {string} field - Field to average
+     * @param {string} [alias] - Optional alias for the result key (default: `avg_<field>`)
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    avg(field, alias) {
+        this.stages.push({ type: 'aggregate', op: 'avg', field, alias: alias || `avg_${field}` });
+        return this;
+    }
+
+    /**
+     * Count aggregate.
+     * @param {string} [alias] - Optional alias for the result key (default: `count`)
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    count(alias) {
+        this.stages.push({ type: 'aggregate', op: 'count', alias: alias || 'count' });
+        return this;
+    }
+
+    /**
+     * Min aggregate on a numeric field.
+     * @param {string} field - Field to find minimum of
+     * @param {string} [alias] - Optional alias for the result key (default: `min_<field>`)
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    min(field, alias) {
+        this.stages.push({ type: 'aggregate', op: 'min', field, alias: alias || `min_${field}` });
+        return this;
+    }
+
+    /**
+     * Max aggregate on a numeric field.
+     * @param {string} field - Field to find maximum of
+     * @param {string} [alias] - Optional alias for the result key (default: `max_<field>`)
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    max(field, alias) {
+        this.stages.push({ type: 'aggregate', op: 'max', field, alias: alias || `max_${field}` });
+        return this;
+    }
+
+    /**
+     * Time-series bucketing - groups data by time intervals.
+     * Acts like a group stage but groups by truncated timestamp buckets.
+     * @param {string} timestampField - Field containing a unix timestamp (seconds)
+     * @param {'minute'|'hour'|'day'|'week'|'month'} interval - Bucket interval
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    bucket(timestampField, interval) {
+        this.stages.push({ type: 'bucket', timestampField, interval });
+        return this;
+    }
+
+    /**
+     * Sort results by a field.
+     * @param {string} field - Field to sort by
+     * @param {'asc'|'desc'} [order='asc'] - Sort order
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    sort(field, order = 'asc') {
+        this.stages.push({ type: 'sort', field, order });
+        return this;
+    }
+
+    /**
+     * Limit the number of results returned.
+     * @param {number} n - Maximum number of results
+     * @returns {AggregationPipeline} this (for chaining)
+     */
+    limit(n) {
+        this.stages.push({ type: 'limit', n });
+        return this;
+    }
+
+    /**
+     * Truncate a Date to the start of the given interval bucket.
+     * @private
+     * @param {Date} date
+     * @param {string} interval
+     * @returns {string} ISO string representing the bucket start
+     */
+    _truncateDate(date, interval) {
+        const d = new Date(date);
+        switch (interval) {
+            case 'minute':
+                d.setUTCSeconds(0, 0);
+                break;
+            case 'hour':
+                d.setUTCMinutes(0, 0, 0);
+                break;
+            case 'day':
+                d.setUTCHours(0, 0, 0, 0);
+                break;
+            case 'week': {
+                d.setUTCHours(0, 0, 0, 0);
+                // Move back to Monday (ISO week start)
+                const day = d.getUTCDay();
+                const diff = (day === 0 ? 6 : day - 1);
+                d.setUTCDate(d.getUTCDate() - diff);
+                break;
+            }
+            case 'month':
+                d.setUTCDate(1);
+                d.setUTCHours(0, 0, 0, 0);
+                break;
+            default:
+                break;
+        }
+        return d.toISOString();
+    }
+
+    /**
+     * Simple query matching, mirroring DataModel.matchesQuery.
+     * @private
+     * @param {Object} document
+     * @param {Object} query
+     * @returns {boolean}
+     */
+    _matchesQuery(document, query) {
+        for (const [field, value] of Object.entries(query)) {
+            if (typeof value === 'object' && value !== null) {
+                for (const [operator, operand] of Object.entries(value)) {
+                    switch (operator) {
+                        case '$gte':
+                            if (!(document[field] >= operand)) return false;
+                            break;
+                        case '$lte':
+                            if (!(document[field] <= operand)) return false;
+                            break;
+                        case '$gt':
+                            if (!(document[field] > operand)) return false;
+                            break;
+                        case '$lt':
+                            if (!(document[field] < operand)) return false;
+                            break;
+                        case '$ne':
+                            if (document[field] === operand) return false;
+                            break;
+                        case '$in':
+                            if (!Array.isArray(operand) || !operand.includes(document[field])) return false;
+                            break;
+                        case '$nin':
+                            if (Array.isArray(operand) && operand.includes(document[field])) return false;
+                            break;
+                        case '$regex':
+                            if (!new RegExp(operand).test(document[field])) return false;
+                            break;
+                        default:
+                            if (document[field] !== value) return false;
+                    }
+                }
+            } else {
+                if (document[field] !== value) return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Compute a single aggregate value over an array of documents.
+     * @private
+     * @param {Array} docs - Array of documents
+     * @param {Object} stage - Aggregate stage descriptor
+     * @returns {number} Computed aggregate value
+     */
+    _computeAggregate(docs, stage) {
+        const { op, field } = stage;
+        switch (op) {
+            case 'count':
+                return docs.length;
+            case 'sum': {
+                let total = 0;
+                for (const doc of docs) {
+                    const v = Number(doc[field]);
+                    if (!isNaN(v)) total += v;
+                }
+                return total;
+            }
+            case 'avg': {
+                let sum = 0;
+                let cnt = 0;
+                for (const doc of docs) {
+                    const v = Number(doc[field]);
+                    if (!isNaN(v)) {
+                        sum += v;
+                        cnt++;
+                    }
+                }
+                return cnt > 0 ? sum / cnt : 0;
+            }
+            case 'min': {
+                let result = Infinity;
+                for (const doc of docs) {
+                    const v = Number(doc[field]);
+                    if (!isNaN(v) && v < result) result = v;
+                }
+                return result === Infinity ? 0 : result;
+            }
+            case 'max': {
+                let result = -Infinity;
+                for (const doc of docs) {
+                    const v = Number(doc[field]);
+                    if (!isNaN(v) && v > result) result = v;
+                }
+                return result === -Infinity ? 0 : result;
+            }
+            default:
+                return 0;
+        }
+    }
+
+    /**
+     * Execute the aggregation pipeline and return results.
+     *
+     * Processing order:
+     *   1. match stages filter the working dataset
+     *   2. group / bucket stages partition data into groups
+     *   3. aggregate stages compute values per group (or globally)
+     *   4. sort / limit are applied to the final result set
+     *
+     * @returns {Object|Array} Aggregation results
+     */
+    execute() {
+        let workingData = [...this.data];
+
+        // Separate stages by type
+        const matchStages = [];
+        const aggregateStages = [];
+        let groupStage = null;
+        let bucketStage = null;
+        const postStages = []; // sort, limit
+
+        for (const stage of this.stages) {
+            switch (stage.type) {
+                case 'match':
+                    matchStages.push(stage);
+                    break;
+                case 'group':
+                    groupStage = stage;
+                    break;
+                case 'bucket':
+                    bucketStage = stage;
+                    break;
+                case 'aggregate':
+                    aggregateStages.push(stage);
+                    break;
+                case 'sort':
+                case 'limit':
+                    postStages.push(stage);
+                    break;
+            }
+        }
+
+        // 1. Apply match stages (filters)
+        for (const stage of matchStages) {
+            workingData = workingData.filter(doc => this._matchesQuery(doc, stage.query));
+        }
+
+        // 2. Determine grouping
+        let groups = null; // Map<string, Array>
+
+        if (bucketStage) {
+            // Bucket by time interval
+            groups = new Map();
+            const { timestampField, interval } = bucketStage;
+            for (const doc of workingData) {
+                const ts = doc[timestampField];
+                if (ts === undefined || ts === null) continue;
+                // Treat as unix seconds; convert to Date
+                const date = new Date(typeof ts === 'number' ? ts * 1000 : ts);
+                if (isNaN(date.getTime())) continue;
+                const bucketKey = this._truncateDate(date, interval);
+                if (!groups.has(bucketKey)) {
+                    groups.set(bucketKey, []);
+                }
+                groups.get(bucketKey).push(doc);
+            }
+        } else if (groupStage) {
+            groups = new Map();
+            const { field } = groupStage;
+            for (const doc of workingData) {
+                const key = doc[field] !== undefined && doc[field] !== null
+                    ? String(doc[field])
+                    : '__null__';
+                if (!groups.has(key)) {
+                    groups.set(key, []);
+                }
+                groups.get(key).push(doc);
+            }
+        }
+
+        // 3. Compute aggregates
+        let results;
+
+        if (groups) {
+            // Grouped / bucketed results
+            results = [];
+            for (const [groupKey, docs] of groups) {
+                const row = { _id: groupKey === '__null__' ? null : groupKey };
+                for (const aggStage of aggregateStages) {
+                    row[aggStage.alias] = this._computeAggregate(docs, aggStage);
+                }
+                results.push(row);
+            }
+        } else if (aggregateStages.length > 0) {
+            // Ungrouped global aggregation
+            const row = {};
+            for (const aggStage of aggregateStages) {
+                row[aggStage.alias] = this._computeAggregate(workingData, aggStage);
+            }
+            results = row;
+        } else {
+            // No aggregation or grouping -- return filtered data
+            results = workingData;
+        }
+
+        // 4. Apply post-processing (sort, limit) -- only meaningful for arrays
+        if (Array.isArray(results)) {
+            for (const stage of postStages) {
+                if (stage.type === 'sort') {
+                    const { field, order } = stage;
+                    const dir = order === 'desc' ? -1 : 1;
+                    results.sort((a, b) => {
+                        const va = a[field];
+                        const vb = b[field];
+                        if (va < vb) return -1 * dir;
+                        if (va > vb) return 1 * dir;
+                        return 0;
+                    });
+                } else if (stage.type === 'limit') {
+                    results = results.slice(0, stage.n);
+                }
+            }
+        }
+
+        return results;
+    }
+}
+
+/**
+ * Main DataTransformer class
+ */
+export default class DataTransformer {
+    constructor(gunInstance, options = {}) {
+        this.gun = gunInstance;
+        this.options = {
+            logLevel: 'info',
+            enableAutoTransform: true,
+            ...options
+        };
+        
+        this.logger = new Logger({
+            context: 'DataTransformer',
+            level: this.options.logLevel
+        });
+        
+        this.models = new Map();
+        this.transformers = new Map();
+        
+        this.logger.info('DataTransformer initialized', { options: this.options });
+    }
+
+    /**
+     * Register a data model
+     * @param {string} name - Model name
+     * @param {Object} schema - Model schema
+     * @param {Object} options - Model options
+     * @returns {DataModel} Created model
+     */
+    model(name, schema, options = {}) {
+        if (this.models.has(name)) {
+            return this.models.get(name);
+        }
+        
+        const model = new DataModel(name, schema, this.gun, this, options);
+        this.models.set(name, model);
+        
+        this.logger.info('Model registered', { name, schema });
+        return model;
+    }
+
+    /**
+     * Register an event transformer
+     * @param {string} eventName - Event name to transform
+     * @param {string} modelName - Target model name
+     * @param {Function} transformFn - Transformation function
+     */
+    registerTransformer(eventName, modelName, transformFn) {
+        if (!this.models.has(modelName)) {
+            throw new ModelError(`Model '${modelName}' not found. Register the model first.`);
+        }
+        
+        this.transformers.set(eventName, {
+            modelName,
+            transformFn
+        });
+        
+        this.logger.info('Transformer registered', { eventName, modelName });
+    }
+
+    /**
+     * Transform and store an event
+     * @param {Object} event - Blockchain event
+     * @returns {Promise<Object|null>} Transformed data or null if no transformer
+     */
+    async transformEvent(event) {
+        if (!event || !event.eventName) {
+            return null;
+        }
+        
+        const transformer = this.transformers.get(event.eventName);
+        if (!transformer) {
+            this.logger.debug('No transformer found for event', { eventName: event.eventName });
+            return null;
+        }
+        
+        try {
+            const { modelName, transformFn } = transformer;
+            const model = this.models.get(modelName);
+            
+            // Transform the event data
+            const transformedData = await transformFn(event);
+            
+            // Save to the model
+            const savedData = await model.save(transformedData);
+            
+            this.logger.info('Event transformed and saved', { 
+                eventName: event.eventName, 
+                modelName,
+                id: savedData.id
+            });
+            
+            return savedData;
+            
+        } catch (error) {
+            this.logger.error('Failed to transform event', { 
+                error: error.message, 
+                eventName: event.eventName 
+            });
+            throw error;
+        }
+    }
+
+    /**
+     * Get a registered model
+     * @param {string} name - Model name
+     * @returns {DataModel} Model instance
+     */
+    getModel(name) {
+        const model = this.models.get(name);
+        if (!model) {
+            throw new ModelError(`Model '${name}' not found`);
+        }
+        return model;
+    }
+
+    /**
+     * Get all registered models
+     * @returns {Map<string, DataModel>} All models
+     */
+    getModels() {
+        return new Map(this.models);
+    }
+
+    /**
+     * Get transformation statistics
+     * @returns {Object} Statistics
+     */
+    getStats() {
+        return {
+            modelsCount: this.models.size,
+            transformersCount: this.transformers.size,
+            registeredModels: Array.from(this.models.keys()),
+            registeredTransformers: Array.from(this.transformers.keys())
+        };
+    }
+
+    /**
+     * Create an AggregationPipeline from raw data.
+     * @param {Array} data - Array of documents to aggregate
+     * @returns {AggregationPipeline} A new AggregationPipeline instance
+     */
+    aggregate(data) {
+        return new AggregationPipeline(data);
+    }
+
+    /**
+     * Create a SchemaMigration instance for a model.
+     * @param {string} modelName - Name of the model
+     * @returns {SchemaMigration} A new SchemaMigration instance
+     */
+    migration(modelName) {
+        return new SchemaMigration(modelName, this.gun);
+    }
+}