divhunt 2.0.0

package/addons/core/queue/README.md

@@ -0,0 +1,216 @@
+# Queue
+
+A lightweight task queue management system for handling asynchronous operations with concurrency control.
+
+## Usage
+
+Create a queue instance:
+
+```javascript
+const myQueue = queue.Item(
+{
+    id: 'my-queue',
+    concurrency: 5,
+    timeout: 3000,
+    onTaskStart: (task, batch) => 
+    {
+        console.log(`Task ${task.id} started`);
+    },
+    onTaskEnd: (task, batch) => 
+    {
+        console.log(`Task ${task.id} ended, success: ${task.success}`);
+    },
+    onEmpty: (batch) => 
+    {
+        console.log('Queue is empty');
+    }
+});
+```
+
+Add tasks to the queue:
+
+```javascript
+myQueue.Fn('add', null, { url: 'https://example.com' }, (data, resolve) => 
+{
+    try 
+    {
+        fetch(data.url).then(response => 
+        {
+            if (!response.ok) throw new Error('Network response failed');
+            return response.json();
+        })
+        .then(result => 
+        {
+            resolve(); // Tasks completed
+        })
+        .catch(error => 
+        {
+            throw new Error(`Fetch failed: ${error.message}`);
+        });
+    } 
+    catch (error) 
+    {
+        // Task wont fail since error hasn't been throw, but timeout will kick in
+        console.error(error);
+    }
+});
+```
+
+## Configuration
+
+The queue accepts the following configuration options:
+
+- `id`: Unique identifier for the queue
+- `concurrency`: Maximum number of tasks to process simultaneously (default: 10)
+- `timeout`: Task timeout in milliseconds (default: 5000)
+- `running`: Whether the queue is actively processing tasks (default: true)
+
+## Events
+
+Callback functions that can be defined:
+
+- `onTaskStart`: Called when a task begins processing
+- `onTaskEnd`: Called when a task completes (success or failure)
+- `onEmpty`: Called when the queue becomes empty
+- `onRun`: Called when the queue starts processing a batch of tasks
+- `whileRunning`: Called periodically while tasks are being processed
+
+## Task Format
+
+Tasks have the following properties:
+
+- `id`: Unique identifier (auto-generated if null is provided)
+- `data`: Custom data passed to the task callback
+- `callback`: Function to execute the task logic
+- `success`: Result status after completion
+- `duration`: Processing time in milliseconds
+
+## Batch Statistics
+
+During processing, the batch object provides:
+
+- `active`: Number of currently running tasks
+- `processed`: Total completed tasks in current batch
+- `succeeded`: Number of successful tasks
+- `failed`: Number of failed tasks
+- `reset()`: Method to reset batch counters
+
+## Error Handling
+
+Tasks automatically fail when an error is thrown within the callback function. Use try/catch blocks and throw appropriate errors:
+
+```javascript
+myQueue.Fn('add', null, data, (data, resolve) => 
+{
+    try 
+    {
+        // Task logic
+        if (!data.isValid) 
+        {
+            throw new Error('Invalid data provided');
+        }
+        resolve();
+    } 
+    catch (error) 
+    {
+        // Will be captured by queue error handling
+        throw new Error(`Task failed: ${error.message}`);
+    }
+});
+```
+
+## Utility Functions
+
+The queue includes utility functions:
+
+### Calculate Requests Per Second
+
+```javascript
+const timestamps = [/* array of task completion timestamps */];
+const rps = myQueue.Fn('calculate.rps', timestamps);
+console.log(rps[100]); // RPS over the last 100 tasks
+```
+
+### Calculate Average Response Time
+
+```javascript
+const durations = [/* array of task durations */];
+const art = myQueue.Fn('calculate.art', durations);
+console.log(art[100]); // Average response time over the last 100 tasks
+```
+
+## Controlling Queue Execution
+
+Pause queue processing:
+
+```javascript
+myQueue.Set('running', false);
+```
+
+Resume queue processing:
+
+```javascript
+myQueue.Set('running', true);
+```
+
+## Advanced Example
+
+Here's a complete example of a queue that processes web requests:
+
+```javascript
+import queue from '#queue/load.js';
+
+const scrapeQueue = queue.Item(
+{
+    id: 'scraper',
+    concurrency: 3,
+    timeout: 10000,
+    onTaskStart: (task, batch) => 
+    {
+        console.log(`Scraping ${task.data.url}`);
+    },
+    onTaskEnd: (task, batch) => 
+    {
+        console.log(`Completed ${task.data.url} in ${task.duration}ms`);
+        console.log(`Batch stats - Active: ${batch.active}, Succeeded: ${batch.succeeded}, Failed: ${batch.failed}`);
+    },
+    onEmpty: (batch) => 
+    {
+        console.log('All scraping tasks completed');
+    },
+    onRun: (batch) => 
+    {
+        console.log('Starting batch processing');
+    },
+    whileRunning: (batch) => 
+    {
+        console.log(`Processing... ${batch.active} tasks active`);
+    }
+});
+
+// Add multiple tasks
+const urls = [
+    'https://example.com',
+    'https://example.org',
+    'https://example.net'
+];
+
+urls.forEach(url => 
+{
+    scrapeQueue.Fn('add', null, { url }, async (data, resolve) => 
+    {
+        try 
+        {
+            const response = await fetch(data.url);
+            if (!response.ok) throw new Error(`HTTP error ${response.status}`);
+            const html = await response.text();
+            // Process HTML
+            resolve();
+        } 
+        catch (error) 
+        {
+            throw new Error(`Scrape failed for ${data.url}: ${error.message}`);
+        }
+    });
+});
+```

package/addons/core/queue/back/addon.js

@@ -0,0 +1,21 @@
+import divhunt from '#framework/load.js';
+
+const queue = divhunt.Addon('queue', (addon) =>
+{
+    addon.Field('id', ['string']);
+
+    addon.Field('concurrency', ['number', 10]);
+    addon.Field('timeout', ['number', 5000]);
+
+    addon.Field('onTaskStart', ['function']);
+    addon.Field('onTaskEnd', ['function']);
+    addon.Field('onEmpty', ['function']);
+    addon.Field('onRun', ['function']);
+    addon.Field('whileRunning', ['function']);
+
+    addon.Field('running', ['boolean', true]);
+
+    addon.Field('tasks', ['array', []]);
+});
+
+export default queue;

package/addons/core/queue/back/functions/calculate/art.js

@@ -0,0 +1,12 @@
+import queue from '#queue/addon.js';
+
+queue.Fn('calculate.art', function(durations = [])
+{
+    return {
+        10: durations.length >= 10 ? parseFloat((durations.slice(-10).reduce((sum, t) => sum + t, 0) / 10).toFixed(2)) : null,
+        100: durations.length >= 100 ? parseFloat((durations.slice(-100).reduce((sum, t) => sum + t, 0) / 100).toFixed(2)) : null,
+        250: durations.length >= 250 ? parseFloat((durations.slice(-250).reduce((sum, t) => sum + t, 0) / 250).toFixed(2)) : null,
+        500: durations.length >= 500 ? parseFloat((durations.slice(-500).reduce((sum, t) => sum + t, 0) / 500).toFixed(2)) : null,
+        1000: durations.length >= 1000 ? parseFloat((durations.slice(-1000).reduce((sum, t) => sum + t, 0) / 1000).toFixed(2)) : null
+    };
+});

package/addons/core/queue/back/functions/calculate/rps.js

@@ -0,0 +1,22 @@
+import queue from '#queue/addon.js';
+
+queue.Fn('calculate.rps', function(timestamps = [])
+{
+    this.methods.calculateRateForWindow = (windowTimestamps) =>
+    {
+        if(windowTimestamps.length < 2) return 0;
+        
+        const timeSpanSeconds = (windowTimestamps[windowTimestamps.length - 1] - windowTimestamps[0]) / 1000;
+        if(timeSpanSeconds === 0) return 0;
+        
+        return parseFloat((windowTimestamps.length / timeSpanSeconds).toFixed(2));
+    };
+
+    return {
+        10: timestamps.length >= 10 ? this.methods.calculateRateForWindow(timestamps.slice(-10)) : null,
+        100: timestamps.length >= 100 ? this.methods.calculateRateForWindow(timestamps.slice(-100)) : null,
+        250: timestamps.length >= 250 ? this.methods.calculateRateForWindow(timestamps.slice(-250)) : null,
+        500: timestamps.length >= 500 ? this.methods.calculateRateForWindow(timestamps.slice(-500)) : null,
+        1000: timestamps.length >= 1000 ? this.methods.calculateRateForWindow(timestamps.slice(-1000)) : null,
+    };
+});

package/addons/core/queue/back/item/catch/add.js

@@ -0,0 +1,6 @@
+import queue from '#queue/addon.js';
+
+queue.ItemOn('add', (item) => 
+{
+    item.Fn('start');
+})

package/addons/core/queue/back/item/catch/remove.js

@@ -0,0 +1,6 @@
+import queue from '#queue/addon.js';
+
+queue.ItemOn('remove', (item) => 
+{
+    item.Set('running', false);
+})

package/addons/core/queue/back/item/functions/add.js

@@ -0,0 +1,15 @@
+import divhunt from '#framework/load.js';
+import queue from '#queue/addon.js';
+
+queue.Fn('item.add', function(item, id, data, callback)
+{
+    const tasks = item.Get('tasks');
+
+    if(id === null)
+    {
+        id = divhunt.GenerateUID();
+    }
+
+    tasks.push({id, data, callback});
+    item.Set('tasks', tasks);
+});

package/addons/core/queue/back/item/functions/start.js

@@ -0,0 +1,135 @@
+import queue from '#queue/addon.js';
+
+queue.Fn('item.start', async function(item)
+{
+    const batch = {
+        active: 0,
+        processed: 0,
+        succeeded: 0, 
+        failed: 0, 
+        reset: function()
+        {
+            item.Get('onReset') && item.Get('onReset')(batch);
+
+            this.active = 0;
+            this.processed = 0;
+            this.succeeded = 0;
+            this.failed = 0;
+        }
+    };
+
+    this.methods.runTask = (task) => 
+    {
+        return new Promise((resolve, reject) => 
+        {
+            let ended = false;
+
+            const timeout = setTimeout(() => 
+            { 
+                ended = true;
+                reject(new Error('Task timeout'));
+            }, item.Get('timeout'));
+
+            try
+            {
+                task.callback(task.data, () => 
+                { 
+                    clearTimeout(timeout); 
+
+                    if(!ended)
+                    {
+                        resolve();
+                    }
+                });
+            }
+            catch(error)
+            {
+                clearTimeout(timeout);
+                reject(error);
+            }
+        });
+    };
+
+    this.methods.processNextTask = async () => 
+    {
+        if(!item.Get('tasks').length) 
+        {
+            return false;
+        }
+        
+        const task = item.Get('tasks').shift();
+        const now = performance.now();
+
+        batch.active++;
+
+        item.Get('onTaskStart') && item.Get('onTaskStart')(task, batch);
+
+        try
+        {
+            await this.methods.runTask(task);
+
+            batch.succeeded++;
+            task.success = true;
+        }
+        catch(error)
+        {
+            batch.failed++;
+            task.success = false;
+        }
+        finally
+        {
+            task.duration = performance.now() - now;
+
+            batch.active--;
+            batch.processed++;
+           
+            item.Get('onTaskEnd') && item.Get('onTaskEnd')(task, batch);
+            
+            if(item.Get('tasks').length && item.Get('running'))
+            {
+                this.methods.processNextTask();
+            }
+        }
+    };
+    
+    this.methods.run = async () => 
+    {
+        batch.reset();
+
+        if(!item.Get('tasks').length)
+        {
+            item.Get('onEmpty') && item.Get('onEmpty')(batch);
+            return false;
+        }
+        
+        item.Get('onRun') && item.Get('onRun')(batch);
+        
+        const concurrency = Math.min(item.Get('concurrency'), item.Get('tasks').length);
+
+        for(let i = 0; i < concurrency; i++)
+        {
+            this.methods.processNextTask();
+        }
+        
+        while(batch.active > 0)
+        {
+            item.Get('whileRunning') && item.Get('whileRunning')(batch);
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+
+        return item.Get('tasks').length > 0;
+    };
+    
+    while(true)
+    {
+        if(!item.Get('running'))
+        {
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+        else
+        {
+            const remain = await this.methods.run();
+            await new Promise(resolve => setTimeout(resolve, remain ? 10 : 100));
+        }
+    }
+});

package/addons/core/queue/back/load.js

@@ -0,0 +1,12 @@
+import queue from '#queue/addon.js';
+
+import '#queue/functions/calculate/art.js';
+import '#queue/functions/calculate/rps.js';
+
+import '#queue/item/functions/start.js';
+import '#queue/item/functions/add.js';
+
+import '#queue/item/catch/add.js';
+import '#queue/item/catch/remove.js';
+
+export default queue;

package/addons/core/queue/index.js

@@ -0,0 +1,66 @@
+import queue from '#queue/load.js';
+
+const mainQueue = queue.Item({
+    id: 'main',
+    onTaskStart: (task, batch) => 
+    {
+    },
+    onTaskEnd: (task, batch) => 
+    {
+    },
+    onRun: () => 
+    {
+    },
+    whileRunning: (batch) => 
+    {
+        console.log('ACTIVE :1', batch.active);
+    }
+});
+
+mainQueue.Fn('add', null, {}, (data, resolve) => 
+{
+    setTimeout(() => {
+        resolve();
+    }, 1500)
+});
+
+setInterval(() => 
+{
+    mainQueue.Fn('add', null, {}, (data, resolve) => 
+    {
+        setTimeout(() => {
+            resolve();
+        }, 1500)
+    });
+
+}, 1);
+
+
+
+// const scrapper = workers.Fn('client.gRPC', '192.168.0.101:50001');
+
+// async function main()
+// {
+//     try 
+//     {
+//         const actions = [{
+//             name: 'navigate',
+//             properties: {
+//                 url: 'https://mondorama.rs',
+//                 waitUntil: 'networkidle'
+//             }
+//         }, {
+//             name: 'seo',
+//         }];
+
+//         const scrape = await scrapper.execute('scrape', {actions});
+
+//         console.log(scrape.data.context.seo);
+//     }
+//     catch(error)
+//     {
+//         console.log(error.message);
+//     }
+// }
+
+// main();

package/addons/core/queue/package.json

@@ -0,0 +1,13 @@
+{
+    "name": "addon-queue",
+    "type": "module",
+    "imports": {
+        "#queue/*": "./back/*",
+        "#framework/*": "./lib/framework/*"
+    },
+    "main": "index.js",
+    "dependencies": {
+        "@grpc/grpc-js": "^1.13.3",
+        "@grpc/proto-loader": "^0.7.15"
+    }
+}

package/addons/core/servers/README.md

@@ -0,0 +1,85 @@
+# Servers Module
+
+HTTP and gRPC server implementations for the application framework providing multiple communication protocols.
+
+## HTTP Server
+
+REST API server with comprehensive request handling and middleware integration.
+
+### Features
+
+**Request Processing:**
+- Automatic data parsing from query parameters and request body
+- Type conversion with hints (`:string`, `:number`, `:boolean`, `:array`, `:object`)
+- URL parameter extraction from dynamic routes
+- JSON body parsing for POST/PUT requests
+
+**Response Formatting:**
+- Multiple content types: JSON, HTML, CSS, JavaScript
+- Automatic response wrapping with data, message, code, and duration
+- Error handling with standardized error responses
+- Request timing and performance metrics
+
+**Middleware Integration:**
+- Command routing through middleware system
+- Request/response lifecycle hooks
+- Error catching and logging
+- Custom callback support for request handling
+
+### Configuration
+
+```javascript
+serversHTTP.Item({
+    id: 'main',
+    port: 3000,
+    onStart: (server) => console.log('HTTP server started'),
+    onRequest: (http) => {
+        console.log(`${http.request.method} ${http.request.url}`);
+    },
+    onError: (error) => console.log('Server error:', error),
+    onComplete: (http) => {
+        console.log(`Request completed in ${http.time}ms`);
+    }
+});
+```
+
+## gRPC Server
+
+High-performance bidirectional streaming server using Protocol Buffers for real-time communication.
+
+### Features
+
+**Streaming Communication:**
+- Universal service with single stream method
+- Bidirectional streaming for real-time data exchange
+- JSON message serialization over Protocol Buffers
+- Stream lifecycle management with connection tracking
+
+**Command Integration:**
+- Direct command execution through streaming interface
+- Promise-based request/response pattern
+- Automatic command resolution and timeout handling
+- Error propagation and handling
+
+**Stream Management:**
+- Connection lifecycle callbacks (connect, data, close, end, error)
+- Stream identification and tracking
+- Request/response correlation with unique IDs
+- Timeout handling for pending requests
+
+### Configuration
+
+```javascript
+serversGRPC.Item({
+    port: 50000,
+    onStart: () => console.log('gRPC server started'),
+    onStreamConnect: (stream) => console.log('Stream connected'),
+    onStreamData: async (stream, payload) => {
+        // Handle incoming command requests
+        const response = await executeCommand(payload);
+        stream.respond(response.data, response.message, response.code);
+    },
+    onStreamClose: (stream) => console.log('Stream closed'),
+    onStreamError: (stream, error) => console.log('Stream error:', error)
+});
+```

package/addons/core/servers/back/grpc/addon.js

@@ -0,0 +1,23 @@
+// © 2025 Divhunt GmbH — Licensed under the Divhunt Framework License. See LICENSE for terms.
+
+import divhunt from '#framework/load.js';
+
+const serversGRPC = divhunt.Addon('servers.grpc', (addon) =>
+{
+    addon.Field('id', ['string']);
+    addon.Field('instance', ['object']);
+    addon.Field('port', ['number', 50000]);
+    addon.Field('host', ['string', '0.0.0.0']);
+
+    addon.Field('onError', ['function']);
+    addon.Field('onStart', ['function']);
+    
+    addon.Field('onStreamConnect', ['function']);
+    addon.Field('onStreamError', ['function']);
+    addon.Field('onStreamEnd', ['function']);
+    addon.Field('onStreamData', ['function']);
+    addon.Field('onStreamRequest', ['function']);
+    addon.Field('onStreamRespond', ['function']);
+});
+
+export default serversGRPC;

package/addons/core/servers/back/grpc/item/catch/add.js

@@ -0,0 +1,6 @@
+import serversGRPC from '#servers/grpc/addon.js';
+
+serversGRPC.ItemOn('add', function(item)
+{
+    item.Fn('start');
+});

package/addons/core/servers/back/grpc/item/functions/resolve.js

@@ -0,0 +1,41 @@
+// © 2025 Divhunt GmbH — Licensed under the Divhunt Framework License. See LICENSE for terms.
+
+import serversGRPC from '#servers/grpc/addon.js';
+
+const promises = {};
+
+serversGRPC.Fn('item.resolve', function(item, id, payload = null, onChunk = null)
+{
+    if(payload === null)
+    {
+        const timeout = setTimeout(() =>
+        {
+            promises[id].resolve({data: null, message: 'Request timed out after 60 seconds. No response received.', code: 500, id, time: 60000});
+            delete(promises[id]);
+        }, 60000);
+
+        return new Promise((resolve) =>
+        {
+            promises[id] = { resolve, timeout, onChunk, time: performance.now() };
+        });
+    }
+
+    if(!promises[id])
+    {
+        return;
+    }
+
+    if(promises[id].onChunk && !payload.end)
+    {
+        promises[id].onChunk(payload);
+    }
+
+    if(payload.end)
+    {
+        payload.time = (performance.now() - promises[id].time).toFixed(2);
+        promises[id].resolve(payload);
+        clearTimeout(promises[id].timeout);
+
+        delete promises[id];
+    }
+});