@pikku/mongodb 0.12.1

package/dist/src/mongodb-workflow-service.js

@@ -0,0 +1,351 @@
+import { PikkuWorkflowService, } from '@pikku/core/workflow';
+import { MongoDBWorkflowRunService } from './mongodb-workflow-run-service.js';
+export class MongoDBWorkflowService extends PikkuWorkflowService {
+    initialized = false;
+    runService;
+    runs;
+    steps;
+    stepHistory;
+    versions;
+    constructor(db) {
+        super();
+        this.runService = new MongoDBWorkflowRunService(db);
+        this.runs = db.collection('workflow_runs');
+        this.steps = db.collection('workflow_step');
+        this.stepHistory = db.collection('workflow_step_history');
+        this.versions = db.collection('workflow_versions');
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        await this.runs.createIndex({ workflow: 1 });
+        await this.runs.createIndex({ status: 1 });
+        await this.runs.createIndex({ createdAt: -1 });
+        await this.steps.createIndex({ workflowRunId: 1, stepName: 1 }, { unique: true });
+        await this.steps.createIndex({ workflowRunId: 1 });
+        await this.stepHistory.createIndex({ workflowStepId: 1 });
+        await this.stepHistory.createIndex({ createdAt: 1 });
+        await this.versions.createIndex({ workflowName: 1, graphHash: 1 }, { unique: true });
+        this.initialized = true;
+    }
+    async createRun(workflowName, input, inline, graphHash, wire) {
+        const id = crypto.randomUUID();
+        const now = new Date();
+        await this.runs.insertOne({
+            _id: id,
+            workflow: workflowName,
+            status: 'running',
+            input,
+            output: null,
+            error: null,
+            state: {},
+            inline,
+            graphHash,
+            wire,
+            createdAt: now,
+            updatedAt: now,
+        });
+        return id;
+    }
+    async getRun(id) {
+        return this.runService.getRun(id);
+    }
+    async updateRunStatus(id, status, output, error) {
+        await this.runs.updateOne({ _id: id }, {
+            $set: {
+                status,
+                output: output ?? null,
+                error: error ?? null,
+                updatedAt: new Date(),
+            },
+        });
+    }
+    async insertStepState(runId, stepName, rpcName, data, stepOptions) {
+        const stepId = crypto.randomUUID();
+        const now = new Date();
+        await this.steps.insertOne({
+            _id: stepId,
+            workflowRunId: runId,
+            stepName,
+            rpcName,
+            data: data ?? null,
+            status: 'pending',
+            result: null,
+            error: null,
+            branchTaken: null,
+            retries: stepOptions?.retries ?? null,
+            retryDelay: stepOptions?.retryDelay?.toString() ?? null,
+            createdAt: now,
+            updatedAt: now,
+        });
+        await this.insertHistoryRecord(stepId, 'pending');
+        return {
+            stepId,
+            status: 'pending',
+            result: undefined,
+            error: undefined,
+            attemptCount: 1,
+            retries: stepOptions?.retries,
+            retryDelay: stepOptions?.retryDelay?.toString(),
+            createdAt: now,
+            updatedAt: now,
+        };
+    }
+    async getStepState(runId, stepName) {
+        const row = await this.steps.findOne({
+            workflowRunId: runId,
+            stepName,
+        });
+        if (!row) {
+            throw new Error(`Step not found: runId=${runId}, stepName=${stepName}. Use insertStepState to create it.`);
+        }
+        const attemptCount = await this.stepHistory.countDocuments({
+            workflowStepId: row._id,
+        });
+        return {
+            stepId: row._id,
+            status: row.status,
+            result: row.result ?? undefined,
+            error: row.error ?? undefined,
+            attemptCount,
+            retries: row.retries != null ? Number(row.retries) : undefined,
+            retryDelay: row.retryDelay ?? undefined,
+            createdAt: new Date(row.createdAt),
+            updatedAt: new Date(row.updatedAt),
+        };
+    }
+    async getRunHistory(runId) {
+        return this.runService.getRunHistory(runId);
+    }
+    async setStepRunning(stepId) {
+        await this.steps.updateOne({ _id: stepId }, { $set: { status: 'running', updatedAt: new Date() } });
+        const latestHistory = await this.stepHistory
+            .find({ workflowStepId: stepId })
+            .sort({ createdAt: -1 })
+            .limit(1)
+            .toArray();
+        if (latestHistory.length > 0) {
+            await this.stepHistory.updateOne({ _id: latestHistory[0]._id }, { $set: { status: 'running' } });
+        }
+    }
+    async setStepScheduled(stepId) {
+        await this.steps.updateOne({ _id: stepId }, { $set: { status: 'scheduled', updatedAt: new Date() } });
+    }
+    async insertHistoryRecord(stepId, status, result, error) {
+        const now = new Date();
+        const doc = {
+            _id: crypto.randomUUID(),
+            workflowStepId: stepId,
+            status,
+            result: result ?? null,
+            error: error ?? null,
+            createdAt: now,
+            runningAt: null,
+            scheduledAt: null,
+            succeededAt: null,
+            failedAt: null,
+        };
+        const timestampField = this.getTimestampFieldForStatus(status);
+        if (timestampField !== 'createdAt') {
+            doc[timestampField] = now;
+        }
+        await this.stepHistory.insertOne(doc);
+    }
+    getTimestampFieldForStatus(status) {
+        switch (status) {
+            case 'running':
+                return 'runningAt';
+            case 'scheduled':
+                return 'scheduledAt';
+            case 'succeeded':
+                return 'succeededAt';
+            case 'failed':
+                return 'failedAt';
+            default:
+                return 'createdAt';
+        }
+    }
+    async setStepResult(stepId, result) {
+        await this.steps.updateOne({ _id: stepId }, {
+            $set: {
+                status: 'succeeded',
+                result,
+                error: null,
+                updatedAt: new Date(),
+            },
+        });
+        const latestHistory = await this.stepHistory
+            .find({ workflowStepId: stepId })
+            .sort({ createdAt: -1 })
+            .limit(1)
+            .toArray();
+        if (latestHistory.length > 0) {
+            await this.stepHistory.updateOne({ _id: latestHistory[0]._id }, { $set: { status: 'succeeded', result } });
+        }
+    }
+    async setStepError(stepId, error) {
+        const serializedError = {
+            message: error.message,
+            stack: error.stack,
+            code: error.code,
+        };
+        await this.steps.updateOne({ _id: stepId }, {
+            $set: {
+                status: 'failed',
+                error: serializedError,
+                result: null,
+                updatedAt: new Date(),
+            },
+        });
+        const latestHistory = await this.stepHistory
+            .find({ workflowStepId: stepId })
+            .sort({ createdAt: -1 })
+            .limit(1)
+            .toArray();
+        if (latestHistory.length > 0) {
+            await this.stepHistory.updateOne({ _id: latestHistory[0]._id }, { $set: { status: 'failed', error: serializedError } });
+        }
+    }
+    async createRetryAttempt(stepId, status) {
+        await this.steps.updateOne({ _id: stepId }, {
+            $set: {
+                status,
+                result: null,
+                error: null,
+                updatedAt: new Date(),
+            },
+        });
+        await this.insertHistoryRecord(stepId, status);
+        const row = await this.steps.findOne({ _id: stepId });
+        if (!row)
+            throw new Error(`Step not found: ${stepId}`);
+        const attemptCount = await this.stepHistory.countDocuments({
+            workflowStepId: stepId,
+        });
+        return {
+            stepId: row._id,
+            status: row.status,
+            result: row.result ?? undefined,
+            error: row.error ?? undefined,
+            attemptCount,
+            retries: row.retries != null ? Number(row.retries) : undefined,
+            retryDelay: row.retryDelay ?? undefined,
+            createdAt: new Date(row.createdAt),
+            updatedAt: new Date(row.updatedAt),
+        };
+    }
+    async withRunLock(id, fn) {
+        return fn();
+    }
+    async withStepLock(_runId, _stepName, fn) {
+        return fn();
+    }
+    async getCompletedGraphState(runId) {
+        const results = await this.steps
+            .find({
+            workflowRunId: runId,
+            status: { $in: ['succeeded', 'failed'] },
+        })
+            .toArray();
+        const completedNodeIds = [];
+        const failedNodeIds = [];
+        const branchKeys = {};
+        for (const row of results) {
+            const nodeId = row.stepName;
+            if (row.status === 'succeeded') {
+                completedNodeIds.push(nodeId);
+                if (row.branchTaken) {
+                    branchKeys[nodeId] = row.branchTaken;
+                }
+            }
+            else if (row.status === 'failed') {
+                const maxAttempts = (row.retries ?? 0) + 1;
+                const attemptCount = await this.stepHistory.countDocuments({
+                    workflowStepId: row._id,
+                });
+                if (attemptCount >= maxAttempts) {
+                    failedNodeIds.push(nodeId);
+                }
+            }
+        }
+        return { completedNodeIds, failedNodeIds, branchKeys };
+    }
+    async getNodesWithoutSteps(runId, nodeIds) {
+        if (nodeIds.length === 0)
+            return [];
+        const result = await this.steps
+            .find({
+            workflowRunId: runId,
+            stepName: { $in: nodeIds },
+        })
+            .project({ stepName: 1 })
+            .toArray();
+        const existingStepNames = new Set(result.map((r) => r.stepName));
+        return nodeIds.filter((id) => !existingStepNames.has(id));
+    }
+    async getNodeResults(runId, nodeIds) {
+        if (nodeIds.length === 0)
+            return {};
+        const result = await this.steps
+            .find({
+            workflowRunId: runId,
+            stepName: { $in: nodeIds },
+            status: 'succeeded',
+        })
+            .toArray();
+        const results = {};
+        for (const row of result) {
+            results[row.stepName] = row.result;
+        }
+        return results;
+    }
+    async setBranchTaken(stepId, branchKey) {
+        await this.steps.updateOne({ _id: stepId }, { $set: { branchTaken: branchKey, updatedAt: new Date() } });
+    }
+    async updateRunState(runId, name, value) {
+        await this.runs.updateOne({ _id: runId }, {
+            $set: {
+                [`state.${name}`]: value,
+                updatedAt: new Date(),
+            },
+        });
+    }
+    async getRunState(runId) {
+        const row = await this.runs.findOne({ _id: runId }, { projection: { state: 1 } });
+        if (!row)
+            return {};
+        return row.state ?? {};
+    }
+    async upsertWorkflowVersion(name, graphHash, graph, source, status) {
+        await this.versions.updateOne({ workflowName: name, graphHash }, {
+            $setOnInsert: {
+                workflowName: name,
+                graphHash,
+                graph,
+                source,
+                status: status ?? 'active',
+                createdAt: new Date(),
+            },
+        }, { upsert: true });
+    }
+    async updateWorkflowVersionStatus(name, graphHash, status) {
+        await this.versions.updateOne({ workflowName: name, graphHash }, { $set: { status } });
+    }
+    async getWorkflowVersion(name, graphHash) {
+        return this.runService.getWorkflowVersion(name, graphHash);
+    }
+    async getAIGeneratedWorkflows(agentName) {
+        const filter = { source: 'ai-agent', status: 'active' };
+        if (agentName) {
+            const escaped = agentName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+            filter.workflowName = { $regex: `^ai:${escaped}:` };
+        }
+        const docs = await this.versions.find(filter).toArray();
+        return docs.map((doc) => ({
+            workflowName: doc.workflowName,
+            graphHash: doc.graphHash,
+            graph: doc.graph,
+        }));
+    }
+    async close() { }
+}

package/dist/src/pikku-mongodb.d.ts

@@ -0,0 +1,12 @@
+import type { Logger } from '@pikku/core/services';
+import { MongoClient, type MongoClientOptions, type Db } from 'mongodb';
+export declare class PikkuMongoDB {
+    private logger;
+    private dbName;
+    db: Db;
+    private client;
+    private ownsConnection;
+    constructor(logger: Logger, clientOrUri: MongoClient | string, dbName: string, options?: MongoClientOptions);
+    init(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/src/pikku-mongodb.js

@@ -0,0 +1,37 @@
+import { MongoClient } from 'mongodb';
+export class PikkuMongoDB {
+    logger;
+    dbName;
+    db;
+    client;
+    ownsConnection;
+    constructor(logger, clientOrUri, dbName, options) {
+        this.logger = logger;
+        this.dbName = dbName;
+        if (typeof clientOrUri === 'string') {
+            this.client = new MongoClient(clientOrUri, options);
+            this.ownsConnection = true;
+        }
+        else {
+            this.client = clientOrUri;
+            this.ownsConnection = false;
+        }
+        this.db = this.client.db(dbName);
+    }
+    async init() {
+        this.logger.info(`Connecting to MongoDB database: ${this.dbName}`);
+        try {
+            await this.client.db('admin').command({ ping: 1 });
+            this.logger.info('MongoDB connection successful');
+        }
+        catch (error) {
+            this.logger.error('Error connecting to MongoDB', error);
+            process.exit(1);
+        }
+    }
+    async close() {
+        if (this.ownsConnection) {
+            await this.client.close();
+        }
+    }
+}

package/dist/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"root":["../src/index.ts","../src/mongodb-agent-run-service.ts","../src/mongodb-ai-storage-service.ts","../src/mongodb-channel-store.ts","../src/mongodb-deployment-service.ts","../src/mongodb-eventhub-store.ts","../src/mongodb-secret-service.ts","../src/mongodb-workflow-run-service.ts","../src/mongodb-workflow-service.ts","../src/pikku-mongodb.ts"],"version":"5.9.3"}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@pikku/mongodb",
+  "version": "0.12.1",
+  "author": "yasser.fadl@gmail.com",
+  "license": "MIT",
+  "module": "dist/src/index.js",
+  "main": "dist/src/index.js",
+  "type": "module",
+  "scripts": {
+    "tsc": "tsc",
+    "ncu": "npx npm-check-updates",
+    "build": "tsc -b",
+    "test": "bash run-tests.sh",
+    "test:watch": "bash run-tests.sh --watch",
+    "test:coverage": "bash run-tests.sh --coverage",
+    "release": "npm run build && npm test"
+  },
+  "peerDependencies": {
+    "@pikku/core": "^0.12.3"
+  },
+  "dependencies": {
+    "mongodb": "^6.12.0"
+  },
+  "devDependencies": {
+    "mongodb": "^6.12.0",
+    "mongodb-memory-server": "^10.4.0",
+    "typescript": "^5.9"
+  }
+}

package/run-tests.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Enable nullglob to handle cases where no files match the pattern
+shopt -s nullglob
+
+# Initialize variables for options
+watch_mode=false
+coverage_mode=false
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --watch)
+      watch_mode=true
+      shift
+      ;;
+    --coverage)
+      coverage_mode=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+# Define the pattern to match your test files
+pattern="src/*.test.ts"
+
+# Expand the pattern into an array of files
+files=($(find src -type f -name "*.test.ts"))
+
+# Check if any files matched the pattern
+if [ ${#files[@]} -eq 0 ]; then
+  echo "No test files found matching pattern: $pattern"
+  exit 0
+fi
+
+# Construct the node command
+node_cmd="node --import tsx --test"
+
+# Append options based on flags
+if [ "$watch_mode" = true ]; then
+  node_cmd="$node_cmd --watch"
+fi
+
+if [ "$coverage_mode" = true ]; then
+  node_cmd="$node_cmd --test-coverage-include=\"src/**/*.{ts,js}\" --test-coverage-exclude=\"**/dist/**\" --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info"
+fi
+
+# Execute the node command with the expanded list of files
+$node_cmd "${files[@]}"

package/src/index.ts

@@ -0,0 +1,14 @@
+export { PikkuMongoDB } from './pikku-mongodb.js'
+
+export { MongoDBChannelStore } from './mongodb-channel-store.js'
+export { MongoDBEventHubStore } from './mongodb-eventhub-store.js'
+export { MongoDBWorkflowService } from './mongodb-workflow-service.js'
+export { MongoDBWorkflowRunService } from './mongodb-workflow-run-service.js'
+export { MongoDBDeploymentService } from './mongodb-deployment-service.js'
+export { MongoDBAIStorageService } from './mongodb-ai-storage-service.js'
+export { MongoDBAgentRunService } from './mongodb-agent-run-service.js'
+export { MongoDBSecretService } from './mongodb-secret-service.js'
+export type { MongoDBSecretServiceConfig } from './mongodb-secret-service.js'
+
+export type { WorkflowRunService } from '@pikku/core/workflow'
+export type { AgentRunService, AgentRunRow } from '@pikku/core/ai-agent'