meta_workflows 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: afd80be43cbf526411e03bbf7ced8c539b12d81e82d19b2e805dc2a85045aa83
+  data.tar.gz: 4ce850bc8e751d2271ea049724bce5ddbf82d55b9d83749e7d06cfb1d0aebb2c
+SHA512:
+  metadata.gz: 4fda18b86bf518cfbace011470987b8aa6fb7d1ebbfbb8e95933498736a6792b9c90183a8e820a2868ed8a34dab8cb0a32e194f054315d6bfbbdd4f6c067feb6
+  data.tar.gz: 45148c5baac802a9c4c330206724b4e8a867df7886a2fc984b9f5b5854a14d926336492c88c0e0f718f84a3d0b8973373f682eaf20d628929dd03d7bc4c5a9ee

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog
+
+All notable changes to the MetaWorkflows engine will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial Rails engine structure
+- Configuration system for engine customization
+- Support for Rails 7.2+
+- Integration with Sidekiq for background job processing
+- Integration with ruby_conversations for AI chat functionality
+- ViewComponent support for UI components
+- Turbo and Stimulus integration for frontend interactivity
+- PostgreSQL database support following host application patterns
+
+### Changed
+- Extracted from course-builder monolith into standalone engine
+- Updated dependencies for Rails 7.2 compatibility
+- Switched from SQLite3 to PostgreSQL for database backend
+- Updated dummy application to use PostgreSQL configuration
+- Aligned database setup with host application (course-builder) patterns
+
+## [0.1.0] - 2025-01-XX
+
+### Added
+- Initial release of MetaWorkflows as a Rails engine
+- Core workflow execution framework
+- AI-powered conversation management
+- Human interaction points in workflows
+- Background job processing with Sidekiq
+- Modern UI components with TailwindCSS and Stimulus
+- PostgreSQL database backend for reliability and scalability 

data/README.md

@@ -0,0 +1,498 @@
+# MetaWorkflows
+
+<div align="center">
+  <img src="spec/dummy/public/icon.svg" alt="MetaWorkflows Logo" width="128" height="128">
+</div>
+
+A Rails engine for managing AI-powered meta workflows with human interaction points, extracted from course-builder into a standalone, reusable Rails gem.
+
+## Overview
+
+MetaWorkflows provides a flexible framework for creating and executing AI-powered workflows that can include human interaction points, background job processing, and modern UI components. Built as a Rails engine for easy integration into existing applications, it enables sophisticated multi-step processes that combine AI automation with human oversight.
+
+Originally developed as part of the course-builder application and extracted to improve modularity and enable reuse across multiple applications. The system orchestrates complex workflows requiring both AI agent and human interactions through a configurable YAML-based workflow definition system.
+
+## Key Features
+
+- **AI-Powered Workflows**: Integration with `ruby_conversations` for AI chat functionality and tool calling
+- **Human Interaction Points**: Pause workflows for human input, review, and feedback with built-in UI components
+- **Background Processing**: Sidekiq integration for reliable, asynchronous job execution
+- **Modern UI**: TailwindCSS styling with Turbo and Stimulus for real-time interactivity
+- **Flexible Configuration**: YAML-based workflow definitions with multiple action types
+- **Rails Engine Architecture**: Clean integration as a mountable engine with proper namespacing
+- **Atomic Operations**: Database transactions ensure data integrity during batch operations
+- **Extensible Service Layer**: Pluggable services for record updates and collection creation
+
+## System Architecture
+
+### Core Components
+
+The MetaWorkflows system consists of several interconnected components:
+
+#### Database Models
+- **MetaWorkflows::Workflow**: Stores workflow definitions with named steps and JSON recipes
+- **MetaWorkflows::WorkflowExecution**: Tracks active workflow instances and their state
+- **MetaWorkflows::WorkflowStep**: Represents individual steps within a workflow execution
+- **MetaWorkflows::Chat**: Manages LLM conversation contexts for workflow steps
+- **MetaWorkflows::Message**: Stores individual messages within workflow conversations
+- **MetaWorkflows::ToolCall**: Records tool calls made during LLM interactions
+
+#### Controllers
+- **MetaWorkflows::MetaController**: Base controller with shared workflow functionality
+- **MetaWorkflows::HumansController**: Manages human interactions within workflows
+
+#### Background Jobs
+- **MetaWorkflows::MetaWorkflowJob**: Entry point for workflow processing and step coordination
+- **MetaWorkflows::MetaJob**: Base job class for workflow-related processing
+- **MetaWorkflows::HumanInputJob**: Handles human input processing within workflows
+- **MetaWorkflows::RecordRedirectJob**: Manages post-workflow redirections
+
+#### Services
+- **Services::MetaWorkflows::MetaWorkflowService**: Central workflow coordinator that processes all workflow steps and actions
+- **Services::MetaWorkflows::Updaters::MetaService**: Generic record update service for modifying model attributes
+
+## Workflow Action Types
+
+MetaWorkflows supports several action types for different workflow needs:
+
+### `agent`
+- Executes LLM interactions using specified prompts and tools
+- Requires `prompt_id` configuration
+- Supports tool calling and complex reasoning tasks
+- Output is extracted and stored for subsequent steps
+
+### `human`
+- Presents UI for human interaction and input collection
+- Requires `prompt_id` for context presentation
+- Collects user feedback through forms
+- Workflow pauses until human input is provided
+
+### `collection_create`
+- Creates collections of related records atomically
+- Requires `collection_creator` service class name
+- Handles batch creation operations with all-or-nothing semantics
+- Automatically halts workflow on any creation failure
+
+### `record_update`
+- Updates the associated record with specified attributes
+- Requires `record_updater` service class name and `attributes_to_update` list
+- Uses values from `workflow_params` to update the record
+- Supports both generic and custom updater services
+
+### `record_redirect`
+- Redirects users to specified paths
+- Marks workflow as complete
+- Requires `redirect_path` or `redirect_method` configuration
+
+## Requirements
+
+- **Rails 7.2+**: Compatible with modern Rails applications
+- **PostgreSQL**: Database backend (9.3 and up supported)
+- **Sidekiq**: Background job processing with Redis
+- **Ruby 3.2+**: Modern Ruby version
+- **TailwindCSS**: For UI styling (configured automatically)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'meta_workflows'
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+Or install it yourself as:
+
+```bash
+gem install meta_workflows
+```
+
+## Setup
+
+### 1. Database Setup
+
+Ensure PostgreSQL is installed and running on your system. The engine uses PostgreSQL for reliable data storage and supports the same configuration as your host Rails application.
+
+### 2. Environment Configuration
+
+For development and testing, configure the required environment variables:
+
+```bash
+# Copy the example environment file
+cp spec/dummy/.env.example spec/dummy/.env
+
+# Edit the .env file with your actual values
+```
+
+The following environment variables are required for StrongMind Identity SSO integration:
+
+| Variable | Description | Example |
+|:--------:|:----------:|:-------:|
+| `IDENTITY_CLIENT_ID` | Your StrongMind Identity client ID | `course-builder-development` |
+| `IDENTITY_CLIENT_SECRET` | Your StrongMind Identity client secret | `your-secret-key` |
+| `IDENTITY_BASE_URL` | StrongMind Identity service URL | `https://devlogin.strongmind.com` |
+
+> 💡 **Tip:** Contact your system administrator for the appropriate values for your environment.
+
+### 3. Install and Run Migrations
+
+```bash
+rails meta_workflows:install:migrations
+rails db:migrate
+```
+
+### 4. Mount the Engine
+
+Add to your `config/routes.rb`:
+
+```ruby
+Rails.application.routes.draw do
+  mount MetaWorkflows::Engine, at: '/meta_workflows'
+  # ... your other routes
+end
+```
+
+### 5. Configure Sidekiq
+
+Ensure Sidekiq is configured in your application for background job processing:
+
+```ruby
+# config/application.rb or config/initializers/sidekiq.rb
+require 'sidekiq'
+
+# Configure Redis connection and other Sidekiq settings
+```
+
+## Usage
+
+### Creating Workflow Definitions
+
+Define workflows in YAML files within your configured workflow definitions path:
+
+```yaml
+# config/workflows/example_workflow.yml
+name: "example_workflow"
+description: "An example workflow demonstrating multiple action types"
+steps:
+  - name: "initial_generation"
+    action: "agent"
+    prompt_id: "generation_prompt"
+    step_progress:
+      - "Analyzing input parameters..."
+      - "Generating initial content..."
+      - "Refining results..."
+    input:
+      title:
+        type: "string"
+        required: true
+        description: "The title for processing"
+    output:
+      generated_content:
+        type: "string"
+        description: "The generated content"
+
+  - name: "human_review"
+    action: "human"
+    prompt_id: "review_prompt"
+    step_progress:
+      - "Preparing content for review..."
+    input:
+      generated_content:
+        type: "string"
+        from_step: "initial_generation"
+    output:
+      user_feedback:
+        type: "string"
+        description: "User feedback on the content"
+
+  - name: "record_update"
+    action: "record_update"
+    record_updater: "Services::MetaWorkflows::Updaters::MetaService"
+    attributes_to_update:
+      - "title"
+      - "topic"
+    step_progress:
+      - "Updating record..."
+      - "Saving changes..."
+    workflow_params:
+      - name: "title"
+        type: string
+        description: "The new title for the record"
+
+  - name: "completion_redirect"
+    action: "record_redirect"
+    redirect_path: "/your/completion/path"
+```
+
+### Executing Workflows
+
+#### From Controller Actions
+
+```ruby
+class YourController < ApplicationController
+  def start_workflow
+    @record = YourModel.find(params[:id])
+    
+    MetaWorkflows::MetaWorkflowJob.perform_later(
+      workflow_name: "example_workflow",
+      record_type: @record.class.name,
+      record_id: @record.id,
+      user_id: current_user.id,
+      inputs: {
+        title: @record.title,
+        # other initial parameters
+      }
+    )
+    
+    redirect_to workflow_path(@record)
+  end
+end
+```
+
+#### Direct Job Execution
+
+```ruby
+# Start a workflow
+MetaWorkflows::MetaWorkflowJob.perform_later(
+  'example_workflow', 
+  context: { 
+    user_id: 1,
+    record_id: 123,
+    record_type: 'YourModel'
+  }
+)
+```
+
+### Using Meta Workflow UI Components
+
+Instead of creating custom views from scratch, use the provided Meta Workflow partials:
+
+#### Complete View Example
+
+```erb
+<%= turbo_stream_from turbo_stream_name(@record) %>
+<div class="max-w-3xl m-auto flex flex-col p-10 gap-6">
+  <!-- Workflow progress and chat messages area -->
+  <%= render partial: 'meta_workflows/loader', locals: {
+    record: @record,
+    step_progress: ["Generating draft content...", "Processing with AI..."]
+  } %>
+  
+  <!-- User input area for human interaction steps -->
+  <%= render partial: 'meta_workflows/response_form', locals: {
+    record: @record,
+    response_enabled: false
+  } %>
+</div>
+```
+
+## File Structure Overview
+
+The MetaWorkflows gem follows a structured organization:
+
+```
+meta_workflows/
+├── app/
+│   ├── controllers/meta_workflows/       # Workflow controllers
+│   ├── jobs/meta_workflows/             # Background jobs
+│   ├── models/meta_workflows/           # Database models
+│   └── views/meta_workflows/            # UI partials and templates
+├── lib/
+│   └── services/meta_workflows/         # Business logic services
+│       ├── updaters/                    # Record update services
+│       └── creators/                    # Collection creation services
+├── config/
+│   └── workflows/                       # YAML workflow definitions
+└── spec/
+    └── dummy/                          # Test Rails application
+```
+
+## Development
+
+### Setting Up Development Environment
+
+After checking out the repo, run:
+
+```bash
+bundle install
+```
+
+Set up the test database:
+
+```bash
+cd spec/dummy
+rails db:create
+rails db:migrate
+```
+
+### Running Tests
+
+To run the test suite:
+
+```bash
+bundle exec rspec
+```
+
+### Dummy Application
+
+The gem includes a complete Rails application in `spec/dummy/` for testing and development:
+
+```bash
+# Set up environment variables
+cp spec/dummy/.env.example spec/dummy/.env
+# Edit spec/dummy/.env with your actual values
+
+# Start the dummy application
+cd spec/dummy
+rails server
+```
+
+The dummy app demonstrates MetaWorkflows integration with:
+- User authentication (Devise)
+- Post and Comment models with associations
+- Complete CRUD operations
+- MetaWorkflows engine mounted at `/meta_workflows`
+- StrongMind Identity SSO integration
+
+## Extending the System
+
+### Creating Custom Record Updater Services
+
+```ruby
+# lib/services/meta_workflows/updaters/your_model_service.rb
+module Services
+  module MetaWorkflows
+    module Updaters
+      class YourModelService < ApplicationService
+        def initialize(record, attributes_to_update, workflow_params)
+          @record = record
+          @attributes_to_update = attributes_to_update
+          @workflow_params = workflow_params
+        end
+
+        def call
+          update_attributes = build_update_attributes
+          record.update!(update_attributes)
+          record
+        end
+
+        private
+
+        def build_update_attributes
+          # Custom logic for building update attributes
+        end
+      end
+    end
+  end
+end
+```
+
+### Creating Custom Collection Creator Services
+
+```ruby
+# lib/services/meta_workflows/creators/your_collection_service.rb
+module Services
+  module MetaWorkflows
+    module Creators
+      class YourCollectionService
+        def self.create_collection(parent, items, options = {})
+          created_items = []
+          
+          ActiveRecord::Base.transaction do
+            items.each do |item_data|
+              created_items << create_item(parent, item_data, options)
+            end
+          end
+          
+          created_items
+        end
+
+        private_class_method
+
+        def self.create_item(parent, data, options = {})
+          YourModel.create!(
+            content: data[:content],
+            parent: parent,
+            organization_id: parent.organization_id
+          )
+        end
+      end
+    end
+  end
+end
+```
+
+### Adding New Action Types
+
+To add a new action type:
+
+1. Add the action to `ACTIONS_WITHOUT_CONVERSATION` in `Services::MetaWorkflows::MetaWorkflowService` if it doesn't require conversation tracking
+2. Implement a `process_[action_name]_action` method in the service
+3. Add the action handling to the `process_action` method
+4. Create any necessary supporting services
+5. Add comprehensive tests for the new action type
+
+## Best Practices
+
+### Workflow Design
+- Keep steps focused and single-purpose
+- Design clear input/output schemas
+- Provide meaningful progress messages
+- Handle error cases gracefully
+- Use `record_update` steps to persist intermediate workflow results
+
+### Performance
+- Use background jobs for long-running processes
+- Implement appropriate caching strategies
+- Monitor workflow execution times
+- Set reasonable timeouts for LLM interactions
+- Use bulk operations in collection creators when possible
+
+### UI/UX Considerations
+- Use the provided meta partials for consistent UI
+- Provide meaningful step progress messages in workflow definitions
+- Ensure proper Turbo Stream configuration for real-time updates
+- Handle edge cases and error states appropriately
+
+## Architecture Details
+
+### Engine Isolation
+- All classes properly namespaced under `MetaWorkflows::`
+- Controllers inherit from `MetaWorkflows::ApplicationController`
+- Jobs inherit from `MetaWorkflows::ApplicationJob`
+- Models use engine-specific associations and validations
+- Services organized under `Services::MetaWorkflows::` namespace
+
+### Database Compatibility
+- Migration files include `table_exists?` checks to prevent conflicts
+- Exact schema compatibility maintained with existing applications
+- Proper foreign key references and indexes included
+- Migration timestamps ordered correctly for dependency resolution
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/strongmind/meta_workflows.
+
+For development contributions:
+1. Fork the repository
+2. Create a feature branch
+3. Add comprehensive tests
+4. Follow the established code patterns and namespacing
+5. Update documentation as needed
+6. Submit a pull request
+
+## Documentation
+
+For detailed information about the MetaWorkflows system:
+- [Setup Guide](https://strongmind.atlassian.net/wiki/spaces/MW/pages/3929833507) - Configuration and workflow setup
+- [File Structure Guide](https://strongmind.atlassian.net/wiki/spaces/MW/pages/3930521627) - Complete file organization reference
+- [Dummy App Guide](https://strongmind.atlassian.net/wiki/spaces/MW/pages/3941072927) - Test application documentation
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+APP_RAKEFILE = File.expand_path('spec/dummy/Rakefile', __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'

data/app/assets/javascripts/meta_workflows/controllers/loading_phrases_controller.js

@@ -0,0 +1,31 @@
+import { Controller } from "@hotwired/stimulus"
+
+// Connects to data-controller="loading-phrases"
+export default class extends Controller {
+  static targets = ["message"]
+  static values = {
+    phrases: Array
+  }
+  
+  connect() {
+    if (!this.hasPhrasesValue || this.phrasesValue.length === 0) {
+      this.phrasesValue = ["Talking to the LLM..."]
+    }
+    
+    this.currentIndex = 0
+    this.startCycling()
+  }
+  
+  disconnect() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId)
+    }
+  }
+  
+  startCycling() {
+    this.intervalId = setInterval(() => {
+      this.currentIndex = (this.currentIndex + 1) % this.phrasesValue.length
+      this.messageTarget.textContent = this.phrasesValue[this.currentIndex]
+    }, 3000) // Change phrase every 3 seconds
+  }
+} 

data/app/assets/javascripts/meta_workflows/controllers/redirect_controller.js

@@ -0,0 +1,15 @@
+import { Controller } from "@hotwired/stimulus"
+
+// Connects to data-controller="redirect"
+export default class extends Controller {
+  static values = {
+    url: String,
+    delay: { type: Number, default: 500 }
+  }
+
+  connect() {
+    setTimeout(() => {
+      Turbo.visit(this.urlValue)
+    }, this.delayValue)
+  }
+} 

data/app/assets/javascripts/meta_workflows/controllers/response_scroll_controller.js

@@ -0,0 +1,67 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  connect() {
+    this.setupObserver()
+    this.setupTurboListener()
+    this.scrollToBottom()
+  }
+
+  disconnect() {
+    if (this.observer) {
+      this.observer.disconnect()
+    }
+    document.removeEventListener("turbo:frame-render", this.handleTurboRender)
+  }
+
+  setupObserver() {
+    this.observer = new MutationObserver((mutations) => {
+      if (mutations.some(mutation => 
+        mutation.type === 'childList' || 
+        mutation.type === 'characterData'
+      )) {
+        setTimeout(() => this.scrollToBottom(), 10)
+      }
+    })
+
+    const contentContainer = this.element.closest('#response-content-container')
+    if (contentContainer) {
+      this.observer.observe(contentContainer, {
+        childList: true,
+        subtree: true,
+        characterData: true,
+        attributes: true
+      })
+    } else {
+      this.observer.observe(this.element, {
+        childList: true,
+        subtree: true,
+        characterData: true,
+        attributes: true
+      })
+    }
+  }
+
+  setupTurboListener() {
+    this.handleTurboRender = () => {
+      setTimeout(() => this.scrollToBottom(), 10)
+    }
+    document.addEventListener("turbo:frame-render", this.handleTurboRender)
+  }
+
+  scrollToBottom() {
+    const mainContainer = document.querySelector('#main-scroll-container')
+    if (!mainContainer) return
+
+    const scrollHeight = mainContainer.scrollHeight
+    const clientHeight = mainContainer.clientHeight
+    const maxScroll = scrollHeight - clientHeight
+
+    if (maxScroll > 0) {
+      mainContainer.scrollTo({
+        top: maxScroll,
+        behavior: 'smooth'
+      })
+    }
+  }
+} 

data/app/assets/javascripts/meta_workflows_manifest.js

@@ -0,0 +1,13 @@
+//= require_tree ./meta_workflows/controllers
+
+// MetaWorkflows Stimulus Controllers
+import LoadingPhrasesController from "./meta_workflows/controllers/loading_phrases_controller"
+import ResponseScrollController from "./meta_workflows/controllers/response_scroll_controller"
+import RedirectController from "./meta_workflows/controllers/redirect_controller"
+
+// Register controllers with Stimulus
+const application = window.Stimulus || Application.start()
+
+application.register("loading-phrases", LoadingPhrasesController)
+application.register("response-scroll", ResponseScrollController)
+application.register("redirect", RedirectController)