@marktoflow/core 2.0.0-alpha.9 → 2.0.2

package/README.md

@@ -1,70 +1,51 @@
 # @marktoflow/core
 
-> **Author:** Scott Glover <scottgl@gmail.com>
+> Workflow engine for parsing, executing, and managing markdown-based automations.
 
-Core engine for marktoflow - parser, executor, and state management.
+[![npm](https://img.shields.io/npm/v/@marktoflow/core)](https://www.npmjs.com/package/@marktoflow/core)
 
-## Overview
+Part of [marktoflow](../../README.md) — open-source markdown workflow automation.
 
-`@marktoflow/core` is the foundation of the marktoflow automation framework. It provides the core workflow engine, state management, scheduling, and execution infrastructure.
-
-## Features
-
-- **Workflow Parser** - Parse markdown + YAML workflow definitions
-- **Execution Engine** - Step-by-step workflow execution with retry and error handling
-- **State Management** - SQLite-based persistent state tracking
-- **Scheduling** - Cron-based workflow scheduling
-- **Queue System** - Support for Redis, RabbitMQ, and in-memory queues
-- **Webhooks** - HTTP webhook trigger support
-- **File Watching** - Monitor files for changes and trigger workflows
-- **Security** - RBAC, approval workflows, and audit logging
-- **Cost Tracking** - Track and manage API usage costs
-- **Plugin System** - Extensible plugin architecture with 17 hook types
-- **Templates** - Reusable workflow templates with variables
-- **Tool Registry** - Support for MCP, OpenAPI, and custom tools
-- **Agent Routing** - Multi-agent workflow support with routing strategies
-
-## Installation
+## Quick Start
 
 ```bash
 npm install @marktoflow/core
 ```
 
-## Usage
-
-### Basic Workflow Execution
-
 ```typescript
 import { WorkflowParser, WorkflowEngine } from '@marktoflow/core';
 
-// Parse workflow
 const parser = new WorkflowParser();
 const workflow = await parser.parseWorkflow('workflow.md');
 
-// Execute workflow
 const engine = new WorkflowEngine();
 const result = await engine.execute(workflow, {
   inputs: { message: 'Hello World' },
 });
-
-console.log(result);
 ```
 
-### With State Management
+## Features
+
+- **Workflow Parser** — Parse markdown + YAML workflow definitions
+- **Execution Engine** — Step-by-step execution with retry, circuit breakers, and error handling
+- **State Management** — SQLite-based persistent state tracking
+- **Plugin System** — Extensible architecture with 17 hook types
+- **Cost Tracking** — Monitor and budget API usage per workflow
+- **Scheduling** — Cron-based workflow scheduling
+- **Security** — RBAC, approval workflows, and audit logging
+- **Queue System** — Distributed execution via Redis, RabbitMQ, or in-memory
+
+## Usage
+
+### State Management
 
 ```typescript
 import { WorkflowEngine, StateManager } from '@marktoflow/core';
 
-// Initialize state manager
-const stateManager = new StateManager({
-  dbPath: '.marktoflow/state.db',
-});
-
-// Execute workflow with state
+const stateManager = new StateManager({ dbPath: '.marktoflow/state.db' });
 const engine = new WorkflowEngine({ stateManager });
 const result = await engine.execute(workflow);
 
-// Query state
 const history = await stateManager.getWorkflowHistory(workflow.id);
 ```
 
@@ -73,234 +54,55 @@ const history = await stateManager.getWorkflowHistory(workflow.id);
 ```typescript
 import { Scheduler } from '@marktoflow/core';
 
-// Create scheduler
 const scheduler = new Scheduler();
-
-// Schedule workflow (cron format)
 await scheduler.schedule({
   workflowId: 'daily-report',
-  cron: '0 9 * * 1-5', // 9 AM weekdays
+  cron: '0 9 * * 1-5',
   workflowPath: './workflows/daily-report.md',
 });
-
-// Start scheduler
 await scheduler.start();
 ```
 
-### Webhooks
-
-```typescript
-import { WebhookServer } from '@marktoflow/core';
-
-// Create webhook server
-const webhookServer = new WebhookServer({ port: 3000 });
-
-// Register webhook
-await webhookServer.registerWebhook({
-  path: '/github',
-  workflowPath: './workflows/github-pr.md',
-  secret: process.env.GITHUB_WEBHOOK_SECRET,
-});
-
-// Start server
-await webhookServer.start();
-```
-
 ### Plugin System
 
 ```typescript
 import { PluginRegistry } from '@marktoflow/core';
 
-// Register plugin
 const registry = new PluginRegistry();
 await registry.register({
   name: 'my-plugin',
   hooks: {
-    beforeWorkflowStart: async (context) => {
-      console.log('Starting workflow:', context.workflow.id);
-    },
-    afterStepComplete: async (context) => {
-      console.log('Completed step:', context.step.action);
-    },
+    beforeWorkflowStart: async (ctx) => console.log('Starting:', ctx.workflow.id),
+    afterStepComplete: async (ctx) => console.log('Done:', ctx.step.action),
   },
 });
 ```
 
-## Workflow Format
-
-Workflows are written in markdown with YAML frontmatter:
-
-```markdown
----
-workflow:
-  id: example
-  name: Example Workflow
-
-tools:
-  slack:
-    sdk: '@slack/web-api'
-    auth:
-      token: '${SLACK_BOT_TOKEN}'
-
-triggers:
-  - type: schedule
-    cron: '0 9 * * *'
-
-inputs:
-  message:
-    type: string
-    required: true
-
-outputs:
-  result:
-    type: string
----
-
-# Example Workflow
-
-This workflow posts a message to Slack.
-
-## Step 1: Post Message
-
-\`\`\`yaml
-action: slack.chat.postMessage
-inputs:
-channel: '#general'
-text: '{{ inputs.message }}'
-output_variable: result
-\`\`\`
-```
-
-## Architecture
-
-### Core Components
-
-1. **Parser** (`parser.ts`) - Parse markdown + YAML workflow definitions
-2. **Engine** (`engine.ts`) - Execute workflows with retry/circuit breaker
-3. **State Manager** (`state.ts`) - SQLite-based state persistence
-4. **Scheduler** (`scheduler.ts`) - Cron-based workflow scheduling
-5. **Queue** (`queue.ts`) - Redis/RabbitMQ/InMemory queue support
-6. **Webhook** (`webhook.ts`) - HTTP webhook triggers
-7. **File Watcher** (`filewatcher.ts`) - File change monitoring
-8. **Security** (`security.ts`) - RBAC and audit logging
-9. **Cost Tracker** (`costs.ts`) - API usage cost management
-10. **Plugin System** (`plugins.ts`) - Extensible hooks
-11. **Templates** (`templates.ts`) - Reusable workflow patterns
-12. **Tool Registry** (`tool-registry.ts`) - MCP/OpenAPI/Custom tools
-13. **Agent Routing** (`routing.ts`) - Multi-agent coordination
-
-### Execution Flow
-
-```
-Parser → Validate → Security Check → Execute Steps → Save State → Output
-           ↓           ↓                    ↓            ↓
-        Schema      RBAC              Retry/Failover  Audit Log
-```
-
-## Configuration
-
-### Environment Variables
-
-```bash
-# Database
-MARKTOFLOW_DB_PATH=.marktoflow/state.db
-
-# Queue (Redis)
-REDIS_HOST=localhost
-REDIS_PORT=6379
-
-# Queue (RabbitMQ)
-RABBITMQ_URL=amqp://localhost
-
-# Security
-MARKTOFLOW_SECRET_KEY=your-secret-key
-```
-
-### Configuration File
-
-Create `.marktoflow/config.yaml`:
-
-```yaml
-state:
-  dbPath: .marktoflow/state.db
-
-queue:
-  type: redis # redis, rabbitmq, or memory
-  redis:
-    host: localhost
-    port: 6379
-
-security:
-  rbac:
-    enabled: true
-  auditLog:
-    enabled: true
-
-costs:
-  budget:
-    daily: 100
-    monthly: 3000
-  alerts:
-    - threshold: 50
-      emails: [admin@example.com]
-```
-
 ## API Reference
 
-### WorkflowParser
-
 ```typescript
 class WorkflowParser {
   parseWorkflow(filePath: string): Promise<Workflow>;
   parseYAML(content: string): Workflow;
   validate(workflow: Workflow): ValidationResult;
 }
-```
-
-### WorkflowEngine
 
-```typescript
 class WorkflowEngine {
   constructor(options?: EngineOptions);
   execute(workflow: Workflow, context?: ExecutionContext): Promise<WorkflowResult>;
   stop(): Promise<void>;
 }
-```
-
-### StateManager
 
-```typescript
 class StateManager {
   constructor(options: StateOptions);
   getWorkflowHistory(workflowId: string): Promise<WorkflowRun[]>;
-  getWorkflowState(runId: string): Promise<WorkflowState>;
   saveWorkflowState(state: WorkflowState): Promise<void>;
 }
 ```
 
-### Scheduler
-
-```typescript
-class Scheduler {
-  schedule(config: ScheduleConfig): Promise<void>;
-  unschedule(workflowId: string): Promise<void>;
-  start(): Promise<void>;
-  stop(): Promise<void>;
-}
-```
-
-## Testing
-
-```bash
-npm test
-```
-
-## Links
+## Contributing
 
-- [Main Repository](https://github.com/scottgl9/marktoflow)
-- [Documentation](https://github.com/scottgl9/marktoflow#readme)
-- [CLI Package](@marktoflow/cli)
-- [Integrations Package](@marktoflow/integrations)
+See the [contributing guide](https://github.com/marktoflow/marktoflow/blob/main/CONTRIBUTING.md).
 
 ## License
 

package/dist/built-in-operations.d.ts

@@ -0,0 +1,150 @@
+/**
+ * Built-in Operations for marktoflow
+ *
+ * Provides common operations that eliminate the need for verbose script blocks:
+ * - core.set: Simple variable assignment
+ * - core.transform: Map/filter/reduce transformations
+ * - core.extract: Nested path access
+ * - core.format: Date/number/string formatting
+ * - file.read: Read files with format conversion (docx, pdf, xlsx)
+ * - file.write: Write files (text or binary)
+ */
+import { ExecutionContext } from './models.js';
+export interface SetOperationInputs {
+    [key: string]: unknown;
+}
+export interface TransformOperationInputs {
+    input: unknown[];
+    operation: 'map' | 'filter' | 'reduce' | 'find' | 'group_by' | 'unique' | 'sort';
+    expression?: string;
+    condition?: string;
+    initialValue?: unknown;
+    key?: string;
+    reverse?: boolean;
+}
+export interface ExtractOperationInputs {
+    input: unknown;
+    path: string;
+    default?: unknown;
+}
+export interface FormatOperationInputs {
+    value: unknown;
+    type: 'date' | 'number' | 'string' | 'currency' | 'json';
+    format?: string;
+    locale?: string;
+    currency?: string;
+    precision?: number;
+}
+/**
+ * Set multiple variables at once with expression resolution.
+ *
+ * Example:
+ * ```yaml
+ * action: core.set
+ * inputs:
+ *   owner: "{{ inputs.repo =~ /^([^\/]+)\// }}"
+ *   repo_name: "{{ inputs.repo =~ /\/(.+)$/ }}"
+ *   timestamp: "{{ now() }}"
+ * ```
+ */
+export declare function executeSet(inputs: SetOperationInputs, context: ExecutionContext): Record<string, unknown>;
+/**
+ * Transform arrays using common operations like map, filter, reduce.
+ *
+ * Examples:
+ *
+ * Map:
+ * ```yaml
+ * action: core.transform
+ * inputs:
+ *   input: "{{ oncall_response.data.oncalls }}"
+ *   operation: map
+ *   expression: "@{{ item.user.name }}"
+ * ```
+ *
+ * Filter:
+ * ```yaml
+ * action: core.transform
+ * inputs:
+ *   input: "{{ issues }}"
+ *   operation: filter
+ *   condition: "item.priority == 'high'"
+ * ```
+ *
+ * Reduce:
+ * ```yaml
+ * action: core.transform
+ * inputs:
+ *   input: "{{ numbers }}"
+ *   operation: reduce
+ *   expression: "{{ accumulator + item }}"
+ *   initialValue: 0
+ * ```
+ */
+export declare function executeTransform(rawInputs: TransformOperationInputs, resolvedInputs: Record<string, unknown>, context: ExecutionContext): unknown;
+/**
+ * Extract values from nested objects safely.
+ *
+ * Example:
+ * ```yaml
+ * action: core.extract
+ * inputs:
+ *   input: "{{ api_response }}"
+ *   path: "data.users[0].email"
+ *   default: "unknown@example.com"
+ * ```
+ */
+export declare function executeExtract(inputs: ExtractOperationInputs, context: ExecutionContext): unknown;
+/**
+ * Format values for display (dates, numbers, strings, currency).
+ *
+ * Examples:
+ *
+ * Date:
+ * ```yaml
+ * action: core.format
+ * inputs:
+ *   value: "{{ now() }}"
+ *   type: date
+ *   format: "YYYY-MM-DD HH:mm:ss"
+ * ```
+ *
+ * Number:
+ * ```yaml
+ * action: core.format
+ * inputs:
+ *   value: 1234.56
+ *   type: number
+ *   precision: 2
+ * ```
+ *
+ * Currency:
+ * ```yaml
+ * action: core.format
+ * inputs:
+ *   value: 1234.56
+ *   type: currency
+ *   currency: USD
+ *   locale: en-US
+ * ```
+ */
+export declare function executeFormat(inputs: FormatOperationInputs, context: ExecutionContext): string;
+/**
+ * Execute a built-in operation based on action name
+ */
+export declare function executeBuiltInOperation(action: string, rawInputs: Record<string, unknown>, resolvedInputs: Record<string, unknown>, context: ExecutionContext): unknown | Promise<unknown>;
+/**
+ * Check if an action is a built-in operation
+ */
+export declare function isBuiltInOperation(action: string): boolean;
+export declare function executeAggregate(inputs: Record<string, unknown>, _context: ExecutionContext): unknown;
+export declare function executeCompare(inputs: Record<string, unknown>, _context: ExecutionContext): unknown;
+export declare function executeRenameKeys(inputs: Record<string, unknown>): unknown;
+export declare function executeLimit(inputs: Record<string, unknown>): unknown;
+export declare function executeSortOperation(inputs: Record<string, unknown>): unknown;
+export declare function executeCrypto(inputs: Record<string, unknown>): unknown;
+export declare function executeDatetime(inputs: Record<string, unknown>): unknown;
+export declare function executeParse(inputs: Record<string, unknown>): unknown;
+export declare function executeCompress(inputs: Record<string, unknown>): unknown;
+export declare function executeDecompress(inputs: Record<string, unknown>): unknown;
+//# sourceMappingURL=built-in-operations.d.ts.map

package/dist/built-in-operations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"built-in-operations.d.ts","sourceRoot":"","sources":["../src/built-in-operations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAQ/C,MAAM,WAAW,kBAAkB;IACjC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,SAAS,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;;;;GAWG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,gBAAgB,GACxB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CASzB;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,wBAAwB,EACnC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,OAAO,EAAE,gBAAgB,GACxB,OAAO,CA4CT;AAsKD;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,sBAAsB,EAC9B,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAkBT;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,qBAAqB,EAC7B,OAAO,EAAE,gBAAgB,GACxB,MAAM,CAsBR;AAiHD;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,OAAO,EAAE,gBAAgB,GACxB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAoD5B;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAQ1D;AAMD,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAuCT;AAMD,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAuBT;AAMD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CA4B1E;AAMD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CASrE;AAMD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAgB7E;AAMD,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAsDtE;AAMD,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CA2DxE;AAMD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAsDrE;AAMD,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAgBxE;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAgB1E"}