@flomatai/core 0.1.0

package/src/pipeline-registry.ts

@@ -0,0 +1,253 @@
+/**
+ * Pipeline Registry — discover, load, and manage pipelines from various sources.
+ *
+ * This module provides:
+ * - PipelineSource: define where a pipeline comes from (npm, local path, git)
+ * - PipelineRegistry: local registry that maps names to sources
+ * - loadPipeline(): resolve a source to a BuiltPipeline
+ */
+
+import { resolve } from 'path';
+import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import type { BuiltPipeline } from './pipeline.js';
+import type { ZodSchema } from 'zod';
+
+// ── Pipeline Metadata ────────────────────────────────────────────────────────
+
+/**
+ * Metadata for a pipeline package.
+ * Exported from the pipeline's main file.
+ */
+export interface PipelinePackageMeta {
+  /** Unique name (e.g., 'content-generation') */
+  name: string;
+  /** Human-readable description */
+  description: string;
+  /** Semantic version */
+  version?: string;
+  /** Tags for discovery */
+  tags?: string[];
+  /** Author information */
+  author?: string;
+  /** Input schema (for display/documentation) */
+  inputSchema?: ZodSchema;
+  /** Output schema (for display/documentation) */
+  outputSchema?: ZodSchema;
+}
+
+/**
+ * Full pipeline export from a pipeline package.
+ */
+export interface PipelinePackageExports {
+  /** The built pipeline */
+  pipeline: BuiltPipeline;
+  /** Optional metadata */
+  metadata?: PipelinePackageMeta;
+  /** Optional config schema for runtime validation */
+  configSchema?: ZodSchema;
+  /** Optional orchestrator override */
+  orchestrator?: unknown;
+}
+
+// ── Pipeline Source ─────────────────────────────────────────────────────────
+
+/** Where a pipeline is loaded from */
+export type PipelineSource =
+  | { type: 'npm'; package: string; export?: string }
+  | { type: 'path'; path: string }
+  | { type: 'git'; url: string; ref?: string };
+
+/** Registered pipeline in the local registry */
+export interface RegisteredPipeline {
+  /** Local name (e.g., 'my-pipeline') */
+  name: string;
+  /** Source where the pipeline is loaded from */
+  source: PipelineSource;
+  /** Optional version constraint (for npm) */
+  version?: string;
+  /** When this was registered */
+  registeredAt: Date;
+  /** Cached loaded exports */
+  exports?: PipelinePackageExports;
+}
+
+// ── Pipeline Registry ────────────────────────────────────────────────────────
+
+/**
+ * Local pipeline registry.
+ * Maps friendly names to pipeline sources.
+ */
+export class PipelineRegistry {
+  private readonly pipelines: Map<string, RegisteredPipeline> = new Map();
+  private readonly pipelineDirs: string[] = [];
+
+  constructor(pipelineDirs: string[] = []) {
+    this.pipelineDirs = pipelineDirs;
+  }
+
+  /**
+   * Register a pipeline from an npm package.
+   */
+  register(name: string, packageName: string, options?: { version?: string }): void {
+    this.pipelines.set(name, {
+      name,
+      source: { type: 'npm', package: packageName },
+      version: options?.version,
+      registeredAt: new Date(),
+    });
+  }
+
+  /**
+   * Register a pipeline from a local path.
+   */
+  registerPath(name: string, path: string): void {
+    this.pipelines.set(name, {
+      name,
+      source: { type: 'path', path: resolve(path) },
+      registeredAt: new Date(),
+    });
+  }
+
+  /**
+   * Register a pipeline from a git repository.
+   */
+  registerGit(name: string, url: string, ref?: string): void {
+    this.pipelines.set(name, {
+      name,
+      source: { type: 'git', url, ref },
+      registeredAt: new Date(),
+    });
+  }
+
+  /**
+   * Get a registered pipeline by name.
+   */
+  get(name: string): RegisteredPipeline | undefined {
+    return this.pipelines.get(name);
+  }
+
+  /**
+   * List all registered pipelines.
+   */
+  list(): RegisteredPipeline[] {
+    return Array.from(this.pipelines.values());
+  }
+
+  /**
+   * Remove a pipeline from the registry.
+   */
+  unregister(name: string): boolean {
+    return this.pipelines.delete(name);
+  }
+
+  /**
+   * Clear all registrations.
+   */
+  clear(): void {
+    this.pipelines.clear();
+  }
+}
+
+// ── Pipeline Loader ─────────────────────────────────────────────────────────
+
+/**
+ * Load a pipeline from a source.
+ */
+export async function loadPipeline(source: PipelineSource): Promise<PipelinePackageExports> {
+  switch (source.type) {
+    case 'path':
+      return loadFromPath(source.path);
+    case 'npm':
+      return loadFromNpm(source.package, source.export);
+    case 'git':
+      // TODO: Implement git loading (clone + package.json lookup)
+      throw new Error('Git pipeline loading not yet implemented');
+  }
+}
+
+/**
+ * Load a pipeline from a local path.
+ */
+async function loadFromPath(absolutePath: string): Promise<PipelinePackageExports> {
+  // Check if it's a directory (pipeline package) or a file
+  const dirExists = existsSync(absolutePath);
+  const fileExists = existsSync(absolutePath + '.ts') || existsSync(absolutePath + '.js');
+
+  if (dirExists) {
+    // It's a directory — look for pipeline.ts or package.json
+    const pkgPath = resolve(absolutePath, 'package.json');
+    if (existsSync(pkgPath)) {
+      const pkg = JSON.parse(await readFile(pkgPath, 'utf-8'));
+      const mainFile = resolve(absolutePath, pkg.main || pkg.exports?.['.'] || 'index.js');
+      return import(mainFile) as Promise<PipelinePackageExports>;
+    }
+    const pipelinePath = resolve(absolutePath, 'pipeline.ts');
+    if (existsSync(pipelinePath)) {
+      return import(pipelinePath) as Promise<PipelinePackageExports>;
+    }
+    throw new Error(`No pipeline.ts or package.json found in ${absolutePath}`);
+  }
+
+  if (fileExists) {
+    // Direct file import
+    const ext = existsSync(absolutePath + '.ts') ? '.ts' : '.js';
+    return import(absolutePath + ext) as Promise<PipelinePackageExports>;
+  }
+
+  throw new Error(`Pipeline not found at ${absolutePath}`);
+}
+
+/**
+ * Load a pipeline from an npm package.
+ * Note: This requires the package to be installed in node_modules.
+ */
+async function loadFromNpm(packageName: string, exportPath?: string): Promise<PipelinePackageExports> {
+  try {
+    const resolved = exportPath
+      ? await import(`${packageName}/${exportPath}`)
+      : await import(packageName);
+
+    if (!resolved.pipeline) {
+      throw new Error(`Package ${packageName} does not export a 'pipeline'`);
+    }
+
+    return resolved as PipelinePackageExports;
+  } catch (err) {
+    if (err instanceof Error && err.message.includes('package')) {
+      throw new Error(`Pipeline package not found: ${packageName}. Did you install it?`);
+    }
+    throw err;
+  }
+}
+
+// ── Registry Resolution ────────────────────────────────────────────────────────
+
+/**
+ * Create a registry from a config object.
+ */
+export function createRegistryFromConfig(
+  config: Record<string, string | { package: string; version?: string }>,
+): PipelineRegistry {
+  const registry = new PipelineRegistry();
+
+  for (const [name, source] of Object.entries(config)) {
+    if (typeof source === 'string') {
+      // Short form: 'my-pipeline': './path' or 'my-pipeline': '@org/pkg'
+      if (source.startsWith('./') || source.startsWith('/')) {
+        registry.registerPath(name, source);
+      } else if (source.startsWith('git+') || source.startsWith('https://')) {
+        const url = source.startsWith('git+') ? source : `git+${source}`;
+        registry.registerGit(name, url);
+      } else {
+        // Assume npm package
+        registry.register(name, source);
+      }
+    } else {
+      // Long form: 'my-pipeline': { package: '@org/pkg', version: '1.0.0' }
+      registry.register(name, source.package, { version: source.version });
+    }
+  }
+
+  return registry;
+}

package/src/pipeline.ts

@@ -0,0 +1,265 @@
+/**
+ * Pipeline DSL — fluent builder for defining execution graphs.
+ *
+ * A Pipeline is a directed sequence of Steps. Each Step is a Skill, Agent,
+ * or another Pipeline. Built pipelines are immutable value objects executed
+ * by the Orchestrator.
+ */
+
+import type { ZodSchema } from 'zod';
+import type { Skill } from './skill.js';
+
+// ── Step Types ────────────────────────────────────────────────────────────────
+
+export type StepKind = 'skill' | 'map' | 'filter' | 'reduce' | 'branch' | 'parallel';
+
+/** A single resolved step in the pipeline graph. */
+export interface PipelineStep {
+  name: string;
+  kind: StepKind;
+  /** For 'skill' steps: the skill to run. */
+  skill?: Skill;
+  /** For 'map' | 'filter' | 'reduce': source field path (e.g. 'parse.services'). */
+  sourceField?: string;
+  /** For 'map': the sub-pipeline to run for each item. */
+  subPipeline?: BuiltPipeline;
+  /** For 'filter': predicate function. */
+  predicate?: (item: unknown, index: number) => boolean;
+  /** For 'reduce': reducer function. */
+  reducer?: (acc: unknown, item: unknown, index: number) => unknown;
+  /** For 'reduce': initial accumulator. */
+  initialValue?: unknown;
+  /** For 'parallel': array of sub-pipelines to run concurrently. */
+  branches?: BuiltPipeline[];
+  /** Concurrency limit for 'map' steps (default 1 = sequential). */
+  concurrency?: number;
+  /** How to handle item errors in 'map' steps. */
+  onItemError?: 'fail' | 'skip' | 'retry';
+  /** Max retries for items in 'map' with onItemError='retry'. */
+  maxItemRetries?: number;
+  /**
+   * Custom input mapper. Receives the current pipeline context and
+   * returns the input for this step. Default: pass entire previous output.
+   */
+  inputMapper?: (ctx: StepInputContext) => unknown;
+  /** Skip this step if this returns true. */
+  skipIf?: (ctx: StepInputContext) => boolean;
+}
+
+export interface StepInputContext {
+  /** The overall pipeline input. */
+  pipelineInput: unknown;
+  /** Outputs from all previously completed steps, keyed by step name. */
+  stepOutputs: Record<string, unknown>;
+  /** The output of the immediately preceding step. */
+  previousOutput: unknown;
+}
+
+// ── Built Pipeline (immutable) ────────────────────────────────────────────────
+
+export interface BuiltPipeline {
+  name: string;
+  steps: PipelineStep[];
+  inputSchema?: ZodSchema;
+  outputSchema?: ZodSchema;
+  /** Whether to save step checkpoints for resume capability. */
+  checkpointing?: boolean;
+}
+
+// ── Pipeline Builder ──────────────────────────────────────────────────────────
+
+export class PipelineBuilder {
+  private readonly _name: string;
+  private _steps: PipelineStep[] = [];
+  private _inputSchema?: ZodSchema;
+  private _outputSchema?: ZodSchema;
+  private _checkpointing = false;
+
+  constructor(name: string) {
+    this._name = name;
+  }
+
+  /** Declare the input schema for validation and documentation. */
+  input(schema: ZodSchema): this {
+    this._inputSchema = schema;
+    return this;
+  }
+
+  /** Declare the output schema for validation and documentation. */
+  output(schema: ZodSchema): this {
+    this._outputSchema = schema;
+    return this;
+  }
+
+  /** Enable checkpoint saving for resume capability. */
+  withCheckpointing(): this {
+    this._checkpointing = true;
+    return this;
+  }
+
+  /**
+   * Add a skill step.
+   *
+   * @param name    Step name (must be unique within the pipeline).
+   * @param skill   The skill to execute.
+   * @param options Optional input mapper and skip condition.
+   */
+  step(
+    name: string,
+    skill: Skill,
+    options?: {
+      input?: (ctx: StepInputContext) => unknown;
+      skipIf?: (ctx: StepInputContext) => boolean;
+    },
+  ): this {
+    this._steps.push({
+      name,
+      kind: 'skill',
+      skill,
+      inputMapper: options?.input,
+      skipIf: options?.skipIf,
+    });
+    return this;
+  }
+
+  /**
+   * Map over an array field, running a sub-pipeline for each item.
+   *
+   * @param sourceField  Dot-path to the array in step outputs (e.g. 'parse.services').
+   *                     Use '*' to use the entire previous step output as array.
+   * @param subPipeline  Sub-pipeline (built or builder) to run per item.
+   * @param options      Concurrency and error handling options.
+   */
+  mapOver(
+    sourceField: string,
+    subPipeline: BuiltPipeline | PipelineBuilder,
+    options?: {
+      concurrency?: number;
+      onItemError?: 'fail' | 'skip' | 'retry';
+      maxItemRetries?: number;
+      name?: string;
+    },
+  ): this {
+    const built =
+      subPipeline instanceof PipelineBuilder ? subPipeline.build() : subPipeline;
+    this._steps.push({
+      name: options?.name ?? `map:${sourceField}`,
+      kind: 'map',
+      sourceField,
+      subPipeline: built,
+      concurrency: options?.concurrency ?? 1,
+      onItemError: options?.onItemError ?? 'fail',
+      maxItemRetries: options?.maxItemRetries ?? 0,
+    });
+    return this;
+  }
+
+  /**
+   * Filter an array field, keeping only items where predicate returns true.
+   */
+  filter(
+    sourceField: string,
+    predicate: (item: unknown, index: number) => boolean,
+    name?: string,
+  ): this {
+    this._steps.push({
+      name: name ?? `filter:${sourceField}`,
+      kind: 'filter',
+      sourceField,
+      predicate,
+    });
+    return this;
+  }
+
+  /**
+   * Reduce an array field to a single value.
+   */
+  reduce(
+    sourceField: string,
+    reducer: (acc: unknown, item: unknown, index: number) => unknown,
+    initialValue: unknown,
+    name?: string,
+  ): this {
+    this._steps.push({
+      name: name ?? `reduce:${sourceField}`,
+      kind: 'reduce',
+      sourceField,
+      reducer,
+      initialValue,
+    });
+    return this;
+  }
+
+  /**
+   * Run multiple sub-pipelines in parallel, collecting all results.
+   */
+  parallel(
+    branches: Array<BuiltPipeline | PipelineBuilder>,
+    name?: string,
+  ): this {
+    this._steps.push({
+      name: name ?? 'parallel',
+      kind: 'parallel',
+      branches: branches.map((b) =>
+        b instanceof PipelineBuilder ? b.build() : b,
+      ),
+    });
+    return this;
+  }
+
+  /**
+   * Conditionally branch: run one of several sub-pipelines based on a condition.
+   */
+  branch(
+    name: string,
+    condition: (ctx: StepInputContext) => string, // returns branch name
+    branches: Record<string, BuiltPipeline | PipelineBuilder>,
+  ): this {
+    // Implemented as a 'branch' step; orchestrator resolves at runtime
+    this._steps.push({
+      name,
+      kind: 'branch',
+      // Store condition and branches in inputMapper/predicate slots
+      // (Orchestrator handles branch resolution specially)
+      inputMapper: condition as unknown as (ctx: StepInputContext) => unknown,
+      branches: Object.values(branches).map((b) =>
+        b instanceof PipelineBuilder ? b.build() : b,
+      ),
+    });
+    return this;
+  }
+
+  /** Finalize and return the immutable pipeline definition. */
+  build(): BuiltPipeline {
+    return {
+      name: this._name,
+      steps: [...this._steps],
+      inputSchema: this._inputSchema,
+      outputSchema: this._outputSchema,
+      checkpointing: this._checkpointing,
+    };
+  }
+}
+
+// ── Public Factory ────────────────────────────────────────────────────────────
+
+export const Pipeline = {
+  /** Create a new pipeline builder. */
+  create: (name: string) => new PipelineBuilder(name),
+};
+
+// ── Utility: resolve a dot-path from step outputs ─────────────────────────────
+
+export function resolvePath(
+  stepOutputs: Record<string, unknown>,
+  path: string,
+): unknown {
+  if (path === '*') return stepOutputs;
+  const parts = path.split('.');
+  let current: unknown = stepOutputs;
+  for (const part of parts) {
+    if (current === null || current === undefined) return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}

package/src/skill.ts

@@ -0,0 +1,127 @@
+/**
+ * Skill — the atomic unit of work in flomatai.
+ *
+ * A Skill is a typed, composable function that can:
+ *  - Call an LLM (LLMSkill)
+ *  - Transform data (TransformSkill)
+ *  - Perform IO (IOSkill)
+ *  - Execute Python (PythonSkill via bridge)
+ *  - Wrap a sub-pipeline (CompositeSkill)
+ */
+
+import type { ZodSchema } from 'zod';
+import type { LLMProvider } from './llm-provider.js';
+import type { StateStore } from './state/types.js';
+import type { Logger } from './logger.js';
+
+// ── MCP Client (minimal interface, keeps core provider-agnostic) ──────────────
+
+/**
+ * Minimal interface that any MCP client must satisfy.
+ * Defined in core so OrchestratorConfig and SkillContext can reference it
+ * without hard-depending on @flomatai/mcp-client.
+ *
+ * @flomatai/mcp-client's MCPClient class implements this interface.
+ */
+export interface MCPClientLike {
+  /**
+   * List all tools exposed by the connected MCP server.
+   * Returns an array of tool descriptors (name, description, inputSchema).
+   */
+  listTools(): Promise<Array<{
+    name: string;
+    description: string;
+    inputSchema: Record<string, unknown>;
+  }>>;
+  /**
+   * Call a named tool on the connected MCP server.
+   * @param name  Tool name as returned by listTools().
+   * @param args  Arguments matching the tool's inputSchema.
+   */
+  callTool(name: string, args: Record<string, unknown>): Promise<unknown>;
+}
+
+// ── Skill Metadata ────────────────────────────────────────────────────────────
+
+export interface SkillMeta {
+  /** Unique identifier for this skill. Used in pipeline step names. */
+  name: string;
+  /** Human-readable description for agent routing and planning prompts. */
+  description: string;
+  /** Semantic version. */
+  version?: string;
+  /** Categorization tags (e.g. 'llm', 'transform', 'io'). */
+  tags?: string[];
+  /**
+   * Retry configuration.
+   * Defaults to 0 retries (fail immediately).
+   */
+  retries?: number;
+  /**
+   * Per-execution timeout in milliseconds.
+   * 0 or undefined means no timeout.
+   */
+  timeout?: number;
+  /**
+   * Cache configuration. If provided, identical inputs return cached results.
+   */
+  cache?: {
+    /** Time-to-live in seconds. */
+    ttl: number;
+    /** Function that derives a cache key from the input. */
+    key: (input: unknown) => string;
+  };
+}
+
+// ── Skill Context ─────────────────────────────────────────────────────────────
+
+export interface SkillContext {
+  /** The LLM bound to this execution context. */
+  llm: LLMProvider;
+  /** Child logger scoped to this skill execution. */
+  logger: Logger;
+  /** State store for caching and cross-step communication. */
+  state: StateStore;
+  /** Emit a named event (for observability / side effects). */
+  emit: (event: string, data: unknown) => void;
+  /** Arbitrary config passed from the orchestrator. */
+  config: Record<string, unknown>;
+  /** Abort signal — check this for graceful cancellation. */
+  abortSignal: AbortSignal;
+  /** ID of the current run. */
+  runId: string;
+  /** LLM registry — retrieve a named LLM by key. */
+  getLLM: (key: string) => LLMProvider;
+  /**
+   * MCP client registry — retrieve a named MCP client by key.
+   * Only present when the Orchestrator is configured with an `mcp` registry.
+   */
+  getMCPClient?: (key: string) => MCPClientLike;
+}
+
+// ── Skill Interface ───────────────────────────────────────────────────────────
+
+export interface Skill<TInput = unknown, TOutput = unknown> {
+  /** Metadata describing this skill. */
+  meta: SkillMeta;
+  /** Zod schema for validating input at runtime. */
+  inputSchema: ZodSchema<TInput>;
+  /** Zod schema for validating output at runtime. */
+  outputSchema: ZodSchema<TOutput>;
+  /**
+   * Execute the skill.
+   * @param input Validated input data.
+   * @param ctx   Execution context (LLM, logger, state, etc.).
+   * @returns     Validated output data.
+   */
+  execute(input: TInput, ctx: SkillContext): Promise<TOutput>;
+}
+
+// ── Skill Result (with metadata) ─────────────────────────────────────────────
+
+export interface SkillResult<T = unknown> {
+  output: T;
+  tokensUsed: number;
+  durationMs: number;
+  cached: boolean;
+}