@pgflow/dsl 0.0.5-prealpha.2 → 0.0.6

package/brainstorming/subflows/subflows-callbacks.ts.md

@@ -1,124 +0,0 @@
-import { Flow } from './dsl.ts';
-
-// Define the input type for our flow
-type Input = {
-  query: string;
-  preferredLanguage?: string;
-};
-
-const AnswerTranslatedFlow = (subflow) =>
-  subflow
-    .step(
-      { slug: 'translate' },
-      async (input) =>
-        await translateText(
-          input.run.query,
-          input.run.preferredLanguage || 'english'
-        )
-    )
-    .step(
-      { slug: 'answerTranslated' },
-      async (input) =>
-        await generateAnswer(
-          input.translate.translatedText,
-          input.run.preferredLanguage || 'english'
-        )
-    );
-
-const AnswerTranslatedFlow = new Flow<string>({})
-  .step(
-    { slug: 'translate' },
-    async (input) =>
-      await translateText(
-        input.run.query,
-        input.run.preferredLanguage || 'english'
-      )
-  )
-  .step(
-    { slug: 'answerTranslated' },
-    async (input) =>
-      await generateAnswer(
-        input.translate.translatedText,
-        input.run.preferredLanguage || 'english'
-      )
-  );
-
-// Using the new subflow DSL method
-export const QueryAnswerFlow = new Flow<Input>({
-  slug: 'query_answer',
-  maxAttempts: 3,
-  baseDelay: 5,
-  timeout: 60,
-})
-  .step(
-    { slug: 'detectLanguage' },
-    async (input) => await detectLanguage(input.run.query)
-  )
-  .subflow(
-    {
-      slug: 'translation',
-      runIf: {
-        detectLanguage: (result) =>
-          result.language !== input.run.preferredLanguage &&
-          input.run.preferredLanguage,
-      },
-    },
-    AnswerTranslatedFlow
-  )
-  .subflow(
-    {
-      slug: 'directAnswer',
-      runIf: {
-        detectLanguage: (result) =>
-          result.language === input.run.preferredLanguage ||
-          !input.run.preferredLanguage,
-      },
-    },
-    (subflow) =>
-      subflow.step(
-        { slug: 'answerDirect' },
-        async (input) =>
-          await generateAnswer(input.run.query, input.detectLanguage.language)
-      )
-  )
-  .subflow(
-    {
-      slug: 'analysis',
-      runIf: {
-        detectLanguage: (result) =>
-          result.language === input.run.preferredLanguage ||
-          !input.run.preferredLanguage,
-      },
-    },
-    new Flow<string>({ slug: 'yolo ' })
-      .step(
-        { slug: 'answerDirect' },
-        async (input) =>
-          await generateAnswer(input.run.query, input.detectLanguage.language)
-      )
-      .step(
-        { slug: 'answerDirect' },
-        async (input) =>
-          await generateAnswer(input.run.query, input.detectLanguage.language)
-      )
-  )
-  .step(
-    {
-      slug: 'formatOutput',
-      dependsOn: ['translation.answerTranslated', 'directAnswer.answerDirect'],
-      optional: ['translation.answerTranslated', 'directAnswer.answerDirect'],
-    },
-    async (input) => {
-      const answerData =
-        input.translation?.answerTranslated || input.directAnswer?.answerDirect;
-      return await formatResponse(answerData.answer, answerData.sources);
-    }
-  )
-  .step(
-    { slug: 'logResponse', dependsOn: ['formatOutput'] },
-    async (input) =>
-      await logResult({
-        query: input.run.query,
-        response: input.formatOutput.formattedResponse,
-      })
-  );

package/brainstorming/subflows/subflows-classes.ts.md

@@ -1,83 +0,0 @@
-import { Flow } from './dsl.ts';
-
-// Define the input type for our flow
-type Input = {
-query: string;
-preferredLanguage?: string;
-};
-
-// Using the new branch DSL method
-export const QueryAnswerFlow = new Flow<Input>({
-slug: 'query_answer',
-maxAttempts: 3,
-baseDelay: 5,
-timeout: 60,
-})
-.step(
-{ slug: 'detectLanguage' },
-async (input) => await detectLanguage(input.run.query)
-)
-.branch(
-{
-slug: 'translation',
-runIf: {
-detectLanguage: (result) =>
-result.language !== input.run.preferredLanguage &&
-input.run.preferredLanguage,
-},
-},
-(branch) =>
-branch
-.step(
-{ slug: 'translate' },
-async (input) =>
-await translateText(
-input.run.query,
-input.run.preferredLanguage || 'english'
-)
-)
-.step(
-{ slug: 'answerTranslated' },
-async (input) =>
-await generateAnswer(
-input.translate.translatedText,
-input.run.preferredLanguage || 'english'
-)
-)
-)
-.branch(
-{
-slug: 'directAnswer',
-runIf: {
-detectLanguage: (result) =>
-result.language === input.run.preferredLanguage ||
-!input.run.preferredLanguage,
-},
-},
-(branch) =>
-branch.step(
-{ slug: 'answerDirect' },
-async (input) =>
-await generateAnswer(input.run.query, input.detectLanguage.language)
-)
-)
-.step(
-{
-slug: 'formatOutput',
-dependsOn: ['translation.answerTranslated', 'directAnswer.answerDirect'],
-optional: ['translation.answerTranslated', 'directAnswer.answerDirect'],
-},
-async (input) => {
-const answerData =
-input.translation?.answerTranslated || input.directAnswer?.answerDirect;
-return await formatResponse(answerData.answer, answerData.sources);
-}
-)
-.step(
-{ slug: 'logResponse', dependsOn: ['formatOutput'] },
-async (input) =>
-await logResult({
-query: input.run.query,
-response: input.formatOutput.formattedResponse,
-})
-);

package/brainstorming/subflows/subflows-flattening-versioned.md

@@ -1,119 +0,0 @@
-# Comparing Two Approaches to Subflows in the Context of Versioning
-
-This document analyzes two different ways of handling subflows (or “branching”) in a pgflow-like system and how they relate to versioning. We will refer to:
-
-1. **Approach A**: Subflows are defined and stored as standalone `Flow` objects in the database, which can be referenced by parent flows.  
-2. **Approach B**: Subflows (or “branches”) are appended (flattened) immediately into the parent flow in code, with slug-prefixing for isolation.
-
-We will compare these approaches, focusing especially on how flow versioning plays out in each scenario.
-
----
-
-## Background on Flow Versioning
-
-pgflow’s versioning strategy is often approached by treating each flow definition as immutable once stored:
-
-- Each flow definition stays unchanged in the database once uploaded.
-- Changing a flow’s logic requires uploading a new flow slug (e.g., `myFlow_v2` instead of `myFlow_v1`).
-- This explicitly separates old runs from new runs.
-- It keeps flows simpler to reason about (no in-place updates that can lead to “half-upgraded” scenarios).
-
-Subflow usage under this scheme means that if the subflow’s logic or steps need to change:
-- Under Approach A, the user would upload a new version of the subflow flow slug.
-- Under Approach B, changes to the subflow are effectively changes to the parent flow itself (flattened in the final DAG).
-
-Below, we’ll walk through how each subflow approach interacts with this versioning model.
-
----
-
-## Approach A: Standalone Subflows
-
-### Description
-
-In this approach:
-1. Each subflow is a standalone `Flow` with its own slug.  
-2. The parent flow references the subflow slug and sets up input/output mappings.  
-3. The engine links them at runtime by storing a reference (e.g., parent run/step → subflow run).
-
-Persistence-wise, subflows might store a `parent_run_id + parent_step_slug` in `pgflow.runs`, indicating that a subflow run is part of a higher-level run. Each subflow can also keep its own version slug.
-
-### Versioning Impact
-
-- **Independent Versioning**  
-  Each subflow gets uploaded to the database under its own slug, say `payment_flow_v3`. If the subflow’s logic changes, you create `payment_flow_v4`. You then refer to `payment_flow_v4` from a new parent flow version if desired.
-  
-- **Reusability**  
-  Multiple parent flows can reference the same subflow version or different versions of the subflow. For example, one parent flow might still use `payment_flow_v3`, while another references `payment_flow_v4`.
-  
-- **Clear Boundaries**  
-  Because each subflow is its own entity with a slug, it is easier to see where the subflow boundary lies and which version is being invoked. You can introspect the table of flows to see exactly which subflows are used by each version of each parent flow.
-
-- **Overhead in Management**  
-  The main downside is that you must manage multiple database entries with potentially many flow slugs. In large systems, you might end up with many versions that you have to keep track of, along with their references.
-
-In short, Approach A fits neatly with pgflow’s “immutable upload” versioning style. Each subflow is simply another flow that is also immutable. Parent flows can upgrade independently by referencing new versions of their subflows.
-
----
-
-## Approach B: Flattened “Branches”
-
-### Description
-
-In this approach (sometimes called `.branch()` or `.subflow()` in the DSL):
-
-1. You call a `.subflow()` (or `.branch()`) method on the parent flow.  
-2. That subflow’s steps are immediately added (flattened) into the parent flow’s step list.  
-3. Within the final DAG, each subflow step slug is internally prefixed (e.g., `branch1.some_step`) to avoid collisions.  
-4. The subflow’s relative dependencies remain consistent, but it’s all stored and tracked as a single flow in the database.
-
-Visually, you still see a subflow in code, but from the database perspective, it’s just one set of steps belonging to the parent flow, each slug forcibly prefixed.
-
-### Versioning Impact
-
-- **Single Flow Slug**  
-  Because the subflow is flattened, you end up with a single flow slug representing the entire DAG. If you need to update the subflow steps, you’re effectively changing the entire parent flow. This means you deploy a new version of the parent flow (e.g., `parent_flow_v2`) to incorporate changed “subflow” steps.
-  
-- **No Independent Subflow Versions**  
-  There isn’t a separate, independently tracked `Flow` object in the database. You cannot keep a stable “payment_flow_v3” that is reused, because it is not actually stored as its own entity. It is always merged into the parent when you `.subflow(...)`.
-  
-- **Simplicity in Code**  
-  From the user’s perspective in TypeScript code, it can be very straightforward: you just define a “branch” in the function syntax. There’s no separate “upload the subflow flow” step.  
-  However, from a pure versioning viewpoint, it’s less flexible. You can’t upgrade the subflow independently—any subflow change triggers a new *parent flow version*.  
-
-- **Potential Duplication**  
-  If multiple parent flows want the same subflow logic, each (parent) flow is storing that subflow’s steps. This can lead to duplication across multiple flows (and multiple places to update if you want consistency among them).
-
----
-
-## Which Approach Is Easier to Use with Immutable Versioning?
-
-It depends on your needs:
-
-1. **Approach A (Standalone Subflows)**  
-   - **Pros**:  
-     - One subflow flow slug can be easily versioned and referenced by multiple parents.  
-     - Easier to see which version of the subflow is used.  
-     - Ideal if you need a truly reusable piece of logic across many flows.  
-   - **Cons**:  
-     - Requires managing references between parent flow versions and subflow versions.  
-     - Slightly more overhead in “flow slug management” in the DB.
-
-2. **Approach B (Flattened Branches)**  
-   - **Pros**:  
-     - Simpler to read or write in code, as you “just add steps” without referencing a second flow.  
-     - No special subflow DB entries or separate subflow run records.  
-   - **Cons**:  
-     - Not reusable as a separate entity (changing subflow requires a new version of the entire parent flow).  
-     - Potential duplication if the same subflow logic is used across many flows.  
-     - Harder to maintain an organized version history of subflows because each subflow is “encapsulated” within a parent flow version.
-
-From a purely versioning standpoint, **Approach A** aligns more directly with pgflow’s principle of immutable flows: You can create and track separate versions of the subflow and clearly see which parent flows reference them. **Approach B** remains attractive for simpler or “one-off” subflows that don’t need to be used anywhere else.
-
----
-
-## Conclusion
-
-- If you want robust, maintainable versioning paths for subflows—where the same subflow logic might change independently and be referenced by multiple parents—Approach A is likely better. You will have a more explicit, consistent way to handle updates across multiple flows.
-- If your subflow is more like a local branch logic snippet, needed only by that parent flow, and you don’t expect to reuse it, Approach B can be simpler. You’ll still keep an immutable “parent flow version,” but you won’t have to track a separate subflow slug.
-
-Ultimately, your choice depends on how often you expect subflows to be reused, how critical their independent versioning is, and whether you want the subflows to appear as first-class flows in the database. For official “reusable building blocks,” a separate flow slug per subflow (Approach A) is typically the most consistent with a versioned, immutable database model.

package/brainstorming/subflows/subflows-flattening.md

@@ -1,138 +0,0 @@
-# Subflows in pgflow: Comparing Two Approaches
-
-This document compares two different ways of implementing *subflows* (or *branching*) within **pgflow**. Specifically, we look at:
-
-1. **Standalone Subflows**  
-   - As described in `subflows.md`, subflows are stored as separate flows in the database.  
-   - They are referenced by special DSL methods and require separate input/output mapping.  
-   - The database would need a particular column (e.g., `parent_run_id` + `step_slug`) in `pgflow.runs` to track a subflow’s parent.
-
-2. **Flattened Branches (`.branch()` or `.subflow()`)**  
-   - Steps are “immediately appended” to the main flow but grouped logically as a “branch.”  
-   - Step slugs in the branch receive an automatic prefix to avoid name collisions.  
-   - Subflows are effectively “flattened” into the parent, but the DSL can preserve isolation through prefixing.  
-   - An alternate design might allow passing an entire `Flow` object into `.subflow()`, flattening and prefixing each step under a single slug.
-
-Below, we analyze these two approaches based on readability, maintainability, ease of implementation, schema changes, and performance.
-
----
-
-## 1. Standalone Subflows
-
-### Description
-- Each subflow is defined as its own flow in the DSL and written to the database as an independent entity.
-- A parent flow that wants to invoke a subflow references it by some special method, passing input and receiving output. 
-- Internally, the database must store extra *relationship fields*, such as `parent_run_id` or `parent_step_slug`.
-
-### Key Points
-- Subflow runs have their own records in the `runs` table (with a field linking them back to the parent run/step).  
-- Each subflow can be reused by referencing it from multiple parent flows without redefinition.  
-- Input/output mapping between parent flow and subflow is explicit and can be edited independently.
-
----
-
-## 2. Flattened Branches (`.branch()` or `.subflow()`)
-
-### Description
-- A separate “branch” object is created in the DSL, but in reality, the steps are appended (flattened) to the main flow at definition time.
-- Steps inside the branch are prefixed with the branch slug to avoid name collisions (e.g., `branch1.some_step`).
-- The “branch” functionality is a DSL abstraction. It allows a more contained experience in code (like a local subflow), but physically they remain within a single flow run and single DAG in the database.
-- The `.branch()` method could accept a callback defining the sub-steps, or it could accept a `Flow` object and flatten it in place.
-
-### Key Points
-- Maintains a single run record in the database: no new `runs` record for each subflow, because everything is part of the parent run.  
-- No need for extra columns for parent-child run relationships; the logical subflow is merely a DSL concept.
-- Each step is stored under a prefixed slug (e.g., `branch_slug.step_slug`) to isolate dependencies and outputs within the parent flow.
-
----
-
-## Comparison
-
-### 1. Readability
-
-**Standalone Subflows**  
-- Potentially clearer when you want a truly modular flow.  
-- A subflow can be understood in isolation, with its own steps, dependencies, and run data.  
-- References to subflows might look more verbose if you have multiple subflows in a single parent flow.
-
-**Flattened Branches**  
-- The DSL usage can be very direct; you add a `.branch()` call inline, which shows the sub-steps right there in the code.  
-- The flow remains conceptually “unified” from top to bottom, although the prefixed step slugs might be less direct if you are reading the flattened representation in the database.  
-- Possibly simpler to grasp as a quick branching or subflow without large structural changes.
-
-**Verdict**:  
-- If you value the ability to treat subflows as self-contained “mini-flows,” free of the parent’s step definitions, the standalone approach is more explicit.  
-- If you prefer everything in one place, flattened branches offer a straightforward inline style.
-
-### 2. Maintainability
-
-**Standalone Subflows**  
-- Fits well if you plan to reuse subflows in multiple places. You maintain the subflow in a separate, self-contained file or DSL definition.  
-- Changes to the subflow do not directly break the parent flow’s definitions (as long as the input/output interface remains compatible).  
-- Potential overhead in updating any references in the parent flows if you change the I/O signature of the subflow.
-
-**Flattened Branches**  
-- Changing or removing a branch typically just changes the portion of the code that defines that segment of the flow.  
-- The steps get stored as if they were part of the main flow, so you do not have an extra subflow entity to manage or version.  
-- Not as easily reusable if you want the same subflow logic in multiple parent flows (though you could replicate the `.branch()` code or attempt a DSL function).
-
-**Verdict**:  
-- Standalone subflows can be more reusable and can reduce duplication in large codebases.  
-- Flattened branches are easier to keep contained within a single flow’s definition, however, and maintain a simpler mental model if you rarely need to share subflows.
-
-### 3. Ease of Implementation
-
-**Standalone Subflows**  
-- Requires additional concepts at the database layer: a subflow run must carry references to its parent run/step.  
-- You need new logic in the engine to start, link, and complete these subflow runs.  
-- The DSL and engine must handle mapping inputs/outputs across flow boundaries.
-
-**Flattened Branches**  
-- Simpler from a runtime perspective, since everything is just one flow run.  
-- The DSL logic that “flattens” the subflow steps is not overly complex; it mostly handles prefixing slugs and hooking up dependencies.  
-- No new database columns or specialized logic for multiple runs.
-
-**Verdict**:  
-- Flattened branches require fewer structural changes and might be quicker to implement.  
-- True subflow runs demand a more robust solution at both the DSL and SQL layer.
-
-### 4. Impact on SQL Schema / Migration Cost
-
-**Standalone Subflows**  
-- Necessitates additional database columns or relationships. For instance, the `runs` table needs to store `parent_run_id`, `parent_step_slug`, or something similar to link subflow runs to their parent.  
-- Removing or changing this schema later may be more involved if it’s deeply integrated into how runs are tracked.
-
-**Flattened Branches**  
-- No major schema changes are required—still the same approach of storing steps, runs, step states, etc.  
-- You only introduce new step slugs that happen to include a “branch prefix” for uniqueness.  
-- Easier to remove or refactor because it doesn’t alter the overarching structure of the data model.
-
-**Verdict**:  
-- Standalone subflows require more schema modifications, making them more complex to revert or adjust.  
-- Flattened branches have minimal, if any, schema changes.
-
-### 5. Performance Impact
-
-**Standalone Subflows**  
-- Potentially more overhead spinning up multiple runs, especially if subflows are large or run in parallel.  
-- Each subflow has its own overhead (e.g., run record, root step tasks, etc.). However, for large workflows, this might help isolate concurrency or error handling at the subflow boundary.
-
-**Flattened Branches**  
-- The entire workflow, including “branch” steps, exists in a single run, so there is no extra overhead from creating new runs.  
-- You treat all steps as part of one DAG, so concurrency and step scheduling is handled in one place. This can be more efficient if the flows share certain resources or data.  
-- For extremely large flows, flattening might produce an enormous DAG, which could be less clear or hamper debugging if you tend to think in separate modules.
-
-**Verdict**:  
-- Flattened branches are simpler to schedule in a single run, with presumably lower overhead for short or mid-sized flows.  
-- If your subflows are big, standalone subflows might be beneficial from a concurrency and debugging standpoint, but they also mean more “runs” exist in the database.
-
----
-
-## Conclusion
-
-Both approaches solve the problem of reusing and modularizing flow logic. The right choice depends on how you plan to structure and reuse workflows:
-
-- **Standalone Subflows** are powerful if you need fully independent runs or if reusability and clear boundaries between subflows are critical. On the other hand, they complicate the SQL schema and add overhead in linking parent and child runs.  
-- **Flattened Branches** (or `.subflow()`) allow you to keep everything in one run and maintain a simpler database structure. This is typically easier to implement and manage, especially if you don’t need separate subflow runs or advanced usage patterns. However, it offers less natural reuse if you need to embed the same subflow logic in multiple parent flows without duplication.
-
-In practice, **flattened branches** often suffice for many workflow needs. If your application demands heavy reuse of subflows across many flows, or if you need logically distinct runs for separate modules, **standalone subflows** might be the more scalable approach.

package/brainstorming/subflows/subflows.md

@@ -1,118 +0,0 @@
-# Implementing Subflows in pgflow
-
-Subflows allow you to create reusable workflow components while maintaining type safety and a clean, declarative syntax. This document outlines how to implement subflows using a type-safe field mapping approach.
-
-## How Subflows Work
-
-A subflow is a complete workflow that is embedded within another workflow. The parent flow must:
-
-1. Map its own inputs/outputs to the subflow's required inputs
-2. Access the subflow's outputs in subsequent steps
-
-## Implementation Example
-
-Let's look at a practical example with a payment processing subflow that's used within an order processing workflow.
-
-### Step 1: Define the Payment Flow
-
-First, we define our reusable payment processing flow:
-
-```typescript
-// Define a payment processing flow
-const PaymentFlow = new Flow<{
-  orderId: string;
-  amount: number;
-  currency: string;
-}>({
-  slug: 'payment_flow',
-})
-  .step({ slug: 'validate_payment' }, async (input) => {
-    return validatePayment(
-      input.run.orderId,
-      input.run.amount,
-      input.run.currency
-    );
-    // Returns { validationToken: string, isValid: boolean }
-  })
-  .step(
-    { slug: 'process_transaction', dependsOn: ['validate_payment'] },
-    async (input) => {
-      if (!input.validate_payment.isValid) {
-        throw new Error('Payment validation failed');
-      }
-      return processTransaction(
-        input.run.orderId,
-        input.run.amount,
-        input.validate_payment.validationToken
-      );
-      // Returns { transactionId: string, status: string, timestamp: Date }
-    }
-  );
-```
-
-### Step 2: Use the Payment Flow as a Subflow
-
-Now we can use this payment flow as a subflow in our order processing workflow:
-
-```typescript
-// Use PaymentFlow as a subflow in an order processing flow
-const OrderFlow = new Flow<{ customerId: string; items: CartItem[] }>({
-  slug: 'order_flow',
-})
-  .step({ slug: 'create_order' }, async (input) => {
-    return createOrder(input.run.customerId, input.run.items);
-    // Returns { orderId: string, totalAmount: number, currency: string }
-  })
-  .subflow({
-    slug: 'process_payment',
-    dependsOn: ['create_order'],
-    flow: PaymentFlow,
-    input: (input) => ({
-      orderId: input.create_order.orderId,
-      amount: input.create_order.totalAmount,
-      currency: input.create_order.currency || 'USD',
-    }),
-    output: (subflowOutput) => ({
-      transactionId: subflowOutput.process_transaction.transactionId,
-      paymentStatus: subflowOutput.process_transaction.status,
-      paymentTimestamp: subflowOutput.process_transaction.timestamp,
-    }),
-  })
-  .step(
-    { slug: 'finalize_order', dependsOn: ['process_payment'] },
-    async (input) => {
-      return finalizeOrder(
-        input.create_order.orderId,
-        input.process_payment.transactionId,
-        input.process_payment.paymentStatus
-      );
-      // Returns { orderStatus: string, completionDate: Date }
-    }
-  );
-```
-
-## Key Properties
-
-The `subflow()` method accepts:
-
-- `slug`: A unique identifier for the subflow step
-- `dependsOn`: The steps that must complete before this subflow can be executed
-- `flow`: The Flow object to be executed as a subflow
-- `input`: A function that maps parent flow data to the subflow's input requirements
-- `output`: A function that transforms the subflow's output for easier consumption by subsequent steps
-
-## Type Safety
-
-The entire subflow implementation maintains full type safety:
-
-1. The `input` function is type-checked against the subflow's input requirements
-2. The `output` function is type-checked against the subflow's available outputs
-3. Subsequent steps in the parent flow can access the transformed subflow outputs with full type information
-
-## Benefits
-
-- **Reusability**: Create workflow components once and reuse them across multiple flows
-- **Maintainability**: Update a subflow in one place and all parent flows benefit
-- **Clean API**: The input/output mappings provide a clean interface between workflows
-- **Type Safety**: Maintain full type checking across flow boundaries
-- **Transparency**: The execution engine handles subflows seamlessly, making them appear as normal steps to users