edsger 0.59.0 → 0.61.0

package/dist/phases/data-flow/mcp-server.d.ts

@@ -0,0 +1,85 @@
+/**
+ * In-process MCP server for the data-flow phase. Exposes a single tool —
+ * `submit_data_flow` — that the agent calls with the structured extraction,
+ * plus `record_progress` for streaming status messages.
+ *
+ * Mirrors the shape of phases/screen-flow/mcp-server.ts; see that file for
+ * the design rationale (zod schema + cross-field consistency + capture state).
+ */
+import { z } from 'zod';
+import type { DataFlowExtraction } from './types.js';
+export interface DataFlowCaptureState {
+    captured: DataFlowExtraction | null;
+}
+export declare function createDataFlowCaptureState(): DataFlowCaptureState;
+export type DataFlowProgressSink = (event: {
+    phase: 'detection' | 'enumeration' | 'nodes' | 'edges' | 'submission';
+    message: string;
+}) => void;
+export declare function validateConsistency(extraction: DataFlowExtraction): {
+    error: string | null;
+};
+export declare function createSubmitDataFlowTool(state: DataFlowCaptureState): import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
+    summary: z.ZodString;
+    nodes: z.ZodArray<z.ZodObject<{
+        slug: z.ZodString;
+        name: z.ZodString;
+        kind: z.ZodEnum<{
+            model: "model";
+            source: "source";
+            dataset: "dataset";
+            transform: "transform";
+            sink: "sink";
+            queue: "queue";
+        }>;
+        file: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        tech: z.ZodOptional<z.ZodString>;
+        schedule: z.ZodOptional<z.ZodString>;
+        inputs: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            type: z.ZodOptional<z.ZodString>;
+            required: z.ZodOptional<z.ZodBoolean>;
+            description: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        outputs: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            type: z.ZodOptional<z.ZodString>;
+            required: z.ZodOptional<z.ZodBoolean>;
+            description: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        sample: z.ZodOptional<z.ZodObject<{
+            columns: z.ZodArray<z.ZodString>;
+            rows: z.ZodArray<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>;
+        stats: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            label: z.ZodString;
+            value: z.ZodString;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>;
+    edges: z.ZodArray<z.ZodObject<{
+        fromSlug: z.ZodString;
+        toSlug: z.ZodString;
+        kind: z.ZodEnum<{
+            data: "data";
+            event: "event";
+            control: "control";
+            derives: "derives";
+        }>;
+        label: z.ZodOptional<z.ZodString>;
+        sourceFile: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}>;
+export declare function createRecordProgressTool(sink?: DataFlowProgressSink): import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
+    phase: z.ZodEnum<{
+        nodes: "nodes";
+        edges: "edges";
+        detection: "detection";
+        enumeration: "enumeration";
+        submission: "submission";
+    }>;
+    message: z.ZodString;
+}>;
+export declare function createDataFlowMcpServer(state: DataFlowCaptureState, options?: {
+    onProgress?: DataFlowProgressSink;
+}): import("@anthropic-ai/claude-agent-sdk").McpSdkServerConfigWithInstance;

package/dist/phases/data-flow/mcp-server.js

@@ -0,0 +1,140 @@
+/**
+ * In-process MCP server for the data-flow phase. Exposes a single tool —
+ * `submit_data_flow` — that the agent calls with the structured extraction,
+ * plus `record_progress` for streaming status messages.
+ *
+ * Mirrors the shape of phases/screen-flow/mcp-server.ts; see that file for
+ * the design rationale (zod schema + cross-field consistency + capture state).
+ */
+import { createSdkMcpServer, tool } from '@anthropic-ai/claude-agent-sdk';
+import { z } from 'zod';
+export function createDataFlowCaptureState() {
+    return { captured: null };
+}
+// ---------------------------------------------------------------------------
+// Zod schemas (mirror types.ts)
+// ---------------------------------------------------------------------------
+const dataFieldSchema = z.object({
+    name: z.string().min(1),
+    type: z.string().optional(),
+    required: z.boolean().optional(),
+    description: z.string().optional(),
+});
+const dataSampleSchema = z.object({
+    columns: z.array(z.string()),
+    rows: z.array(z.array(z.string())),
+});
+const dataStatSchema = z.object({
+    label: z.string(),
+    value: z.string(),
+});
+const dataNodeSchema = z.object({
+    slug: z.string().min(1),
+    name: z.string().min(1),
+    kind: z.enum(['source', 'dataset', 'transform', 'sink', 'queue', 'model']),
+    file: z.string().optional(),
+    description: z.string().optional(),
+    tech: z.string().optional(),
+    schedule: z.string().optional(),
+    inputs: z.array(dataFieldSchema).optional(),
+    outputs: z.array(dataFieldSchema).optional(),
+    sample: dataSampleSchema.optional(),
+    stats: z.array(dataStatSchema).optional(),
+});
+const dataEdgeSchema = z.object({
+    fromSlug: z.string().min(1),
+    toSlug: z.string().min(1),
+    kind: z.enum(['data', 'event', 'control', 'derives']),
+    label: z.string().optional(),
+    sourceFile: z.string().optional(),
+});
+export function validateConsistency(extraction) {
+    const slugs = new Set();
+    for (const node of extraction.nodes) {
+        if (slugs.has(node.slug)) {
+            return {
+                error: `Duplicate node slug "${node.slug}". Each node.slug MUST be unique within the flow. Re-call submit_data_flow with deduplicated nodes.`,
+            };
+        }
+        slugs.add(node.slug);
+    }
+    for (const edge of extraction.edges) {
+        if (!slugs.has(edge.fromSlug)) {
+            return {
+                error: `Edge fromSlug "${edge.fromSlug}" → "${edge.toSlug}" does not match any node slug. Either add the missing node or drop the edge, then re-call submit_data_flow.`,
+            };
+        }
+        if (!slugs.has(edge.toSlug)) {
+            return {
+                error: `Edge fromSlug "${edge.fromSlug}" → toSlug "${edge.toSlug}" does not match any node slug. Either add the missing node or drop the edge, then re-call submit_data_flow.`,
+            };
+        }
+    }
+    return { error: null };
+}
+export function createSubmitDataFlowTool(state) {
+    return tool('submit_data_flow', [
+        'Submit the final data flow extraction. Call this EXACTLY once,',
+        'when you have finished mapping every data node and connection. Pass',
+        'the full structured flow as the argument. After this call succeeds,',
+        'end your turn — do NOT also paste the same data as a fenced code',
+        'block. If validation fails, the error message tells you what to fix;',
+        'call the tool again with corrected data.',
+    ].join(' '), {
+        summary: z
+            .string()
+            .min(1)
+            .describe('1-3 sentence narrative of what this system does with data and the primary pipelines.'),
+        nodes: z
+            .array(dataNodeSchema)
+            .describe('Every data node: source / dataset / transform / sink / queue / model. node.slug MUST be unique within the flow.'),
+        edges: z
+            .array(dataEdgeSchema)
+            .describe('Connections. fromSlug = upstream, toSlug = downstream. Every fromSlug / toSlug MUST reference a slug present in nodes; drop edges whose endpoints you did not emit.'),
+    }, async (args) => {
+        const extraction = {
+            summary: args.summary,
+            nodes: args.nodes,
+            edges: args.edges,
+        };
+        const { error } = validateConsistency(extraction);
+        if (error) {
+            return {
+                content: [{ type: 'text', text: error }],
+                isError: true,
+            };
+        }
+        state.captured = extraction;
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Captured ${extraction.nodes.length} data nodes / ${extraction.edges.length} connections. End your turn now.`,
+                },
+            ],
+        };
+    });
+}
+export function createRecordProgressTool(sink) {
+    return tool('record_progress', 'Send a short status update to the user. Does not affect the extraction. Call it at each phase boundary so the user sees progress.', {
+        phase: z
+            .enum(['detection', 'enumeration', 'nodes', 'edges', 'submission'])
+            .describe('Which phase the message belongs to.'),
+        message: z.string().min(1).describe('Human-readable status update.'),
+    }, async (args) => {
+        sink?.({ phase: args.phase, message: args.message });
+        return {
+            content: [{ type: 'text', text: 'ok' }],
+        };
+    });
+}
+export function createDataFlowMcpServer(state, options) {
+    return createSdkMcpServer({
+        name: 'data-flow',
+        version: '1.0.0',
+        tools: [
+            createSubmitDataFlowTool(state),
+            createRecordProgressTool(options?.onProgress),
+        ],
+    });
+}

package/dist/phases/data-flow/prompts.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Prompts for the data-flow phase. Loads the system prompt body from
+ * `skills/phase/data-flow/SKILL.md` (with optional project override) and
+ * appends the JSON output contract.
+ */
+export declare function createDataFlowSystemPrompt(options?: {
+    projectDir?: string;
+    hasCodebase?: boolean;
+}): Promise<string>;
+export declare function createDataFlowUserPrompt(args: {
+    productName: string;
+    productDescription?: string;
+    guidance?: string;
+}): string;

package/dist/phases/data-flow/prompts.js

@@ -0,0 +1,36 @@
+/**
+ * Prompts for the data-flow phase. Loads the system prompt body from
+ * `skills/phase/data-flow/SKILL.md` (with optional project override) and
+ * appends the JSON output contract.
+ */
+import { processConditionals, resolveSkill, } from '../../services/skill-resolver.js';
+import { OUTPUT_CONTRACTS } from '../output-contracts.js';
+export async function createDataFlowSystemPrompt(options) {
+    const skill = await resolveSkill('phase/data-flow', {
+        projectDir: options?.projectDir,
+    });
+    if (!skill) {
+        throw new Error('Failed to load skill: phase/data-flow');
+    }
+    const prompt = processConditionals(skill.prompt, {
+        hasCodebase: options?.hasCodebase ?? true,
+    });
+    return `${prompt}
+
+${OUTPUT_CONTRACTS['data-flow']}`;
+}
+export function createDataFlowUserPrompt(args) {
+    const guidanceBlock = args.guidance
+        ? `\n\n**Human guidance for this run** (focus or exclude as instructed):\n${args.guidance}`
+        : '';
+    const descBlock = args.productDescription
+        ? `\n**Product description**: ${args.productDescription}`
+        : '';
+    return `Map the data flow for **${args.productName}**.${descBlock}${guidanceBlock}
+
+Start by detecting the stack (check package.json / pyproject.toml / go.mod / Cargo.toml / requirements.txt etc.), then look for: ETL/pipeline definitions, database migrations or schema files, queue/topic configs, model invocation sites, file ingest scripts. Read just enough source per node to fill in a useful DataNodeSchema — do not need to read everything.
+
+Call \`mcp__data-flow__record_progress\` at each phase boundary so the user can see your progress (otherwise the CLI looks frozen).
+
+When you are done, return the result by **calling the \`mcp__data-flow__submit_data_flow\` tool exactly once** with \`summary\`, \`nodes\`, and \`edges\` as arguments. Do not paste the JSON as a fenced text block — the tool call is the deliverable. If the tool returns an error, fix the issue it describes and call the tool again.`;
+}

package/dist/phases/data-flow/types.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Data Flow domain types.
+ *
+ * A DataNodeSchema is a structured description of one unit of data state or
+ * computation in a product — a source, dataset, transform, sink, queue, or
+ * model. The CLI extracts these from source code (pipeline definitions,
+ * schema files, queue handlers, etc.) and the desktop renders them with a
+ * unified <DataNodePreview> component.
+ *
+ * Companion to ScreenSchema: same flow-graph shape (nodes + edges with a
+ * shared `flows` table storing the JSONB schema), different domain. Data
+ * flow edges describe how data moves between nodes, not user navigation.
+ */
+export type DataNodeKind = 'source' | 'dataset' | 'transform' | 'sink' | 'queue' | 'model';
+export interface DataField {
+    name: string;
+    type?: string;
+    required?: boolean;
+    description?: string;
+}
+export interface DataSample {
+    columns: string[];
+    rows: string[][];
+}
+export interface DataStat {
+    label: string;
+    value: string;
+}
+export interface DataNodeSchema {
+    /** Stable slug within the flow (e.g. 'raw-events', 'enrich-user'). */
+    slug: string;
+    /** Human-readable name. */
+    name: string;
+    kind: DataNodeKind;
+    /** Source file path (jump anchor); for datasets, the schema/migration file. */
+    file?: string;
+    /** One-sentence description. */
+    description?: string;
+    /** Technology/format hint: 'postgres', 'parquet', 'kafka', 'openai-api', etc. */
+    tech?: string;
+    /** For transforms: 'cron 0 0 * * *', 'on-event', 'manual', 'continuous'. */
+    schedule?: string;
+    /** Schema of inputs the node consumes. */
+    inputs?: DataField[];
+    /** Schema of outputs the node produces. */
+    outputs?: DataField[];
+    /** Tiny realistic sample (≤ 4 rows) for datasets. */
+    sample?: DataSample;
+    /** Volume / latency hints — free-form key/value pairs. */
+    stats?: DataStat[];
+}
+/**
+ * Edge kinds. Direction is always "data movement": fromSlug = upstream,
+ * toSlug = downstream. The kind describes the *nature* of the connection.
+ */
+export type DataEdgeKind = 'data' | 'event' | 'control' | 'derives';
+export interface DataEdge {
+    fromSlug: string;
+    toSlug: string;
+    kind: DataEdgeKind;
+    /** Free-form descriptor: 'nightly batch', 'on user signup', 'embedding'. */
+    label?: string;
+    /** File containing the connection definition (when distinct from from-node's file). */
+    sourceFile?: string;
+}
+export interface DataFlowExtraction {
+    summary: string;
+    nodes: DataNodeSchema[];
+    edges: DataEdge[];
+}
+export declare function isDataFlowExtraction(value: unknown): value is DataFlowExtraction;

package/dist/phases/data-flow/types.js

@@ -0,0 +1,86 @@
+/**
+ * Data Flow domain types.
+ *
+ * A DataNodeSchema is a structured description of one unit of data state or
+ * computation in a product — a source, dataset, transform, sink, queue, or
+ * model. The CLI extracts these from source code (pipeline definitions,
+ * schema files, queue handlers, etc.) and the desktop renders them with a
+ * unified <DataNodePreview> component.
+ *
+ * Companion to ScreenSchema: same flow-graph shape (nodes + edges with a
+ * shared `flows` table storing the JSONB schema), different domain. Data
+ * flow edges describe how data moves between nodes, not user navigation.
+ */
+// ============================================================================
+// Runtime validation for AI-produced extraction
+// ============================================================================
+const NODE_KINDS = new Set([
+    'source',
+    'dataset',
+    'transform',
+    'sink',
+    'queue',
+    'model',
+]);
+const EDGE_KINDS = new Set([
+    'data',
+    'event',
+    'control',
+    'derives',
+]);
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+function isDataNodeSchema(value) {
+    if (!isRecord(value)) {
+        return false;
+    }
+    if (typeof value.slug !== 'string' || value.slug.length === 0) {
+        return false;
+    }
+    if (typeof value.name !== 'string' || value.name.length === 0) {
+        return false;
+    }
+    if (typeof value.kind !== 'string' ||
+        !NODE_KINDS.has(value.kind)) {
+        return false;
+    }
+    return true;
+}
+function isDataEdge(value) {
+    if (!isRecord(value)) {
+        return false;
+    }
+    if (typeof value.fromSlug !== 'string') {
+        return false;
+    }
+    if (typeof value.toSlug !== 'string') {
+        return false;
+    }
+    if (typeof value.kind !== 'string' ||
+        !EDGE_KINDS.has(value.kind)) {
+        return false;
+    }
+    return true;
+}
+export function isDataFlowExtraction(value) {
+    if (!isRecord(value)) {
+        return false;
+    }
+    if (typeof value.summary !== 'string') {
+        return false;
+    }
+    if (!Array.isArray(value.nodes)) {
+        return false;
+    }
+    if (!Array.isArray(value.edges)) {
+        return false;
+    }
+    if (!value.nodes.every(isDataNodeSchema)) {
+        return false;
+    }
+    if (!value.edges.every(isDataEdge)) {
+        return false;
+    }
+    return true;
+}

package/dist/phases/output-contracts.js

@@ -949,5 +949,76 @@ submit_screen_flow({
   ]
 })
 \`\`\`
+`,
+    'data-flow': `
+**CRITICAL — How to return the result**:
+
+Return the extraction by calling the MCP tool
+\`mcp__data-flow__submit_data_flow\` **exactly once** with three arguments:
+
+- \`summary\` — 1-3 sentence narrative of what this system does with data and its primary pipelines
+- \`nodes\` — array of DataNodeSchema objects (every source / dataset / transform / sink / queue / model)
+- \`edges\` — array of DataEdge objects (connections, with direction = data movement)
+
+The tool validates the arguments against the schema. If it returns an error,
+fix the issue it describes and call the tool again. After a successful call,
+end your turn — do not also paste the same data as a fenced text block.
+
+You can also call \`mcp__data-flow__record_progress({ phase, message })\` at
+each phase boundary (detection / enumeration / nodes / edges / submission)
+to keep the user informed during long runs. This is observability only — it
+does not affect the extraction.
+
+DataNodeSchema fields:
+- \`slug\` (unique within the flow), \`name\`, \`kind\`, \`file?\`
+- \`kind\`: one of \`source\`, \`dataset\`, \`transform\`, \`sink\`, \`queue\`, \`model\`
+- \`description?\`: one-sentence summary
+- \`tech?\`: technology / format hint (e.g. \`postgres\`, \`parquet\`, \`kafka\`, \`openai-api\`)
+- \`schedule?\`: for transforms (e.g. \`cron 0 0 * * *\`, \`on-event\`, \`manual\`, \`continuous\`)
+- \`inputs?\` / \`outputs?\`: arrays of \`{ name, type?, required?, description? }\`
+- \`sample?\`: \`{ columns: [string], rows: [[string]] }\` — at most 4 sample rows for datasets
+- \`stats?\`: array of \`{ label, value }\` (volume, latency, count hints)
+
+DataEdge fields:
+- \`fromSlug\` (upstream), \`toSlug\` (downstream) — both MUST appear in nodes
+- \`kind\`: one of \`data\`, \`event\`, \`control\`, \`derives\`
+- \`label?\`: free-form descriptor (e.g. \`nightly batch\`, \`on user signup\`)
+- \`sourceFile?\`: file containing the read/write/trigger code
+
+Edge direction convention: fromSlug is upstream (data origin), toSlug is downstream
+(data destination). A transform that reads from a dataset and writes to a queue
+produces two edges: \`dataset → transform\` (kind: data) and \`transform → queue\`
+(kind: event).
+
+Schematic example of the tool call:
+
+\`\`\`
+submit_data_flow({
+  summary: "Nightly product-feed pipeline: scrape vendor sites, normalize, write to Postgres, publish change events to Kafka.",
+  nodes: [
+    { slug: "vendor-scrape", name: "Vendor scraper", kind: "source",
+      file: "src/scrape/vendor.ts", tech: "playwright",
+      schedule: "cron 0 0 * * *",
+      outputs: [{ name: "html", type: "string" }, { name: "url", type: "string" }] },
+    { slug: "normalize", name: "Normalize products", kind: "transform",
+      file: "src/etl/normalize.ts", tech: "node",
+      inputs: [{ name: "html", type: "string" }],
+      outputs: [{ name: "sku", type: "string" }, { name: "price", type: "decimal" }] },
+    { slug: "products", name: "products", kind: "dataset",
+      file: "supabase/migrations/0001_products.sql", tech: "postgres",
+      sample: { columns: ["sku", "price"], rows: [["ABC-1", "9.99"]] } },
+    { slug: "change-events", name: "product.changed", kind: "queue",
+      file: "src/queues/products.ts", tech: "kafka" }
+  ],
+  edges: [
+    { fromSlug: "vendor-scrape", toSlug: "normalize", kind: "data",
+      label: "raw HTML", sourceFile: "src/etl/normalize.ts" },
+    { fromSlug: "normalize", toSlug: "products", kind: "data",
+      label: "upsert", sourceFile: "src/etl/normalize.ts" },
+    { fromSlug: "products", toSlug: "change-events", kind: "event",
+      label: "on row change", sourceFile: "src/queues/products.ts" }
+  ]
+})
+\`\`\`
 `,
 };

package/dist/phases/recipes/index.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Recipes phase: clone the product's repo, ask Claude to identify each
+ * non-trivial capability the product implements, and persist HOW it's built
+ * via the recipes / product_recipes tables.
+ *
+ * Production-grade behaviours layered on top of the basic agent loop:
+ *
+ *   - Heartbeat: `last_heartbeat_at` on the recipe_scans row is refreshed
+ *     on every assistant message so the reader can detect stalled / crashed
+ *     runs (see desktop-app/.../services/db/recipe-scans.ts for the lazy
+ *     reaper).
+ *   - Cancellation-safe writes: markRunning / markSuccess / markFailed only
+ *     touch rows whose status is in {pending, running}. If the user clicked
+ *     Stop and the row is now 'cancelled', the final write no-ops.
+ *   - Per-call MCP writes: agent commits each create / update / link /
+ *     unlink as it goes. There is no "submit at the end" buffer — partial
+ *     progress survives even if the agent later errors out.
+ */
+import type { SupabaseClient } from '@supabase/supabase-js';
+import type { RecipeSummary } from './types.js';
+export interface RecipesPhaseOptions {
+    productId: string;
+    scanId: string;
+    guidance?: string;
+    verbose?: boolean;
+}
+export interface RecipesPhaseResult {
+    status: 'success' | 'error' | 'cancelled';
+    message: string;
+    counts?: {
+        created: number;
+        updated: number;
+        linked: number;
+        unlinked: number;
+    };
+}
+export declare function runRecipesPhase(options: RecipesPhaseOptions): Promise<RecipesPhaseResult>;
+export declare function getProductTeamId(supabase: SupabaseClient, productId: string): Promise<string | null>;
+export declare function getScanCreator(supabase: SupabaseClient, scanId: string): Promise<{
+    created_by: string;
+} | null>;
+export declare function listTeamRecipes(supabase: SupabaseClient, teamId: string): Promise<RecipeSummary[]>;
+export declare function listProductRecipeLinks(supabase: SupabaseClient, productId: string): Promise<{
+    recipe_id: string;
+    name: string;
+}[]>;
+/**
+ * Claim the row by flipping `pending` → `running`. Returns true on success
+ * (we won the claim) and false when the row has already moved on (e.g. user
+ * cancelled before the CLI started). Bounded by the status filter so we
+ * can't accidentally resurrect a 'cancelled' row.
+ */
+export declare function markRunning(supabase: SupabaseClient, scanId: string): Promise<boolean>;
+export declare function heartbeat(supabase: SupabaseClient, scanId: string): Promise<void>;
+export declare function markFailed(supabase: SupabaseClient, scanId: string, errorMessage: string): Promise<boolean>;
+export declare function markSuccess(supabase: SupabaseClient, scanId: string): Promise<boolean>;