edsger 0.59.0 → 0.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/auth/env-store.js +3 -0
  2. package/dist/commands/data-flow/index.d.ts +17 -0
  3. package/dist/commands/data-flow/index.js +46 -0
  4. package/dist/commands/recipes/index.d.ts +15 -0
  5. package/dist/commands/recipes/index.js +34 -0
  6. package/dist/commands/screen-flow/index.d.ts +4 -4
  7. package/dist/commands/screen-flow/index.js +5 -5
  8. package/dist/commands/sync-aws/index.d.ts +16 -0
  9. package/dist/commands/sync-aws/index.js +184 -0
  10. package/dist/commands/sync-datadog/index.d.ts +16 -0
  11. package/dist/commands/sync-datadog/index.js +199 -0
  12. package/dist/commands/sync-terraform/index.d.ts +16 -0
  13. package/dist/commands/sync-terraform/index.js +211 -0
  14. package/dist/index.js +99 -8
  15. package/dist/phases/data-flow/index.d.ts +25 -0
  16. package/dist/phases/data-flow/index.js +257 -0
  17. package/dist/phases/data-flow/mcp-server.d.ts +85 -0
  18. package/dist/phases/data-flow/mcp-server.js +140 -0
  19. package/dist/phases/data-flow/prompts.d.ts +14 -0
  20. package/dist/phases/data-flow/prompts.js +36 -0
  21. package/dist/phases/data-flow/types.d.ts +71 -0
  22. package/dist/phases/data-flow/types.js +86 -0
  23. package/dist/phases/output-contracts.js +71 -0
  24. package/dist/phases/recipes/index.d.ts +56 -0
  25. package/dist/phases/recipes/index.js +301 -0
  26. package/dist/phases/recipes/mcp-server.d.ts +63 -0
  27. package/dist/phases/recipes/mcp-server.js +204 -0
  28. package/dist/phases/recipes/prompts.d.ts +35 -0
  29. package/dist/phases/recipes/prompts.js +105 -0
  30. package/dist/phases/recipes/types.d.ts +42 -0
  31. package/dist/phases/recipes/types.js +16 -0
  32. package/dist/phases/screen-flow/index.d.ts +2 -2
  33. package/dist/phases/screen-flow/index.js +27 -15
  34. package/dist/phases/screen-flow/mcp-server.d.ts +1 -1
  35. package/dist/skills/phase/data-flow/SKILL.md +82 -0
  36. package/package.json +3 -3
  37. package/vitest.config.ts +1 -1
@@ -0,0 +1,211 @@
1
+ /**
2
+ * CLI command: `edsger sync-terraform <teamId>`
3
+ *
4
+ * Reads the team's terraform_repo_full_name from the DB, clones the repo,
5
+ * and lets the agent (Claude SDK) analyse .tf files to extract service
6
+ * identities, AWS resources, module dependencies, and tier signals.
7
+ *
8
+ * Unlike sync-datadog (structured API → deterministic mapping), Terraform
9
+ * HCL is too varied for regex parsing — the agent reads files and reasons
10
+ * about module structure, variable passing, resource tagging, etc.
11
+ */
12
+ import { execSync } from 'child_process';
13
+ import { mkdtempSync, readFileSync, readdirSync, rmSync, statSync } from 'fs';
14
+ import { tmpdir } from 'os';
15
+ import { join, relative } from 'path';
16
+ import { callMcpEndpoint } from '../../api/mcp-client.js';
17
+ import { logError, logInfo, logSuccess, logWarning } from '../../utils/logger.js';
18
+ export async function runSyncTerraform(teamId, options = {}) {
19
+ const { verbose } = options;
20
+ logInfo(`Starting Terraform sync for team ${teamId}`);
21
+ // 1. Get team's terraform repo from DB
22
+ let repoFullName = null;
23
+ let localDir = options.dir ?? null;
24
+ if (!localDir) {
25
+ try {
26
+ const result = (await callMcpEndpoint('agents.get_team', {
27
+ team_id: teamId,
28
+ }));
29
+ repoFullName = result?.team?.terraform_repo_full_name ?? null;
30
+ }
31
+ catch {
32
+ // Fallback: read directly from supabase if MCP doesn't have the endpoint
33
+ logWarning('Could not fetch team via MCP, attempting to read terraform_repo_full_name from env');
34
+ }
35
+ if (!repoFullName) {
36
+ logError('No Terraform repo configured for this team. ' +
37
+ 'Go to Team Settings and set a Terraform repo, or use --dir to point at a local directory.');
38
+ process.exit(1);
39
+ }
40
+ // 2. Clone the repo
41
+ const tmpDir = mkdtempSync(join(tmpdir(), 'edsger-tf-'));
42
+ localDir = tmpDir;
43
+ logInfo(`Cloning ${repoFullName}...`);
44
+ try {
45
+ execSync(`git clone --depth=1 https://github.com/${repoFullName}.git "${tmpDir}/repo"`, { stdio: verbose ? 'inherit' : 'pipe', timeout: 60_000 });
46
+ localDir = join(tmpDir, 'repo');
47
+ }
48
+ catch (err) {
49
+ logError(`Failed to clone ${repoFullName}: ${err instanceof Error ? err.message : String(err)}`);
50
+ rmSync(tmpDir, { recursive: true, force: true });
51
+ process.exit(1);
52
+ }
53
+ }
54
+ logInfo(`Scanning Terraform files in ${localDir}`);
55
+ // 3. Find all .tf files
56
+ const tfFiles = [];
57
+ function walk(dir) {
58
+ try {
59
+ for (const entry of readdirSync(dir)) {
60
+ if (entry.startsWith('.') || entry === 'node_modules')
61
+ continue;
62
+ const full = join(dir, entry);
63
+ try {
64
+ const stat = statSync(full);
65
+ if (stat.isDirectory()) {
66
+ walk(full);
67
+ }
68
+ else if (entry.endsWith('.tf')) {
69
+ tfFiles.push(full);
70
+ }
71
+ }
72
+ catch {
73
+ // skip inaccessible files
74
+ }
75
+ }
76
+ }
77
+ catch {
78
+ // skip inaccessible dirs
79
+ }
80
+ }
81
+ walk(localDir);
82
+ if (tfFiles.length === 0) {
83
+ logWarning('No .tf files found in the repository');
84
+ cleanup(localDir, options.dir);
85
+ return;
86
+ }
87
+ logInfo(`Found ${tfFiles.length} .tf files`);
88
+ // 4. Read and group by module directory
89
+ const moduleFiles = new Map();
90
+ for (const tf of tfFiles) {
91
+ const relPath = relative(localDir, tf);
92
+ const moduleDir = relPath.includes('/')
93
+ ? relPath.split('/').slice(0, -1).join('/')
94
+ : '.';
95
+ if (!moduleFiles.has(moduleDir)) {
96
+ moduleFiles.set(moduleDir, []);
97
+ }
98
+ try {
99
+ const content = readFileSync(tf, 'utf8');
100
+ moduleFiles.get(moduleDir).push({ path: relPath, content });
101
+ }
102
+ catch {
103
+ if (verbose)
104
+ logWarning(`Could not read ${relPath}`);
105
+ }
106
+ }
107
+ logInfo(`Found ${moduleFiles.size} module directories`);
108
+ // 5. For each module, extract resource info and upsert service
109
+ let created = 0;
110
+ let updated = 0;
111
+ let components = 0;
112
+ for (const [moduleDir, files] of moduleFiles) {
113
+ // Skip root-level files that are just backend/provider config
114
+ if (moduleDir === '.' && files.every((f) => /^(backend|provider|versions|terraform)\b/.test(f.path))) {
115
+ continue;
116
+ }
117
+ // Extract service name from module directory
118
+ const moduleName = moduleDir === '.'
119
+ ? repoFullName?.split('/')[1] ?? 'root'
120
+ : moduleDir.split('/').pop() ?? moduleDir;
121
+ // Extract resource ARNs and types from .tf content
122
+ const resources = [];
123
+ const tags = {};
124
+ let hasMultiAz = false;
125
+ let hasAutoscaling = false;
126
+ const moduleRefs = [];
127
+ for (const file of files) {
128
+ // Resource blocks
129
+ for (const m of file.content.matchAll(/resource\s+"([^"]+)"\s+"([^"]+)"/g)) {
130
+ resources.push({ type: m[1], name: m[2] });
131
+ }
132
+ // Tag extraction
133
+ for (const m of file.content.matchAll(/Service\s*=\s*"([^"]+)"/g)) {
134
+ tags.Service = m[1];
135
+ }
136
+ for (const m of file.content.matchAll(/Team\s*=\s*"([^"]+)"/g)) {
137
+ tags.Team = m[1];
138
+ }
139
+ // Tier signals
140
+ if (/multi_az\s*=\s*true/i.test(file.content))
141
+ hasMultiAz = true;
142
+ if (/aws_appautoscaling|autoscaling_group/i.test(file.content))
143
+ hasAutoscaling = true;
144
+ // Module references (dependencies)
145
+ for (const m of file.content.matchAll(/module\.([a-zA-Z0-9_-]+)\./g)) {
146
+ if (!moduleRefs.includes(m[1]))
147
+ moduleRefs.push(m[1]);
148
+ }
149
+ }
150
+ if (resources.length === 0)
151
+ continue;
152
+ const serviceName = tags.Service ?? moduleName;
153
+ const tier = hasMultiAz || hasAutoscaling ? 'critical' : 'standard';
154
+ if (verbose) {
155
+ logInfo(` ${serviceName}: ${resources.length} resources, tier=${tier}${moduleRefs.length > 0 ? `, deps: ${moduleRefs.join(', ')}` : ''}`);
156
+ }
157
+ try {
158
+ const result = (await callMcpEndpoint('services.upsert_draft', {
159
+ team_id: teamId,
160
+ name: serviceName,
161
+ kind: 'service',
162
+ tier,
163
+ source: 'terraform',
164
+ source_ref: `terraform:${repoFullName ?? localDir}:${moduleDir}`,
165
+ field_sources: {
166
+ name: { value: serviceName, confidence: tags.Service ? 1.0 : 0.7, source: 'terraform', source_detail: moduleDir },
167
+ tier: { value: tier, confidence: hasMultiAz ? 0.8 : 0.5, source: 'terraform' },
168
+ ...(tags.Team ? { owner: { value: tags.Team, confidence: 0.8, source: 'terraform_tag' } } : {}),
169
+ },
170
+ needs_review: !tags.Service,
171
+ }));
172
+ if (result.action === 'created')
173
+ created++;
174
+ else
175
+ updated++;
176
+ // Add AWS resource components
177
+ for (const res of resources) {
178
+ try {
179
+ await callMcpEndpoint('services.add_component', {
180
+ service_id: result.service.id,
181
+ kind: 'aws_resource',
182
+ provider: 'terraform',
183
+ external_id: `${res.type}.${res.name}`,
184
+ metadata: { resource_type: res.type, resource_name: res.name, module: moduleDir },
185
+ });
186
+ components++;
187
+ }
188
+ catch {
189
+ // ignore duplicate
190
+ }
191
+ }
192
+ }
193
+ catch (err) {
194
+ logWarning(`Failed to sync module "${moduleName}": ${err instanceof Error ? err.message : String(err)}`);
195
+ }
196
+ }
197
+ cleanup(localDir, options.dir);
198
+ logSuccess(`Terraform sync completed: ${created} created, ${updated} updated, ${components} resource components`);
199
+ }
200
+ function cleanup(dir, userDir) {
201
+ if (!userDir && dir) {
202
+ try {
203
+ // Clean up the temp clone directory (parent of repo/)
204
+ const parent = dir.endsWith('/repo') ? dir.replace(/\/repo$/, '') : dir;
205
+ rmSync(parent, { recursive: true, force: true });
206
+ }
207
+ catch {
208
+ // ignore
209
+ }
210
+ }
211
+ }
package/dist/index.js CHANGED
@@ -13,6 +13,7 @@ import { runBuild } from './commands/build/index.js';
13
13
  import { runChecklists } from './commands/checklists/index.js';
14
14
  import { runCodeReview } from './commands/code-review/index.js';
15
15
  import { runConfigGet, runConfigList, runConfigSet, runConfigUnset, } from './commands/config/index.js';
16
+ import { runDataFlow } from './commands/data-flow/index.js';
16
17
  import { runFinancingDeck } from './commands/financing-deck/index.js';
17
18
  import { runFindArchitecture } from './commands/find-architecture/index.js';
18
19
  import { runFindBugs } from './commands/find-bugs/index.js';
@@ -24,15 +25,18 @@ import { runIntelligence } from './commands/intelligence/index.js';
24
25
  import { runIssueAnalysisCommand } from './commands/issue-analysis/index.js';
25
26
  import { runPRResolve } from './commands/pr-resolve/index.js';
26
27
  import { runPRReview } from './commands/pr-review/index.js';
27
- import { runProductTechniques } from './commands/product-techniques/index.js';
28
28
  import { runProductTestCases } from './commands/product-test-cases/index.js';
29
29
  import { runQualityBenchmarkCli } from './commands/quality-benchmark/index.js';
30
+ import { runRecipes } from './commands/recipes/index.js';
30
31
  import { runRefactor } from './commands/refactor/refactor.js';
31
32
  import { runReleaseSyncCommand } from './commands/release-sync/index.js';
32
33
  import { runRunSheetCommand } from './commands/run-sheet/index.js';
33
34
  import { runScreenFlow } from './commands/screen-flow/index.js';
34
35
  import { runSmokeTestCommand } from './commands/smoke-test/index.js';
35
36
  import { runSyncGithubIssues } from './commands/sync-github-issues/index.js';
37
+ import { runSyncAws } from './commands/sync-aws/index.js';
38
+ import { runSyncDatadog } from './commands/sync-datadog/index.js';
39
+ import { runSyncTerraform } from './commands/sync-terraform/index.js';
36
40
  import { runSyncSentryIssues } from './commands/sync-sentry-issues/index.js';
37
41
  import { runTaskWorker } from './commands/task-worker/index.js';
38
42
  import { runTechnicalDesignCommand } from './commands/technical-design/index.js';
@@ -166,7 +170,7 @@ program
166
170
  program
167
171
  .command('screen-flow <productId>')
168
172
  .description('Generate a structured screen flow (screens + transitions) for a product from its source code')
169
- .requiredOption('--flow-id <id>', 'Pending screen_flows row id to populate')
173
+ .requiredOption('--flow-id <id>', 'Pending flows row id to populate')
170
174
  .option('-g, --guidance <text>', 'Human direction for the AI (focus areas, exclusions)')
171
175
  .option('-v, --verbose', 'Verbose output')
172
176
  .action(async (productId, opts) => {
@@ -183,6 +187,28 @@ program
183
187
  }
184
188
  });
185
189
  // ============================================================
190
+ // Subcommand: edsger data-flow <productId>
191
+ // ============================================================
192
+ program
193
+ .command('data-flow <productId>')
194
+ .description('Generate a structured data flow (sources, datasets, transforms, sinks, queues, models) for a product from its source code')
195
+ .requiredOption('--flow-id <id>', 'Pending flows row id to populate')
196
+ .option('-g, --guidance <text>', 'Human direction for the AI (focus areas, exclusions)')
197
+ .option('-v, --verbose', 'Verbose output')
198
+ .action(async (productId, opts) => {
199
+ try {
200
+ await runDataFlow(productId, {
201
+ flowId: opts.flowId,
202
+ guidance: opts.guidance,
203
+ verbose: opts.verbose,
204
+ });
205
+ }
206
+ catch (error) {
207
+ logError(error instanceof Error ? error.message : String(error));
208
+ process.exit(1);
209
+ }
210
+ });
211
+ // ============================================================
186
212
  // Subcommand: edsger user-psychology <productId>
187
213
  // ============================================================
188
214
  program
@@ -594,6 +620,71 @@ program
594
620
  }
595
621
  });
596
622
  // ============================================================
623
+ // Subcommand: edsger sync-datadog <teamId>
624
+ //
625
+ // Requires DD_API_KEY, DD_APP_KEY in env (set via `edsger config set`
626
+ // or desktop Settings → Credentials → Datadog). Optional DD_SITE.
627
+ // ============================================================
628
+ program
629
+ .command('sync-datadog <teamId>')
630
+ .description('Import services from Datadog Service Catalog into the edsger service catalog (requires DD_API_KEY, DD_APP_KEY env)')
631
+ .option('-v, --verbose', 'Verbose output')
632
+ .action(async (teamId, opts) => {
633
+ try {
634
+ await runSyncDatadog(teamId, opts);
635
+ }
636
+ catch (error) {
637
+ logError(error instanceof Error ? error.message : String(error));
638
+ process.exit(1);
639
+ }
640
+ });
641
+ // ============================================================
642
+ // ============================================================
643
+ // Subcommand: edsger sync-aws <teamId>
644
+ //
645
+ // Uses the standard AWS credential chain to discover services by
646
+ // their Service tag and optionally pull cost data.
647
+ // ============================================================
648
+ program
649
+ .command('sync-aws <teamId>')
650
+ .description('Discover services from AWS by Service tag and pull cost data from Cost Explorer')
651
+ .option('-v, --verbose', 'Verbose output')
652
+ .option('--region <region>', 'AWS region (default: AWS_REGION env or us-east-1)')
653
+ .option('--no-cost', 'Skip Cost Explorer query')
654
+ .action(async (teamId, opts) => {
655
+ try {
656
+ await runSyncAws(teamId, {
657
+ verbose: opts.verbose,
658
+ region: opts.region,
659
+ noCost: opts.cost === false,
660
+ });
661
+ }
662
+ catch (error) {
663
+ logError(error instanceof Error ? error.message : String(error));
664
+ process.exit(1);
665
+ }
666
+ });
667
+ // Subcommand: edsger sync-terraform <teamId>
668
+ //
669
+ // Clones the team's terraform repo (from teams.terraform_repo_full_name)
670
+ // and lets the agent analyse .tf files to extract services, resources,
671
+ // and dependencies. Alternatively, --dir for a local directory.
672
+ // ============================================================
673
+ program
674
+ .command('sync-terraform <teamId>')
675
+ .description('Scan a Terraform repo to discover services, AWS resources, and infrastructure dependencies')
676
+ .option('-v, --verbose', 'Verbose output')
677
+ .option('--dir <path>', 'Local directory to scan instead of cloning from GitHub')
678
+ .action(async (teamId, opts) => {
679
+ try {
680
+ await runSyncTerraform(teamId, opts);
681
+ }
682
+ catch (error) {
683
+ logError(error instanceof Error ? error.message : String(error));
684
+ process.exit(1);
685
+ }
686
+ });
687
+ // ============================================================
597
688
  // Subcommand: edsger find-features <productId>
598
689
  // ============================================================
599
690
  program
@@ -695,18 +786,18 @@ program
695
786
  }
696
787
  });
697
788
  // ============================================================
698
- // Subcommand: edsger product-techniques <productId>
789
+ // Subcommand: edsger recipes <productId>
699
790
  // ============================================================
700
791
  program
701
- .command('product-techniques <productId>')
702
- .description("Generate a catalogue of the techniques a product's repo uses (languages, frameworks, patterns, state idioms, build & deploy, notable bits). Writes the result to the pending product_techniques row identified by --techniques-id.")
703
- .requiredOption('--techniques-id <id>', 'Pending product_techniques row id to populate')
792
+ .command('recipes <productId>')
793
+ .description("Scan a product's repo for the implementation recipes it uses (which services/tools are chained together to deliver each capability) and persist them via the recipes / product_recipes tables. Writes against the pending recipe_scans row identified by --scan-id.")
794
+ .requiredOption('--scan-id <id>', 'Pending recipe_scans row id to drive (created by the desktop UI before invocation)')
704
795
  .option('-g, --guidance <text>', 'Human direction for the AI (focus areas, exclusions)')
705
796
  .option('-v, --verbose', 'Verbose output')
706
797
  .action(async (productId, opts) => {
707
798
  try {
708
- await runProductTechniques(productId, {
709
- techniquesId: opts.techniquesId,
799
+ await runRecipes(productId, {
800
+ scanId: opts.scanId,
710
801
  guidance: opts.guidance,
711
802
  verbose: opts.verbose,
712
803
  });
@@ -0,0 +1,25 @@
1
+ /**
2
+ * data-flow phase: clone the product's repo, ask Claude to map every data
3
+ * node (source / dataset / transform / sink / queue / model) and the
4
+ * connections between them into a structured DataFlowExtraction, then
5
+ * persist the result to flows / flow_nodes / flow_edges (rows tagged
6
+ * `type = 'data'`) via the Supabase SDK.
7
+ *
8
+ * Companion to screen-flow: same generation pattern (workspace clone +
9
+ * Claude Agent SDK + in-process MCP server), same storage tables, different
10
+ * domain.
11
+ */
12
+ export interface DataFlowPhaseOptions {
13
+ productId: string;
14
+ flowId: string;
15
+ guidance?: string;
16
+ verbose?: boolean;
17
+ }
18
+ export interface DataFlowPhaseResult {
19
+ status: 'success' | 'error';
20
+ message: string;
21
+ nodesCreated?: number;
22
+ edgesCreated?: number;
23
+ summary?: string;
24
+ }
25
+ export declare function runDataFlowPhase(options: DataFlowPhaseOptions): Promise<DataFlowPhaseResult>;
@@ -0,0 +1,257 @@
1
+ /**
2
+ * data-flow phase: clone the product's repo, ask Claude to map every data
3
+ * node (source / dataset / transform / sink / queue / model) and the
4
+ * connections between them into a structured DataFlowExtraction, then
5
+ * persist the result to flows / flow_nodes / flow_edges (rows tagged
6
+ * `type = 'data'`) via the Supabase SDK.
7
+ *
8
+ * Companion to screen-flow: same generation pattern (workspace clone +
9
+ * Claude Agent SDK + in-process MCP server), same storage tables, different
10
+ * domain.
11
+ */
12
+ import { query } from '@anthropic-ai/claude-agent-sdk';
13
+ import { getGitHubConfigByProduct } from '../../api/github.js';
14
+ import { DEFAULT_MODEL } from '../../constants.js';
15
+ import { getSupabase } from '../../supabase/client.js';
16
+ import { logError, logInfo, logSuccess, logWarning } from '../../utils/logger.js';
17
+ import { cleanupIssueRepo, cloneIssueRepo, ensureWorkspaceDir, } from '../../workspace/workspace-manager.js';
18
+ import { fetchProductBasics } from '../find-shared/mcp.js';
19
+ import { createPromptGenerator, extractTextFromContent, tryExtractResult, } from '../pr-shared/agent-utils.js';
20
+ import { createDataFlowCaptureState, createDataFlowMcpServer, validateConsistency, } from './mcp-server.js';
21
+ import { createDataFlowSystemPrompt, createDataFlowUserPrompt, } from './prompts.js';
22
+ import { isDataFlowExtraction, } from './types.js';
23
+ const WORKSPACE_KEY = 'data-flow';
24
+ const MAX_TURNS = 150;
25
+ // Auto-layout: simple grid; users can drag afterwards and we persist positions.
26
+ const COLUMN_WIDTH = 320;
27
+ const ROW_HEIGHT = 220;
28
+ const COLUMNS = 4;
29
+ export async function runDataFlowPhase(options) {
30
+ const { productId, flowId, guidance, verbose } = options;
31
+ logInfo(`Starting data-flow generation for product ${productId}`);
32
+ const supabase = getSupabase();
33
+ await markFlowRunning(supabase, flowId);
34
+ const githubConfig = await getGitHubConfigByProduct(productId, verbose);
35
+ if (!githubConfig.configured ||
36
+ !githubConfig.token ||
37
+ !githubConfig.owner ||
38
+ !githubConfig.repo) {
39
+ const msg = githubConfig.message ||
40
+ 'GitHub repository not configured for this product. Connect a repo first.';
41
+ await markFlowFailed(supabase, flowId, msg);
42
+ return { status: 'error', message: msg };
43
+ }
44
+ let repoPath;
45
+ let succeeded = false;
46
+ try {
47
+ const workspaceRoot = ensureWorkspaceDir();
48
+ const repoKey = `${WORKSPACE_KEY}-${productId}`;
49
+ ({ repoPath } = cloneIssueRepo(workspaceRoot, repoKey, githubConfig.owner, githubConfig.repo, githubConfig.token));
50
+ const product = await fetchProductBasics(productId);
51
+ const systemPrompt = await createDataFlowSystemPrompt({
52
+ projectDir: repoPath,
53
+ hasCodebase: true,
54
+ });
55
+ const userPrompt = createDataFlowUserPrompt({
56
+ productName: product.name,
57
+ productDescription: product.description,
58
+ guidance,
59
+ });
60
+ logInfo('Running Claude data-flow extraction...');
61
+ const captureState = createDataFlowCaptureState();
62
+ const mcpServer = createDataFlowMcpServer(captureState, {
63
+ onProgress: ({ phase, message }) => {
64
+ logInfo(`[${phase}] ${message}`);
65
+ },
66
+ });
67
+ let lastAssistantResponse = '';
68
+ let extraction = null;
69
+ for await (const message of query({
70
+ prompt: createPromptGenerator(userPrompt),
71
+ options: {
72
+ systemPrompt: {
73
+ type: 'preset',
74
+ preset: 'claude_code',
75
+ append: systemPrompt,
76
+ },
77
+ model: DEFAULT_MODEL,
78
+ maxTurns: MAX_TURNS,
79
+ permissionMode: 'bypassPermissions',
80
+ cwd: repoPath,
81
+ mcpServers: {
82
+ 'data-flow': mcpServer,
83
+ },
84
+ },
85
+ })) {
86
+ const { assistantBuffer, extraction: nextExtraction } = processSdkMessage(message, lastAssistantResponse, captureState, verbose);
87
+ lastAssistantResponse = assistantBuffer;
88
+ if (nextExtraction) {
89
+ extraction = nextExtraction;
90
+ }
91
+ }
92
+ if (!extraction) {
93
+ const msg = 'Data flow extraction failed: agent did not call submit_data_flow and no parseable data_flow block was found in the response';
94
+ await markFlowFailed(supabase, flowId, msg);
95
+ return { status: 'error', message: msg };
96
+ }
97
+ logInfo(`Extraction produced ${extraction.nodes.length} nodes / ${extraction.edges.length} connections`);
98
+ const { nodesCreated, edgesCreated } = await persistFlow(supabase, flowId, extraction);
99
+ await markFlowSuccess(supabase, flowId, extraction.summary);
100
+ succeeded = true;
101
+ logSuccess(`Data flow generated: ${nodesCreated} nodes, ${edgesCreated} connections`);
102
+ return {
103
+ status: 'success',
104
+ message: `Data flow generated (${nodesCreated} nodes, ${edgesCreated} connections)`,
105
+ nodesCreated,
106
+ edgesCreated,
107
+ summary: extraction.summary,
108
+ };
109
+ }
110
+ catch (error) {
111
+ const errorMessage = error instanceof Error ? error.message : String(error);
112
+ logError(`Data flow failed: ${errorMessage}`);
113
+ await markFlowFailed(supabase, flowId, errorMessage);
114
+ return { status: 'error', message: errorMessage };
115
+ }
116
+ finally {
117
+ if (succeeded) {
118
+ cleanupIssueRepo(repoPath);
119
+ }
120
+ }
121
+ }
122
+ function processSdkMessage(
123
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
124
+ message, assistantBuffer, captureState, verbose) {
125
+ if (message.type === 'assistant') {
126
+ const next = assistantBuffer +
127
+ extractTextFromContent(message.message?.content ?? [], verbose);
128
+ return { assistantBuffer: next, extraction: null };
129
+ }
130
+ if (message.type === 'user' && verbose) {
131
+ const userContent = message.message?.content;
132
+ if (Array.isArray(userContent)) {
133
+ extractTextFromContent(userContent, verbose);
134
+ }
135
+ return { assistantBuffer, extraction: null };
136
+ }
137
+ if (message.type !== 'result') {
138
+ return { assistantBuffer, extraction: null };
139
+ }
140
+ if (captureState.captured) {
141
+ return { assistantBuffer, extraction: captureState.captured };
142
+ }
143
+ const fallback = tryFallbackParse(message, assistantBuffer);
144
+ if (fallback) {
145
+ logWarning('Agent emitted a fenced data_flow block instead of calling submit_data_flow; using the parsed text as a fallback.');
146
+ return { assistantBuffer, extraction: fallback };
147
+ }
148
+ if (message.subtype !== 'success') {
149
+ logError(`Extraction incomplete: ${message.subtype}`);
150
+ }
151
+ return { assistantBuffer, extraction: null };
152
+ }
153
+ function tryFallbackParse(resultMessage, assistantText) {
154
+ const responseText = resultMessage.subtype === 'success'
155
+ ? resultMessage.result || assistantText
156
+ : assistantText;
157
+ const parsed = tryExtractResult(responseText, 'data_flow');
158
+ if (!isDataFlowExtraction(parsed)) {
159
+ return null;
160
+ }
161
+ const { error } = validateConsistency(parsed);
162
+ if (error) {
163
+ logWarning(`Fallback extraction failed consistency check: ${error}`);
164
+ return null;
165
+ }
166
+ return parsed;
167
+ }
168
+ // ============================================================================
169
+ // Persistence
170
+ // ============================================================================
171
+ async function markFlowRunning(supabase, flowId) {
172
+ const { error } = await supabase
173
+ .from('flows')
174
+ .update({ status: 'running', error: null })
175
+ .eq('id', flowId);
176
+ if (error) {
177
+ logWarning(`Could not mark flow as running: ${error.message}`);
178
+ }
179
+ }
180
+ async function markFlowFailed(supabase, flowId, errorMessage) {
181
+ await supabase
182
+ .from('flows')
183
+ .update({
184
+ status: 'failed',
185
+ error: errorMessage,
186
+ completed_at: new Date().toISOString(),
187
+ })
188
+ .eq('id', flowId);
189
+ }
190
+ async function markFlowSuccess(supabase, flowId, summary) {
191
+ await supabase
192
+ .from('flows')
193
+ .update({
194
+ status: 'success',
195
+ summary,
196
+ error: null,
197
+ completed_at: new Date().toISOString(),
198
+ })
199
+ .eq('id', flowId);
200
+ }
201
+ async function persistFlow(supabase, flowId, extraction) {
202
+ await supabase.from('flow_edges').delete().eq('flow_id', flowId);
203
+ await supabase.from('flow_nodes').delete().eq('flow_id', flowId);
204
+ if (extraction.nodes.length === 0) {
205
+ return { nodesCreated: 0, edgesCreated: 0 };
206
+ }
207
+ const nodeRows = extraction.nodes.map((n, i) => buildNodeRow(flowId, n, i));
208
+ const { data: insertedNodes, error: nodesError } = await supabase
209
+ .from('flow_nodes')
210
+ .insert(nodeRows)
211
+ .select('id, slug');
212
+ if (nodesError) {
213
+ throw new Error(`Failed to insert nodes: ${nodesError.message}`);
214
+ }
215
+ const slugToId = new Map((insertedNodes ?? []).map((n) => [n.slug, n.id]));
216
+ const edgeRows = extraction.edges
217
+ .map((e) => buildEdgeRow(flowId, e, slugToId))
218
+ .filter((e) => e !== null);
219
+ if (edgeRows.length > 0) {
220
+ const { error: edgesError } = await supabase
221
+ .from('flow_edges')
222
+ .insert(edgeRows);
223
+ if (edgesError) {
224
+ throw new Error(`Failed to insert edges: ${edgesError.message}`);
225
+ }
226
+ }
227
+ return {
228
+ nodesCreated: nodeRows.length,
229
+ edgesCreated: edgeRows.length,
230
+ };
231
+ }
232
+ function buildNodeRow(flowId, node, index) {
233
+ return {
234
+ flow_id: flowId,
235
+ slug: node.slug,
236
+ name: node.name,
237
+ kind: node.kind,
238
+ schema: node,
239
+ position_x: (index % COLUMNS) * COLUMN_WIDTH,
240
+ position_y: Math.floor(index / COLUMNS) * ROW_HEIGHT,
241
+ };
242
+ }
243
+ function buildEdgeRow(flowId, edge, slugToId) {
244
+ const fromId = slugToId.get(edge.fromSlug);
245
+ const toId = slugToId.get(edge.toSlug);
246
+ if (!fromId || !toId) {
247
+ return null;
248
+ }
249
+ return {
250
+ flow_id: flowId,
251
+ from_node_id: fromId,
252
+ to_node_id: toId,
253
+ label: edge.label ?? null,
254
+ source_anchor: edge.sourceFile ?? null,
255
+ kind: edge.kind,
256
+ };
257
+ }