edsger 0.60.0 → 0.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/env-store.js +3 -0
- package/dist/commands/data-flow/index.d.ts +17 -0
- package/dist/commands/data-flow/index.js +46 -0
- package/dist/commands/screen-flow/index.d.ts +4 -4
- package/dist/commands/screen-flow/index.js +5 -5
- package/dist/commands/sync-aws/index.d.ts +16 -0
- package/dist/commands/sync-aws/index.js +184 -0
- package/dist/commands/sync-datadog/index.d.ts +16 -0
- package/dist/commands/sync-datadog/index.js +199 -0
- package/dist/commands/sync-org-repos/index.d.ts +11 -0
- package/dist/commands/sync-org-repos/index.js +59 -0
- package/dist/commands/sync-terraform/index.d.ts +16 -0
- package/dist/commands/sync-terraform/index.js +211 -0
- package/dist/index.js +111 -2
- package/dist/phases/data-flow/index.d.ts +25 -0
- package/dist/phases/data-flow/index.js +257 -0
- package/dist/phases/data-flow/mcp-server.d.ts +85 -0
- package/dist/phases/data-flow/mcp-server.js +140 -0
- package/dist/phases/data-flow/prompts.d.ts +14 -0
- package/dist/phases/data-flow/prompts.js +36 -0
- package/dist/phases/data-flow/types.d.ts +71 -0
- package/dist/phases/data-flow/types.js +86 -0
- package/dist/phases/output-contracts.js +71 -0
- package/dist/phases/screen-flow/index.d.ts +2 -2
- package/dist/phases/screen-flow/index.js +27 -15
- package/dist/phases/screen-flow/mcp-server.d.ts +1 -1
- package/dist/phases/sync-org-repos/index.d.ts +24 -0
- package/dist/phases/sync-org-repos/index.js +143 -0
- package/dist/skills/phase/data-flow/SKILL.md +82 -0
- package/package.json +3 -3
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI command: `edsger sync-terraform <teamId>`
|
|
3
|
+
*
|
|
4
|
+
* Reads the team's terraform_repo_full_name from the DB, clones the repo,
|
|
5
|
+
* and lets the agent (Claude SDK) analyse .tf files to extract service
|
|
6
|
+
* identities, AWS resources, module dependencies, and tier signals.
|
|
7
|
+
*
|
|
8
|
+
* Unlike sync-datadog (structured API → deterministic mapping), Terraform
|
|
9
|
+
* HCL is too varied for regex parsing — the agent reads files and reasons
|
|
10
|
+
* about module structure, variable passing, resource tagging, etc.
|
|
11
|
+
*/
|
|
12
|
+
import { execSync } from 'child_process';
|
|
13
|
+
import { mkdtempSync, readFileSync, readdirSync, rmSync, statSync } from 'fs';
|
|
14
|
+
import { tmpdir } from 'os';
|
|
15
|
+
import { join, relative } from 'path';
|
|
16
|
+
import { callMcpEndpoint } from '../../api/mcp-client.js';
|
|
17
|
+
import { logError, logInfo, logSuccess, logWarning } from '../../utils/logger.js';
|
|
18
|
+
export async function runSyncTerraform(teamId, options = {}) {
|
|
19
|
+
const { verbose } = options;
|
|
20
|
+
logInfo(`Starting Terraform sync for team ${teamId}`);
|
|
21
|
+
// 1. Get team's terraform repo from DB
|
|
22
|
+
let repoFullName = null;
|
|
23
|
+
let localDir = options.dir ?? null;
|
|
24
|
+
if (!localDir) {
|
|
25
|
+
try {
|
|
26
|
+
const result = (await callMcpEndpoint('agents.get_team', {
|
|
27
|
+
team_id: teamId,
|
|
28
|
+
}));
|
|
29
|
+
repoFullName = result?.team?.terraform_repo_full_name ?? null;
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// Fallback: read directly from supabase if MCP doesn't have the endpoint
|
|
33
|
+
logWarning('Could not fetch team via MCP, attempting to read terraform_repo_full_name from env');
|
|
34
|
+
}
|
|
35
|
+
if (!repoFullName) {
|
|
36
|
+
logError('No Terraform repo configured for this team. ' +
|
|
37
|
+
'Go to Team Settings and set a Terraform repo, or use --dir to point at a local directory.');
|
|
38
|
+
process.exit(1);
|
|
39
|
+
}
|
|
40
|
+
// 2. Clone the repo
|
|
41
|
+
const tmpDir = mkdtempSync(join(tmpdir(), 'edsger-tf-'));
|
|
42
|
+
localDir = tmpDir;
|
|
43
|
+
logInfo(`Cloning ${repoFullName}...`);
|
|
44
|
+
try {
|
|
45
|
+
execSync(`git clone --depth=1 https://github.com/${repoFullName}.git "${tmpDir}/repo"`, { stdio: verbose ? 'inherit' : 'pipe', timeout: 60_000 });
|
|
46
|
+
localDir = join(tmpDir, 'repo');
|
|
47
|
+
}
|
|
48
|
+
catch (err) {
|
|
49
|
+
logError(`Failed to clone ${repoFullName}: ${err instanceof Error ? err.message : String(err)}`);
|
|
50
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
51
|
+
process.exit(1);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
logInfo(`Scanning Terraform files in ${localDir}`);
|
|
55
|
+
// 3. Find all .tf files
|
|
56
|
+
const tfFiles = [];
|
|
57
|
+
function walk(dir) {
|
|
58
|
+
try {
|
|
59
|
+
for (const entry of readdirSync(dir)) {
|
|
60
|
+
if (entry.startsWith('.') || entry === 'node_modules')
|
|
61
|
+
continue;
|
|
62
|
+
const full = join(dir, entry);
|
|
63
|
+
try {
|
|
64
|
+
const stat = statSync(full);
|
|
65
|
+
if (stat.isDirectory()) {
|
|
66
|
+
walk(full);
|
|
67
|
+
}
|
|
68
|
+
else if (entry.endsWith('.tf')) {
|
|
69
|
+
tfFiles.push(full);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
// skip inaccessible files
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// skip inaccessible dirs
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
walk(localDir);
|
|
82
|
+
if (tfFiles.length === 0) {
|
|
83
|
+
logWarning('No .tf files found in the repository');
|
|
84
|
+
cleanup(localDir, options.dir);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
logInfo(`Found ${tfFiles.length} .tf files`);
|
|
88
|
+
// 4. Read and group by module directory
|
|
89
|
+
const moduleFiles = new Map();
|
|
90
|
+
for (const tf of tfFiles) {
|
|
91
|
+
const relPath = relative(localDir, tf);
|
|
92
|
+
const moduleDir = relPath.includes('/')
|
|
93
|
+
? relPath.split('/').slice(0, -1).join('/')
|
|
94
|
+
: '.';
|
|
95
|
+
if (!moduleFiles.has(moduleDir)) {
|
|
96
|
+
moduleFiles.set(moduleDir, []);
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const content = readFileSync(tf, 'utf8');
|
|
100
|
+
moduleFiles.get(moduleDir).push({ path: relPath, content });
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
if (verbose)
|
|
104
|
+
logWarning(`Could not read ${relPath}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
logInfo(`Found ${moduleFiles.size} module directories`);
|
|
108
|
+
// 5. For each module, extract resource info and upsert service
|
|
109
|
+
let created = 0;
|
|
110
|
+
let updated = 0;
|
|
111
|
+
let components = 0;
|
|
112
|
+
for (const [moduleDir, files] of moduleFiles) {
|
|
113
|
+
// Skip root-level files that are just backend/provider config
|
|
114
|
+
if (moduleDir === '.' && files.every((f) => /^(backend|provider|versions|terraform)\b/.test(f.path))) {
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
// Extract service name from module directory
|
|
118
|
+
const moduleName = moduleDir === '.'
|
|
119
|
+
? repoFullName?.split('/')[1] ?? 'root'
|
|
120
|
+
: moduleDir.split('/').pop() ?? moduleDir;
|
|
121
|
+
// Extract resource ARNs and types from .tf content
|
|
122
|
+
const resources = [];
|
|
123
|
+
const tags = {};
|
|
124
|
+
let hasMultiAz = false;
|
|
125
|
+
let hasAutoscaling = false;
|
|
126
|
+
const moduleRefs = [];
|
|
127
|
+
for (const file of files) {
|
|
128
|
+
// Resource blocks
|
|
129
|
+
for (const m of file.content.matchAll(/resource\s+"([^"]+)"\s+"([^"]+)"/g)) {
|
|
130
|
+
resources.push({ type: m[1], name: m[2] });
|
|
131
|
+
}
|
|
132
|
+
// Tag extraction
|
|
133
|
+
for (const m of file.content.matchAll(/Service\s*=\s*"([^"]+)"/g)) {
|
|
134
|
+
tags.Service = m[1];
|
|
135
|
+
}
|
|
136
|
+
for (const m of file.content.matchAll(/Team\s*=\s*"([^"]+)"/g)) {
|
|
137
|
+
tags.Team = m[1];
|
|
138
|
+
}
|
|
139
|
+
// Tier signals
|
|
140
|
+
if (/multi_az\s*=\s*true/i.test(file.content))
|
|
141
|
+
hasMultiAz = true;
|
|
142
|
+
if (/aws_appautoscaling|autoscaling_group/i.test(file.content))
|
|
143
|
+
hasAutoscaling = true;
|
|
144
|
+
// Module references (dependencies)
|
|
145
|
+
for (const m of file.content.matchAll(/module\.([a-zA-Z0-9_-]+)\./g)) {
|
|
146
|
+
if (!moduleRefs.includes(m[1]))
|
|
147
|
+
moduleRefs.push(m[1]);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (resources.length === 0)
|
|
151
|
+
continue;
|
|
152
|
+
const serviceName = tags.Service ?? moduleName;
|
|
153
|
+
const tier = hasMultiAz || hasAutoscaling ? 'critical' : 'standard';
|
|
154
|
+
if (verbose) {
|
|
155
|
+
logInfo(` ${serviceName}: ${resources.length} resources, tier=${tier}${moduleRefs.length > 0 ? `, deps: ${moduleRefs.join(', ')}` : ''}`);
|
|
156
|
+
}
|
|
157
|
+
try {
|
|
158
|
+
const result = (await callMcpEndpoint('services.upsert_draft', {
|
|
159
|
+
team_id: teamId,
|
|
160
|
+
name: serviceName,
|
|
161
|
+
kind: 'service',
|
|
162
|
+
tier,
|
|
163
|
+
source: 'terraform',
|
|
164
|
+
source_ref: `terraform:${repoFullName ?? localDir}:${moduleDir}`,
|
|
165
|
+
field_sources: {
|
|
166
|
+
name: { value: serviceName, confidence: tags.Service ? 1.0 : 0.7, source: 'terraform', source_detail: moduleDir },
|
|
167
|
+
tier: { value: tier, confidence: hasMultiAz ? 0.8 : 0.5, source: 'terraform' },
|
|
168
|
+
...(tags.Team ? { owner: { value: tags.Team, confidence: 0.8, source: 'terraform_tag' } } : {}),
|
|
169
|
+
},
|
|
170
|
+
needs_review: !tags.Service,
|
|
171
|
+
}));
|
|
172
|
+
if (result.action === 'created')
|
|
173
|
+
created++;
|
|
174
|
+
else
|
|
175
|
+
updated++;
|
|
176
|
+
// Add AWS resource components
|
|
177
|
+
for (const res of resources) {
|
|
178
|
+
try {
|
|
179
|
+
await callMcpEndpoint('services.add_component', {
|
|
180
|
+
service_id: result.service.id,
|
|
181
|
+
kind: 'aws_resource',
|
|
182
|
+
provider: 'terraform',
|
|
183
|
+
external_id: `${res.type}.${res.name}`,
|
|
184
|
+
metadata: { resource_type: res.type, resource_name: res.name, module: moduleDir },
|
|
185
|
+
});
|
|
186
|
+
components++;
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
// ignore duplicate
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
catch (err) {
|
|
194
|
+
logWarning(`Failed to sync module "${moduleName}": ${err instanceof Error ? err.message : String(err)}`);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
cleanup(localDir, options.dir);
|
|
198
|
+
logSuccess(`Terraform sync completed: ${created} created, ${updated} updated, ${components} resource components`);
|
|
199
|
+
}
|
|
200
|
+
function cleanup(dir, userDir) {
|
|
201
|
+
if (!userDir && dir) {
|
|
202
|
+
try {
|
|
203
|
+
// Clean up the temp clone directory (parent of repo/)
|
|
204
|
+
const parent = dir.endsWith('/repo') ? dir.replace(/\/repo$/, '') : dir;
|
|
205
|
+
rmSync(parent, { recursive: true, force: true });
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
// ignore
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -13,6 +13,7 @@ import { runBuild } from './commands/build/index.js';
|
|
|
13
13
|
import { runChecklists } from './commands/checklists/index.js';
|
|
14
14
|
import { runCodeReview } from './commands/code-review/index.js';
|
|
15
15
|
import { runConfigGet, runConfigList, runConfigSet, runConfigUnset, } from './commands/config/index.js';
|
|
16
|
+
import { runDataFlow } from './commands/data-flow/index.js';
|
|
16
17
|
import { runFinancingDeck } from './commands/financing-deck/index.js';
|
|
17
18
|
import { runFindArchitecture } from './commands/find-architecture/index.js';
|
|
18
19
|
import { runFindBugs } from './commands/find-bugs/index.js';
|
|
@@ -25,14 +26,18 @@ import { runIssueAnalysisCommand } from './commands/issue-analysis/index.js';
|
|
|
25
26
|
import { runPRResolve } from './commands/pr-resolve/index.js';
|
|
26
27
|
import { runPRReview } from './commands/pr-review/index.js';
|
|
27
28
|
import { runProductTestCases } from './commands/product-test-cases/index.js';
|
|
28
|
-
import { runRecipes } from './commands/recipes/index.js';
|
|
29
29
|
import { runQualityBenchmarkCli } from './commands/quality-benchmark/index.js';
|
|
30
|
+
import { runRecipes } from './commands/recipes/index.js';
|
|
30
31
|
import { runRefactor } from './commands/refactor/refactor.js';
|
|
31
32
|
import { runReleaseSyncCommand } from './commands/release-sync/index.js';
|
|
32
33
|
import { runRunSheetCommand } from './commands/run-sheet/index.js';
|
|
33
34
|
import { runScreenFlow } from './commands/screen-flow/index.js';
|
|
34
35
|
import { runSmokeTestCommand } from './commands/smoke-test/index.js';
|
|
35
36
|
import { runSyncGithubIssues } from './commands/sync-github-issues/index.js';
|
|
37
|
+
import { runSyncOrgRepos } from './commands/sync-org-repos/index.js';
|
|
38
|
+
import { runSyncAws } from './commands/sync-aws/index.js';
|
|
39
|
+
import { runSyncDatadog } from './commands/sync-datadog/index.js';
|
|
40
|
+
import { runSyncTerraform } from './commands/sync-terraform/index.js';
|
|
36
41
|
import { runSyncSentryIssues } from './commands/sync-sentry-issues/index.js';
|
|
37
42
|
import { runTaskWorker } from './commands/task-worker/index.js';
|
|
38
43
|
import { runTechnicalDesignCommand } from './commands/technical-design/index.js';
|
|
@@ -166,7 +171,7 @@ program
|
|
|
166
171
|
program
|
|
167
172
|
.command('screen-flow <productId>')
|
|
168
173
|
.description('Generate a structured screen flow (screens + transitions) for a product from its source code')
|
|
169
|
-
.requiredOption('--flow-id <id>', 'Pending
|
|
174
|
+
.requiredOption('--flow-id <id>', 'Pending flows row id to populate')
|
|
170
175
|
.option('-g, --guidance <text>', 'Human direction for the AI (focus areas, exclusions)')
|
|
171
176
|
.option('-v, --verbose', 'Verbose output')
|
|
172
177
|
.action(async (productId, opts) => {
|
|
@@ -183,6 +188,28 @@ program
|
|
|
183
188
|
}
|
|
184
189
|
});
|
|
185
190
|
// ============================================================
|
|
191
|
+
// Subcommand: edsger data-flow <productId>
|
|
192
|
+
// ============================================================
|
|
193
|
+
program
|
|
194
|
+
.command('data-flow <productId>')
|
|
195
|
+
.description('Generate a structured data flow (sources, datasets, transforms, sinks, queues, models) for a product from its source code')
|
|
196
|
+
.requiredOption('--flow-id <id>', 'Pending flows row id to populate')
|
|
197
|
+
.option('-g, --guidance <text>', 'Human direction for the AI (focus areas, exclusions)')
|
|
198
|
+
.option('-v, --verbose', 'Verbose output')
|
|
199
|
+
.action(async (productId, opts) => {
|
|
200
|
+
try {
|
|
201
|
+
await runDataFlow(productId, {
|
|
202
|
+
flowId: opts.flowId,
|
|
203
|
+
guidance: opts.guidance,
|
|
204
|
+
verbose: opts.verbose,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
catch (error) {
|
|
208
|
+
logError(error instanceof Error ? error.message : String(error));
|
|
209
|
+
process.exit(1);
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
// ============================================================
|
|
186
213
|
// Subcommand: edsger user-psychology <productId>
|
|
187
214
|
// ============================================================
|
|
188
215
|
program
|
|
@@ -575,6 +602,23 @@ program
|
|
|
575
602
|
}
|
|
576
603
|
});
|
|
577
604
|
// ============================================================
|
|
605
|
+
// Subcommand: edsger sync-org-repos <teamId>
|
|
606
|
+
// ============================================================
|
|
607
|
+
program
|
|
608
|
+
.command('sync-org-repos <teamId>')
|
|
609
|
+
.description("Sync a GitHub org's repos as products under a team. Uses the local gh CLI.")
|
|
610
|
+
.option('-v, --verbose', 'Verbose output')
|
|
611
|
+
.option('--org <name>', 'GitHub org name (defaults to the team\'s configured github_org)')
|
|
612
|
+
.action(async (teamId, opts) => {
|
|
613
|
+
try {
|
|
614
|
+
await runSyncOrgRepos(teamId, opts);
|
|
615
|
+
}
|
|
616
|
+
catch (error) {
|
|
617
|
+
logError(error instanceof Error ? error.message : String(error));
|
|
618
|
+
process.exit(1);
|
|
619
|
+
}
|
|
620
|
+
});
|
|
621
|
+
// ============================================================
|
|
578
622
|
// Subcommand: edsger sync-sentry-issues <productId>
|
|
579
623
|
//
|
|
580
624
|
// Requires SENTRY_AUTH_TOKEN, SENTRY_ORG, SENTRY_PROJECT in env (set via
|
|
@@ -594,6 +638,71 @@ program
|
|
|
594
638
|
}
|
|
595
639
|
});
|
|
596
640
|
// ============================================================
|
|
641
|
+
// Subcommand: edsger sync-datadog <teamId>
|
|
642
|
+
//
|
|
643
|
+
// Requires DD_API_KEY, DD_APP_KEY in env (set via `edsger config set`
|
|
644
|
+
// or desktop Settings → Credentials → Datadog). Optional DD_SITE.
|
|
645
|
+
// ============================================================
|
|
646
|
+
program
|
|
647
|
+
.command('sync-datadog <teamId>')
|
|
648
|
+
.description('Import services from Datadog Service Catalog into the edsger service catalog (requires DD_API_KEY, DD_APP_KEY env)')
|
|
649
|
+
.option('-v, --verbose', 'Verbose output')
|
|
650
|
+
.action(async (teamId, opts) => {
|
|
651
|
+
try {
|
|
652
|
+
await runSyncDatadog(teamId, opts);
|
|
653
|
+
}
|
|
654
|
+
catch (error) {
|
|
655
|
+
logError(error instanceof Error ? error.message : String(error));
|
|
656
|
+
process.exit(1);
|
|
657
|
+
}
|
|
658
|
+
});
|
|
659
|
+
// ============================================================
|
|
660
|
+
// ============================================================
|
|
661
|
+
// Subcommand: edsger sync-aws <teamId>
|
|
662
|
+
//
|
|
663
|
+
// Uses the standard AWS credential chain to discover services by
|
|
664
|
+
// their Service tag and optionally pull cost data.
|
|
665
|
+
// ============================================================
|
|
666
|
+
program
|
|
667
|
+
.command('sync-aws <teamId>')
|
|
668
|
+
.description('Discover services from AWS by Service tag and pull cost data from Cost Explorer')
|
|
669
|
+
.option('-v, --verbose', 'Verbose output')
|
|
670
|
+
.option('--region <region>', 'AWS region (default: AWS_REGION env or us-east-1)')
|
|
671
|
+
.option('--no-cost', 'Skip Cost Explorer query')
|
|
672
|
+
.action(async (teamId, opts) => {
|
|
673
|
+
try {
|
|
674
|
+
await runSyncAws(teamId, {
|
|
675
|
+
verbose: opts.verbose,
|
|
676
|
+
region: opts.region,
|
|
677
|
+
noCost: opts.cost === false,
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
catch (error) {
|
|
681
|
+
logError(error instanceof Error ? error.message : String(error));
|
|
682
|
+
process.exit(1);
|
|
683
|
+
}
|
|
684
|
+
});
|
|
685
|
+
// Subcommand: edsger sync-terraform <teamId>
|
|
686
|
+
//
|
|
687
|
+
// Clones the team's terraform repo (from teams.terraform_repo_full_name)
|
|
688
|
+
// and lets the agent analyse .tf files to extract services, resources,
|
|
689
|
+
// and dependencies. Alternatively, --dir for a local directory.
|
|
690
|
+
// ============================================================
|
|
691
|
+
program
|
|
692
|
+
.command('sync-terraform <teamId>')
|
|
693
|
+
.description('Scan a Terraform repo to discover services, AWS resources, and infrastructure dependencies')
|
|
694
|
+
.option('-v, --verbose', 'Verbose output')
|
|
695
|
+
.option('--dir <path>', 'Local directory to scan instead of cloning from GitHub')
|
|
696
|
+
.action(async (teamId, opts) => {
|
|
697
|
+
try {
|
|
698
|
+
await runSyncTerraform(teamId, opts);
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
logError(error instanceof Error ? error.message : String(error));
|
|
702
|
+
process.exit(1);
|
|
703
|
+
}
|
|
704
|
+
});
|
|
705
|
+
// ============================================================
|
|
597
706
|
// Subcommand: edsger find-features <productId>
|
|
598
707
|
// ============================================================
|
|
599
708
|
program
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* data-flow phase: clone the product's repo, ask Claude to map every data
|
|
3
|
+
* node (source / dataset / transform / sink / queue / model) and the
|
|
4
|
+
* connections between them into a structured DataFlowExtraction, then
|
|
5
|
+
* persist the result to flows / flow_nodes / flow_edges (rows tagged
|
|
6
|
+
* `type = 'data'`) via the Supabase SDK.
|
|
7
|
+
*
|
|
8
|
+
* Companion to screen-flow: same generation pattern (workspace clone +
|
|
9
|
+
* Claude Agent SDK + in-process MCP server), same storage tables, different
|
|
10
|
+
* domain.
|
|
11
|
+
*/
|
|
12
|
+
export interface DataFlowPhaseOptions {
|
|
13
|
+
productId: string;
|
|
14
|
+
flowId: string;
|
|
15
|
+
guidance?: string;
|
|
16
|
+
verbose?: boolean;
|
|
17
|
+
}
|
|
18
|
+
export interface DataFlowPhaseResult {
|
|
19
|
+
status: 'success' | 'error';
|
|
20
|
+
message: string;
|
|
21
|
+
nodesCreated?: number;
|
|
22
|
+
edgesCreated?: number;
|
|
23
|
+
summary?: string;
|
|
24
|
+
}
|
|
25
|
+
export declare function runDataFlowPhase(options: DataFlowPhaseOptions): Promise<DataFlowPhaseResult>;
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* data-flow phase: clone the product's repo, ask Claude to map every data
|
|
3
|
+
* node (source / dataset / transform / sink / queue / model) and the
|
|
4
|
+
* connections between them into a structured DataFlowExtraction, then
|
|
5
|
+
* persist the result to flows / flow_nodes / flow_edges (rows tagged
|
|
6
|
+
* `type = 'data'`) via the Supabase SDK.
|
|
7
|
+
*
|
|
8
|
+
* Companion to screen-flow: same generation pattern (workspace clone +
|
|
9
|
+
* Claude Agent SDK + in-process MCP server), same storage tables, different
|
|
10
|
+
* domain.
|
|
11
|
+
*/
|
|
12
|
+
import { query } from '@anthropic-ai/claude-agent-sdk';
|
|
13
|
+
import { getGitHubConfigByProduct } from '../../api/github.js';
|
|
14
|
+
import { DEFAULT_MODEL } from '../../constants.js';
|
|
15
|
+
import { getSupabase } from '../../supabase/client.js';
|
|
16
|
+
import { logError, logInfo, logSuccess, logWarning } from '../../utils/logger.js';
|
|
17
|
+
import { cleanupIssueRepo, cloneIssueRepo, ensureWorkspaceDir, } from '../../workspace/workspace-manager.js';
|
|
18
|
+
import { fetchProductBasics } from '../find-shared/mcp.js';
|
|
19
|
+
import { createPromptGenerator, extractTextFromContent, tryExtractResult, } from '../pr-shared/agent-utils.js';
|
|
20
|
+
import { createDataFlowCaptureState, createDataFlowMcpServer, validateConsistency, } from './mcp-server.js';
|
|
21
|
+
import { createDataFlowSystemPrompt, createDataFlowUserPrompt, } from './prompts.js';
|
|
22
|
+
import { isDataFlowExtraction, } from './types.js';
|
|
23
|
+
const WORKSPACE_KEY = 'data-flow';
|
|
24
|
+
const MAX_TURNS = 150;
|
|
25
|
+
// Auto-layout: simple grid; users can drag afterwards and we persist positions.
|
|
26
|
+
const COLUMN_WIDTH = 320;
|
|
27
|
+
const ROW_HEIGHT = 220;
|
|
28
|
+
const COLUMNS = 4;
|
|
29
|
+
export async function runDataFlowPhase(options) {
|
|
30
|
+
const { productId, flowId, guidance, verbose } = options;
|
|
31
|
+
logInfo(`Starting data-flow generation for product ${productId}`);
|
|
32
|
+
const supabase = getSupabase();
|
|
33
|
+
await markFlowRunning(supabase, flowId);
|
|
34
|
+
const githubConfig = await getGitHubConfigByProduct(productId, verbose);
|
|
35
|
+
if (!githubConfig.configured ||
|
|
36
|
+
!githubConfig.token ||
|
|
37
|
+
!githubConfig.owner ||
|
|
38
|
+
!githubConfig.repo) {
|
|
39
|
+
const msg = githubConfig.message ||
|
|
40
|
+
'GitHub repository not configured for this product. Connect a repo first.';
|
|
41
|
+
await markFlowFailed(supabase, flowId, msg);
|
|
42
|
+
return { status: 'error', message: msg };
|
|
43
|
+
}
|
|
44
|
+
let repoPath;
|
|
45
|
+
let succeeded = false;
|
|
46
|
+
try {
|
|
47
|
+
const workspaceRoot = ensureWorkspaceDir();
|
|
48
|
+
const repoKey = `${WORKSPACE_KEY}-${productId}`;
|
|
49
|
+
({ repoPath } = cloneIssueRepo(workspaceRoot, repoKey, githubConfig.owner, githubConfig.repo, githubConfig.token));
|
|
50
|
+
const product = await fetchProductBasics(productId);
|
|
51
|
+
const systemPrompt = await createDataFlowSystemPrompt({
|
|
52
|
+
projectDir: repoPath,
|
|
53
|
+
hasCodebase: true,
|
|
54
|
+
});
|
|
55
|
+
const userPrompt = createDataFlowUserPrompt({
|
|
56
|
+
productName: product.name,
|
|
57
|
+
productDescription: product.description,
|
|
58
|
+
guidance,
|
|
59
|
+
});
|
|
60
|
+
logInfo('Running Claude data-flow extraction...');
|
|
61
|
+
const captureState = createDataFlowCaptureState();
|
|
62
|
+
const mcpServer = createDataFlowMcpServer(captureState, {
|
|
63
|
+
onProgress: ({ phase, message }) => {
|
|
64
|
+
logInfo(`[${phase}] ${message}`);
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
let lastAssistantResponse = '';
|
|
68
|
+
let extraction = null;
|
|
69
|
+
for await (const message of query({
|
|
70
|
+
prompt: createPromptGenerator(userPrompt),
|
|
71
|
+
options: {
|
|
72
|
+
systemPrompt: {
|
|
73
|
+
type: 'preset',
|
|
74
|
+
preset: 'claude_code',
|
|
75
|
+
append: systemPrompt,
|
|
76
|
+
},
|
|
77
|
+
model: DEFAULT_MODEL,
|
|
78
|
+
maxTurns: MAX_TURNS,
|
|
79
|
+
permissionMode: 'bypassPermissions',
|
|
80
|
+
cwd: repoPath,
|
|
81
|
+
mcpServers: {
|
|
82
|
+
'data-flow': mcpServer,
|
|
83
|
+
},
|
|
84
|
+
},
|
|
85
|
+
})) {
|
|
86
|
+
const { assistantBuffer, extraction: nextExtraction } = processSdkMessage(message, lastAssistantResponse, captureState, verbose);
|
|
87
|
+
lastAssistantResponse = assistantBuffer;
|
|
88
|
+
if (nextExtraction) {
|
|
89
|
+
extraction = nextExtraction;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (!extraction) {
|
|
93
|
+
const msg = 'Data flow extraction failed: agent did not call submit_data_flow and no parseable data_flow block was found in the response';
|
|
94
|
+
await markFlowFailed(supabase, flowId, msg);
|
|
95
|
+
return { status: 'error', message: msg };
|
|
96
|
+
}
|
|
97
|
+
logInfo(`Extraction produced ${extraction.nodes.length} nodes / ${extraction.edges.length} connections`);
|
|
98
|
+
const { nodesCreated, edgesCreated } = await persistFlow(supabase, flowId, extraction);
|
|
99
|
+
await markFlowSuccess(supabase, flowId, extraction.summary);
|
|
100
|
+
succeeded = true;
|
|
101
|
+
logSuccess(`Data flow generated: ${nodesCreated} nodes, ${edgesCreated} connections`);
|
|
102
|
+
return {
|
|
103
|
+
status: 'success',
|
|
104
|
+
message: `Data flow generated (${nodesCreated} nodes, ${edgesCreated} connections)`,
|
|
105
|
+
nodesCreated,
|
|
106
|
+
edgesCreated,
|
|
107
|
+
summary: extraction.summary,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
112
|
+
logError(`Data flow failed: ${errorMessage}`);
|
|
113
|
+
await markFlowFailed(supabase, flowId, errorMessage);
|
|
114
|
+
return { status: 'error', message: errorMessage };
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
if (succeeded) {
|
|
118
|
+
cleanupIssueRepo(repoPath);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function processSdkMessage(
|
|
123
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
124
|
+
message, assistantBuffer, captureState, verbose) {
|
|
125
|
+
if (message.type === 'assistant') {
|
|
126
|
+
const next = assistantBuffer +
|
|
127
|
+
extractTextFromContent(message.message?.content ?? [], verbose);
|
|
128
|
+
return { assistantBuffer: next, extraction: null };
|
|
129
|
+
}
|
|
130
|
+
if (message.type === 'user' && verbose) {
|
|
131
|
+
const userContent = message.message?.content;
|
|
132
|
+
if (Array.isArray(userContent)) {
|
|
133
|
+
extractTextFromContent(userContent, verbose);
|
|
134
|
+
}
|
|
135
|
+
return { assistantBuffer, extraction: null };
|
|
136
|
+
}
|
|
137
|
+
if (message.type !== 'result') {
|
|
138
|
+
return { assistantBuffer, extraction: null };
|
|
139
|
+
}
|
|
140
|
+
if (captureState.captured) {
|
|
141
|
+
return { assistantBuffer, extraction: captureState.captured };
|
|
142
|
+
}
|
|
143
|
+
const fallback = tryFallbackParse(message, assistantBuffer);
|
|
144
|
+
if (fallback) {
|
|
145
|
+
logWarning('Agent emitted a fenced data_flow block instead of calling submit_data_flow; using the parsed text as a fallback.');
|
|
146
|
+
return { assistantBuffer, extraction: fallback };
|
|
147
|
+
}
|
|
148
|
+
if (message.subtype !== 'success') {
|
|
149
|
+
logError(`Extraction incomplete: ${message.subtype}`);
|
|
150
|
+
}
|
|
151
|
+
return { assistantBuffer, extraction: null };
|
|
152
|
+
}
|
|
153
|
+
function tryFallbackParse(resultMessage, assistantText) {
|
|
154
|
+
const responseText = resultMessage.subtype === 'success'
|
|
155
|
+
? resultMessage.result || assistantText
|
|
156
|
+
: assistantText;
|
|
157
|
+
const parsed = tryExtractResult(responseText, 'data_flow');
|
|
158
|
+
if (!isDataFlowExtraction(parsed)) {
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
const { error } = validateConsistency(parsed);
|
|
162
|
+
if (error) {
|
|
163
|
+
logWarning(`Fallback extraction failed consistency check: ${error}`);
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
return parsed;
|
|
167
|
+
}
|
|
168
|
+
// ============================================================================
|
|
169
|
+
// Persistence
|
|
170
|
+
// ============================================================================
|
|
171
|
+
async function markFlowRunning(supabase, flowId) {
|
|
172
|
+
const { error } = await supabase
|
|
173
|
+
.from('flows')
|
|
174
|
+
.update({ status: 'running', error: null })
|
|
175
|
+
.eq('id', flowId);
|
|
176
|
+
if (error) {
|
|
177
|
+
logWarning(`Could not mark flow as running: ${error.message}`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
async function markFlowFailed(supabase, flowId, errorMessage) {
|
|
181
|
+
await supabase
|
|
182
|
+
.from('flows')
|
|
183
|
+
.update({
|
|
184
|
+
status: 'failed',
|
|
185
|
+
error: errorMessage,
|
|
186
|
+
completed_at: new Date().toISOString(),
|
|
187
|
+
})
|
|
188
|
+
.eq('id', flowId);
|
|
189
|
+
}
|
|
190
|
+
async function markFlowSuccess(supabase, flowId, summary) {
|
|
191
|
+
await supabase
|
|
192
|
+
.from('flows')
|
|
193
|
+
.update({
|
|
194
|
+
status: 'success',
|
|
195
|
+
summary,
|
|
196
|
+
error: null,
|
|
197
|
+
completed_at: new Date().toISOString(),
|
|
198
|
+
})
|
|
199
|
+
.eq('id', flowId);
|
|
200
|
+
}
|
|
201
|
+
async function persistFlow(supabase, flowId, extraction) {
|
|
202
|
+
await supabase.from('flow_edges').delete().eq('flow_id', flowId);
|
|
203
|
+
await supabase.from('flow_nodes').delete().eq('flow_id', flowId);
|
|
204
|
+
if (extraction.nodes.length === 0) {
|
|
205
|
+
return { nodesCreated: 0, edgesCreated: 0 };
|
|
206
|
+
}
|
|
207
|
+
const nodeRows = extraction.nodes.map((n, i) => buildNodeRow(flowId, n, i));
|
|
208
|
+
const { data: insertedNodes, error: nodesError } = await supabase
|
|
209
|
+
.from('flow_nodes')
|
|
210
|
+
.insert(nodeRows)
|
|
211
|
+
.select('id, slug');
|
|
212
|
+
if (nodesError) {
|
|
213
|
+
throw new Error(`Failed to insert nodes: ${nodesError.message}`);
|
|
214
|
+
}
|
|
215
|
+
const slugToId = new Map((insertedNodes ?? []).map((n) => [n.slug, n.id]));
|
|
216
|
+
const edgeRows = extraction.edges
|
|
217
|
+
.map((e) => buildEdgeRow(flowId, e, slugToId))
|
|
218
|
+
.filter((e) => e !== null);
|
|
219
|
+
if (edgeRows.length > 0) {
|
|
220
|
+
const { error: edgesError } = await supabase
|
|
221
|
+
.from('flow_edges')
|
|
222
|
+
.insert(edgeRows);
|
|
223
|
+
if (edgesError) {
|
|
224
|
+
throw new Error(`Failed to insert edges: ${edgesError.message}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
return {
|
|
228
|
+
nodesCreated: nodeRows.length,
|
|
229
|
+
edgesCreated: edgeRows.length,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
function buildNodeRow(flowId, node, index) {
|
|
233
|
+
return {
|
|
234
|
+
flow_id: flowId,
|
|
235
|
+
slug: node.slug,
|
|
236
|
+
name: node.name,
|
|
237
|
+
kind: node.kind,
|
|
238
|
+
schema: node,
|
|
239
|
+
position_x: (index % COLUMNS) * COLUMN_WIDTH,
|
|
240
|
+
position_y: Math.floor(index / COLUMNS) * ROW_HEIGHT,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
function buildEdgeRow(flowId, edge, slugToId) {
|
|
244
|
+
const fromId = slugToId.get(edge.fromSlug);
|
|
245
|
+
const toId = slugToId.get(edge.toSlug);
|
|
246
|
+
if (!fromId || !toId) {
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
return {
|
|
250
|
+
flow_id: flowId,
|
|
251
|
+
from_node_id: fromId,
|
|
252
|
+
to_node_id: toId,
|
|
253
|
+
label: edge.label ?? null,
|
|
254
|
+
source_anchor: edge.sourceFile ?? null,
|
|
255
|
+
kind: edge.kind,
|
|
256
|
+
};
|
|
257
|
+
}
|