@dataparade/cli 0.0.1

package/dist/src/ai-enrichment/agent-orchestrator.js

@@ -0,0 +1,476 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateAgenticProposals = generateAgenticProposals;
+const agents_1 = require("./agents");
+const provider_prompt_1 = require("./provider-prompt");
+const providers_1 = require("./providers");
+const tools_1 = require("./tools");
+const langsmith_tracing_1 = require("../tracing/langsmith-tracing");
+const third_party_evidence_1 = require("./third-party-evidence");
+const provider_contract_1 = require("./providers/provider-contract");
+const scan_paths_1 = require("./scan-paths");
+const platform_proxy_provider_1 = require("./providers/platform-proxy-provider");
+const PROPERTY_AGENT_SLICE_SIZE = 12;
+const TP_AGENT_MAX_ROUNDS = 3;
+const TP_AGENT_MAX_FILES = 8;
+const TP_AGENT_SEARCH_LIMIT = 6;
+const IMPORT_RE = /\bimport\s+(?:[^"']+from\s+)?["']([^"']+)["']|\brequire\(["']([^"']+)["']\)/g;
+function chunkCandidates(queue, size) {
+    const n = Math.max(1, Math.floor(size));
+    const out = [];
+    for (let i = 0; i < queue.length; i += n) {
+        out.push(queue.slice(i, i + n));
+    }
+    return out;
+}
+function shouldCallProviderForAgent(agent) {
+    // Flow patches are now merge-gated to intra-section changes only, so provider
+    // calls can safely support both property and flow-oriented queues.
+    return (agent === "tpAgent" ||
+        agent === "propertyAgent" ||
+        agent === "directionAgent" ||
+        agent === "interactionAgent");
+}
+function estimateTokens(text) {
+    // Rough but stable approximation used only for budget gating.
+    return Math.max(1, Math.ceil(text.length / 4));
+}
+async function mapWithConcurrency(items, concurrency, mapper) {
+    if (items.length === 0)
+        return [];
+    const limit = Math.max(1, Math.floor(concurrency));
+    const out = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (true) {
+            const index = next;
+            next += 1;
+            if (index >= items.length)
+                return;
+            out[index] = await mapper(items[index], index);
+        }
+    };
+    const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+    await Promise.all(workers);
+    return out;
+}
+function gatherSeedPathsForCandidate(candidate, components) {
+    const component = components.find((c) => c.id === candidate.componentId);
+    if (!component)
+        return [];
+    const out = new Set();
+    const sectionId = component.properties.section_id;
+    if (typeof sectionId === "string" && sectionId.trim())
+        out.add(sectionId.trim());
+    for (const loc of component.sourceLocations ?? []) {
+        if (loc.filePath?.trim())
+            out.add(loc.filePath.trim());
+    }
+    for (const ref of component.detectedFrom ?? []) {
+        const fp = ref.sourceLocation?.filePath;
+        if (fp?.trim())
+            out.add(fp.trim());
+    }
+    return [...out];
+}
+function gatherSearchTerms(componentName) {
+    const base = componentName.trim().toLowerCase();
+    const terms = new Set([base, "auth", "token", "api", "storage", "database"]);
+    if (base.includes("supabase")) {
+        terms.add("supabase");
+        terms.add("storage.from");
+        terms.add("create_client");
+    }
+    return [...terms].filter((t) => t.length >= 3);
+}
+function pickUniqueSearchMatches(matches, alreadyReviewed, limit) {
+    const max = Math.max(1, Math.floor(limit));
+    const preferred = [];
+    const fallback = [];
+    const seen = new Set();
+    for (const match of matches) {
+        const filePath = match.filePath?.trim();
+        if (!filePath || seen.has(filePath))
+            continue;
+        seen.add(filePath);
+        if (!alreadyReviewed.has(filePath))
+            preferred.push(match);
+        else
+            fallback.push(match);
+    }
+    return [...preferred, ...fallback].slice(0, max);
+}
+function resolveRelativeImportPath(fromFile, rawImport) {
+    if (!rawImport.startsWith("."))
+        return undefined;
+    const normalizedFrom = fromFile.replace(/\\/g, "/");
+    const parts = normalizedFrom.split("/");
+    parts.pop();
+    for (const piece of rawImport.split("/")) {
+        if (piece === "." || piece === "")
+            continue;
+        if (piece === "..")
+            parts.pop();
+        else
+            parts.push(piece);
+    }
+    const base = parts.join("/");
+    const candidates = [base, `${base}.ts`, `${base}.js`, `${base}.py`, `${base}/index.ts`, `${base}/index.js`];
+    return candidates[0];
+}
+function expandImportPaths(filePath, content) {
+    const out = [];
+    for (const match of content.matchAll(IMPORT_RE)) {
+        const rawImport = (match[1] ?? match[2] ?? "").trim();
+        if (!rawImport)
+            continue;
+        const rel = resolveRelativeImportPath(filePath, rawImport);
+        if (rel)
+            out.push(rel);
+    }
+    return out;
+}
+function appendTraceStep(list, step) {
+    list.push(step);
+}
+function logToolStep(candidateId, step) {
+    if (!(0, provider_contract_1.aiDebugEnabled)())
+        return;
+    const touched = step.filesTouched?.length ?? 0;
+    const stats = step.stats && Object.keys(step.stats).length > 0
+        ? ` stats=${JSON.stringify(step.stats)}`
+        : "";
+    console.warn(`[dataparade-ai] tpAgent tool call candidate=${candidateId} round=${step.round} action=${step.action} files_touched=${touched} detail="${step.detail}"${stats}`);
+}
+async function generateAgenticProposals(input, plan, candidates, options) {
+    const scanContext = {
+        components: input.components,
+        dataFlows: input.dataFlows,
+        findings: input.findings ?? [],
+        files: input.files ?? [],
+        sections: input.sections ?? [],
+    };
+    const localProposals = [];
+    const tools = (0, tools_1.createAgentTooling)(scanContext, candidates, localProposals);
+    const llmEnabled = options.llmEnabled !== false;
+    const skipStructural = options.skipStructuralHeuristics === true;
+    const provider = options.platformProxy
+        ? new platform_proxy_provider_1.PlatformProxyProvider(options.platformProxy)
+        : (0, providers_1.createAiProvider)(options.provider, {
+            endpoint: options.endpoint,
+            model: options.model,
+            apiKey: options.apiKey,
+        });
+    const generated = [];
+    const agenticTrace = [];
+    const aggregateUsage = {
+        providerCalls: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        estimatedCostUsd: 0,
+    };
+    let sequence = 0;
+    for (const queue of plan.queues) {
+        if (!skipStructural && queue.agent === "tpAgent") {
+            for (const candidate of queue.queue) {
+                if (!candidate.componentId)
+                    continue;
+                const component = input.components.find((item) => item.id === candidate.componentId);
+                if (!component || component.type !== "third_party")
+                    continue;
+                const patch = (0, third_party_evidence_1.buildThirdPartyHeuristicProposal)({
+                    candidate,
+                    component,
+                    files: input.files ?? [],
+                });
+                if (!patch)
+                    continue;
+                const verification = (0, third_party_evidence_1.verifyThirdPartyProposal)(patch);
+                if (!verification.ok)
+                    continue;
+                sequence += 1;
+                generated.push({ id: `heuristic_${sequence}`, proposal: patch });
+            }
+        }
+        if (!skipStructural) {
+            const heuristic = await (0, agents_1.runAgentQueue)(queue, tools);
+            for (const proposal of heuristic) {
+                sequence += 1;
+                generated.push({ id: `heuristic_${sequence}`, proposal });
+            }
+        }
+        if (!llmEnabled || !shouldCallProviderForAgent(queue.agent)) {
+            continue;
+        }
+        if (queue.agent === "tpAgent") {
+            const maxCalls = Math.max(0, Math.floor(queue.maxModelCalls));
+            const maxCandidates = maxCalls === 0 ? 0 : Math.min(queue.queue.length, maxCalls);
+            const tpCandidates = queue.queue.slice(0, maxCandidates);
+            const maxRounds = Math.max(1, options.toolLoopMaxRounds ?? TP_AGENT_MAX_ROUNDS);
+            const maxFiles = Math.max(1, options.toolLoopMaxFiles ?? TP_AGENT_MAX_FILES);
+            const maxSearches = Math.max(1, options.toolLoopMaxSearches ?? TP_AGENT_SEARCH_LIMIT);
+            const queueBudget = Math.max(0, Math.floor(queue.budgetTokens));
+            let queueBudgetSpent = 0;
+            for (const candidate of tpCandidates) {
+                const component = input.components.find((c) => c.id === candidate.componentId);
+                if (!component)
+                    continue;
+                const traceSteps = [];
+                const reviewed = new Set();
+                const seed = gatherSeedPathsForCandidate(candidate, input.components);
+                for (const p of seed)
+                    reviewed.add(p);
+                appendTraceStep(traceSteps, {
+                    round: 0,
+                    action: "seed_files",
+                    detail: "Seed files from detectedFrom/sourceLocations/section_id",
+                    filesTouched: [...reviewed],
+                    stats: { seedCount: reviewed.size },
+                });
+                logToolStep(candidate.id, traceSteps[traceSteps.length - 1]);
+                const searchTerms = gatherSearchTerms(component.name);
+                for (const term of searchTerms.slice(0, maxSearches)) {
+                    const rawMatches = tools.searchInFiles(term, seed.length > 0 ? seed : undefined);
+                    const matches = pickUniqueSearchMatches(rawMatches, reviewed, 8);
+                    for (const match of matches)
+                        reviewed.add(match.filePath);
+                    appendTraceStep(traceSteps, {
+                        round: 0,
+                        action: "search_text",
+                        detail: `Search term: ${term}`,
+                        filesTouched: matches.map((m) => m.filePath),
+                        stats: { hits: matches.length },
+                    });
+                    logToolStep(candidate.id, traceSteps[traceSteps.length - 1]);
+                }
+                let finalProposals = [];
+                let executedRounds = 0;
+                for (let round = 1; round <= maxRounds; round += 1) {
+                    executedRounds = round;
+                    const preferredPaths = [...reviewed].slice(0, maxFiles);
+                    const prompt = JSON.stringify((0, provider_prompt_1.buildProviderPromptPayload)({
+                        agent: queue.agent,
+                        queue: [candidate],
+                        components: input.components,
+                        dataFlows: input.dataFlows,
+                        sections: input.sections,
+                        files: input.files,
+                        preferredPaths,
+                    }));
+                    const estimatedTokens = estimateTokens(prompt) + Math.max(1, options.maxTokens ?? 8192);
+                    if (queueBudget > 0 && queueBudgetSpent + estimatedTokens > queueBudget) {
+                        appendTraceStep(traceSteps, {
+                            round,
+                            action: "provider_infer",
+                            detail: "Skipped provider inference due to queue budget limit",
+                            filesTouched: preferredPaths,
+                            stats: { estimatedTokens, queueBudget, queueBudgetSpent },
+                        });
+                        logToolStep(candidate.id, traceSteps[traceSteps.length - 1]);
+                        break;
+                    }
+                    const providerResult = await (0, langsmith_tracing_1.traceDataparadeProviderInfer)({
+                        providerId: options.provider,
+                        model: options.model,
+                        run: () => provider.infer({
+                            prompt,
+                            model: options.model,
+                            endpoint: options.endpoint,
+                            apiKey: options.apiKey,
+                            maxTokens: options.maxTokens,
+                            temperature: options.temperature,
+                        }),
+                    });
+                    if (queueBudget > 0)
+                        queueBudgetSpent += estimatedTokens;
+                    if (providerResult.usage) {
+                        aggregateUsage.providerCalls += 1;
+                        aggregateUsage.inputTokens += providerResult.usage.inputTokens;
+                        aggregateUsage.outputTokens += providerResult.usage.outputTokens;
+                        aggregateUsage.totalTokens += providerResult.usage.totalTokens;
+                        if (typeof providerResult.usage.estimatedCostUsd === "number") {
+                            aggregateUsage.estimatedCostUsd =
+                                (aggregateUsage.estimatedCostUsd ?? 0) + providerResult.usage.estimatedCostUsd;
+                        }
+                        else {
+                            aggregateUsage.estimatedCostUsd = undefined;
+                        }
+                    }
+                    appendTraceStep(traceSteps, {
+                        round,
+                        action: "provider_infer",
+                        detail: "Provider inference call for current reviewed file set",
+                        filesTouched: preferredPaths,
+                        stats: { reviewedCount: preferredPaths.length, proposals: providerResult.proposals.length },
+                    });
+                    logToolStep(candidate.id, traceSteps[traceSteps.length - 1]);
+                    const acceptedThisRound = [];
+                    for (const proposal of providerResult.proposals) {
+                        const verification = (0, third_party_evidence_1.verifyThirdPartyProposal)(proposal);
+                        if (!verification.ok) {
+                            if ((0, provider_contract_1.aiDebugEnabled)()) {
+                                console.warn(`[dataparade-ai] tpAgent proposal rejected by verifyThirdPartyProposal: ${verification.reason}`);
+                            }
+                            continue;
+                        }
+                        const validation = tools.validateProposal(proposal);
+                        if (!validation.ok) {
+                            if ((0, provider_contract_1.aiDebugEnabled)()) {
+                                console.warn(`[dataparade-ai] provider proposal rejected by tooling validation: ${validation.reason}`);
+                            }
+                            continue;
+                        }
+                        acceptedThisRound.push(proposal);
+                    }
+                    if (acceptedThisRound.length > 0)
+                        finalProposals = acceptedThisRound;
+                    const beforeExpand = reviewed.size;
+                    const filesForExpand = [...reviewed].slice(0, maxFiles);
+                    for (const fp of filesForExpand) {
+                        const fi = input.files ? (0, scan_paths_1.resolveScannedFileExact)(input.files, fp) : undefined;
+                        if (!fi)
+                            continue;
+                        const imports = expandImportPaths(fi.path, fi.content);
+                        for (const imp of imports) {
+                            if (reviewed.size >= maxFiles)
+                                break;
+                            const resolved = input.files
+                                ? (0, scan_paths_1.resolveScannedFileExact)(input.files, imp)
+                                : undefined;
+                            if (resolved)
+                                reviewed.add(resolved.path);
+                        }
+                        if (reviewed.size >= maxFiles)
+                            break;
+                    }
+                    const added = reviewed.size - beforeExpand;
+                    appendTraceStep(traceSteps, {
+                        round,
+                        action: "expand_imports",
+                        detail: "Expand import graph from reviewed files",
+                        filesTouched: [...reviewed],
+                        stats: { addedFiles: added, totalReviewed: reviewed.size },
+                    });
+                    logToolStep(candidate.id, traceSteps[traceSteps.length - 1]);
+                    if (added === 0 && finalProposals.length > 0)
+                        break;
+                }
+                appendTraceStep(traceSteps, {
+                    round: executedRounds,
+                    action: "finalize",
+                    detail: "Finalize accepted proposals for candidate",
+                    filesTouched: [...reviewed],
+                    stats: { finalProposalCount: finalProposals.length },
+                });
+                logToolStep(candidate.id, traceSteps[traceSteps.length - 1]);
+                for (const proposal of finalProposals) {
+                    sequence += 1;
+                    generated.push({ id: `provider_${sequence}`, proposal });
+                    if ((0, provider_contract_1.aiDebugEnabled)()) {
+                        console.warn(`[dataparade-ai] provider proposal accepted: kind=${proposal.kind} candidate=${proposal.candidateType} agent=${queue.agent}`);
+                    }
+                }
+                agenticTrace.push({
+                    candidateId: candidate.id,
+                    componentId: candidate.componentId,
+                    filesReviewed: [...reviewed],
+                    rounds: executedRounds,
+                    toolCalls: traceSteps,
+                    finalProposalCount: finalProposals.length,
+                });
+            }
+            continue;
+        }
+        // Non-tp agents stay in bounded prompt batches.
+        const providerSlices = chunkCandidates(queue.queue, PROPERTY_AGENT_SLICE_SIZE);
+        const preparedProviderCalls = providerSlices.map((slice) => {
+            const prompt = JSON.stringify((0, provider_prompt_1.buildProviderPromptPayload)({
+                agent: queue.agent,
+                queue: slice,
+                components: input.components,
+                dataFlows: input.dataFlows,
+                sections: input.sections,
+                files: input.files,
+            }));
+            const estimatedTokens = estimateTokens(prompt) + Math.max(1, options.maxTokens ?? 8192);
+            return { prompt, estimatedTokens };
+        });
+        const maxCalls = Math.max(0, Math.floor(queue.maxModelCalls));
+        const callsWithinLimit = maxCalls === 0
+            ? []
+            : preparedProviderCalls.slice(0, Math.min(maxCalls, preparedProviderCalls.length));
+        const budget = Math.max(0, Math.floor(queue.budgetTokens));
+        const budgetedCalls = [];
+        if (budget > 0) {
+            let spent = 0;
+            for (const call of callsWithinLimit) {
+                if (spent + call.estimatedTokens > budget)
+                    break;
+                budgetedCalls.push(call);
+                spent += call.estimatedTokens;
+            }
+            // Avoid starving property enrichment when estimated tokens are conservative.
+            if (queue.agent === "propertyAgent" &&
+                budgetedCalls.length === 0 &&
+                callsWithinLimit.length > 0) {
+                budgetedCalls.push(callsWithinLimit[0]);
+            }
+        }
+        else {
+            // Zero/negative budget means no provider calls for this queue.
+            budgetedCalls.push(...[]);
+        }
+        const providerConcurrency = Math.max(1, Math.floor(options.providerConcurrency ?? 1));
+        const providerResults = await mapWithConcurrency(budgetedCalls, providerConcurrency, (call) => (0, langsmith_tracing_1.traceDataparadeProviderInfer)({
+            providerId: options.provider,
+            model: options.model,
+            run: () => provider.infer({
+                prompt: call.prompt,
+                model: options.model,
+                endpoint: options.endpoint,
+                apiKey: options.apiKey,
+                maxTokens: options.maxTokens,
+                temperature: options.temperature,
+            }),
+        }));
+        for (const providerResult of providerResults) {
+            if (providerResult.usage) {
+                aggregateUsage.providerCalls += 1;
+                aggregateUsage.inputTokens += providerResult.usage.inputTokens;
+                aggregateUsage.outputTokens += providerResult.usage.outputTokens;
+                aggregateUsage.totalTokens += providerResult.usage.totalTokens;
+                if (typeof providerResult.usage.estimatedCostUsd === "number") {
+                    aggregateUsage.estimatedCostUsd =
+                        (aggregateUsage.estimatedCostUsd ?? 0) + providerResult.usage.estimatedCostUsd;
+                }
+                else {
+                    aggregateUsage.estimatedCostUsd = undefined;
+                }
+            }
+            for (const proposal of providerResult.proposals) {
+                const validation = tools.validateProposal(proposal);
+                if (!validation.ok) {
+                    if ((0, provider_contract_1.aiDebugEnabled)()) {
+                        console.warn(`[dataparade-ai] provider proposal rejected by tooling validation: ${validation.reason}`);
+                    }
+                    continue;
+                }
+                sequence += 1;
+                generated.push({ id: `provider_${sequence}`, proposal });
+                if ((0, provider_contract_1.aiDebugEnabled)()) {
+                    console.warn(`[dataparade-ai] provider proposal accepted: kind=${proposal.kind} candidate=${proposal.candidateType} agent=${queue.agent}`);
+                }
+            }
+        }
+    }
+    const hasUsage = aggregateUsage.providerCalls > 0 ||
+        aggregateUsage.inputTokens > 0 ||
+        aggregateUsage.outputTokens > 0 ||
+        aggregateUsage.totalTokens > 0 ||
+        agenticTrace.length > 0;
+    return {
+        proposals: generated,
+        usageSummary: hasUsage ? { ...aggregateUsage, agenticTrace } : undefined,
+    };
+}

package/dist/src/ai-enrichment/agents/index.d.ts

@@ -0,0 +1,4 @@
+import type { AiProposal } from "../types";
+import type { AgentTooling } from "../tools";
+import type { AiQueuePlan } from "../types";
+export declare function runAgentQueue(queuePlan: AiQueuePlan, tools: AgentTooling): Promise<AiProposal[]>;

package/dist/src/ai-enrichment/agents/index.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAgentQueue = runAgentQueue;
+function defaultEvidence(reason) {
+    return [{ filePath: "unknown", startLine: 1, endLine: 1, reason }];
+}
+async function runAgentQueue(queuePlan, tools) {
+    const proposals = [];
+    for (const candidate of queuePlan.queue) {
+        if (queuePlan.agent === "tpAgent") {
+            continue;
+        }
+        if (queuePlan.agent === "directionAgent" && candidate.flowId) {
+            proposals.push({
+                kind: "flow_patch",
+                candidateType: candidate.candidateType,
+                targetFlowId: candidate.flowId,
+                setDirection: "reverse",
+                confidence: { score: 0.8, band: "medium" },
+                evidence: defaultEvidence(candidate.rationale),
+                provider: "mock",
+                model: "heuristic",
+                agent: "directionAgent",
+            });
+            continue;
+        }
+        if (queuePlan.agent === "interactionAgent" && candidate.hints.length >= 2) {
+            const context = tools.getScanContext();
+            const sourceComponent = context.components.find((component) => String(component.properties.section_id ?? "") === candidate.hints[0]);
+            const targetComponent = context.components.find((component) => String(component.properties.section_id ?? "") === candidate.hints[1]);
+            if (sourceComponent && targetComponent) {
+                proposals.push({
+                    kind: "flow_patch",
+                    candidateType: candidate.candidateType,
+                    insertIfMissing: true,
+                    sourceComponentId: sourceComponent.id,
+                    targetComponentId: targetComponent.id,
+                    setType: "api_call",
+                    confidence: { score: 0.86, band: "high" },
+                    evidence: defaultEvidence(candidate.rationale),
+                    provider: "mock",
+                    model: "heuristic",
+                    agent: "interactionAgent",
+                });
+            }
+            continue;
+        }
+    }
+    return proposals;
+}

package/dist/src/ai-enrichment/candidate-selector.d.ts

@@ -0,0 +1,10 @@
+import type { ServiceSection } from "../core/sectioning/discover-service-sections";
+import type { DetectedComponent } from "../core/types/component";
+import type { DetectedDataFlow } from "../core/types/data-flow";
+import type { AiInferenceCandidate, AiInferenceScope } from "./types";
+export declare function selectInferenceCandidates(input: {
+    components: DetectedComponent[];
+    dataFlows: DetectedDataFlow[];
+    sections?: ServiceSection[];
+    inferenceScope?: AiInferenceScope;
+}): AiInferenceCandidate[];