@todoist/doistbot-cli 0.0.0-stage → 1.0.16

package/sandbox/dist/tasks/review/engines/dedupe.js

@@ -0,0 +1,291 @@
+import { SpanStatusCode } from '@opentelemetry/api';
+import { logger } from '../../../core/logging.js';
+import { invokePiPrompt } from '../../../core/pi.js';
+import { logPiProgressEvent } from '../../../core/pi-progress-logging.js';
+import { genAiProviderName, withPhase } from '../../../core/tracing.js';
+import { resolveReviewProviderCredential } from '../../../providers/credentials.js';
+import { extractJsonFromResponse, readProviderErrorDetails } from '../../../providers/helpers.js';
+import { isDedupeResponse } from '../../../providers/schemas.js';
+const DEDUPE_TIMEOUT_MS = 5 * 60 * 1000;
+const DEDUPE_PROVIDER = 'gemini';
+const DEDUPE_MODEL_BY_PI_PROVIDER = {
+    google: 'gemini-3-flash-preview',
+    openrouter: 'google/gemini-3-flash-preview',
+};
+const DEDUPE_THINKING_LEVEL = 'medium';
+const DEDUPE_MAX_ATTEMPTS = 2;
+const DEDUPE_RETRY_DELAY_MS = 1_000;
+const DEDUPE_SYSTEM_PROMPT = 'You are a JSON-only duplicate-finding classifier. Use only the findings JSON in the user prompt. Do not inspect files, do not search the repository, and do not use tools. Return only valid JSON matching the requested schema.';
+const DEDUPE_PHASE = 'review.dedupe';
+const DEDUPE_PROVIDER_UNAVAILABLE_LOG_TYPE = 'review.dedupe.provider_unavailable';
+function getDedupeProviderRoute(providerConfig) {
+    return providerConfig.piProviderId ?? 'google';
+}
+function toDedupeProviderConfig(credential, piProviderId) {
+    return {
+        provider: DEDUPE_PROVIDER,
+        apiKey: credential.apiKey,
+        ...(piProviderId === 'google' ? {} : { piProviderId }),
+        modelName: DEDUPE_MODEL_BY_PI_PROVIDER[piProviderId] ?? DEDUPE_MODEL_BY_PI_PROVIDER.google,
+    };
+}
+function buildOpenRouterDedupeProviderConfig(credential) {
+    const resolvedCredential = resolveReviewProviderCredential(credential);
+    if (!resolvedCredential) {
+        return undefined;
+    }
+    return toDedupeProviderConfig(resolvedCredential, 'openrouter');
+}
+function buildDedupeProviderConfigs({ geminiCredential, openRouterCredential, }) {
+    const primaryPiProviderId = geminiCredential.piProviderId ?? 'google';
+    const primaryConfig = toDedupeProviderConfig(geminiCredential, primaryPiProviderId);
+    const openRouterConfig = primaryPiProviderId === 'openrouter'
+        ? undefined
+        : buildOpenRouterDedupeProviderConfig(openRouterCredential);
+    return openRouterConfig ? [primaryConfig, openRouterConfig] : [primaryConfig];
+}
+function logDedupeProviderUnavailable({ providerConfig, attempt, error, shouldRetry, timedOut, }) {
+    const details = readProviderErrorDetails(error);
+    function log(message, meta) {
+        if (shouldRetry) {
+            logger.warn(message, meta);
+        }
+        else {
+            logger.error(message, meta);
+        }
+    }
+    log('Multi-focus dedupe provider unavailable', {
+        type: DEDUPE_PROVIDER_UNAVAILABLE_LOG_TYPE,
+        provider: providerConfig.provider,
+        provider_route: getDedupeProviderRoute(providerConfig),
+        model: providerConfig.modelName,
+        attempt,
+        attempts_total: DEDUPE_MAX_ATTEMPTS,
+        retrying: shouldRetry,
+        error_code: details.errorCode,
+        status: details.status,
+        reason: details.reason,
+        error: details.rawError,
+        timed_out: timedOut ?? false,
+    });
+}
+function buildDedupePrompt(findings) {
+    const payload = {
+        findings: findings.map((finding) => ({
+            id: finding.id,
+            focus: finding.focus,
+            model: finding.sourceModel,
+            path: finding.path,
+            line: finding.line,
+            side: finding.side,
+            body: finding.body,
+        })),
+    };
+    return `You are identifying duplicate code review findings produced by multiple focused review passes.
+
+Input findings JSON:
+${JSON.stringify(payload, null, 2)}
+
+Group findings only when they describe the same underlying issue and would lead to materially the same fix.
+
+Guidelines:
+- Treat wording differences as duplicates if the root cause is the same.
+- Treat nearby line references in the same file as duplicates if they refer to the same issue.
+- Do not group related-but-distinct concerns.
+- Every finding id may appear in at most one group.
+- Do not include singleton groups.
+- If there are no duplicates, return {"groups":[]}.
+
+Return only valid JSON in this exact shape:
+{"groups":[{"ids":["finding-1","finding-2"],"reason":"same issue"}]}`;
+}
+function parseDedupeResponse(raw) {
+    const parsed = JSON.parse(extractJsonFromResponse(raw));
+    if (!isDedupeResponse(parsed)) {
+        throw new Error('Dedupe output did not match schema');
+    }
+    return {
+        groups: parsed.groups
+            .filter((group) => Array.isArray(group?.ids))
+            .map((group) => ({
+            ids: Array.from(new Set(group.ids.filter((id) => typeof id === 'string' && id.length > 0))),
+            reason: typeof group.reason === 'string' ? group.reason : undefined,
+        }))
+            .filter((group) => group.ids.length >= 2),
+    };
+}
+function logDedupeProgress(providerConfig, event) {
+    logPiProgressEvent({
+        event,
+        messages: {
+            agentStart: 'Multi-focus dedupe agent started',
+            agentEnd: 'Multi-focus dedupe agent finished',
+            thinking: 'Multi-focus dedupe still thinking',
+            responding: 'Multi-focus dedupe streaming response',
+            toolStart: 'Multi-focus dedupe attempted to use a tool',
+            toolUpdate: 'Multi-focus dedupe tool still running',
+            toolEnd: 'Multi-focus dedupe tool finished',
+        },
+        meta: { provider: providerConfig.provider, model: providerConfig.modelName },
+        logResponding: true,
+        logTools: true,
+        toolLogLevel: 'warn',
+    });
+}
+function hasToolUsage(toolUsage) {
+    return toolUsage.some((tool) => tool.calls > 0);
+}
+async function waitForDedupeRetry(attempt) {
+    await new Promise((resolve) => setTimeout(resolve, DEDUPE_RETRY_DELAY_MS * attempt));
+}
+export function applyDedupeGroups(findings, groups) {
+    if (groups.length === 0) {
+        return findings;
+    }
+    const byId = new Map(findings.map((finding) => [finding.id, finding]));
+    const dropped = new Set();
+    const assigned = new Set();
+    for (const group of groups) {
+        const existingIds = Array.from(new Set(group.ids)).filter((id) => byId.has(id) && !assigned.has(id));
+        if (existingIds.length < 2) {
+            continue;
+        }
+        const [keepId, ...rest] = existingIds;
+        assigned.add(keepId);
+        for (const id of rest) {
+            dropped.add(id);
+            assigned.add(id);
+        }
+    }
+    logger.info('Applying multi-focus dedupe groups', {
+        originalCount: findings.length,
+        groupsCount: groups.length,
+        droppedCount: dropped.size,
+        assignedCount: assigned.size,
+    });
+    return findings.filter((finding) => !dropped.has(finding.id));
+}
+export async function dedupeMultiFocusFindings({ findings, availableModels, workspaceDir, geminiCredential, openRouterCredential, }) {
+    if (findings.length <= 1) {
+        return findings;
+    }
+    if (!availableModels.includes(DEDUPE_PROVIDER)) {
+        logger.error('Multi-focus dedupe unavailable because Gemini is not enabled', {
+            provider: DEDUPE_PROVIDER,
+            model: DEDUPE_MODEL_BY_PI_PROVIDER.google,
+        });
+        throw new Error('Multi-focus dedupe requires Gemini to be enabled');
+    }
+    const credential = resolveReviewProviderCredential(geminiCredential);
+    if (!credential) {
+        logger.error('Multi-focus dedupe unavailable because Gemini review credential is unavailable', {
+            provider: DEDUPE_PROVIDER,
+            model: DEDUPE_MODEL_BY_PI_PROVIDER.google,
+        });
+        throw new Error('Multi-focus dedupe requires a Gemini review credential');
+    }
+    return runDedupeAttempts({
+        findings,
+        workspaceDir,
+        providerConfigs: buildDedupeProviderConfigs({
+            geminiCredential: credential,
+            openRouterCredential,
+        }),
+    });
+}
+async function runDedupeAttempts({ findings, workspaceDir, providerConfigs, }) {
+    for (let attempt = 1; attempt <= DEDUPE_MAX_ATTEMPTS; attempt += 1) {
+        const providerConfig = providerConfigs[Math.min(attempt - 1, providerConfigs.length - 1)];
+        let result;
+        try {
+            // One span per actual invoke: a fail-then-succeed retry shows up as a
+            // separate error span and ok span, not one inflated span. Provider/
+            // model are known here, so the span carries full gen_ai.*.
+            result = await withPhase(DEDUPE_PHASE, {
+                'gen_ai.operation.name': 'chat',
+                'gen_ai.provider.name': genAiProviderName(providerConfig.provider),
+                'gen_ai.request.model': providerConfig.modelName,
+                'review.provider_route': getDedupeProviderRoute(providerConfig),
+            }, async (span) => {
+                const invocation = await invokePiPrompt({
+                    provider: providerConfig.provider,
+                    apiKey: providerConfig.apiKey,
+                    ...(providerConfig.piProviderId
+                        ? { piProviderId: providerConfig.piProviderId }
+                        : {}),
+                    prompt: buildDedupePrompt(findings),
+                    cwd: workspaceDir,
+                    timeoutMs: DEDUPE_TIMEOUT_MS,
+                    capabilityPreset: 'read-only',
+                    modelName: providerConfig.modelName,
+                    thinkingLevel: DEDUPE_THINKING_LEVEL,
+                    systemPrompt: DEDUPE_SYSTEM_PROMPT,
+                    onProgress: (event) => {
+                        logDedupeProgress(providerConfig, event);
+                    },
+                });
+                if (!invocation.ok) {
+                    span.setStatus({
+                        code: SpanStatusCode.ERROR,
+                        message: invocation.error ?? 'dedupe request failed',
+                    });
+                }
+                return invocation;
+            });
+        }
+        catch (error) {
+            const shouldRetry = attempt < DEDUPE_MAX_ATTEMPTS;
+            logDedupeProviderUnavailable({ providerConfig, attempt, error, shouldRetry });
+            if (shouldRetry) {
+                await waitForDedupeRetry(attempt);
+                continue;
+            }
+            throw new Error(`Multi-focus dedupe request threw: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (!result.ok) {
+            const shouldRetry = attempt < DEDUPE_MAX_ATTEMPTS;
+            logDedupeProviderUnavailable({
+                providerConfig,
+                attempt,
+                error: result.error ?? 'dedupe request failed',
+                shouldRetry,
+                timedOut: result.timedOut,
+            });
+            if (shouldRetry) {
+                await waitForDedupeRetry(attempt);
+                continue;
+            }
+            throw new Error(`Multi-focus dedupe request failed: ${result.error}`);
+        }
+        if (hasToolUsage(result.toolUsage)) {
+            logger.error('Multi-focus dedupe used tools', {
+                provider: providerConfig.provider,
+                model: providerConfig.modelName,
+                tool_usage: result.toolUsage,
+            });
+            throw new Error('Multi-focus dedupe used tools unexpectedly');
+        }
+        let parsed;
+        try {
+            parsed = parseDedupeResponse(result.output);
+        }
+        catch (error) {
+            logger.error('Multi-focus dedupe parsing failed', {
+                provider: providerConfig.provider,
+                model: providerConfig.modelName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw new Error(`Multi-focus dedupe parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        const deduped = applyDedupeGroups(findings, parsed.groups);
+        logger.info('Multi-focus dedupe completed', {
+            provider: providerConfig.provider,
+            model: providerConfig.modelName,
+            originalCount: findings.length,
+            dedupedCount: deduped.length,
+            groupsCount: parsed.groups.length,
+        });
+        return deduped;
+    }
+    throw new Error('Multi-focus dedupe exhausted attempts without a result');
+}