@superblocksteam/vite-plugin-file-sync 2.0.159-next.0 → 2.0.160-next.0

package/dist/ai-service/index.js

@@ -6,14 +6,14 @@ import { hasToolCall, stepCountIs, } from "ai";
 import { z } from "zod";
 import { AiGenerationState, AiMode, resolveAttachmentFileName, } from "@superblocksteam/library-shared/types";
 import { isToolPermissionClarification } from "@superblocksteam/library-shared/types/tool-permission-response";
-import { addTracingToMethods, AiSystemTriggerType, classifyAttachmentDelivery, FactAccessType, FactCreatedSource, FactEntityScope, TracedEventEmitter, getAiQuotaPaywallReasonFromMessage, POLICY_GATE_MAX_TOTAL_TIME_MS, POLICY_GATE_STREAM_IDLE_TIMEOUT_MS, } from "@superblocksteam/shared";
+import { addTracingToMethods, AiSystemTriggerType, classifyAttachmentDelivery, FactAccessType, FactCreatedSource, FactEntityScope, TracedEventEmitter, getAiQuotaPaywallReasonFromMessage, } from "@superblocksteam/shared";
 import { npmRegistryEmitter } from "@superblocksteam/telemetry";
 import { applyTestApiDenialSideEffects, buildMigrationTestApiDenialContinuationPrompt, } from "../migration/mark-api-testing-denied.js";
 import { buildTranslationPrompt } from "../migration/translation-prompt.js";
 import { POLICY_GATE_RUN_IN_PROGRESS_ERROR, policyGateRunRegistry, } from "../policy-gate-run-registry.js";
 import { getErrorMeta, getLogger, setDefaultLogger } from "../util/logger.js";
 import { composeSecurityAgentPrompt, } from "./agent/prompts/build-security-scan-prompt.js";
-import { getToolCallArguments, getToolErrorCode, getToolErrorMessage, getToolOutput, } from "./agent/tool-message-utils.js";
+import { getToolCallArguments, getToolErrorCode, getToolOutput, } from "./agent/tool-message-utils.js";
 import { recordToolPermissionDeliveryOutcome } from "./agent/tool-permission-delivery-metrics.js";
 import { createExpiredPermissionPromptDeliveryError, flushPendingToolPermissionPrompt, inFlightToolPermissionReplay, } from "./agent/tool-permission-delivery.js";
 import { buildSecurityScanTools, isBackgroundTaskFrameworkEnabled, } from "./agent/tools.js";
@@ -31,7 +31,6 @@ import { NpmPackageLookup } from "./app-interface/npm-package-lookup.js";
 import { NpmRegistryClient, createDiskRegistryPolicyStore, } from "./app-interface/npm-registry.js";
 import { AppShell } from "./app-interface/shell.js";
 import { AppSkillsManager } from "./app-skills/manager.js";
-import { parseDataUrl } from "./attachments/parse-data-url.js";
 import { isUploadedAttachment, toUploadedAttachmentContentPart, } from "./attachments/uploaded-content-part.js";
 import { ChatSessionStore } from "./chat/chat-session-store.js";
 import { normalizeTextAttachmentForModel } from "./chat/utils.js";
@@ -70,7 +69,6 @@ import { buildServerMigrationLlmConfig, isMigrationAutoDispatchEnabled, } from "
 import { isMigrationSessionActive } from "./migration-session-active.js";
 import { hasSettledNormalMigrationTurn, recordMigrationTurnDispatched, recordMigrationTurnSettled, } from "./migration-turns-log.js";
 import { recordPolicyAgentExecution, } from "./policy-agent-metrics.js";
-import { classifyPolicyGateError } from "./policy-gate-error.js";
 import { policyGateStreamPartToAgentStep } from "./policy-gate-step.js";
 import { ClarkProfiler } from "./profiler/clark-profiler.js";
 import { buildSummarizeApiUsagePrompt } from "./prompts/summarize-api-usage.js";
@@ -146,6 +144,7 @@ const MAX_POLICY_GATE_STREAM_ERROR_ARRAY_LENGTH = 10;
 const MAX_POLICY_GATE_STREAM_ERROR_FIELD_LENGTH = 500;
 const MAX_POLICY_GATE_STREAM_ERROR_OBJECT_DEPTH = 3;
 const MAX_POLICY_GATE_STREAM_ERROR_OBJECT_KEYS = 20;
+const POLICY_GATE_MAX_TOTAL_TIME_MS = 15 * 60 * 1000;
 // Ceiling above the historical 30-step cap; 100 is an existing policy-agent
 // histogram bucket. Tighten from production p99 if scans complete far below this.
 const POLICY_GATE_MAX_STEPS = 100;
@@ -162,54 +161,6 @@ const POLICY_GATE_STREAM_ERROR_SECRET_VALUE_PATTERN_SOURCES = [
     String.raw `\bsk-[a-z0-9_-]{8,}\b`,
     String.raw `\bxox[baprs]-[a-z0-9-]{10,}\b`,
 ];
-/**
- * Fails a policy gate stream that goes idle, independent of whatever
- * abort/timeout wiring the stream itself does or doesn't honor. A dropped
- * dev-server connection can stall the stream without ever rejecting it
- * (APPS-5642), which leaves the run "running" forever since nothing throws
- * for the existing catch/classification logic to act on.
- */
-async function* withPolicyGateIdleTimeout(stream, idleTimeoutMs) {
-    const iterator = stream[Symbol.asyncIterator]();
-    try {
-        while (true) {
-            const nextResult = iterator.next();
-            // If the idle timeout wins the race below, this promise is abandoned
-            // mid-flight; attach a handler now so its eventual settlement (once the
-            // dead connection is cleaned up, if ever) never surfaces as an
-            // unhandled rejection. Promise.race still gets its own notification
-            // from this same promise independently.
-            nextResult.catch(() => { });
-            let timer;
-            const idleTimeout = new Promise((_, reject) => {
-                timer = setTimeout(() => {
-                    reject(Object.assign(new Error("Policy gate stream went idle"), {
-                        name: "TimeoutError",
-                    }));
-                }, idleTimeoutMs);
-            });
-            let result;
-            try {
-                result = await Promise.race([nextResult, idleTimeout]);
-            }
-            finally {
-                clearTimeout(timer);
-            }
-            if (result.done) {
-                return;
-            }
-            yield result.value;
-        }
-    }
-    finally {
-        // Best-effort only: never await this. On the exact connection this
-        // wrapper exists to fail past — one that stopped responding without
-        // erroring or closing — iterator.return() can hang the same way
-        // .next() did, and blocking cleanup on it here would defeat the whole
-        // point of a timeout that's supposed to guarantee termination.
-        void iterator.return?.()?.catch(() => { });
-    }
-}
 function truncatePolicyGateStreamErrorField(value) {
     if (value.length <= MAX_POLICY_GATE_STREAM_ERROR_FIELD_LENGTH) {
         return value;
@@ -820,19 +771,6 @@ export class AiService extends TracedEventEmitter {
                     this.getLogger().debug("[ai-service] Failed to report npm-install-blocked audit event", getErrorMeta(error));
                 });
             },
-            // Privacy-safe install-gate USER_EVENT: outcome + runner only. Same
-            // fire-and-forget socket path; not gated by audit-v2 or the blocked-
-            // install throttle. Prefetch/uninstall never call this reporter.
-            reportNpmInstallOutcome: (report) => {
-                void this.config.rpcClient
-                    .call(async (client) => {
-                    const rpcClient = client;
-                    return await rpcClient.call.v1.audit.npmInstallOutcome(report);
-                })
-                    .catch((error) => {
-                    this.getLogger().debug("[ai-service] Failed to report npm-install-outcome user event", getErrorMeta(error));
-                });
-            },
             virtualFileSystem: createDefaultVirtualFileSystem({
                 useSystemSkills: config.features.useSystemSkills,
                 knowledgeManager: this.knowledgeManager,
@@ -2185,9 +2123,7 @@ export class AiService extends TracedEventEmitter {
                     // logged so we can audit if this ever fires in prod.
                     let promptForContext = request.prompt;
                     try {
-                        const scanResult = await scanContentForSecrets(request.prompt, {
-                            consumer: "context_persistence",
-                        });
+                        const scanResult = await scanContentForSecrets(request.prompt);
                         if (scanResult.success && scanResult.hasSecrets) {
                             this.getLogger().warn(`[secret-scan] Redacting ${scanResult.findings.length} secret(s) from pre-call paywall prompt before ContextV2 persist`, {
                                 findingCount: scanResult.findings.length,
@@ -2197,6 +2133,15 @@ export class AiService extends TracedEventEmitter {
                             const redactor = new SecretRedactor(scanResult.findings);
                             promptForContext = redactor.redact(request.prompt);
                         }
+                        else if (!scanResult.success) {
+                            // Do NOT log `scanResult.error` directly — it can contain
+                            // raw TruffleHog stdout/stderr payloads, which on some
+                            // failure paths include excerpts of the scanned prompt
+                            // (i.e. unredacted user secret material). Log only that the
+                            // scan failed; operators can correlate via timestamp +
+                            // peerId. (APPS-4080 security review)
+                            this.getLogger().warn("[secret-scan] Secret scan failed on pre-call paywall prompt; persisting unredacted (fail-open, matches mainline)");
+                        }
                     }
                     catch (scanErr) {
                         // Same reason — don't pipe raw scanner error meta to logs.
@@ -2729,12 +2674,12 @@ export class AiService extends TracedEventEmitter {
             };
         }
         else if (attachment.type === "pdf" || attachment.type === "archive") {
-            const parsed = parseDataUrl(attachment.data);
-            const data = parsed?.data ?? attachment.data;
+            const match = attachment.data.match(/^data:([^;]+);base64,(.+)$/);
+            const data = match?.[2] ?? attachment.data;
             const defaultMediaType = attachment.type === "pdf"
                 ? "application/pdf"
                 : "application/octet-stream";
-            const mediaType = parsed?.mediaType ?? defaultMediaType;
+            const mediaType = match?.[1] ?? defaultMediaType;
             return {
                 type: "file",
                 data,
@@ -3358,7 +3303,7 @@ export class AiService extends TracedEventEmitter {
                         type: "tool-error",
                         toolCallId,
                         toolName,
-                        error: getToolErrorMessage(result.error.cause ?? result.error),
+                        error: result.error.message,
                         errorCode,
                     });
                 }
@@ -4025,16 +3970,13 @@ export class AiService extends TracedEventEmitter {
         // Invalidate context errors cache since runtime errors changed
         debugCache.invalidateContextErrors();
     }
-    handleClearBuildSystemErrors(source) {
+    handleClearBuildSystemErrors() {
         this.clark.updateContext((context) => ({
             ...context,
-            buildSystemErrors: source
-                ? (context.buildSystemErrors ?? []).filter((error) => error.source !== source)
-                : [],
+            buildSystemErrors: [],
         }));
-        this.getLogger().info(source
-            ? `[ai-service] Cleared ${source} build system errors`
-            : `[ai-service] Cleared all build system errors`);
+        this.getLogger().info(`[ai-service] Cleared all build system errors`);
+        // Invalidate context errors cache since build system errors changed
         debugCache.invalidateContextErrors();
     }
     async handlePromptInterpretation(request) {
@@ -4147,11 +4089,8 @@ export class AiService extends TracedEventEmitter {
         const streamPartTypeCounts = {};
         let stepCount = 0;
         let textDeltaChars = 0;
-        let failureErrorCode;
-        let maxOutputTokens;
         let modelName;
         let promptHash;
-        let quotaReason;
         // Hoisted so the outer catch can attribute a failed run to the same model as
         // the attempt it belongs to. Usage arrives from the client's usage observer,
         // which also flushes on abort, on error, and for compaction calls.
@@ -4188,10 +4127,8 @@ export class AiService extends TracedEventEmitter {
             finishReasons,
             grepMatchCounts,
             grepPatterns,
-            maxOutputTokens,
             model: modelName,
             promptHash,
-            quotaReason,
             reportToolCalled: Boolean(scanResult),
             scanComplete: scanResult?.scanComplete === true,
             scanRoot,
@@ -4201,26 +4138,13 @@ export class AiService extends TracedEventEmitter {
             textDeltaChars,
             toolCallCounts,
         });
-        const failPolicyGateExecution = (message, errorCode) => {
-            failureErrorCode = errorCode;
-            recordExecutionMetrics(errorCode);
-            throw Object.assign(new Error(message), {
-                errorCode,
-                runtimeMetadataJson: buildRuntimeMetadata(message),
-            });
+        const failPolicyGateExecution = (message, outcome) => {
+            recordExecutionMetrics(outcome);
+            const error = new Error(message);
+            error.runtimeMetadataJson = buildRuntimeMetadata(message);
+            throw error;
         };
         try {
-            try {
-                await this.ensureAiQuotaAvailable(request.jwt);
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : "AI quota unavailable";
-                const errorCode = classifyPolicyGateError(error);
-                if (errorCode === "out_of_credits") {
-                    quotaReason = getAiQuotaPaywallReasonFromMessage(message);
-                }
-                failPolicyGateExecution(message, errorCode);
-            }
             const requestLlmConfig = request.llmConfig;
             const contextLlmConfig = this.clark.context.llmConfig;
             const llmConfig = requestLlmConfig ?? contextLlmConfig;
@@ -4249,13 +4173,16 @@ export class AiService extends TracedEventEmitter {
             let scanScopeJson = request.scanScopeJson;
             if (scanScopeJson?.unifiedDiff) {
                 try {
-                    const scanResult = await scanContentForSecrets(scanScopeJson.unifiedDiff, { consumer: "policy_gate" });
+                    const scanResult = await scanContentForSecrets(scanScopeJson.unifiedDiff);
                     if (scanResult.success && scanResult.hasSecrets) {
                         scanScopeJson = {
                             ...scanScopeJson,
                             unifiedDiff: new SecretRedactor(scanResult.findings).redact(scanScopeJson.unifiedDiff),
                         };
                     }
+                    else if (!scanResult.success) {
+                        logger.warn("[secret-scan] Secret scan failed on policy gate diff; using the original diff");
+                    }
                 }
                 catch {
                     logger.warn("[secret-scan] Secret scan threw on policy gate diff; using the original diff");
@@ -4265,8 +4192,7 @@ export class AiService extends TracedEventEmitter {
             const systemPrompt = composeSecurityAgentPrompt(appContext, request.adminPrompt, scanScopeJson);
             promptHash = createHash("sha256").update(systemPrompt).digest("hex");
             const scanContext = {};
-            const { headers, interleavedThinking, maxOutputTokens: configuredMaxOutputTokens, providerOptions, thinkingBudgetTokens, thinkingEnabled, } = processLLMConfig(llmConfig);
-            maxOutputTokens = configuredMaxOutputTokens;
+            const { headers, interleavedThinking, maxOutputTokens, providerOptions, thinkingBudgetTokens, thinkingEnabled, } = processLLMConfig(llmConfig);
             // Scan-owned client: gate JWT, own usage sink, built first so scan tools
             // (webFetch generateObject) use this client instead of Clark's.
             const policyGateLlmClient = new LLMClient({
@@ -4300,11 +4226,8 @@ export class AiService extends TracedEventEmitter {
                 },
             };
             const tools = await buildSecurityScanTools(this.clark, scanToolsDeps);
-            const timeoutSignal = AbortSignal.timeout(POLICY_GATE_MAX_TOTAL_TIME_MS);
             const result = await policyGateLlmClient.streamText({
-                abortSignal: request.abortSignal
-                    ? AbortSignal.any([request.abortSignal, timeoutSignal])
-                    : timeoutSignal,
+                abortSignal: AbortSignal.timeout(POLICY_GATE_MAX_TOTAL_TIME_MS),
                 context: {
                     contextManagement: {
                         summarizationModel,
@@ -4319,7 +4242,7 @@ export class AiService extends TracedEventEmitter {
                 headers,
                 logging: false,
                 // processLLMConfig returns the same MAX_OUTPUT_TOKENS cap Clark uses.
-                maxOutputTokens: configuredMaxOutputTokens,
+                maxOutputTokens,
                 model,
                 onContextReady: (ctx) => {
                     scanContext.current = ctx;
@@ -4369,7 +4292,7 @@ export class AiService extends TracedEventEmitter {
                 });
             };
             try {
-                for await (const part of withPolicyGateIdleTimeout(result.fullStream, POLICY_GATE_STREAM_IDLE_TIMEOUT_MS)) {
+                for await (const part of result.fullStream) {
                     emitAgentStep(part);
                     const partType = typeof part.type === "string"
                         ? part.type
@@ -4487,9 +4410,6 @@ export class AiService extends TracedEventEmitter {
                 }
             }
             catch (error) {
-                if (failureErrorCode !== undefined) {
-                    throw error;
-                }
                 if (error instanceof Error &&
                     (error.name === "AbortError" ||
                         error.name === "AbortedError" ||
@@ -4504,7 +4424,7 @@ export class AiService extends TracedEventEmitter {
                             ...getErrorMeta(error),
                             elapsedMs,
                         });
-                        failPolicyGateExecution("Policy gate agent timed out", "llm_timeout");
+                        failPolicyGateExecution("Policy gate agent timed out", "timeout");
                     }
                     logger.warn("[ai-service] Policy gate agent aborted", {
                         ...policyRunLogContext,
@@ -4513,32 +4433,22 @@ export class AiService extends TracedEventEmitter {
                     });
                     failPolicyGateExecution("Policy gate agent aborted", "aborted");
                 }
-                const streamError = serializePolicyGateStreamError(error);
-                if (streamErrors.length < MAX_POLICY_GATE_STREAM_ERRORS) {
-                    streamErrors.push(streamError);
-                }
-                logger.warn("[ai-service] Policy gate stream error", {
-                    ...policyRunLogContext,
-                    streamError,
-                });
-                const errorCode = classifyPolicyGateError(error);
-                if (errorCode === "out_of_credits") {
-                    quotaReason =
-                        error instanceof Error
-                            ? getAiQuotaPaywallReasonFromMessage(error.message)
-                            : undefined;
+                if (error instanceof Error &&
+                    (error.name === "LLMProviderError" ||
+                        error.name === "RetriesExhaustedError")) {
+                    const streamError = serializePolicyGateStreamError(error);
+                    if (streamErrors.length < MAX_POLICY_GATE_STREAM_ERRORS) {
+                        streamErrors.push(streamError);
+                    }
+                    logger.warn("[ai-service] Policy gate stream error", {
+                        ...policyRunLogContext,
+                        streamError,
+                    });
                 }
-                failPolicyGateExecution(error instanceof Error ? error.message : "Policy gate stream failed", errorCode);
+                throw error;
             }
             if (!scanResult) {
                 logger.warn("[ai-service] Policy gate agent did not call reportReviewRun", policyRunLogContext);
-                if (finishReasons.includes("length")) {
-                    const outputTokens = llmUsage.at(-1)?.outputTokens ?? 0;
-                    failPolicyGateExecution(`Policy gate agent reached the output token limit of ${maxOutputTokens ?? 0} after producing ${outputTokens} output tokens`, "output_token_limit");
-                }
-                if (finishReasons.includes("content-filter")) {
-                    failPolicyGateExecution("Policy gate agent output was refused by the provider content filter", "llm_refusal");
-                }
                 failPolicyGateExecution("Policy gate agent did not complete a report", "missing_report");
             }
             const completedScanResult = scanResult;
@@ -4579,7 +4489,7 @@ export class AiService extends TracedEventEmitter {
                     aiAgentRunId: request.aiAgentRunId,
                     chatSessionStore: this.chatSessionStore,
                     coreModel,
-                    failure: policyGateFailureFromStream(failureErrorCode, quotaReason),
+                    failure: policyGateFailureFromStream(streamErrors),
                     hadFallback,
                     llmUsage,
                     logger,