nolo-cli 0.1.29 → 0.1.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-runtime/agentRecordConfig.ts +0 -2
- package/agent-runtime/hostAdapter.ts +0 -1
- package/agent-runtime/hostRuntimeCapabilities.ts +7 -0
- package/agent-runtime/hostRuntimeConfig.ts +86 -0
- package/agent-runtime/index.ts +13 -0
- package/agent-runtime/localToolPolicy.ts +13 -6
- package/agent-runtime/localWorkspaceTools.ts +333 -30
- package/agent-runtime/platformChatProvider.ts +4 -1
- package/agent-runtime/runtimeToolPolicy.ts +32 -47
- package/agent-runtime/shellCommandPolicy.ts +117 -0
- package/agent-runtime/toolGate.ts +38 -0
- package/agentCreateCustomCommands.ts +9 -4
- package/agentDialogsCommands.ts +4 -3
- package/agentDoctorCommands.ts +3 -3
- package/agentListCommands.ts +3 -3
- package/agentMachineCommands.ts +6 -6
- package/agentPullCommand.ts +4 -12
- package/agentRecordCommands.ts +5 -3
- package/agentRecordHelpers.ts +42 -23
- package/agentRunCommand.ts +2 -6
- package/agentRunParsing.ts +18 -5
- package/agentRuntimeLocal.ts +10 -0
- package/agentUnpublishCommands.ts +3 -2
- package/ai/agent/agentRunDialogLineage.ts +56 -0
- package/ai/agent/agentSlice.ts +45 -10
- package/ai/agent/avatarUtils.ts +1 -1
- package/ai/agent/buildSystemPrompt.ts +1 -2
- package/ai/agent/cliExecutor.ts +330 -19
- package/ai/agent/createAgentSchema.ts +22 -0
- package/ai/agent/hooks/fetchPublicAgents.ts +14 -39
- package/ai/agent/hooks/usePublicAgents.ts +90 -192
- package/ai/agent/publicAgentCatalog.ts +359 -0
- package/ai/agent/publicAgentIdentity.ts +76 -0
- package/ai/agent/server/fetchPublicAgents.ts +21 -46
- package/ai/agent/serverOrigin.ts +7 -0
- package/ai/agent/streamAgentChatTurn.ts +23 -21
- package/ai/agent/streamAgentChatTurnUtils.ts +292 -6
- package/ai/agent/types.ts +2 -0
- package/ai/agent/web/agentDisplayUtils.ts +171 -0
- package/ai/agent/web/useAgentModelAvatarComponent.ts +32 -0
- package/ai/ai.locale.ts +8 -0
- package/ai/chat/sendOpenAICompletionsRequest.native.ts +15 -0
- package/ai/chat/sendOpenAICompletionsRequest.ts +93 -1
- package/ai/chat/sendOpenAIResponseRequest.ts +86 -9
- package/ai/chat/updateTotalUsage.ts +7 -0
- package/ai/llm/deepinfra.ts +30 -15
- package/ai/llm/mistral.ts +3 -4
- package/ai/llm/providers.ts +2 -0
- package/ai/llm/types.ts +15 -0
- package/ai/memory/capture.ts +3 -3
- package/ai/memory/consolidate.ts +2 -2
- package/ai/memory/delete.ts +4 -3
- package/ai/memory/query.ts +3 -3
- package/ai/memory/remember.ts +2 -2
- package/ai/memory/store.ts +3 -3
- package/ai/memory/storeShared.ts +3 -2
- package/ai/policy/commercialMultiplier.ts +16 -0
- package/ai/token/billingAdminAuditEvent.ts +68 -0
- package/ai/token/billingAdminAuditEventWriter.ts +41 -0
- package/ai/token/billingAnomaly.ts +90 -0
- package/ai/token/billingAnomalyLifecycle.ts +54 -0
- package/ai/token/billingAnomalyLifecycleWriter.ts +44 -0
- package/ai/token/billingAnomalyWriter.ts +41 -0
- package/ai/token/calculatePrice.ts +3 -2
- package/ai/token/deepinfraReconciliation.ts +153 -0
- package/ai/token/deepinfraRequestIdQuery.ts +92 -0
- package/ai/token/externalToolCost.ts +54 -41
- package/ai/token/missingUsageEstimate.ts +12 -3
- package/ai/token/normalizeUsage.ts +45 -5
- package/ai/token/openaiReconciliation.ts +188 -0
- package/ai/token/providerBillableEvent.ts +84 -0
- package/ai/token/providerBillableEventWriter.ts +36 -0
- package/ai/token/providerBillingAnomalyDrilldown.ts +308 -0
- package/ai/token/providerBillingDrilldown.ts +361 -0
- package/ai/token/providerBillingHealthReport.ts +366 -0
- package/ai/token/providerCall.ts +144 -0
- package/ai/token/providerCallSweeper.ts +168 -0
- package/ai/token/providerCallWriter.ts +33 -0
- package/ai/token/providerCredential.ts +89 -0
- package/ai/token/providerCredentialAnomaly.ts +154 -0
- package/ai/token/providerCredentialLifecycle.ts +59 -0
- package/ai/token/providerCredentialLifecycleWriter.ts +44 -0
- package/ai/token/providerCredentialRegistry.ts +83 -0
- package/ai/token/providerCredentialRegistryCommand.ts +97 -0
- package/ai/token/providerCredentialRegistryList.ts +124 -0
- package/ai/token/providerCredentialRegistryResolver.ts +150 -0
- package/ai/token/providerCredentialRegistryWriter.ts +41 -0
- package/ai/token/providerCredentialRotationCommand.ts +102 -0
- package/ai/token/providerCredentialRuntime.ts +56 -0
- package/ai/token/providerDispatchIntent.ts +67 -0
- package/ai/token/providerDispatchIntentWriter.ts +36 -0
- package/ai/token/providerGateway.ts +260 -0
- package/ai/token/providerReconciliation.ts +148 -0
- package/ai/token/providerReconciliationLocal.ts +97 -0
- package/ai/token/providerReconciliationRunner.ts +165 -0
- package/ai/token/providerReconciliationWriter.ts +41 -0
- package/ai/token/ratingResult.ts +202 -0
- package/ai/token/ratingResultWriter.ts +33 -0
- package/ai/token/serverDialogProjection.ts +5 -3
- package/ai/token/serverTokenWriter.ts +11 -3
- package/ai/token/types.ts +10 -0
- package/ai/tools/agent/callAgentTool.ts +3 -4
- package/ai/tools/agent/presets/appBuilderPreset.ts +5 -7
- package/ai/tools/agent/runStreamingAgentTool.ts +1 -2
- package/ai/tools/agent/streamParallelAgentsTool.ts +2 -3
- package/ai/tools/readXPostTool.ts +1 -5
- package/app/admin/adminPages.ts +9 -0
- package/app/i18n/i18n.config.ts +5 -0
- package/app/i18n/translations/pricing.locale.ts +397 -0
- package/app/utils/desktopAgentRuntimeStatusClient.ts +5 -4
- package/app/utils/desktopAgentRuntimeTurnClient.ts +4 -1
- package/app/utils/imageUtils.ts +64 -34
- package/auth/adminPermissions.ts +18 -0
- package/auth/authSlice.ts +26 -12
- package/auth/hooks/useFetchUsers.ts +3 -0
- package/auth/routes.ts +46 -0
- package/auth/server/accessActivity.ts +4 -3
- package/auth/server/adminPermissions.ts +43 -0
- package/auth/server/adminUsageReport.ts +375 -0
- package/auth/server/billingAnomalies.ts +235 -0
- package/auth/server/billingUsageReport.ts +41 -2
- package/auth/server/creatorEarningCreditConversion.ts +10 -2
- package/auth/server/deduct.ts +14 -7
- package/auth/server/delete.ts +10 -2
- package/auth/server/disable.ts +4 -3
- package/auth/server/emailAutomation.ts +7 -5
- package/auth/server/emailDbConfig.ts +4 -2
- package/auth/server/emailDelivery.ts +24 -20
- package/auth/server/emailDeliveryGuards.ts +10 -7
- package/auth/server/emailPreferences.ts +6 -4
- package/auth/server/emailReport.ts +6 -4
- package/auth/server/emailRuntimeConfig.ts +3 -3
- package/auth/server/emailScenarioNotifications.ts +9 -8
- package/auth/server/enable.ts +4 -3
- package/auth/server/getRechargeHistory.ts +3 -2
- package/auth/server/getUser.ts +3 -2
- package/auth/server/gptProAccess.ts +44 -0
- package/auth/server/inviteReward.ts +10 -7
- package/auth/server/listusers.ts +6 -4
- package/auth/server/login.ts +2 -2
- package/auth/server/productEntitlements.ts +7 -4
- package/auth/server/productPurchase.ts +10 -2
- package/auth/server/providerBillingDrilldown.ts +47 -0
- package/auth/server/providerBillingHealth.ts +71 -0
- package/auth/server/providerCredentials.ts +136 -0
- package/auth/server/recharge.ts +14 -8
- package/auth/server/revokeSessions.ts +4 -3
- package/auth/server/route.ts +77 -0
- package/auth/server/sendEmail.ts +5 -4
- package/auth/server/signup.ts +10 -3
- package/auth/server/token.ts +53 -55
- package/auth/server/tokenUsageBilling.ts +18 -7
- package/auth/server/usageManagementAccess.ts +43 -0
- package/auth/server/usageReport.ts +59 -0
- package/auth/types.ts +12 -0
- package/auth/utils.ts +24 -22
- package/authCommands.ts +36 -2
- package/buildPublish.ts +3 -1
- package/buildPublishArtifactAssembly.ts +62 -1
- package/buildPublishImportAnalysis.ts +21 -16
- package/chat/dialog/dialogLaunchScope.ts +32 -0
- package/chat/dialog/objectAssistantRegistry.ts +2 -2
- package/chat/dialog/useCreateDialog.ts +1 -1
- package/chat/hooks/useChatInput.ts +41 -10
- package/chat/messages/messageSlice.ts +5 -1
- package/chat/messages/pendingAttachmentParts.ts +28 -2
- package/chat/messages/sendFirstMessage.ts +76 -79
- package/chat/messages/types.ts +9 -0
- package/chat/messages/web/streamingMarkdownModel.ts +41 -0
- package/chat/messages/web/useStreamingReveal.ts +91 -0
- package/chat/web/messageInputAgentUi.ts +94 -6
- package/cliEnvHelpers.ts +68 -13
- package/cliRuntimeDoctorReport.ts +193 -0
- package/client/agentConfigResolver.test.ts +0 -2
- package/client/agentRun.test.ts +171 -10
- package/client/agentRun.ts +44 -637
- package/client/agentRunInput.ts +10 -0
- package/client/agentRunLocalExecutor.ts +179 -0
- package/client/agentRunServerTransport.ts +394 -0
- package/client/agentRunTaskRunRecorder.ts +176 -0
- package/client/localRuntimeAdapter.test.ts +236 -9
- package/client/localRuntimeAdapter.ts +221 -23
- package/client/localToolPolicy.test.ts +28 -4
- package/connector-experimental/daemonLauncher.ts +38 -4
- package/connectorRunArtifact.ts +34 -10
- package/connectorWebSocketTarget.ts +40 -15
- package/core/builtinAgents.ts +155 -11
- package/create/space/contentLabels.ts +1 -1
- package/create/space/contentTypeMeta.ts +3 -2
- package/create/space/sidebarVisibleTypes.ts +27 -0
- package/create/space/space.locale.ts +76 -4
- package/database/actions/agentReadResolution.ts +2 -8
- package/database/actions/read.ts +11 -9
- package/database/actions/readAndWait.ts +10 -5
- package/database/server/agentDelegation.ts +3 -2
- package/database/server/authorityStoreConformance.ts +209 -0
- package/database/server/authorityStoreTypes.ts +31 -0
- package/database/server/coreDataProxy.ts +1 -1
- package/database/server/dataHandlers.ts +13 -5
- package/database/server/db.ts +15 -34
- package/database/server/delete.ts +88 -71
- package/database/server/emailRepository.ts +11 -9
- package/database/server/fileService.ts +2 -1
- package/database/server/legacyServerDb.ts +58 -0
- package/database/server/levelAuthorityStore.ts +75 -0
- package/database/server/memoryAuthorityStore.ts +133 -0
- package/database/server/patch.ts +13 -12
- package/database/server/query.ts +3 -2
- package/database/server/read.ts +33 -30
- package/database/server/resourceAccess.ts +7 -5
- package/database/server/serverStoreFactory.ts +124 -0
- package/database/server/spaceMemberAuthority.ts +3 -2
- package/database/server/storeBatchCompat.ts +37 -0
- package/database/server/testAuthorityStore.ts +76 -0
- package/database/server/write.ts +29 -27
- package/dialogReadCommands.ts +2 -1
- package/dialogStatusCommands.ts +2 -1
- package/integrations/anthropic/anthropicModels.ts +12 -12
- package/integrations/deepseek/models.ts +67 -67
- package/integrations/google/models.ts +111 -53
- package/integrations/openai/models.ts +22 -22
- package/integrations/xai/models.ts +1 -1
- package/localRuntimeDb.ts +39 -2
- package/machineCommandPrompts.ts +2 -0
- package/machineCommands.ts +193 -62
- package/machineStatusCommands.ts +4 -12
- package/machineWsRunDispatch.ts +80 -15
- package/machineWsSession.ts +46 -2
- package/noloReadBridgePrompt.ts +16 -0
- package/package.json +114 -21
- package/pageCreateCommands.ts +2 -2
- package/pageDeleteCommands.ts +3 -3
- package/pageReadCommands.ts +3 -3
- package/pageUpdateCommands.ts +3 -3
- package/render/layout/mainLayoutViewMode.ts +25 -0
- package/render/page/server/createPage.ts +3 -1
- package/render/page/server/handleCreatePage.ts +2 -11
- package/runtimeDoctorCommands.ts +18 -69
- package/server/api/methods.ts +34 -12
- package/server/api/tableHandlers.ts +6 -3
- package/server/authenticateToken.ts +47 -2
- package/server/billingLedgerRoutes.ts +15 -0
- package/server/buildManifest.ts +54 -0
- package/server/coreReadiness.ts +34 -25
- package/server/coreRuntime.ts +15 -10
- package/server/handlers/activityStatsHandler.ts +5 -2
- package/server/handlers/agentEmailRegistrationE2EHandler.ts +25 -23
- package/server/handlers/agentRun/agentLookup.ts +1 -7
- package/server/handlers/agentRun/autonomousRuntime.ts +11 -10
- package/server/handlers/agentRun/builtinAgents.ts +12 -88
- package/server/handlers/agentRun/continueDialogContext.ts +15 -3
- package/server/handlers/agentRun/{inheritedDialogPrompt.ts → conversationReference.ts} +3 -3
- package/server/handlers/agentRun/dialogLookup.ts +4 -3
- package/server/handlers/agentRun/index.ts +864 -404
- package/server/handlers/agentRun/localRuntimeBootstrap.ts +17 -1
- package/server/handlers/agentRun/loop.ts +920 -508
- package/server/handlers/agentRun/machineConnector.ts +56 -8
- package/server/handlers/agentRun/machineRunAudit.ts +3 -2
- package/server/handlers/agentRun/modelUsageToolHandlers.ts +13 -10
- package/server/handlers/agentRun/requestContext.ts +12 -8
- package/server/handlers/agentRun/runtimeExecutionContext.ts +15 -3
- package/server/handlers/agentRun/runtimePreparation.ts +26 -24
- package/server/handlers/agentRun/runtimeRouting.ts +25 -40
- package/server/handlers/agentRun/runtimeSystemMessages.ts +65 -0
- package/server/handlers/agentRun/saveDialog.ts +97 -87
- package/server/handlers/agentRun/taskRunToolHandler.ts +9 -7
- package/server/handlers/agentRun/toolExecutor.ts +42 -78
- package/server/handlers/agentRun/types.ts +2 -1
- package/server/handlers/appDeployStatusStore.ts +17 -10
- package/server/handlers/appDomainHandler.ts +21 -15
- package/server/handlers/appLegacyInfraCleanup.ts +5 -2
- package/server/handlers/appSandboxHandler.ts +29 -8
- package/server/handlers/appSpaceAccess.ts +15 -8
- package/server/handlers/chatHandler.ts +22 -2
- package/server/handlers/chatUpstreamRetry.ts +225 -2
- package/server/handlers/connector/connectorConnectionRegistry.ts +55 -7
- package/server/handlers/connector/connectorWebSocket.ts +14 -10
- package/server/handlers/desktopAgentRuntimeAdapter.ts +4 -4
- package/server/handlers/desktopAgentRuntimeHostCapabilities.ts +25 -0
- package/server/handlers/desktopAgentRuntimeReadiness.ts +62 -0
- package/server/handlers/desktopAgentRuntimeStatusHandler.ts +38 -30
- package/server/handlers/desktopAgentRuntimeTurnHandler.ts +7 -3
- package/server/handlers/desktopAgentRuntimeTurnService.ts +39 -45
- package/server/handlers/desktopLlamaRuntimeHandler.ts +85 -51
- package/server/handlers/desktopUpdaterHandler.ts +132 -24
- package/server/handlers/favoriteHandlers.ts +8 -2
- package/server/handlers/getDialogBillingHandler.ts +4 -3
- package/server/handlers/getTransactionsHandler.ts +5 -2
- package/server/handlers/machines/machineBinding.ts +1 -0
- package/server/handlers/machines/machineHandlers.ts +0 -14
- package/server/handlers/machines/machineInstallAssets.ts +4 -2
- package/server/handlers/notificationEmitter.ts +5 -3
- package/server/handlers/notificationsHandler.ts +5 -2
- package/server/handlers/providerCallsRecentHandler.ts +234 -0
- package/server/handlers/providerReconciliationRunHandler.ts +180 -0
- package/server/handlers/providerRequestIdsHandler.ts +105 -0
- package/server/handlers/tableAccess.ts +6 -2
- package/server/handlers/tableToolHandlers.ts +28 -14
- package/server/handlers/userSecretsHandler.ts +17 -6
- package/server/handlers/versionHandler.ts +9 -4
- package/server/handlers/versionStore.ts +18 -10
- package/server/html/ssrPreload.ts +5 -3
- package/server/payments/crypto/cryptoUsdcBasePaymentProvider.ts +11 -8
- package/server/payments/crypto/cryptoUsdcBaseScanner.ts +12 -7
- package/server/payments/crypto/cryptoUsdtTronPaymentProvider.ts +11 -8
- package/server/payments/crypto/cryptoUsdtTronScanner.ts +9 -7
- package/server/payments/waffo/waffoPaymentProvider.ts +12 -9
- package/server/runtimeRole.ts +15 -0
- package/server/scheduler/scheduledTaskRuntime.ts +4 -2
- package/server/scheduler/schedulerService.ts +2 -2
- package/server/taskRunBridgeRoutes.ts +8 -5
- package/share/server/fetchCommunityShares.ts +1 -1
- package/share/server/helpers.ts +19 -17
- package/share/server/importShare.ts +9 -13
- package/share/server/listCommunityShares.ts +4 -4
- package/share/server/purchaseDoc.ts +8 -16
- package/share/server/readCommunityShareCover.ts +3 -2
- package/share/server/readSharedTable.ts +14 -13
- package/share/server/tableReplication.ts +42 -39
- package/spaceAttachCommands.ts +135 -0
- package/spaceCategoryCommands.ts +6 -6
- package/spaceCommands.ts +3 -3
- package/spaceDeleteCommands.ts +2 -2
- package/spaceInternalCommandEntries.ts +2 -0
- package/spaceReadCommands.ts +3 -3
- package/tableMetaCommands.ts +2 -2
- package/taskRunCommand.ts +11 -0
- package/ai/agent/runtimeServerBase.ts +0 -37
- package/server/handlers/desktopAgentRuntimeHostFacts.ts +0 -40
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
import type { BillingAnomaly, BillingAnomalySeverity } from "./billingAnomaly";
|
|
2
|
+
import type {
|
|
3
|
+
BillingAnomalyLifecycleEvent,
|
|
4
|
+
} from "./billingAnomalyLifecycle";
|
|
5
|
+
import type { BillingAnomalyStatus } from "./billingAnomaly";
|
|
6
|
+
import type {
|
|
7
|
+
ProviderCallCompletedEvent,
|
|
8
|
+
ProviderCallEvent,
|
|
9
|
+
} from "./providerCall";
|
|
10
|
+
import type {
|
|
11
|
+
ProviderReconciliationBucket,
|
|
12
|
+
ProviderReconciliationStatus,
|
|
13
|
+
} from "./providerReconciliation";
|
|
14
|
+
|
|
15
|
+
export type ProviderBillingHealthStore = {
|
|
16
|
+
iterator(options: { gte?: string; lte?: string }): AsyncIterable<[string, unknown]>;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export type ProviderBillingHealthTotals = {
|
|
20
|
+
bucketCount: number;
|
|
21
|
+
openAnomalyCount: number;
|
|
22
|
+
totalOfficialRawProviderCostUsd: number;
|
|
23
|
+
totalLocalRawProviderCostUsd: number;
|
|
24
|
+
signedRawProviderCostDiffUsd: number;
|
|
25
|
+
absoluteRawProviderCostDiffUsd: number;
|
|
26
|
+
riskPlatformCredits: number;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type ProviderBillingAccountTotals = {
|
|
30
|
+
provider: string;
|
|
31
|
+
providerAccountKey: string;
|
|
32
|
+
providerAccountAlias?: string;
|
|
33
|
+
officialBillingAccountId?: string;
|
|
34
|
+
environment?: string;
|
|
35
|
+
requestCount: number;
|
|
36
|
+
inputTokens: number;
|
|
37
|
+
outputTokens: number;
|
|
38
|
+
rawProviderCostUsd: number;
|
|
39
|
+
platformCredits: number;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export type ProviderBillingHealthReport = {
|
|
43
|
+
since: string;
|
|
44
|
+
until: string;
|
|
45
|
+
summary: ProviderBillingHealthTotals;
|
|
46
|
+
statusCounts: Partial<Record<ProviderReconciliationStatus, number>>;
|
|
47
|
+
severityCounts: Partial<Record<BillingAnomalySeverity, number>>;
|
|
48
|
+
anomalyStatusCounts: Partial<Record<BillingAnomalyStatus, number>>;
|
|
49
|
+
byProvider: Record<
|
|
50
|
+
string,
|
|
51
|
+
ProviderBillingHealthTotals & {
|
|
52
|
+
statusCounts: Partial<Record<ProviderReconciliationStatus, number>>;
|
|
53
|
+
}
|
|
54
|
+
>;
|
|
55
|
+
byProviderAccount: Record<string, ProviderBillingAccountTotals>;
|
|
56
|
+
topBuckets: ProviderReconciliationBucket[];
|
|
57
|
+
openAnomalies: BillingAnomaly[];
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const RECONCILIATION_PREFIX = "provider-reconciliation-bucket-";
|
|
61
|
+
const RECONCILIATION_RANGE_END = `${RECONCILIATION_PREFIX}\uffff`;
|
|
62
|
+
const ANOMALY_PREFIX = "billing-anomaly-";
|
|
63
|
+
const ANOMALY_RANGE_END = `${ANOMALY_PREFIX}\uffff`;
|
|
64
|
+
const ANOMALY_LIFECYCLE_PREFIX = "billing-anomaly-lifecycle-";
|
|
65
|
+
const ANOMALY_LIFECYCLE_RANGE_END = `${ANOMALY_LIFECYCLE_PREFIX}\uffff`;
|
|
66
|
+
const PROVIDER_CALL_PREFIX = "provider-call-";
|
|
67
|
+
const PROVIDER_CALL_RANGE_END = `${PROVIDER_CALL_PREFIX}\uffff`;
|
|
68
|
+
|
|
69
|
+
const roundMoney = (value: number) => Number(value.toFixed(12));
|
|
70
|
+
|
|
71
|
+
const emptyTotals = (): ProviderBillingHealthTotals => ({
|
|
72
|
+
bucketCount: 0,
|
|
73
|
+
openAnomalyCount: 0,
|
|
74
|
+
totalOfficialRawProviderCostUsd: 0,
|
|
75
|
+
totalLocalRawProviderCostUsd: 0,
|
|
76
|
+
signedRawProviderCostDiffUsd: 0,
|
|
77
|
+
absoluteRawProviderCostDiffUsd: 0,
|
|
78
|
+
riskPlatformCredits: 0,
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
const isObject = (value: unknown): value is Record<string, unknown> =>
|
|
82
|
+
Boolean(value && typeof value === "object");
|
|
83
|
+
|
|
84
|
+
const isProviderReconciliationBucket = (
|
|
85
|
+
value: unknown
|
|
86
|
+
): value is ProviderReconciliationBucket =>
|
|
87
|
+
isObject(value) &&
|
|
88
|
+
typeof value.id === "string" &&
|
|
89
|
+
typeof value.provider === "string" &&
|
|
90
|
+
typeof value.bucketStart === "string" &&
|
|
91
|
+
typeof value.bucketEnd === "string" &&
|
|
92
|
+
isObject(value.official) &&
|
|
93
|
+
isObject(value.local) &&
|
|
94
|
+
isObject(value.diff);
|
|
95
|
+
|
|
96
|
+
const isBillingAnomaly = (value: unknown): value is BillingAnomaly =>
|
|
97
|
+
isObject(value) &&
|
|
98
|
+
typeof value.id === "string" &&
|
|
99
|
+
typeof value.kind === "string" &&
|
|
100
|
+
typeof value.severity === "string" &&
|
|
101
|
+
typeof value.status === "string" &&
|
|
102
|
+
typeof value.createdAt === "string";
|
|
103
|
+
|
|
104
|
+
const isBillingAnomalyLifecycleEvent = (
|
|
105
|
+
value: unknown
|
|
106
|
+
): value is BillingAnomalyLifecycleEvent =>
|
|
107
|
+
isObject(value) &&
|
|
108
|
+
value.recordType === "billing_anomaly_lifecycle_event" &&
|
|
109
|
+
typeof value.anomalyId === "string" &&
|
|
110
|
+
typeof value.eventId === "string" &&
|
|
111
|
+
typeof value.status === "string" &&
|
|
112
|
+
typeof value.createdAt === "string";
|
|
113
|
+
|
|
114
|
+
const isCompletedProviderCall = (
|
|
115
|
+
value: unknown
|
|
116
|
+
): value is ProviderCallCompletedEvent =>
|
|
117
|
+
isObject(value) &&
|
|
118
|
+
(value as ProviderCallEvent).status === "completed" &&
|
|
119
|
+
typeof value.provider === "string" &&
|
|
120
|
+
typeof value.model === "string" &&
|
|
121
|
+
typeof value.completedAt === "string" &&
|
|
122
|
+
isObject(value.usage);
|
|
123
|
+
|
|
124
|
+
const overlapsWindow = ({
|
|
125
|
+
start,
|
|
126
|
+
end,
|
|
127
|
+
sinceMs,
|
|
128
|
+
untilMs,
|
|
129
|
+
}: {
|
|
130
|
+
start: string;
|
|
131
|
+
end: string;
|
|
132
|
+
sinceMs: number;
|
|
133
|
+
untilMs: number;
|
|
134
|
+
}) => {
|
|
135
|
+
const startMs = Date.parse(start);
|
|
136
|
+
const endMs = Date.parse(end);
|
|
137
|
+
if (!Number.isFinite(startMs) || !Number.isFinite(endMs)) return false;
|
|
138
|
+
return startMs < untilMs && endMs > sinceMs;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const insideWindow = ({
|
|
142
|
+
createdAt,
|
|
143
|
+
sinceMs,
|
|
144
|
+
untilMs,
|
|
145
|
+
}: {
|
|
146
|
+
createdAt: string;
|
|
147
|
+
sinceMs: number;
|
|
148
|
+
untilMs: number;
|
|
149
|
+
}) => {
|
|
150
|
+
const createdMs = Date.parse(createdAt);
|
|
151
|
+
return Number.isFinite(createdMs) && createdMs >= sinceMs && createdMs < untilMs;
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
const increment = <K extends string>(
|
|
155
|
+
target: Partial<Record<K, number>>,
|
|
156
|
+
key: K
|
|
157
|
+
) => {
|
|
158
|
+
target[key] = (target[key] ?? 0) + 1;
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
const addBucketToTotals = (
|
|
162
|
+
totals: ProviderBillingHealthTotals,
|
|
163
|
+
bucket: ProviderReconciliationBucket
|
|
164
|
+
) => {
|
|
165
|
+
totals.bucketCount += 1;
|
|
166
|
+
totals.totalOfficialRawProviderCostUsd = roundMoney(
|
|
167
|
+
totals.totalOfficialRawProviderCostUsd + bucket.official.rawProviderCostUsd
|
|
168
|
+
);
|
|
169
|
+
totals.totalLocalRawProviderCostUsd = roundMoney(
|
|
170
|
+
totals.totalLocalRawProviderCostUsd + bucket.local.rawProviderCostUsd
|
|
171
|
+
);
|
|
172
|
+
totals.signedRawProviderCostDiffUsd = roundMoney(
|
|
173
|
+
totals.signedRawProviderCostDiffUsd + bucket.diff.rawProviderCostUsd
|
|
174
|
+
);
|
|
175
|
+
totals.absoluteRawProviderCostDiffUsd = roundMoney(
|
|
176
|
+
totals.absoluteRawProviderCostDiffUsd + bucket.diff.absoluteRawProviderCostUsd
|
|
177
|
+
);
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
const addAnomalyToTotals = (
|
|
181
|
+
totals: ProviderBillingHealthTotals,
|
|
182
|
+
anomaly: BillingAnomaly
|
|
183
|
+
) => {
|
|
184
|
+
totals.openAnomalyCount += 1;
|
|
185
|
+
totals.riskPlatformCredits = roundMoney(
|
|
186
|
+
totals.riskPlatformCredits + (anomaly.riskPlatformCredits ?? 0)
|
|
187
|
+
);
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
const addProviderCallToAccountTotals = (
|
|
191
|
+
target: ProviderBillingAccountTotals,
|
|
192
|
+
event: ProviderCallCompletedEvent
|
|
193
|
+
) => {
|
|
194
|
+
target.requestCount += 1;
|
|
195
|
+
target.inputTokens += event.usage.inputTokens;
|
|
196
|
+
target.outputTokens += event.usage.outputTokens;
|
|
197
|
+
if (event.cost.rawProviderCurrency === "USD") {
|
|
198
|
+
target.rawProviderCostUsd = roundMoney(
|
|
199
|
+
target.rawProviderCostUsd + (event.cost.rawProviderCost ?? 0)
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
target.platformCredits = roundMoney(
|
|
203
|
+
target.platformCredits + (event.cost.platformCredits ?? 0)
|
|
204
|
+
);
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
export async function buildProviderBillingHealthReport({
|
|
208
|
+
store,
|
|
209
|
+
since,
|
|
210
|
+
until,
|
|
211
|
+
limit = 20,
|
|
212
|
+
}: {
|
|
213
|
+
store: ProviderBillingHealthStore;
|
|
214
|
+
since: string;
|
|
215
|
+
until: string;
|
|
216
|
+
limit?: number;
|
|
217
|
+
}): Promise<ProviderBillingHealthReport> {
|
|
218
|
+
const sinceMs = Date.parse(since);
|
|
219
|
+
const untilMs = Date.parse(until);
|
|
220
|
+
if (!Number.isFinite(sinceMs) || !Number.isFinite(untilMs) || untilMs <= sinceMs) {
|
|
221
|
+
throw new Error("since and until must be valid ISO timestamps");
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const maxRows = Math.min(Math.max(Math.floor(limit), 1), 200);
|
|
225
|
+
const summary = emptyTotals();
|
|
226
|
+
const statusCounts: Partial<Record<ProviderReconciliationStatus, number>> = {};
|
|
227
|
+
const severityCounts: Partial<Record<BillingAnomalySeverity, number>> = {};
|
|
228
|
+
const anomalyStatusCounts: Partial<Record<BillingAnomalyStatus, number>> = {};
|
|
229
|
+
const byProvider: ProviderBillingHealthReport["byProvider"] = {};
|
|
230
|
+
const byProviderAccount: ProviderBillingHealthReport["byProviderAccount"] = {};
|
|
231
|
+
const buckets: ProviderReconciliationBucket[] = [];
|
|
232
|
+
const openAnomalies: BillingAnomaly[] = [];
|
|
233
|
+
const anomalyLifecycleById = new Map<string, BillingAnomalyLifecycleEvent>();
|
|
234
|
+
|
|
235
|
+
for await (const [, value] of store.iterator({
|
|
236
|
+
gte: ANOMALY_LIFECYCLE_PREFIX,
|
|
237
|
+
lte: ANOMALY_LIFECYCLE_RANGE_END,
|
|
238
|
+
})) {
|
|
239
|
+
if (!isBillingAnomalyLifecycleEvent(value)) continue;
|
|
240
|
+
const existing = anomalyLifecycleById.get(value.anomalyId);
|
|
241
|
+
if (!existing || Date.parse(value.createdAt) >= Date.parse(existing.createdAt)) {
|
|
242
|
+
anomalyLifecycleById.set(value.anomalyId, value);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
for await (const [, value] of store.iterator({
|
|
247
|
+
gte: PROVIDER_CALL_PREFIX,
|
|
248
|
+
lte: PROVIDER_CALL_RANGE_END,
|
|
249
|
+
})) {
|
|
250
|
+
if (!isCompletedProviderCall(value)) continue;
|
|
251
|
+
if (
|
|
252
|
+
!insideWindow({
|
|
253
|
+
createdAt: value.completedAt,
|
|
254
|
+
sinceMs,
|
|
255
|
+
untilMs,
|
|
256
|
+
})
|
|
257
|
+
) {
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
const accountKey = value.credential?.providerAccountKey;
|
|
261
|
+
if (!accountKey) continue;
|
|
262
|
+
const key = `${value.provider}:${accountKey}`;
|
|
263
|
+
const totals =
|
|
264
|
+
byProviderAccount[key] ??
|
|
265
|
+
({
|
|
266
|
+
provider: value.provider,
|
|
267
|
+
providerAccountKey: accountKey,
|
|
268
|
+
providerAccountAlias: value.credential?.providerAccountAlias,
|
|
269
|
+
officialBillingAccountId: value.credential?.officialBillingAccountId,
|
|
270
|
+
environment: value.credential?.environment,
|
|
271
|
+
requestCount: 0,
|
|
272
|
+
inputTokens: 0,
|
|
273
|
+
outputTokens: 0,
|
|
274
|
+
rawProviderCostUsd: 0,
|
|
275
|
+
platformCredits: 0,
|
|
276
|
+
} satisfies ProviderBillingAccountTotals);
|
|
277
|
+
addProviderCallToAccountTotals(totals, value);
|
|
278
|
+
byProviderAccount[key] = totals;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
for await (const [, value] of store.iterator({
|
|
282
|
+
gte: RECONCILIATION_PREFIX,
|
|
283
|
+
lte: RECONCILIATION_RANGE_END,
|
|
284
|
+
})) {
|
|
285
|
+
if (!isProviderReconciliationBucket(value)) continue;
|
|
286
|
+
if (
|
|
287
|
+
!overlapsWindow({
|
|
288
|
+
start: value.bucketStart,
|
|
289
|
+
end: value.bucketEnd,
|
|
290
|
+
sinceMs,
|
|
291
|
+
untilMs,
|
|
292
|
+
})
|
|
293
|
+
) {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
buckets.push(value);
|
|
298
|
+
addBucketToTotals(summary, value);
|
|
299
|
+
increment(statusCounts, value.status);
|
|
300
|
+
|
|
301
|
+
const providerTotals =
|
|
302
|
+
byProvider[value.provider] ??
|
|
303
|
+
({
|
|
304
|
+
...emptyTotals(),
|
|
305
|
+
statusCounts: {},
|
|
306
|
+
} satisfies ProviderBillingHealthReport["byProvider"][string]);
|
|
307
|
+
addBucketToTotals(providerTotals, value);
|
|
308
|
+
increment(providerTotals.statusCounts, value.status);
|
|
309
|
+
byProvider[value.provider] = providerTotals;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
for await (const [, value] of store.iterator({
|
|
313
|
+
gte: ANOMALY_PREFIX,
|
|
314
|
+
lte: ANOMALY_RANGE_END,
|
|
315
|
+
})) {
|
|
316
|
+
if (!isBillingAnomaly(value)) continue;
|
|
317
|
+
const lifecycle = anomalyLifecycleById.get(value.id);
|
|
318
|
+
const effectiveStatus = lifecycle?.status ?? value.status;
|
|
319
|
+
if (!insideWindow({ createdAt: value.createdAt, sinceMs, untilMs })) continue;
|
|
320
|
+
increment(anomalyStatusCounts, effectiveStatus);
|
|
321
|
+
if (effectiveStatus !== "open") continue;
|
|
322
|
+
openAnomalies.push({ ...value, status: effectiveStatus });
|
|
323
|
+
addAnomalyToTotals(summary, value);
|
|
324
|
+
increment(severityCounts, value.severity);
|
|
325
|
+
|
|
326
|
+
const providerTotals =
|
|
327
|
+
byProvider[value.provider] ??
|
|
328
|
+
({
|
|
329
|
+
...emptyTotals(),
|
|
330
|
+
statusCounts: {},
|
|
331
|
+
} satisfies ProviderBillingHealthReport["byProvider"][string]);
|
|
332
|
+
addAnomalyToTotals(providerTotals, value);
|
|
333
|
+
byProvider[value.provider] = providerTotals;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
const topBuckets = buckets
|
|
337
|
+
.sort(
|
|
338
|
+
(left, right) =>
|
|
339
|
+
right.diff.absoluteRawProviderCostUsd - left.diff.absoluteRawProviderCostUsd
|
|
340
|
+
)
|
|
341
|
+
.slice(0, maxRows);
|
|
342
|
+
const severityRank: Record<BillingAnomalySeverity, number> = {
|
|
343
|
+
critical: 4,
|
|
344
|
+
high: 3,
|
|
345
|
+
medium: 2,
|
|
346
|
+
low: 1,
|
|
347
|
+
};
|
|
348
|
+
openAnomalies.sort((left, right) => {
|
|
349
|
+
const severityDiff = severityRank[right.severity] - severityRank[left.severity];
|
|
350
|
+
if (severityDiff !== 0) return severityDiff;
|
|
351
|
+
return Date.parse(right.createdAt) - Date.parse(left.createdAt);
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
return {
|
|
355
|
+
since,
|
|
356
|
+
until,
|
|
357
|
+
summary,
|
|
358
|
+
statusCounts,
|
|
359
|
+
severityCounts,
|
|
360
|
+
anomalyStatusCounts,
|
|
361
|
+
byProvider,
|
|
362
|
+
byProviderAccount,
|
|
363
|
+
topBuckets,
|
|
364
|
+
openAnomalies: openAnomalies.slice(0, maxRows),
|
|
365
|
+
};
|
|
366
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import type { ProviderCredentialIdentity } from "./providerCredential";
|
|
2
|
+
|
|
3
|
+
export type ProviderCallStatus =
|
|
4
|
+
| "pending"
|
|
5
|
+
| "completed"
|
|
6
|
+
| "failed"
|
|
7
|
+
| "interrupted"
|
|
8
|
+
| "anomaly";
|
|
9
|
+
|
|
10
|
+
export type ProviderCallBillingStatus =
|
|
11
|
+
| "unpriced"
|
|
12
|
+
| "pending_ledger"
|
|
13
|
+
| "charged"
|
|
14
|
+
| "zero_policy"
|
|
15
|
+
| "failed";
|
|
16
|
+
|
|
17
|
+
export type ProviderCallBase = {
|
|
18
|
+
providerCallId: string;
|
|
19
|
+
eventId: string;
|
|
20
|
+
userId: string;
|
|
21
|
+
dialogId?: string;
|
|
22
|
+
agentId?: string;
|
|
23
|
+
provider: string;
|
|
24
|
+
model: string;
|
|
25
|
+
endpoint?: string;
|
|
26
|
+
serviceTier?: string;
|
|
27
|
+
credential?: ProviderCredentialIdentity;
|
|
28
|
+
startedAt: string;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export type ProviderCallPendingEvent = ProviderCallBase & {
|
|
32
|
+
status: "pending";
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
export type ProviderCallCompletedEvent = ProviderCallBase & {
|
|
36
|
+
status: "completed";
|
|
37
|
+
completedAt: string;
|
|
38
|
+
usage: {
|
|
39
|
+
inputTokens: number;
|
|
40
|
+
outputTokens: number;
|
|
41
|
+
cacheCreationInputTokens: number;
|
|
42
|
+
cacheReadInputTokens: number;
|
|
43
|
+
};
|
|
44
|
+
cost: {
|
|
45
|
+
rawProviderCost?: number;
|
|
46
|
+
rawProviderCurrency?: string;
|
|
47
|
+
platformCredits?: number;
|
|
48
|
+
pricingVersion?: string;
|
|
49
|
+
};
|
|
50
|
+
billingStatus: ProviderCallBillingStatus;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export type ProviderCallFailedEvent = ProviderCallBase & {
|
|
54
|
+
status: "failed";
|
|
55
|
+
failedAt: string;
|
|
56
|
+
error: {
|
|
57
|
+
name?: string;
|
|
58
|
+
message: string;
|
|
59
|
+
};
|
|
60
|
+
billingStatus: "failed";
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export type ProviderCallEvent =
|
|
64
|
+
| ProviderCallPendingEvent
|
|
65
|
+
| ProviderCallCompletedEvent
|
|
66
|
+
| ProviderCallFailedEvent;
|
|
67
|
+
|
|
68
|
+
export const buildProviderCallKey = (providerCallId: string, eventId: string) =>
|
|
69
|
+
`provider-call-${providerCallId}-event-${eventId}`;
|
|
70
|
+
|
|
71
|
+
export const createProviderCallPendingEvent = (
|
|
72
|
+
input: ProviderCallBase
|
|
73
|
+
): ProviderCallPendingEvent => ({
|
|
74
|
+
...input,
|
|
75
|
+
status: "pending",
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
export const createProviderCallCompletedEvent = (
|
|
79
|
+
input: ProviderCallBase & {
|
|
80
|
+
completedAt: string;
|
|
81
|
+
inputTokens: number;
|
|
82
|
+
outputTokens: number;
|
|
83
|
+
cacheCreationInputTokens: number;
|
|
84
|
+
cacheReadInputTokens: number;
|
|
85
|
+
rawProviderCost?: number;
|
|
86
|
+
rawProviderCurrency?: string;
|
|
87
|
+
platformCredits?: number;
|
|
88
|
+
pricingVersion?: string;
|
|
89
|
+
billingStatus: ProviderCallBillingStatus;
|
|
90
|
+
}
|
|
91
|
+
): ProviderCallCompletedEvent => ({
|
|
92
|
+
providerCallId: input.providerCallId,
|
|
93
|
+
eventId: input.eventId,
|
|
94
|
+
userId: input.userId,
|
|
95
|
+
dialogId: input.dialogId,
|
|
96
|
+
agentId: input.agentId,
|
|
97
|
+
provider: input.provider,
|
|
98
|
+
model: input.model,
|
|
99
|
+
endpoint: input.endpoint,
|
|
100
|
+
serviceTier: input.serviceTier,
|
|
101
|
+
credential: input.credential,
|
|
102
|
+
startedAt: input.startedAt,
|
|
103
|
+
status: "completed",
|
|
104
|
+
completedAt: input.completedAt,
|
|
105
|
+
usage: {
|
|
106
|
+
inputTokens: input.inputTokens,
|
|
107
|
+
outputTokens: input.outputTokens,
|
|
108
|
+
cacheCreationInputTokens: input.cacheCreationInputTokens,
|
|
109
|
+
cacheReadInputTokens: input.cacheReadInputTokens,
|
|
110
|
+
},
|
|
111
|
+
cost: {
|
|
112
|
+
rawProviderCost: input.rawProviderCost,
|
|
113
|
+
rawProviderCurrency: input.rawProviderCurrency,
|
|
114
|
+
platformCredits: input.platformCredits,
|
|
115
|
+
pricingVersion: input.pricingVersion,
|
|
116
|
+
},
|
|
117
|
+
billingStatus: input.billingStatus,
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
export const createProviderCallFailedEvent = (
|
|
121
|
+
input: ProviderCallBase & {
|
|
122
|
+
failedAt: string;
|
|
123
|
+
error: {
|
|
124
|
+
name?: string;
|
|
125
|
+
message: string;
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
): ProviderCallFailedEvent => ({
|
|
129
|
+
providerCallId: input.providerCallId,
|
|
130
|
+
eventId: input.eventId,
|
|
131
|
+
userId: input.userId,
|
|
132
|
+
dialogId: input.dialogId,
|
|
133
|
+
agentId: input.agentId,
|
|
134
|
+
provider: input.provider,
|
|
135
|
+
model: input.model,
|
|
136
|
+
endpoint: input.endpoint,
|
|
137
|
+
serviceTier: input.serviceTier,
|
|
138
|
+
credential: input.credential,
|
|
139
|
+
startedAt: input.startedAt,
|
|
140
|
+
status: "failed",
|
|
141
|
+
failedAt: input.failedAt,
|
|
142
|
+
error: input.error,
|
|
143
|
+
billingStatus: "failed",
|
|
144
|
+
});
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ProviderCallEvent,
|
|
3
|
+
type ProviderCallPendingEvent,
|
|
4
|
+
} from "./providerCall";
|
|
5
|
+
import type { ProviderDispatchIntent } from "./providerDispatchIntent";
|
|
6
|
+
import { createBillingAnomaly } from "./billingAnomaly";
|
|
7
|
+
import {
|
|
8
|
+
writeBillingAnomaly,
|
|
9
|
+
type BillingAnomalyStore,
|
|
10
|
+
} from "./billingAnomalyWriter";
|
|
11
|
+
|
|
12
|
+
export const STALE_PROVIDER_CALL_DEFAULT_TIMEOUT_MS = 10 * 60_000;
|
|
13
|
+
|
|
14
|
+
type ProviderCallSweeperStore = BillingAnomalyStore & {
|
|
15
|
+
iterator(options: { gte?: string; lte?: string }): AsyncIterable<[string, unknown]>;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export type SweepStaleProviderCallsResult = {
|
|
19
|
+
scannedProviderCalls: number;
|
|
20
|
+
stalePendingCalls: number;
|
|
21
|
+
anomaliesWritten: number;
|
|
22
|
+
anomaliesAlreadyExisted: number;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
type ProviderCallState = {
|
|
26
|
+
pending?: ProviderCallPendingEvent;
|
|
27
|
+
dispatchIntent?: ProviderDispatchIntent;
|
|
28
|
+
hasTerminal: boolean;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const PROVIDER_CALL_PREFIX = "provider-call-";
|
|
32
|
+
const PROVIDER_CALL_RANGE_END = `${PROVIDER_CALL_PREFIX}\uffff`;
|
|
33
|
+
const PROVIDER_DISPATCH_INTENT_PREFIX = "provider-dispatch-intent-";
|
|
34
|
+
const PROVIDER_DISPATCH_INTENT_RANGE_END = `${PROVIDER_DISPATCH_INTENT_PREFIX}\uffff`;
|
|
35
|
+
|
|
36
|
+
const isProviderCallEvent = (value: unknown): value is ProviderCallEvent =>
|
|
37
|
+
Boolean(
|
|
38
|
+
value &&
|
|
39
|
+
typeof value === "object" &&
|
|
40
|
+
typeof (value as { providerCallId?: unknown }).providerCallId === "string" &&
|
|
41
|
+
typeof (value as { eventId?: unknown }).eventId === "string" &&
|
|
42
|
+
typeof (value as { status?: unknown }).status === "string"
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
const isProviderDispatchIntent = (value: unknown): value is ProviderDispatchIntent =>
|
|
46
|
+
Boolean(
|
|
47
|
+
value &&
|
|
48
|
+
typeof value === "object" &&
|
|
49
|
+
typeof (value as { providerCallId?: unknown }).providerCallId === "string" &&
|
|
50
|
+
typeof (value as { intentId?: unknown }).intentId === "string" &&
|
|
51
|
+
(value as { status?: unknown }).status === "dispatching"
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
const isAlreadyExistsError = (error: unknown) =>
|
|
55
|
+
error instanceof Error &&
|
|
56
|
+
error.message.startsWith("billing anomaly already exists:");
|
|
57
|
+
|
|
58
|
+
const pendingTimeoutAnomalyId = (providerCallId: string) =>
|
|
59
|
+
`anom_provider_call_pending_timeout_${providerCallId}`;
|
|
60
|
+
|
|
61
|
+
const sentUnknownAnomalyId = (providerCallId: string) =>
|
|
62
|
+
`anom_provider_call_sent_unknown_${providerCallId}`;
|
|
63
|
+
|
|
64
|
+
export async function sweepStaleProviderCalls({
|
|
65
|
+
store,
|
|
66
|
+
nowMs = Date.now(),
|
|
67
|
+
timeoutMs = STALE_PROVIDER_CALL_DEFAULT_TIMEOUT_MS,
|
|
68
|
+
}: {
|
|
69
|
+
store: ProviderCallSweeperStore;
|
|
70
|
+
nowMs?: number;
|
|
71
|
+
timeoutMs?: number;
|
|
72
|
+
}): Promise<SweepStaleProviderCallsResult> {
|
|
73
|
+
const states = new Map<string, ProviderCallState>();
|
|
74
|
+
|
|
75
|
+
for await (const [, value] of store.iterator({
|
|
76
|
+
gte: PROVIDER_CALL_PREFIX,
|
|
77
|
+
lte: PROVIDER_CALL_RANGE_END,
|
|
78
|
+
})) {
|
|
79
|
+
if (!isProviderCallEvent(value)) continue;
|
|
80
|
+
const existing = states.get(value.providerCallId) ?? { hasTerminal: false };
|
|
81
|
+
if (value.status === "pending") {
|
|
82
|
+
existing.pending = value;
|
|
83
|
+
}
|
|
84
|
+
if (value.status === "completed" || value.status === "failed") {
|
|
85
|
+
existing.hasTerminal = true;
|
|
86
|
+
}
|
|
87
|
+
states.set(value.providerCallId, existing);
|
|
88
|
+
}
|
|
89
|
+
for await (const [, value] of store.iterator({
|
|
90
|
+
gte: PROVIDER_DISPATCH_INTENT_PREFIX,
|
|
91
|
+
lte: PROVIDER_DISPATCH_INTENT_RANGE_END,
|
|
92
|
+
})) {
|
|
93
|
+
if (!isProviderDispatchIntent(value)) continue;
|
|
94
|
+
const existing = states.get(value.providerCallId) ?? { hasTerminal: false };
|
|
95
|
+
existing.dispatchIntent = value;
|
|
96
|
+
states.set(value.providerCallId, existing);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const result: SweepStaleProviderCallsResult = {
|
|
100
|
+
scannedProviderCalls: states.size,
|
|
101
|
+
stalePendingCalls: 0,
|
|
102
|
+
anomaliesWritten: 0,
|
|
103
|
+
anomaliesAlreadyExisted: 0,
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
for (const [providerCallId, state] of states) {
|
|
107
|
+
if (!state.pending || state.hasTerminal) continue;
|
|
108
|
+
const startedAtMs = Date.parse(state.pending.startedAt);
|
|
109
|
+
if (!Number.isFinite(startedAtMs)) continue;
|
|
110
|
+
const pendingAgeMs = nowMs - startedAtMs;
|
|
111
|
+
if (pendingAgeMs < timeoutMs) continue;
|
|
112
|
+
|
|
113
|
+
result.stalePendingCalls += 1;
|
|
114
|
+
const isSentUnknown = Boolean(state.dispatchIntent);
|
|
115
|
+
const anomalyId = isSentUnknown
|
|
116
|
+
? sentUnknownAnomalyId(providerCallId)
|
|
117
|
+
: pendingTimeoutAnomalyId(providerCallId);
|
|
118
|
+
const kind = isSentUnknown
|
|
119
|
+
? "provider_call_sent_unknown"
|
|
120
|
+
: "provider_call_pending_timeout";
|
|
121
|
+
const message = isSentUnknown
|
|
122
|
+
? "Provider call was dispatched but no terminal billing evidence was recorded"
|
|
123
|
+
: "Provider call stayed pending past the billing SLA";
|
|
124
|
+
try {
|
|
125
|
+
await writeBillingAnomaly({
|
|
126
|
+
store,
|
|
127
|
+
anomaly: createBillingAnomaly({
|
|
128
|
+
anomalyId,
|
|
129
|
+
kind,
|
|
130
|
+
severity: "critical",
|
|
131
|
+
stage: "reconciliation",
|
|
132
|
+
userId: state.pending.userId,
|
|
133
|
+
dialogId: state.pending.dialogId,
|
|
134
|
+
agentId: state.pending.agentId,
|
|
135
|
+
provider: state.pending.provider,
|
|
136
|
+
model: state.pending.model,
|
|
137
|
+
providerCallId,
|
|
138
|
+
message,
|
|
139
|
+
evidence: {
|
|
140
|
+
pendingEventId: state.pending.eventId,
|
|
141
|
+
endpoint: state.pending.endpoint,
|
|
142
|
+
serviceTier: state.pending.serviceTier,
|
|
143
|
+
startedAt: state.pending.startedAt,
|
|
144
|
+
pendingAgeMs,
|
|
145
|
+
timeoutMs,
|
|
146
|
+
...(state.dispatchIntent
|
|
147
|
+
? {
|
|
148
|
+
dispatchIntentId: state.dispatchIntent.intentId,
|
|
149
|
+
dispatchCreatedAt: state.dispatchIntent.createdAt,
|
|
150
|
+
request: state.dispatchIntent.request,
|
|
151
|
+
}
|
|
152
|
+
: {}),
|
|
153
|
+
},
|
|
154
|
+
createdAt: new Date(nowMs).toISOString(),
|
|
155
|
+
}),
|
|
156
|
+
});
|
|
157
|
+
result.anomaliesWritten += 1;
|
|
158
|
+
} catch (error) {
|
|
159
|
+
if (isAlreadyExistsError(error)) {
|
|
160
|
+
result.anomaliesAlreadyExisted += 1;
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
throw error;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return result;
|
|
168
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { buildProviderCallKey, type ProviderCallEvent } from "./providerCall";
|
|
2
|
+
|
|
3
|
+
export type ProviderCallStore = {
|
|
4
|
+
get(key: string): Promise<unknown>;
|
|
5
|
+
put(key: string, value: unknown): Promise<void>;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
const isMissingRecordError = (error: unknown) => {
|
|
9
|
+
if (!error || typeof error !== "object") return false;
|
|
10
|
+
const code = (error as { code?: unknown }).code;
|
|
11
|
+
return code === "LEVEL_NOT_FOUND" || code === "NOT_FOUND";
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export async function writeProviderCallEvent({
|
|
15
|
+
store,
|
|
16
|
+
event,
|
|
17
|
+
}: {
|
|
18
|
+
store: ProviderCallStore;
|
|
19
|
+
event: ProviderCallEvent;
|
|
20
|
+
}): Promise<{ key: string }> {
|
|
21
|
+
const key = buildProviderCallKey(event.providerCallId, event.eventId);
|
|
22
|
+
let existing: unknown;
|
|
23
|
+
try {
|
|
24
|
+
existing = await store.get(key);
|
|
25
|
+
} catch (error) {
|
|
26
|
+
if (!isMissingRecordError(error)) throw error;
|
|
27
|
+
}
|
|
28
|
+
if (existing) {
|
|
29
|
+
throw new Error(`provider-call event already exists: ${key}`);
|
|
30
|
+
}
|
|
31
|
+
await store.put(key, event);
|
|
32
|
+
return { key };
|
|
33
|
+
}
|