@app-vitals/agent-admin 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +43 -0
- package/README.md +72 -2
- package/e2e/agents-page.e2e.ts +338 -0
- package/e2e/chat-degraded.e2e.ts +153 -0
- package/e2e/chat-page.e2e.ts +169 -0
- package/e2e/chat-progress.e2e.ts +206 -0
- package/e2e/chat-thread-page.e2e.ts +444 -0
- package/e2e/login-page.e2e.ts +148 -0
- package/e2e/test-server.ts +427 -0
- package/openapi.json +6098 -0
- package/openapi.platform.json +5553 -0
- package/package.json +64 -4
- package/playwright.config.ts +25 -0
- package/prisma/client/browser.ts +134 -0
- package/prisma/client/client.ts +158 -0
- package/prisma/client/commonInputTypes.ts +578 -0
- package/prisma/client/enums.ts +15 -0
- package/prisma/client/internal/class.ts +424 -0
- package/prisma/client/internal/prismaNamespace.ts +2898 -0
- package/prisma/client/internal/prismaNamespaceBrowser.ts +476 -0
- package/prisma/client/models/Account.ts +1774 -0
- package/prisma/client/models/AccountInvite.ts +1386 -0
- package/prisma/client/models/AccountMember.ts +1351 -0
- package/prisma/client/models/Agent.ts +4554 -0
- package/prisma/client/models/AgentAsk.ts +1373 -0
- package/prisma/client/models/AgentChatTokenUsageDailyByModel.ts +1638 -0
- package/prisma/client/models/AgentCronJob.ts +2416 -0
- package/prisma/client/models/AgentCronRun.ts +2858 -0
- package/prisma/client/models/AgentCronRunModelBreakdown.ts +1532 -0
- package/prisma/client/models/AgentCronRunSkillUsage.ts +1642 -0
- package/prisma/client/models/AgentEnv.ts +1421 -0
- package/prisma/client/models/AgentGitHubInstallationsSnapshot.ts +1289 -0
- package/prisma/client/models/AgentMember.ts +1316 -0
- package/prisma/client/models/AgentPhaseMethodology.ts +1355 -0
- package/prisma/client/models/AgentPlugin.ts +1421 -0
- package/prisma/client/models/AgentToken.ts +1380 -0
- package/prisma/client/models/AgentTool.ts +1351 -0
- package/prisma/client/models/AgentWorkQueueSnapshot.ts +1297 -0
- package/prisma/client/models/ChatThreadWatch.ts +1183 -0
- package/prisma/client/models/PushSubscription.ts +1233 -0
- package/prisma/client/models/SessionAlertState.ts +1187 -0
- package/prisma/client/models/SessionFollow.ts +1187 -0
- package/prisma/client/models/UserNotificationPrefs.ts +1221 -0
- package/prisma/client/models.ts +34 -0
- package/prisma/migrations/20260608000000_init/migration.sql +112 -0
- package/prisma/migrations/20260616000000_agent_members/migration.sql +18 -0
- package/prisma/migrations/20260624000000_add_agent_self_hosted/migration.sql +2 -0
- package/prisma/migrations/20260624120000_add_agent_repos/migration.sql +3 -0
- package/prisma/migrations/20260625000000_add_agent_cron_run/migration.sql +26 -0
- package/prisma/migrations/20260626000000_add_cron_run_token_fields/migration.sql +10 -0
- package/prisma/migrations/20260627000000_add_agent_chat_token_usage_daily/migration.sql +18 -0
- package/prisma/migrations/20260627010000_add_agent_chat_token_usage_fk/migration.sql +2 -0
- package/prisma/migrations/20260701000000_add_chat_threads_messages/migration.sql +37 -0
- package/prisma/migrations/20260701010000_drop_chat_threads_messages/migration.sql +17 -0
- package/prisma/migrations/20260701020000_add_cron_run_model_breakdown/migration.sql +23 -0
- package/prisma/migrations/20260701020000_chat_token_by_model/migration.sql +25 -0
- package/prisma/migrations/20260701030000_add_secret_flag_to_agent_env/migration.sql +2 -0
- package/prisma/migrations/20260701030000_drop_cron_run_legacy_columns/migration.sql +5 -0
- package/prisma/migrations/20260706000000_consolidate_cron_run_token_accounting/migration.sql +54 -0
- package/prisma/migrations/20260709000000_add_phase_to_agent_cron_run/migration.sql +8 -0
- package/prisma/migrations/20260717000000_add_item_tracking_to_agent_cron_run/migration.sql +10 -0
- package/prisma/migrations/20260717010000_add_agent_work_queue_snapshot/migration.sql +20 -0
- package/prisma/migrations/20260718000000_add_cron_phase_columns/migration.sql +25 -0
- package/prisma/migrations/20260718100000_drop_agent_cron_run_phase_column/migration.sql +8 -0
- package/prisma/migrations/20260724000000_add_agent_type_name/migration.sql +3 -0
- package/prisma/migrations/20260725232433_add_agent_author_allowlist/migration.sql +2 -0
- package/prisma/migrations/20260727120000_add_not_null_to_agent_author_allowlist/migration.sql +3 -0
- package/prisma/migrations/20260731000000_add_session_id_to_agent_cron_run/migration.sql +2 -0
- package/prisma/migrations/20260821000000_add_agent_restrict_slack_to_members/migration.sql +3 -0
- package/prisma/migrations/20260901000000_add_push_subscription_and_chat_thread_watch/migration.sql +40 -0
- package/prisma/migrations/20260903000000_add_agent_patch_author_allowlist/migration.sql +2 -0
- package/prisma/migrations/20260903120000_add_agent_review_author_allowlist/migration.sql +18 -0
- package/prisma/migrations/20260903150000_drop_agent_author_allowlist/migration.sql +14 -0
- package/prisma/migrations/20260905000000_add_started_at_index_to_agent_cron_run/migration.sql +11 -0
- package/prisma/migrations/20260906000000_add_session_follow_notification_prefs_alert_state/migration.sql +47 -0
- package/prisma/migrations/20260911000000_backfill_auto_follow_since/migration.sql +16 -0
- package/prisma/migrations/20260923115814_add_last_heartbeat_at_to_agent_cron_run/migration.sql +2 -0
- package/prisma/migrations/20260924120000_add_trial_expiry_fields_to_agent/migration.sql +6 -0
- package/prisma/migrations/20260929000000_add_agent_phase_methodology/migration.sql +21 -0
- package/prisma/migrations/20260929120000_add_agent_policy_fields/migration.sql +15 -0
- package/prisma/migrations/20261005000000_add_agent_github_installations_snapshot/migration.sql +21 -0
- package/prisma/migrations/20261008000000_add_cron_run_telemetry/migration.sql +50 -0
- package/prisma/migrations/20261008010000_add_account_models/migration.sql +48 -0
- package/prisma/migrations/20261008020000_add_cron_lockdown_disabled_at/migration.sql +3 -0
- package/prisma/migrations/20261009000000_chat_token_usage_bigint/migration.sql +7 -0
- package/prisma/migrations/20261010000000_add_cron_run_command_variant/migration.sql +2 -0
- package/prisma/migrations/20261019000000_add_agent_anthropic_identity/migration.sql +9 -0
- package/prisma/migrations/20261025000000_add_agent_ask/migration.sql +23 -0
- package/prisma/migrations/migration_lock.toml +3 -0
- package/prisma/schema/coding.prisma +85 -0
- package/prisma/schema/schema.prisma +506 -0
- package/prisma.config.ts +34 -0
- package/pwa-assets/icons/apple-touch-icon.png +0 -0
- package/pwa-assets/icons/favicon-32.png +0 -0
- package/pwa-assets/icons/icon-192.png +0 -0
- package/pwa-assets/icons/icon-512.png +0 -0
- package/pwa-assets/icons/icon-maskable-192.png +0 -0
- package/pwa-assets/icons/icon-maskable-512.png +0 -0
- package/src/account-agent-create.ts +42 -0
- package/src/account-agent-create.unit.test.ts +28 -0
- package/src/account-created-notifier.ts +71 -0
- package/src/account-created-notifier.unit.test.ts +70 -0
- package/src/account-email.ts +9 -0
- package/src/account-email.unit.test.ts +21 -0
- package/src/account-invites.ts +70 -0
- package/src/account-lifecycle.integration.test.ts +329 -0
- package/src/account-lifecycle.ts +206 -0
- package/src/account-lifecycle.unit.test.ts +108 -0
- package/src/account-members.ts +128 -0
- package/src/account-onboarding.integration.test.ts +101 -0
- package/src/account-onboarding.ts +95 -0
- package/src/account-trial-expiry-sweeper.ts +85 -0
- package/src/account-trial-warning-sweeper.integration.test.ts +228 -0
- package/src/account-trial-warning-sweeper.ts +216 -0
- package/src/accounts-api.integration.test.ts +196 -0
- package/src/accounts-api.smoke.test.ts +292 -0
- package/src/accounts-api.ts +289 -0
- package/src/accounts-api.unit.test.ts +93 -0
- package/src/accounts.integration.test.ts +178 -0
- package/src/accounts.ts +208 -0
- package/src/admin-boot-config.ts +574 -0
- package/src/admin-boot-config.unit.test.ts +28 -0
- package/src/admin-boot.boot-order.unit.test.ts +91 -0
- package/src/admin-boot.ts +690 -0
- package/src/admin-dev-login.smoke.test.ts +317 -0
- package/src/admin-extension.ts +432 -0
- package/src/admin-extension.unit.test.ts +532 -0
- package/src/admin-local-agent.smoke.test.ts +1643 -0
- package/src/admin-spec.smoke.test.ts +96 -0
- package/src/admin-spec.ts +186 -0
- package/src/admin-ui-account-nav.ts +20 -0
- package/src/admin-ui-account-pages.ts +131 -0
- package/src/admin-ui-account-pages.unit.test.ts +112 -0
- package/src/admin-ui-account.smoke.test.ts +286 -0
- package/src/admin-ui-account.ts +185 -0
- package/src/admin-ui-accounts-pages.ts +202 -0
- package/src/admin-ui-accounts-pages.unit.test.ts +138 -0
- package/src/admin-ui-accounts.smoke.test.ts +434 -0
- package/src/admin-ui-accounts.ts +266 -0
- package/src/admin-ui-layout.ts +60 -0
- package/src/admin-ui-layout.unit.test.ts +140 -0
- package/src/admin-ui-pages.ts +3489 -0
- package/src/admin-ui-pages.unit.test.ts +4355 -0
- package/src/admin-ui-push-test.smoke.test.ts +125 -0
- package/src/admin-ui-push-test.ts +102 -0
- package/src/admin-ui-styles.ts +656 -0
- package/src/admin-ui-styles.unit.test.ts +103 -0
- package/src/admin-ui.fixtures.ts +457 -0
- package/src/admin-ui.smoke.test.ts +7227 -0
- package/src/admin-ui.ts +3824 -0
- package/src/agent-account-assignment.ts +49 -0
- package/src/agent-account-assignment.unit.test.ts +99 -0
- package/src/agent-anthropic-identity.integration.test.ts +99 -0
- package/src/agent-anthropic-identity.ts +78 -0
- package/src/agent-anthropic-identity.unit.test.ts +77 -0
- package/src/agent-ask-config.ts +59 -0
- package/src/agent-ask-config.unit.test.ts +70 -0
- package/src/agent-ask.integration.test.ts +175 -0
- package/src/agent-ask.smoke.test.ts +457 -0
- package/src/agent-ask.ts +378 -0
- package/src/agent-chat-tokens.integration.test.ts +420 -0
- package/src/agent-chat-tokens.smoke.test.ts +551 -0
- package/src/agent-chat-tokens.ts +259 -0
- package/src/agent-cron-jobs.integration.test.ts +582 -0
- package/src/agent-cron-jobs.ts +615 -0
- package/src/agent-cron-jobs.unit.test.ts +420 -0
- package/src/agent-cron-run-stats.integration.test.ts +1551 -0
- package/src/agent-cron-run-stats.smoke.test.ts +953 -0
- package/src/agent-cron-run-stats.ts +752 -0
- package/src/agent-cron-runs.integration.test.ts +1491 -0
- package/src/agent-cron-runs.ts +489 -0
- package/src/agent-deletion-checklist.ts +128 -0
- package/src/agent-deletion-checklist.unit.test.ts +201 -0
- package/src/agent-deletion-per-agent.integration.test.ts +323 -0
- package/src/agent-deletion.integration.test.ts +323 -0
- package/src/agent-deletion.ts +329 -0
- package/src/agent-deletion.unit.test.ts +412 -0
- package/src/agent-envs.integration.test.ts +323 -0
- package/src/agent-envs.ts +244 -0
- package/src/agent-github-installations.integration.test.ts +115 -0
- package/src/agent-github-installations.ts +52 -0
- package/src/agent-manifest.ts +685 -0
- package/src/agent-manifest.unit.test.ts +870 -0
- package/src/agent-members.ts +69 -0
- package/src/agent-members.unit.test.ts +250 -0
- package/src/agent-patch-author-allowlist.integration.test.ts +218 -0
- package/src/agent-plugins.integration.test.ts +157 -0
- package/src/agent-plugins.ts +95 -0
- package/src/agent-plugins.unit.test.ts +43 -0
- package/src/agent-provisioner-per-agent.integration.test.ts +949 -0
- package/src/agent-provisioner.integration.test.ts +1245 -0
- package/src/agent-provisioner.ts +1650 -0
- package/src/agent-provisioner.unit.test.ts +1852 -0
- package/src/agent-tokens.integration.test.ts +150 -0
- package/src/agent-tokens.ts +112 -0
- package/src/agent-tools.integration.test.ts +129 -0
- package/src/agent-tools.ts +111 -0
- package/src/agents-api.integration.test.ts +705 -0
- package/src/agents-api.sentry.smoke.test.ts +478 -0
- package/src/agents-api.smoke.test.ts +5986 -0
- package/src/agents-api.ts +2826 -0
- package/src/agents.integration.test.ts +512 -0
- package/src/agents.ts +1097 -0
- package/src/agents.unit.test.ts +1736 -0
- package/src/anthropic-credential-guard.ts +44 -0
- package/src/anthropic-credential-guard.unit.test.ts +88 -0
- package/src/anthropic-federated-token-provider.integration.test.ts +248 -0
- package/src/anthropic-federated-token-provider.ts +249 -0
- package/src/anthropic-identity-client.fixtures.ts +237 -0
- package/src/anthropic-identity-client.integration.test.ts +342 -0
- package/src/anthropic-identity-client.ts +473 -0
- package/src/anthropic-identity-fixture-spec.integration.test.ts +169 -0
- package/src/anthropic-identity-name.integration.test.ts +83 -0
- package/src/anthropic-identity-name.ts +163 -0
- package/src/anthropic-identity-name.unit.test.ts +345 -0
- package/src/anthropic-openapi-contract.fixtures.ts +192 -0
- package/src/api-auth.ts +383 -0
- package/src/api-auth.unit.test.ts +837 -0
- package/src/api.smoke.test.ts +1225 -0
- package/src/api.ts +376 -0
- package/src/attachment-validation.ts +124 -0
- package/src/attachment-validation.unit.test.ts +218 -0
- package/src/caller-scope.ts +131 -0
- package/src/caller-scope.unit.test.ts +116 -0
- package/src/chat-credential-config.ts +22 -0
- package/src/chat-credentials-logging.unit.test.ts +92 -0
- package/src/chat-markers.ts +114 -0
- package/src/chat-markers.unit.test.ts +116 -0
- package/src/chat-service-provisioning-client.integration.test.ts +412 -0
- package/src/chat-service-provisioning-client.ts +221 -0
- package/src/chat.smoke.test.ts +1875 -0
- package/src/clock.ts +47 -0
- package/src/coding-task-store-provisioning-client.integration.test.ts +372 -0
- package/src/coding-task-store-provisioning-client.ts +156 -0
- package/src/crypto.ts +83 -0
- package/src/dev-auth-guard.ts +29 -0
- package/src/dev-auth-guard.unit.test.ts +51 -0
- package/src/errors.ts +77 -0
- package/src/errors.unit.test.ts +52 -0
- package/src/fixtures/anthropic-identity-recorded-fixtures.json +850 -0
- package/src/fixtures/anthropic-openapi/SDK_VERSION +1 -0
- package/src/fixtures/anthropic-openapi/mock-spec.json.gz +0 -0
- package/src/fixtures/chat-service-provisioning-cassette.json +156 -0
- package/src/fixtures/github-app-manifest-cassette.json +44 -0
- package/src/fixtures/google-auth-cassette.json +36 -0
- package/src/fixtures/http-chat-client-cassette.json +226 -0
- package/src/fixtures/k8s-cassette.json +470 -0
- package/src/fixtures/okta-auth-cassette.json +36 -0
- package/src/fixtures/slack-provision-cassette.json +12 -0
- package/src/fixtures/slack-provisioning-http-cassette.json +98 -0
- package/src/fixtures/task-store-provisioning-cassette.json +85 -0
- package/src/github-app-provisioning-client.integration.test.ts +149 -0
- package/src/github-app-provisioning-client.ts +161 -0
- package/src/github-app-provisioning-client.unit.test.ts +114 -0
- package/src/github-provisioning-service.ts +569 -0
- package/src/github-provisioning-service.unit.test.ts +123 -0
- package/src/google-auth-client.integration.test.ts +195 -0
- package/src/google-auth-client.ts +121 -0
- package/src/http-chat-client.integration.test.ts +554 -0
- package/src/http-chat-client.ts +533 -0
- package/src/http-chat-client.unit.test.ts +119 -0
- package/src/import-boundary.unit.test.ts +240 -0
- package/src/kubernetes-client.integration.test.ts +697 -0
- package/src/kubernetes-client.ts +1027 -0
- package/src/kubernetes-client.unit.test.ts +726 -0
- package/src/main.smoke.test.ts +91 -0
- package/src/main.ts +21 -0
- package/src/main.unit.test.ts +801 -0
- package/src/okta-auth-client.integration.test.ts +263 -0
- package/src/okta-auth-client.ts +152 -0
- package/src/openapi-schemas.ts +1310 -0
- package/src/openapi-schemas.unit.test.ts +348 -0
- package/src/per-agent-identity.fixtures.ts +305 -0
- package/src/platform-doc-route.ts +40 -0
- package/src/prisma-client.integration.test.ts +57 -0
- package/src/prisma-client.ts +187 -0
- package/src/prisma-client.unit.test.ts +178 -0
- package/src/prisma-tx.ts +23 -0
- package/src/push-content.ts +177 -0
- package/src/push-content.unit.test.ts +213 -0
- package/src/push-crypto.ts +135 -0
- package/src/push-crypto.unit.test.ts +66 -0
- package/src/push-sender.ts +169 -0
- package/src/push-sender.unit.test.ts +107 -0
- package/src/push-service.ts +332 -0
- package/src/push-service.unit.test.ts +679 -0
- package/src/push-toggle.ts +135 -0
- package/src/push-toggle.unit.test.ts +78 -0
- package/src/pwa-https-gating.smoke.test.ts +229 -0
- package/src/pwa-icons.unit.test.ts +41 -0
- package/src/pwa-routes.smoke.test.ts +276 -0
- package/src/pwa.ts +380 -0
- package/src/pwa.unit.test.ts +321 -0
- package/src/schema.unit.test.ts +122 -0
- package/src/self-serve-config.ts +52 -0
- package/src/self-serve-config.unit.test.ts +79 -0
- package/src/server.smoke.test.ts +273 -0
- package/src/server.ts +50 -0
- package/src/service-integration.ts +67 -0
- package/src/service-integration.unit.test.ts +129 -0
- package/src/slack-provisioning-client.integration.test.ts +308 -0
- package/src/slack-provisioning-client.ts +411 -0
- package/src/slack-provisioning-client.unit.test.ts +80 -0
- package/src/slack-provisioning-service.integration.test.ts +637 -0
- package/src/slack-provisioning-service.ts +451 -0
- package/src/slack-provisioning.integration.test.ts +395 -0
- package/src/specialty-manifest-loader.fixture.integration.test.ts +63 -0
- package/src/specialty-manifest-loader.ts +255 -0
- package/src/specialty-manifest-loader.unit.test.ts +301 -0
- package/src/specialty-registry.schema.unit.test.ts +35 -0
- package/src/specialty-registry.ts +236 -0
- package/src/specialty-registry.unit.test.ts +272 -0
- package/src/startup-policy-reconcile.ts +77 -0
- package/src/startup-policy-reconcile.unit.test.ts +251 -0
- package/src/theme-tokens.json +12 -0
- package/src/token-crypto.ts +60 -0
- package/src/token-crypto.unit.test.ts +182 -0
- package/src/trial-expiry-sweeper.integration.test.ts +318 -0
- package/src/trial-expiry-sweeper.ts +132 -0
- package/src/trial-expiry-sweeper.unit.test.ts +391 -0
- package/src/trial-expiry-warning-sweeper.integration.test.ts +468 -0
- package/src/trial-expiry-warning-sweeper.ts +447 -0
- package/src/trial-expiry-warning-sweeper.unit.test.ts +148 -0
- package/src/voice-contract.unit.test.ts +102 -0
- package/tsconfig.json +9 -0
|
@@ -0,0 +1,1551 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* admin/src/agent-cron-run-stats.integration.test.ts
|
|
3
|
+
* Integration tests for AgentCronRunStatsService — cron-run token aggregation.
|
|
4
|
+
*
|
|
5
|
+
* Requires DATABASE_URL_ADMIN_TEST to be set; skips otherwise.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
9
|
+
import type { PrismaClient } from "../prisma/client/client.ts";
|
|
10
|
+
import { AgentCronJobService } from "./agent-cron-jobs.ts";
|
|
11
|
+
import { AgentCronRunStatsService } from "./agent-cron-run-stats.ts";
|
|
12
|
+
import { AgentCronRunService } from "./agent-cron-runs.ts";
|
|
13
|
+
import { FixedClock } from "./clock.ts";
|
|
14
|
+
import { createAdminPrismaClient } from "./prisma-client.ts";
|
|
15
|
+
|
|
16
|
+
const TEST_DB = process.env.DATABASE_URL_ADMIN_TEST;
|
|
17
|
+
const describeOrSkip = TEST_DB ? describe : describe.skip;
|
|
18
|
+
|
|
19
|
+
function makePrisma(): PrismaClient {
|
|
20
|
+
// TEST_DB is guaranteed set — the describe block is skipped otherwise.
|
|
21
|
+
return createAdminPrismaClient(TEST_DB as string);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const FIXED_NOW = new Date("2026-01-15T12:00:00Z");
|
|
25
|
+
|
|
26
|
+
async function createAgent(
|
|
27
|
+
prisma: PrismaClient,
|
|
28
|
+
name = "Test Agent",
|
|
29
|
+
): Promise<string> {
|
|
30
|
+
const agent = await prisma.agent.create({ data: { name } });
|
|
31
|
+
return agent.id;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function createCron(
|
|
35
|
+
cronJobService: AgentCronJobService,
|
|
36
|
+
agentId: string,
|
|
37
|
+
name?: string,
|
|
38
|
+
): Promise<string> {
|
|
39
|
+
const job = await cronJobService.create(agentId, {
|
|
40
|
+
schedule: "0 9 * * *",
|
|
41
|
+
prompt: "Test prompt",
|
|
42
|
+
silent: true,
|
|
43
|
+
name,
|
|
44
|
+
});
|
|
45
|
+
return job.id;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Creates a phase child AgentCronJob row (name "shipwright-<phase>") and
|
|
50
|
+
* returns its id — the value tests pass as `phaseId` on AgentCronRun fixture
|
|
51
|
+
* rows, so the stats service's LEFT JOIN on r."phaseId" = p.id resolves the
|
|
52
|
+
* same phase label the old `r.phase` string column used to carry directly
|
|
53
|
+
* (LPC-3.1). Not wired up with parentCronId — the stats queries join on id
|
|
54
|
+
* alone, mirroring how phaseId is a denormalized pointer independent of the
|
|
55
|
+
* parent/child cron-scheduling relationship.
|
|
56
|
+
*/
|
|
57
|
+
async function createPhaseCron(
|
|
58
|
+
cronJobService: AgentCronJobService,
|
|
59
|
+
agentId: string,
|
|
60
|
+
phase: "dev-task" | "review" | "patch" | "deploy",
|
|
61
|
+
): Promise<string> {
|
|
62
|
+
const job = await cronJobService.create(agentId, {
|
|
63
|
+
schedule: "0 9 * * *",
|
|
64
|
+
prompt: "Test prompt",
|
|
65
|
+
silent: true,
|
|
66
|
+
name: `shipwright-${phase}`,
|
|
67
|
+
});
|
|
68
|
+
return job.id;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Seed token usage for a run by inserting an AgentCronRunModelBreakdown row.
|
|
73
|
+
*
|
|
74
|
+
* Since AgentCronRun no longer stores token columns directly, all token data
|
|
75
|
+
* flows through the breakdown table. This mirrors what the agents-api PATCH
|
|
76
|
+
* handler does when it receives a modelBreakdown payload.
|
|
77
|
+
*/
|
|
78
|
+
async function seedTokens(
|
|
79
|
+
prisma: PrismaClient,
|
|
80
|
+
runId: string,
|
|
81
|
+
tokens: {
|
|
82
|
+
input: number;
|
|
83
|
+
output: number;
|
|
84
|
+
cacheRead?: number;
|
|
85
|
+
cacheCreation?: number;
|
|
86
|
+
model?: string;
|
|
87
|
+
costUsd?: number;
|
|
88
|
+
},
|
|
89
|
+
): Promise<void> {
|
|
90
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
91
|
+
data: {
|
|
92
|
+
cronRunId: runId,
|
|
93
|
+
model: tokens.model ?? "legacy-unattributed",
|
|
94
|
+
inputTokens: tokens.input,
|
|
95
|
+
outputTokens: tokens.output,
|
|
96
|
+
cacheReadTokens: tokens.cacheRead ?? 0,
|
|
97
|
+
cacheCreationTokens: tokens.cacheCreation ?? 0,
|
|
98
|
+
costUsd: tokens.costUsd ?? 0,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
describeOrSkip("AgentCronRunStatsService (integration)", () => {
|
|
104
|
+
let prisma: PrismaClient;
|
|
105
|
+
let cronJobService: AgentCronJobService;
|
|
106
|
+
let runService: AgentCronRunService;
|
|
107
|
+
let statsService: AgentCronRunStatsService;
|
|
108
|
+
|
|
109
|
+
beforeEach(async () => {
|
|
110
|
+
prisma = makePrisma();
|
|
111
|
+
await prisma.agentCronRunSkillUsage.deleteMany();
|
|
112
|
+
await prisma.agentCronRunModelBreakdown.deleteMany();
|
|
113
|
+
await prisma.agentCronRun.deleteMany();
|
|
114
|
+
await prisma.agentToken.deleteMany();
|
|
115
|
+
await prisma.agentCronJob.deleteMany();
|
|
116
|
+
await prisma.agentTool.deleteMany();
|
|
117
|
+
await prisma.agentEnv.deleteMany();
|
|
118
|
+
await prisma.agent.deleteMany();
|
|
119
|
+
cronJobService = new AgentCronJobService(prisma, FixedClock(FIXED_NOW));
|
|
120
|
+
runService = new AgentCronRunService(prisma);
|
|
121
|
+
statsService = new AgentCronRunStatsService(prisma);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
afterEach(async () => {
|
|
125
|
+
await prisma.$disconnect();
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
// ─── totals ──────────────────────────────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
it("query() aggregates totals correctly across multiple runs", async () => {
|
|
131
|
+
const agentId = await createAgent(prisma);
|
|
132
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
133
|
+
|
|
134
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
135
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
136
|
+
skipped: false,
|
|
137
|
+
});
|
|
138
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
139
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
140
|
+
skipped: false,
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
await seedTokens(prisma, run1.id, {
|
|
144
|
+
input: 100,
|
|
145
|
+
output: 50,
|
|
146
|
+
cacheRead: 10,
|
|
147
|
+
cacheCreation: 5,
|
|
148
|
+
});
|
|
149
|
+
await seedTokens(prisma, run2.id, {
|
|
150
|
+
input: 200,
|
|
151
|
+
output: 100,
|
|
152
|
+
cacheRead: 20,
|
|
153
|
+
cacheCreation: 10,
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const stats = await statsService.query();
|
|
157
|
+
|
|
158
|
+
expect(stats.totals.input).toBe(300);
|
|
159
|
+
expect(stats.totals.output).toBe(150);
|
|
160
|
+
expect(stats.totals.cacheRead).toBe(30);
|
|
161
|
+
expect(stats.totals.cacheCreation).toBe(15);
|
|
162
|
+
expect(stats.totals.total).toBe(300 + 150 + 30 + 15);
|
|
163
|
+
// Breakdown rows exist (with costUsd 0), so the SUM resolves to 0 rather
|
|
164
|
+
// than NULL — cost is present and zero, not undefined.
|
|
165
|
+
expect(stats.totals.costUsd).toBe(0);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("query() excludes skipped runs from totals", async () => {
|
|
169
|
+
const agentId = await createAgent(prisma);
|
|
170
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
171
|
+
|
|
172
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
173
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
174
|
+
skipped: false,
|
|
175
|
+
});
|
|
176
|
+
await seedTokens(prisma, run1.id, { input: 100, output: 50 });
|
|
177
|
+
|
|
178
|
+
// Skipped run with tokens — must be excluded
|
|
179
|
+
await runService.create(cronId, agentId, {
|
|
180
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
181
|
+
skipped: true,
|
|
182
|
+
skipReason: "pre-check returned false",
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
const stats = await statsService.query();
|
|
186
|
+
|
|
187
|
+
expect(stats.totals.input).toBe(100);
|
|
188
|
+
expect(stats.totals.output).toBe(50);
|
|
189
|
+
expect(stats.totals.total).toBe(150);
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
it("query() returns zero totals when no runs exist", async () => {
|
|
193
|
+
const stats = await statsService.query();
|
|
194
|
+
|
|
195
|
+
expect(stats.totals.input).toBe(0);
|
|
196
|
+
expect(stats.totals.output).toBe(0);
|
|
197
|
+
expect(stats.totals.cacheRead).toBe(0);
|
|
198
|
+
expect(stats.totals.cacheCreation).toBe(0);
|
|
199
|
+
expect(stats.totals.total).toBe(0);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
// ─── byAgent ─────────────────────────────────────────────────────────────────
|
|
203
|
+
|
|
204
|
+
it("query() groups byAgent correctly across 2 agents", async () => {
|
|
205
|
+
const agentId1 = await createAgent(prisma, "Agent Alpha");
|
|
206
|
+
const agentId2 = await createAgent(prisma, "Agent Beta");
|
|
207
|
+
const cronId1 = await createCron(cronJobService, agentId1);
|
|
208
|
+
const cronId2 = await createCron(cronJobService, agentId2);
|
|
209
|
+
|
|
210
|
+
const run1 = await runService.create(cronId1, agentId1, {
|
|
211
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
212
|
+
skipped: false,
|
|
213
|
+
});
|
|
214
|
+
const run2 = await runService.create(cronId2, agentId2, {
|
|
215
|
+
startedAt: new Date("2026-01-10T10:00:00Z"),
|
|
216
|
+
skipped: false,
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
await seedTokens(prisma, run1.id, { input: 100, output: 50 });
|
|
220
|
+
await seedTokens(prisma, run2.id, { input: 300, output: 150 });
|
|
221
|
+
|
|
222
|
+
const stats = await statsService.query();
|
|
223
|
+
|
|
224
|
+
expect(stats.byAgent).toHaveLength(2);
|
|
225
|
+
|
|
226
|
+
const a1 = stats.byAgent.find((a) => a.key === agentId1);
|
|
227
|
+
const a2 = stats.byAgent.find((a) => a.key === agentId2);
|
|
228
|
+
|
|
229
|
+
expect(a1).toBeDefined();
|
|
230
|
+
expect(a1?.input).toBe(100);
|
|
231
|
+
expect(a1?.output).toBe(50);
|
|
232
|
+
|
|
233
|
+
expect(a2).toBeDefined();
|
|
234
|
+
expect(a2?.input).toBe(300);
|
|
235
|
+
expect(a2?.output).toBe(150);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it("query() byAgent excludes skipped runs", async () => {
|
|
239
|
+
const agentId = await createAgent(prisma);
|
|
240
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
241
|
+
|
|
242
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
243
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
244
|
+
skipped: false,
|
|
245
|
+
});
|
|
246
|
+
await seedTokens(prisma, run1.id, { input: 50, output: 25 });
|
|
247
|
+
|
|
248
|
+
// Skipped run — excluded
|
|
249
|
+
await runService.create(cronId, agentId, {
|
|
250
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
251
|
+
skipped: true,
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
const stats = await statsService.query();
|
|
255
|
+
|
|
256
|
+
expect(stats.byAgent).toHaveLength(1);
|
|
257
|
+
expect(stats.byAgent[0].key).toBe(agentId);
|
|
258
|
+
expect(stats.byAgent[0].input).toBe(50);
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
// ─── byCron ──────────────────────────────────────────────────────────────────
|
|
262
|
+
|
|
263
|
+
it("byCron uses cron name (key2) when AgentCronJob.name is set", async () => {
|
|
264
|
+
const agentId = await createAgent(prisma);
|
|
265
|
+
const cronId = await createCron(cronJobService, agentId, "morning-brief");
|
|
266
|
+
|
|
267
|
+
const run = await runService.create(cronId, agentId, {
|
|
268
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
269
|
+
skipped: false,
|
|
270
|
+
});
|
|
271
|
+
await seedTokens(prisma, run.id, { input: 100, output: 50 });
|
|
272
|
+
|
|
273
|
+
const stats = await statsService.query();
|
|
274
|
+
|
|
275
|
+
expect(stats.byCron).toHaveLength(1);
|
|
276
|
+
expect(stats.byCron[0].key1).toBe(agentId);
|
|
277
|
+
expect(stats.byCron[0].key2).toBe("morning-brief");
|
|
278
|
+
expect(stats.byCron[0].input).toBe(100);
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
it("byCron falls back to cronId when AgentCronJob.name is null", async () => {
|
|
282
|
+
const agentId = await createAgent(prisma);
|
|
283
|
+
const cronId = await createCron(cronJobService, agentId); // no name
|
|
284
|
+
|
|
285
|
+
const run = await runService.create(cronId, agentId, {
|
|
286
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
287
|
+
skipped: false,
|
|
288
|
+
});
|
|
289
|
+
await seedTokens(prisma, run.id, { input: 80, output: 40 });
|
|
290
|
+
|
|
291
|
+
const stats = await statsService.query();
|
|
292
|
+
|
|
293
|
+
expect(stats.byCron).toHaveLength(1);
|
|
294
|
+
expect(stats.byCron[0].key2).toBe(cronId);
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
it("byCron excludes skipped runs", async () => {
|
|
298
|
+
const agentId = await createAgent(prisma);
|
|
299
|
+
const cronId = await createCron(cronJobService, agentId, "review-cron");
|
|
300
|
+
|
|
301
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
302
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
303
|
+
skipped: false,
|
|
304
|
+
});
|
|
305
|
+
await seedTokens(prisma, run1.id, { input: 60, output: 30 });
|
|
306
|
+
|
|
307
|
+
// Skipped run — excluded
|
|
308
|
+
await runService.create(cronId, agentId, {
|
|
309
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
310
|
+
skipped: true,
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
const stats = await statsService.query();
|
|
314
|
+
|
|
315
|
+
expect(stats.byCron).toHaveLength(1);
|
|
316
|
+
expect(stats.byCron[0].input).toBe(60);
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
it("byCron groups a single cron into separate rows per phase (WL-3.5, LPC-3.1)", async () => {
|
|
320
|
+
const agentId = await createAgent(prisma);
|
|
321
|
+
const cronId = await createCron(cronJobService, agentId, "shipwright-loop");
|
|
322
|
+
const devTaskPhaseId = await createPhaseCron(
|
|
323
|
+
cronJobService,
|
|
324
|
+
agentId,
|
|
325
|
+
"dev-task",
|
|
326
|
+
);
|
|
327
|
+
const reviewPhaseId = await createPhaseCron(
|
|
328
|
+
cronJobService,
|
|
329
|
+
agentId,
|
|
330
|
+
"review",
|
|
331
|
+
);
|
|
332
|
+
|
|
333
|
+
const devTaskRun = await runService.create(cronId, agentId, {
|
|
334
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
335
|
+
skipped: false,
|
|
336
|
+
phaseId: devTaskPhaseId,
|
|
337
|
+
});
|
|
338
|
+
await seedTokens(prisma, devTaskRun.id, { input: 100, output: 50 });
|
|
339
|
+
|
|
340
|
+
const reviewRun = await runService.create(cronId, agentId, {
|
|
341
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
342
|
+
skipped: false,
|
|
343
|
+
phaseId: reviewPhaseId,
|
|
344
|
+
});
|
|
345
|
+
await seedTokens(prisma, reviewRun.id, { input: 200, output: 100 });
|
|
346
|
+
|
|
347
|
+
const stats = await statsService.query();
|
|
348
|
+
|
|
349
|
+
// Same (agentId, cronId) but two phases → two distinct byCron rows.
|
|
350
|
+
expect(stats.byCron).toHaveLength(2);
|
|
351
|
+
const devTaskRow = stats.byCron.find((r) => r.phase === "dev-task");
|
|
352
|
+
const reviewRow = stats.byCron.find((r) => r.phase === "review");
|
|
353
|
+
|
|
354
|
+
expect(devTaskRow).toBeDefined();
|
|
355
|
+
expect(devTaskRow?.key1).toBe(agentId);
|
|
356
|
+
expect(devTaskRow?.key2).toBe("shipwright-loop");
|
|
357
|
+
expect(devTaskRow?.input).toBe(100);
|
|
358
|
+
|
|
359
|
+
expect(reviewRow).toBeDefined();
|
|
360
|
+
expect(reviewRow?.key1).toBe(agentId);
|
|
361
|
+
expect(reviewRow?.key2).toBe("shipwright-loop");
|
|
362
|
+
expect(reviewRow?.input).toBe(200);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
it("byCron collapses legacy runs with a null phase into a single row per cron (no fragmentation)", async () => {
|
|
366
|
+
const agentId = await createAgent(prisma);
|
|
367
|
+
const cronId = await createCron(cronJobService, agentId, "morning-brief");
|
|
368
|
+
|
|
369
|
+
// Two legacy runs, neither with a phase set — must collapse into ONE row,
|
|
370
|
+
// identical to today's cronId-only grouping (AC#1).
|
|
371
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
372
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
373
|
+
skipped: false,
|
|
374
|
+
});
|
|
375
|
+
await seedTokens(prisma, run1.id, { input: 40, output: 20 });
|
|
376
|
+
|
|
377
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
378
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
379
|
+
skipped: false,
|
|
380
|
+
});
|
|
381
|
+
await seedTokens(prisma, run2.id, { input: 60, output: 30 });
|
|
382
|
+
|
|
383
|
+
const stats = await statsService.query();
|
|
384
|
+
|
|
385
|
+
expect(stats.byCron).toHaveLength(1);
|
|
386
|
+
expect(stats.byCron[0].key1).toBe(agentId);
|
|
387
|
+
expect(stats.byCron[0].key2).toBe("morning-brief");
|
|
388
|
+
expect(stats.byCron[0].phase).toBeNull();
|
|
389
|
+
// Both legacy runs summed into the single collapsed row.
|
|
390
|
+
expect(stats.byCron[0].input).toBe(100);
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
// ─── byModel ─────────────────────────────────────────────────────────────────
|
|
394
|
+
|
|
395
|
+
it("query() groups byModel correctly across 2 models", async () => {
|
|
396
|
+
const agentId = await createAgent(prisma);
|
|
397
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
398
|
+
|
|
399
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
400
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
401
|
+
skipped: false,
|
|
402
|
+
});
|
|
403
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
404
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
405
|
+
skipped: false,
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
409
|
+
data: {
|
|
410
|
+
cronRunId: run1.id,
|
|
411
|
+
model: "claude-sonnet-4-5",
|
|
412
|
+
inputTokens: 100,
|
|
413
|
+
outputTokens: 50,
|
|
414
|
+
cacheReadTokens: 0,
|
|
415
|
+
cacheCreationTokens: 0,
|
|
416
|
+
costUsd: 0,
|
|
417
|
+
},
|
|
418
|
+
});
|
|
419
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
420
|
+
data: {
|
|
421
|
+
cronRunId: run2.id,
|
|
422
|
+
model: "claude-opus-4-5",
|
|
423
|
+
inputTokens: 200,
|
|
424
|
+
outputTokens: 100,
|
|
425
|
+
cacheReadTokens: 0,
|
|
426
|
+
cacheCreationTokens: 0,
|
|
427
|
+
costUsd: 0,
|
|
428
|
+
},
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
const stats = await statsService.query();
|
|
432
|
+
|
|
433
|
+
expect(stats.byModel).toHaveLength(2);
|
|
434
|
+
|
|
435
|
+
const sonnet = stats.byModel.find((m) => m.key2 === "claude-sonnet-4-5");
|
|
436
|
+
const opus = stats.byModel.find((m) => m.key2 === "claude-opus-4-5");
|
|
437
|
+
|
|
438
|
+
expect(sonnet).toBeDefined();
|
|
439
|
+
expect(sonnet?.key1).toBe(agentId);
|
|
440
|
+
expect(sonnet?.input).toBe(100);
|
|
441
|
+
|
|
442
|
+
expect(opus).toBeDefined();
|
|
443
|
+
expect(opus?.key1).toBe(agentId);
|
|
444
|
+
expect(opus?.input).toBe(200);
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
it("query() excludes runs without breakdown rows from all token sums", async () => {
|
|
448
|
+
const agentId = await createAgent(prisma);
|
|
449
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
450
|
+
|
|
451
|
+
// Run without any breakdown rows — now the sole source of token data is
|
|
452
|
+
// AgentCronRunModelBreakdown, so a run with no breakdown contributes zero
|
|
453
|
+
// to every aggregation and never appears in byModel.
|
|
454
|
+
await runService.create(cronId, agentId, {
|
|
455
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
456
|
+
skipped: false,
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
// Run with a breakdown row — contributes to totals and byModel.
|
|
460
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
461
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
462
|
+
skipped: false,
|
|
463
|
+
});
|
|
464
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
465
|
+
data: {
|
|
466
|
+
cronRunId: run2.id,
|
|
467
|
+
model: "claude-sonnet-4-5",
|
|
468
|
+
inputTokens: 200,
|
|
469
|
+
outputTokens: 100,
|
|
470
|
+
cacheReadTokens: 0,
|
|
471
|
+
cacheCreationTokens: 0,
|
|
472
|
+
costUsd: 0,
|
|
473
|
+
},
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
const stats = await statsService.query();
|
|
477
|
+
|
|
478
|
+
// Totals come only from the run with a breakdown row.
|
|
479
|
+
expect(stats.totals.input).toBe(200);
|
|
480
|
+
// byModel only includes the run with breakdown rows.
|
|
481
|
+
expect(stats.byModel).toHaveLength(1);
|
|
482
|
+
expect(stats.byModel[0].key2).toBe("claude-sonnet-4-5");
|
|
483
|
+
expect(stats.byModel[0].input).toBe(200);
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
// ─── daily ───────────────────────────────────────────────────────────────────
|
|
487
|
+
|
|
488
|
+
it("query() groups runs into daily buckets by startedAt date", async () => {
|
|
489
|
+
const agentId = await createAgent(prisma);
|
|
490
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
491
|
+
|
|
492
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
493
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
494
|
+
skipped: false,
|
|
495
|
+
});
|
|
496
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
497
|
+
startedAt: new Date("2026-01-10T15:00:00Z"),
|
|
498
|
+
skipped: false,
|
|
499
|
+
});
|
|
500
|
+
const run3 = await runService.create(cronId, agentId, {
|
|
501
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
502
|
+
skipped: false,
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
await seedTokens(prisma, run1.id, { input: 100, output: 50 });
|
|
506
|
+
await seedTokens(prisma, run2.id, { input: 150, output: 75 });
|
|
507
|
+
await seedTokens(prisma, run3.id, { input: 200, output: 100 });
|
|
508
|
+
|
|
509
|
+
const stats = await statsService.query();
|
|
510
|
+
|
|
511
|
+
expect(stats.daily).toHaveLength(2);
|
|
512
|
+
|
|
513
|
+
const day1 = stats.daily.find((d) => d.period === "2026-01-10");
|
|
514
|
+
const day2 = stats.daily.find((d) => d.period === "2026-01-11");
|
|
515
|
+
|
|
516
|
+
expect(day1).toBeDefined();
|
|
517
|
+
expect(day1?.input).toBe(250); // 100 + 150
|
|
518
|
+
expect(day1?.output).toBe(125); // 50 + 75
|
|
519
|
+
|
|
520
|
+
expect(day2).toBeDefined();
|
|
521
|
+
expect(day2?.input).toBe(200);
|
|
522
|
+
expect(day2?.output).toBe(100);
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it("query() daily excludes skipped runs", async () => {
|
|
526
|
+
const agentId = await createAgent(prisma);
|
|
527
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
528
|
+
|
|
529
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
530
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
531
|
+
skipped: false,
|
|
532
|
+
});
|
|
533
|
+
await seedTokens(prisma, run1.id, { input: 100, output: 50 });
|
|
534
|
+
|
|
535
|
+
// Skipped run on same day — excluded
|
|
536
|
+
await runService.create(cronId, agentId, {
|
|
537
|
+
startedAt: new Date("2026-01-10T10:00:00Z"),
|
|
538
|
+
skipped: true,
|
|
539
|
+
});
|
|
540
|
+
|
|
541
|
+
const stats = await statsService.query();
|
|
542
|
+
|
|
543
|
+
expect(stats.daily).toHaveLength(1);
|
|
544
|
+
expect(stats.daily[0].period).toBe("2026-01-10");
|
|
545
|
+
expect(stats.daily[0].input).toBe(100);
|
|
546
|
+
});
|
|
547
|
+
|
|
548
|
+
// ─── from/to filtering ───────────────────────────────────────────────────────
|
|
549
|
+
|
|
550
|
+
it("query(from, to) filters runs by startedAt", async () => {
|
|
551
|
+
const agentId = await createAgent(prisma);
|
|
552
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
553
|
+
|
|
554
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
555
|
+
startedAt: new Date("2026-01-05T09:00:00Z"),
|
|
556
|
+
skipped: false,
|
|
557
|
+
});
|
|
558
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
559
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
560
|
+
skipped: false,
|
|
561
|
+
});
|
|
562
|
+
const run3 = await runService.create(cronId, agentId, {
|
|
563
|
+
startedAt: new Date("2026-01-20T09:00:00Z"),
|
|
564
|
+
skipped: false,
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
await seedTokens(prisma, run1.id, { input: 10, output: 5 });
|
|
568
|
+
await seedTokens(prisma, run2.id, { input: 100, output: 50 });
|
|
569
|
+
await seedTokens(prisma, run3.id, { input: 1000, output: 500 });
|
|
570
|
+
|
|
571
|
+
// Only include run2
|
|
572
|
+
const stats = await statsService.query(
|
|
573
|
+
"2026-01-08T00:00:00Z",
|
|
574
|
+
"2026-01-15T00:00:00Z",
|
|
575
|
+
);
|
|
576
|
+
|
|
577
|
+
expect(stats.totals.input).toBe(100);
|
|
578
|
+
expect(stats.totals.output).toBe(50);
|
|
579
|
+
expect(stats.daily).toHaveLength(1);
|
|
580
|
+
expect(stats.daily[0].period).toBe("2026-01-10");
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
it("query(from) without to includes all runs from that date", async () => {
|
|
584
|
+
const agentId = await createAgent(prisma);
|
|
585
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
586
|
+
|
|
587
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
588
|
+
startedAt: new Date("2026-01-05T09:00:00Z"),
|
|
589
|
+
skipped: false,
|
|
590
|
+
});
|
|
591
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
592
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
593
|
+
skipped: false,
|
|
594
|
+
});
|
|
595
|
+
|
|
596
|
+
await seedTokens(prisma, run1.id, { input: 10, output: 5 });
|
|
597
|
+
await seedTokens(prisma, run2.id, { input: 100, output: 50 });
|
|
598
|
+
|
|
599
|
+
const stats = await statsService.query("2026-01-08T00:00:00Z");
|
|
600
|
+
|
|
601
|
+
// Only run2 is in range
|
|
602
|
+
expect(stats.totals.input).toBe(100);
|
|
603
|
+
});
|
|
604
|
+
|
|
605
|
+
// ─── byModel with breakdown rows ─────────────────────────────────────────────
|
|
606
|
+
|
|
607
|
+
it("byModel uses breakdown rows when present, splitting tokens across models", async () => {
|
|
608
|
+
const agentId = await createAgent(prisma);
|
|
609
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
610
|
+
|
|
611
|
+
// Run with breakdown rows: one run used both sonnet and haiku
|
|
612
|
+
const run = await runService.create(cronId, agentId, {
|
|
613
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
614
|
+
skipped: false,
|
|
615
|
+
});
|
|
616
|
+
// Write breakdown rows directly (simulating what agents-api PATCH handler does)
|
|
617
|
+
await prisma.agentCronRunModelBreakdown.createMany({
|
|
618
|
+
data: [
|
|
619
|
+
{
|
|
620
|
+
cronRunId: run.id,
|
|
621
|
+
model: "claude-sonnet-4-5",
|
|
622
|
+
inputTokens: 200,
|
|
623
|
+
outputTokens: 100,
|
|
624
|
+
cacheReadTokens: 8,
|
|
625
|
+
cacheCreationTokens: 4,
|
|
626
|
+
costUsd: 0.002,
|
|
627
|
+
},
|
|
628
|
+
{
|
|
629
|
+
cronRunId: run.id,
|
|
630
|
+
model: "claude-haiku-4-5",
|
|
631
|
+
inputTokens: 100,
|
|
632
|
+
outputTokens: 50,
|
|
633
|
+
cacheReadTokens: 2,
|
|
634
|
+
cacheCreationTokens: 1,
|
|
635
|
+
costUsd: 0.001,
|
|
636
|
+
},
|
|
637
|
+
],
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
const stats = await statsService.query();
|
|
641
|
+
|
|
642
|
+
// totals must not double-count tokens from the multi-model run
|
|
643
|
+
expect(stats.totals.input).toBe(300);
|
|
644
|
+
expect(stats.totals.output).toBe(150);
|
|
645
|
+
|
|
646
|
+
// byModel should show 2 entries (from breakdown), not 1 (from dominant model)
|
|
647
|
+
expect(stats.byModel).toHaveLength(2);
|
|
648
|
+
|
|
649
|
+
const sonnet = stats.byModel.find((m) => m.key2 === "claude-sonnet-4-5");
|
|
650
|
+
const haiku = stats.byModel.find((m) => m.key2 === "claude-haiku-4-5");
|
|
651
|
+
|
|
652
|
+
expect(sonnet).toBeDefined();
|
|
653
|
+
expect(sonnet?.key1).toBe(agentId);
|
|
654
|
+
expect(sonnet?.input).toBe(200);
|
|
655
|
+
expect(sonnet?.output).toBe(100);
|
|
656
|
+
|
|
657
|
+
expect(haiku).toBeDefined();
|
|
658
|
+
expect(haiku?.key1).toBe(agentId);
|
|
659
|
+
expect(haiku?.input).toBe(100);
|
|
660
|
+
expect(haiku?.output).toBe(50);
|
|
661
|
+
});
|
|
662
|
+
|
|
663
|
+
// ─── byCronModel ─────────────────────────────────────────────────────────────
|
|
664
|
+
|
|
665
|
+
it("query() includes byCronModel breakdown by agentId:cronName and model", async () => {
|
|
666
|
+
const agentId1 = await createAgent(prisma, "Agent One");
|
|
667
|
+
const agentId2 = await createAgent(prisma, "Agent Two");
|
|
668
|
+
const cronId1 = await createCron(cronJobService, agentId1, "morning-brief");
|
|
669
|
+
const cronId2 = await createCron(cronJobService, agentId2, "review-cron");
|
|
670
|
+
|
|
671
|
+
// Run A: agent1/cron1 with sonnet breakdown
|
|
672
|
+
const runA = await runService.create(cronId1, agentId1, {
|
|
673
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
674
|
+
skipped: false,
|
|
675
|
+
});
|
|
676
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
677
|
+
data: {
|
|
678
|
+
cronRunId: runA.id,
|
|
679
|
+
model: "claude-sonnet-4-5",
|
|
680
|
+
inputTokens: 100,
|
|
681
|
+
outputTokens: 50,
|
|
682
|
+
cacheReadTokens: 0,
|
|
683
|
+
cacheCreationTokens: 0,
|
|
684
|
+
costUsd: 0.001,
|
|
685
|
+
},
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
// Run B: agent1/cron1 again with haiku breakdown (same cron, different model)
|
|
689
|
+
const runB = await runService.create(cronId1, agentId1, {
|
|
690
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
691
|
+
skipped: false,
|
|
692
|
+
});
|
|
693
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
694
|
+
data: {
|
|
695
|
+
cronRunId: runB.id,
|
|
696
|
+
model: "claude-haiku-4-5",
|
|
697
|
+
inputTokens: 200,
|
|
698
|
+
outputTokens: 100,
|
|
699
|
+
cacheReadTokens: 0,
|
|
700
|
+
cacheCreationTokens: 0,
|
|
701
|
+
costUsd: 0.0005,
|
|
702
|
+
},
|
|
703
|
+
});
|
|
704
|
+
|
|
705
|
+
// Run C: agent2/cron2 with opus breakdown
|
|
706
|
+
const runC = await runService.create(cronId2, agentId2, {
|
|
707
|
+
startedAt: new Date("2026-01-10T10:00:00Z"),
|
|
708
|
+
skipped: false,
|
|
709
|
+
});
|
|
710
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
711
|
+
data: {
|
|
712
|
+
cronRunId: runC.id,
|
|
713
|
+
model: "claude-opus-4-5",
|
|
714
|
+
inputTokens: 300,
|
|
715
|
+
outputTokens: 150,
|
|
716
|
+
cacheReadTokens: 0,
|
|
717
|
+
cacheCreationTokens: 0,
|
|
718
|
+
costUsd: 0.003,
|
|
719
|
+
},
|
|
720
|
+
});
|
|
721
|
+
|
|
722
|
+
const stats = await statsService.query();
|
|
723
|
+
|
|
724
|
+
// byCronModel keys: key1 = "agentId:cronName", key2 = model
|
|
725
|
+
expect(stats.byCronModel).toHaveLength(3);
|
|
726
|
+
|
|
727
|
+
const key1A = `${agentId1}:morning-brief`;
|
|
728
|
+
const key1C = `${agentId2}:review-cron`;
|
|
729
|
+
|
|
730
|
+
const sonnetRow = stats.byCronModel.find(
|
|
731
|
+
(r) => r.key1 === key1A && r.key2 === "claude-sonnet-4-5",
|
|
732
|
+
);
|
|
733
|
+
expect(sonnetRow).toBeDefined();
|
|
734
|
+
expect(sonnetRow?.input).toBe(100);
|
|
735
|
+
expect(sonnetRow?.output).toBe(50);
|
|
736
|
+
|
|
737
|
+
const haikuRow = stats.byCronModel.find(
|
|
738
|
+
(r) => r.key1 === key1A && r.key2 === "claude-haiku-4-5",
|
|
739
|
+
);
|
|
740
|
+
expect(haikuRow).toBeDefined();
|
|
741
|
+
expect(haikuRow?.input).toBe(200);
|
|
742
|
+
expect(haikuRow?.output).toBe(100);
|
|
743
|
+
|
|
744
|
+
const opusRow = stats.byCronModel.find(
|
|
745
|
+
(r) => r.key1 === key1C && r.key2 === "claude-opus-4-5",
|
|
746
|
+
);
|
|
747
|
+
expect(opusRow).toBeDefined();
|
|
748
|
+
expect(opusRow?.input).toBe(300);
|
|
749
|
+
expect(opusRow?.output).toBe(150);
|
|
750
|
+
});
|
|
751
|
+
|
|
752
|
+
it("byCronModel groups a single cron+model into separate rows per phase (WL-3.5, LPC-3.1)", async () => {
|
|
753
|
+
const agentId = await createAgent(prisma);
|
|
754
|
+
const cronId = await createCron(cronJobService, agentId, "shipwright-loop");
|
|
755
|
+
const devTaskPhaseId = await createPhaseCron(
|
|
756
|
+
cronJobService,
|
|
757
|
+
agentId,
|
|
758
|
+
"dev-task",
|
|
759
|
+
);
|
|
760
|
+
const reviewPhaseId = await createPhaseCron(
|
|
761
|
+
cronJobService,
|
|
762
|
+
agentId,
|
|
763
|
+
"review",
|
|
764
|
+
);
|
|
765
|
+
|
|
766
|
+
const devTaskRun = await runService.create(cronId, agentId, {
|
|
767
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
768
|
+
skipped: false,
|
|
769
|
+
phaseId: devTaskPhaseId,
|
|
770
|
+
});
|
|
771
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
772
|
+
data: {
|
|
773
|
+
cronRunId: devTaskRun.id,
|
|
774
|
+
model: "claude-sonnet-4-5",
|
|
775
|
+
inputTokens: 100,
|
|
776
|
+
outputTokens: 50,
|
|
777
|
+
cacheReadTokens: 0,
|
|
778
|
+
cacheCreationTokens: 0,
|
|
779
|
+
costUsd: 0.001,
|
|
780
|
+
},
|
|
781
|
+
});
|
|
782
|
+
|
|
783
|
+
const reviewRun = await runService.create(cronId, agentId, {
|
|
784
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
785
|
+
skipped: false,
|
|
786
|
+
phaseId: reviewPhaseId,
|
|
787
|
+
});
|
|
788
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
789
|
+
data: {
|
|
790
|
+
cronRunId: reviewRun.id,
|
|
791
|
+
model: "claude-sonnet-4-5",
|
|
792
|
+
inputTokens: 200,
|
|
793
|
+
outputTokens: 100,
|
|
794
|
+
cacheReadTokens: 0,
|
|
795
|
+
cacheCreationTokens: 0,
|
|
796
|
+
costUsd: 0.002,
|
|
797
|
+
},
|
|
798
|
+
});
|
|
799
|
+
|
|
800
|
+
const stats = await statsService.query();
|
|
801
|
+
|
|
802
|
+
// Same key1 (agentId:cronName) and key2 (model) but different phases →
|
|
803
|
+
// two distinct byCronModel rows.
|
|
804
|
+
expect(stats.byCronModel).toHaveLength(2);
|
|
805
|
+
const key1 = `${agentId}:shipwright-loop`;
|
|
806
|
+
const devTaskRow = stats.byCronModel.find(
|
|
807
|
+
(r) => r.key1 === key1 && r.phase === "dev-task",
|
|
808
|
+
);
|
|
809
|
+
const reviewRow = stats.byCronModel.find(
|
|
810
|
+
(r) => r.key1 === key1 && r.phase === "review",
|
|
811
|
+
);
|
|
812
|
+
|
|
813
|
+
expect(devTaskRow).toBeDefined();
|
|
814
|
+
expect(devTaskRow?.key2).toBe("claude-sonnet-4-5");
|
|
815
|
+
expect(devTaskRow?.input).toBe(100);
|
|
816
|
+
|
|
817
|
+
expect(reviewRow).toBeDefined();
|
|
818
|
+
expect(reviewRow?.key2).toBe("claude-sonnet-4-5");
|
|
819
|
+
expect(reviewRow?.input).toBe(200);
|
|
820
|
+
});
|
|
821
|
+
|
|
822
|
+
it("byCronModel collapses legacy runs with a null phase into a single row (no fragmentation)", async () => {
|
|
823
|
+
const agentId = await createAgent(prisma);
|
|
824
|
+
const cronId = await createCron(cronJobService, agentId, "legacy-cron");
|
|
825
|
+
|
|
826
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
827
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
828
|
+
skipped: false,
|
|
829
|
+
});
|
|
830
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
831
|
+
data: {
|
|
832
|
+
cronRunId: run1.id,
|
|
833
|
+
model: "claude-sonnet-4-5",
|
|
834
|
+
inputTokens: 50,
|
|
835
|
+
outputTokens: 25,
|
|
836
|
+
cacheReadTokens: 0,
|
|
837
|
+
cacheCreationTokens: 0,
|
|
838
|
+
costUsd: 0.0005,
|
|
839
|
+
},
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
843
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
844
|
+
skipped: false,
|
|
845
|
+
});
|
|
846
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
847
|
+
data: {
|
|
848
|
+
cronRunId: run2.id,
|
|
849
|
+
model: "claude-sonnet-4-5",
|
|
850
|
+
inputTokens: 70,
|
|
851
|
+
outputTokens: 35,
|
|
852
|
+
cacheReadTokens: 0,
|
|
853
|
+
cacheCreationTokens: 0,
|
|
854
|
+
costUsd: 0.0007,
|
|
855
|
+
},
|
|
856
|
+
});
|
|
857
|
+
|
|
858
|
+
const stats = await statsService.query();
|
|
859
|
+
|
|
860
|
+
expect(stats.byCronModel).toHaveLength(1);
|
|
861
|
+
expect(stats.byCronModel[0].key1).toBe(`${agentId}:legacy-cron`);
|
|
862
|
+
expect(stats.byCronModel[0].key2).toBe("claude-sonnet-4-5");
|
|
863
|
+
expect(stats.byCronModel[0].phase).toBeNull();
|
|
864
|
+
expect(stats.byCronModel[0].input).toBe(120);
|
|
865
|
+
});
|
|
866
|
+
|
|
867
|
+
// ─── 3+ runs across 2 agents and 2 crons (acceptance test) ──────────────────
|
|
868
|
+
|
|
869
|
+
it("seed 3+ runs across 2 agents and 2 crons; each dimension aggregates correctly; skipped excluded", async () => {
|
|
870
|
+
const agentId1 = await createAgent(prisma, "Agent One");
|
|
871
|
+
const agentId2 = await createAgent(prisma, "Agent Two");
|
|
872
|
+
const cronId1 = await createCron(cronJobService, agentId1, "cron-alpha");
|
|
873
|
+
const cronId2 = await createCron(cronJobService, agentId2, "cron-beta");
|
|
874
|
+
|
|
875
|
+
// Run A: agent1, cron1, sonnet model
|
|
876
|
+
const runA = await runService.create(cronId1, agentId1, {
|
|
877
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
878
|
+
skipped: false,
|
|
879
|
+
});
|
|
880
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
881
|
+
data: {
|
|
882
|
+
cronRunId: runA.id,
|
|
883
|
+
model: "claude-sonnet-4-5",
|
|
884
|
+
inputTokens: 100,
|
|
885
|
+
outputTokens: 50,
|
|
886
|
+
cacheReadTokens: 10,
|
|
887
|
+
cacheCreationTokens: 5,
|
|
888
|
+
costUsd: 0.001,
|
|
889
|
+
},
|
|
890
|
+
});
|
|
891
|
+
|
|
892
|
+
// Run B: agent2, cron2, opus model
|
|
893
|
+
const runB = await runService.create(cronId2, agentId2, {
|
|
894
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
895
|
+
skipped: false,
|
|
896
|
+
});
|
|
897
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
898
|
+
data: {
|
|
899
|
+
cronRunId: runB.id,
|
|
900
|
+
model: "claude-opus-4-5",
|
|
901
|
+
inputTokens: 200,
|
|
902
|
+
outputTokens: 100,
|
|
903
|
+
cacheReadTokens: 20,
|
|
904
|
+
cacheCreationTokens: 10,
|
|
905
|
+
costUsd: 0.002,
|
|
906
|
+
},
|
|
907
|
+
});
|
|
908
|
+
|
|
909
|
+
// Run C: agent1, cron1, sonnet model (second run same cron)
|
|
910
|
+
const runC = await runService.create(cronId1, agentId1, {
|
|
911
|
+
startedAt: new Date("2026-01-12T09:00:00Z"),
|
|
912
|
+
skipped: false,
|
|
913
|
+
});
|
|
914
|
+
await prisma.agentCronRunModelBreakdown.create({
|
|
915
|
+
data: {
|
|
916
|
+
cronRunId: runC.id,
|
|
917
|
+
model: "claude-sonnet-4-5",
|
|
918
|
+
inputTokens: 300,
|
|
919
|
+
outputTokens: 150,
|
|
920
|
+
cacheReadTokens: 30,
|
|
921
|
+
cacheCreationTokens: 15,
|
|
922
|
+
costUsd: 0.003,
|
|
923
|
+
},
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
// Skipped run (should be excluded from everything)
|
|
927
|
+
await runService.create(cronId1, agentId1, {
|
|
928
|
+
startedAt: new Date("2026-01-12T10:00:00Z"),
|
|
929
|
+
skipped: true,
|
|
930
|
+
skipReason: "pre-check false",
|
|
931
|
+
});
|
|
932
|
+
|
|
933
|
+
const stats = await statsService.query();
|
|
934
|
+
|
|
935
|
+
// totals: runs A+B+C only (skipped excluded)
|
|
936
|
+
expect(stats.totals.input).toBe(600);
|
|
937
|
+
expect(stats.totals.output).toBe(300);
|
|
938
|
+
expect(stats.totals.cacheRead).toBe(60);
|
|
939
|
+
expect(stats.totals.cacheCreation).toBe(30);
|
|
940
|
+
expect(stats.totals.total).toBe(600 + 300 + 60 + 30);
|
|
941
|
+
// costUsd is aggregated from breakdown rows via LEFT JOIN
|
|
942
|
+
expect(stats.totals.costUsd).toBeCloseTo(0.001 + 0.002 + 0.003);
|
|
943
|
+
|
|
944
|
+
// byAgent: 2 entries
|
|
945
|
+
expect(stats.byAgent).toHaveLength(2);
|
|
946
|
+
expect(stats.byAgent.find((a) => a.key === agentId1)?.input).toBe(400); // runA + runC
|
|
947
|
+
expect(stats.byAgent.find((a) => a.key === agentId2)?.input).toBe(200); // runB
|
|
948
|
+
|
|
949
|
+
// byCron: 2 entries with names
|
|
950
|
+
expect(stats.byCron).toHaveLength(2);
|
|
951
|
+
const cAlpha = stats.byCron.find((c) => c.key2 === "cron-alpha");
|
|
952
|
+
const cBeta = stats.byCron.find((c) => c.key2 === "cron-beta");
|
|
953
|
+
expect(cAlpha).toBeDefined();
|
|
954
|
+
expect(cAlpha?.key1).toBe(agentId1);
|
|
955
|
+
expect(cAlpha?.input).toBe(400); // runA + runC
|
|
956
|
+
expect(cBeta).toBeDefined();
|
|
957
|
+
expect(cBeta?.key1).toBe(agentId2);
|
|
958
|
+
expect(cBeta?.input).toBe(200);
|
|
959
|
+
|
|
960
|
+
// byModel: 2 entries
|
|
961
|
+
expect(stats.byModel).toHaveLength(2);
|
|
962
|
+
expect(
|
|
963
|
+
stats.byModel.find((m) => m.key2 === "claude-sonnet-4-5")?.input,
|
|
964
|
+
).toBe(400); // runA + runC
|
|
965
|
+
expect(stats.byModel.find((m) => m.key2 === "claude-opus-4-5")?.input).toBe(
|
|
966
|
+
200,
|
|
967
|
+
);
|
|
968
|
+
|
|
969
|
+
// daily: 3 days
|
|
970
|
+
expect(stats.daily).toHaveLength(3);
|
|
971
|
+
expect(stats.daily.find((d) => d.period === "2026-01-10")?.input).toBe(100);
|
|
972
|
+
expect(stats.daily.find((d) => d.period === "2026-01-11")?.input).toBe(200);
|
|
973
|
+
expect(stats.daily.find((d) => d.period === "2026-01-12")?.input).toBe(300); // only runC; skipped run excluded
|
|
974
|
+
|
|
975
|
+
// byCronModel: 2 entries — agentId1:cron-alpha × sonnet, agentId2:cron-beta × opus
|
|
976
|
+
// key1 = agentId:cronName, key2 = model
|
|
977
|
+
expect(stats.byCronModel).toHaveLength(2);
|
|
978
|
+
const byCMSonnet = stats.byCronModel.find(
|
|
979
|
+
(m) =>
|
|
980
|
+
m.key1 === `${agentId1}:cron-alpha` && m.key2 === "claude-sonnet-4-5",
|
|
981
|
+
);
|
|
982
|
+
const byCMOpus = stats.byCronModel.find(
|
|
983
|
+
(m) => m.key1 === `${agentId2}:cron-beta` && m.key2 === "claude-opus-4-5",
|
|
984
|
+
);
|
|
985
|
+
expect(byCMSonnet).toBeDefined();
|
|
986
|
+
expect(byCMSonnet?.input).toBe(400); // runA(100) + runC(300)
|
|
987
|
+
expect(byCMSonnet?.costUsd).toBeCloseTo(0.001 + 0.003);
|
|
988
|
+
expect(byCMOpus).toBeDefined();
|
|
989
|
+
expect(byCMOpus?.input).toBe(200); // runB only
|
|
990
|
+
expect(byCMOpus?.costUsd).toBeCloseTo(0.002);
|
|
991
|
+
});
|
|
992
|
+
|
|
993
|
+
// ─── byPhase ─────────────────────────────────────────────────────────────────
|
|
994
|
+
|
|
995
|
+
it("query() groups byPhase correctly across runs with different phases", async () => {
|
|
996
|
+
const agentId = await createAgent(prisma);
|
|
997
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
998
|
+
const devTaskPhaseId = await createPhaseCron(
|
|
999
|
+
cronJobService,
|
|
1000
|
+
agentId,
|
|
1001
|
+
"dev-task",
|
|
1002
|
+
);
|
|
1003
|
+
const reviewPhaseId = await createPhaseCron(
|
|
1004
|
+
cronJobService,
|
|
1005
|
+
agentId,
|
|
1006
|
+
"review",
|
|
1007
|
+
);
|
|
1008
|
+
|
|
1009
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
1010
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1011
|
+
skipped: false,
|
|
1012
|
+
phaseId: devTaskPhaseId,
|
|
1013
|
+
});
|
|
1014
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
1015
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
1016
|
+
skipped: false,
|
|
1017
|
+
phaseId: reviewPhaseId,
|
|
1018
|
+
});
|
|
1019
|
+
|
|
1020
|
+
await seedTokens(prisma, run1.id, { input: 100, output: 50 });
|
|
1021
|
+
await seedTokens(prisma, run2.id, { input: 200, output: 100 });
|
|
1022
|
+
|
|
1023
|
+
const stats = await statsService.query();
|
|
1024
|
+
|
|
1025
|
+
expect(stats.byPhase).toHaveLength(2);
|
|
1026
|
+
|
|
1027
|
+
const devTask = stats.byPhase.find((p) => p.key === "dev-task");
|
|
1028
|
+
const review = stats.byPhase.find((p) => p.key === "review");
|
|
1029
|
+
|
|
1030
|
+
expect(devTask).toBeDefined();
|
|
1031
|
+
expect(devTask?.input).toBe(100);
|
|
1032
|
+
expect(devTask?.output).toBe(50);
|
|
1033
|
+
|
|
1034
|
+
expect(review).toBeDefined();
|
|
1035
|
+
expect(review?.input).toBe(200);
|
|
1036
|
+
expect(review?.output).toBe(100);
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
it("query() byPhase excludes runs with a null phaseId (legacy runs)", async () => {
|
|
1040
|
+
const agentId = await createAgent(prisma);
|
|
1041
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
1042
|
+
const deployPhaseId = await createPhaseCron(
|
|
1043
|
+
cronJobService,
|
|
1044
|
+
agentId,
|
|
1045
|
+
"deploy",
|
|
1046
|
+
);
|
|
1047
|
+
|
|
1048
|
+
// Legacy run — no phaseId set
|
|
1049
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
1050
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1051
|
+
skipped: false,
|
|
1052
|
+
});
|
|
1053
|
+
await seedTokens(prisma, run1.id, { input: 100, output: 50 });
|
|
1054
|
+
|
|
1055
|
+
// Phase-tagged run
|
|
1056
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
1057
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
1058
|
+
skipped: false,
|
|
1059
|
+
phaseId: deployPhaseId,
|
|
1060
|
+
});
|
|
1061
|
+
await seedTokens(prisma, run2.id, { input: 200, output: 100 });
|
|
1062
|
+
|
|
1063
|
+
const stats = await statsService.query();
|
|
1064
|
+
|
|
1065
|
+
// byPhase only surfaces runs with a non-null phaseId
|
|
1066
|
+
expect(stats.byPhase).toHaveLength(1);
|
|
1067
|
+
expect(stats.byPhase[0].key).toBe("deploy");
|
|
1068
|
+
expect(stats.byPhase[0].input).toBe(200);
|
|
1069
|
+
|
|
1070
|
+
// totals still reflect both runs — phase grouping is additive, not exclusionary
|
|
1071
|
+
expect(stats.totals.input).toBe(300);
|
|
1072
|
+
});
|
|
1073
|
+
|
|
1074
|
+
it("query() byPhase excludes skipped runs", async () => {
|
|
1075
|
+
const agentId = await createAgent(prisma);
|
|
1076
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
1077
|
+
const patchPhaseId = await createPhaseCron(
|
|
1078
|
+
cronJobService,
|
|
1079
|
+
agentId,
|
|
1080
|
+
"patch",
|
|
1081
|
+
);
|
|
1082
|
+
|
|
1083
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
1084
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1085
|
+
skipped: false,
|
|
1086
|
+
phaseId: patchPhaseId,
|
|
1087
|
+
});
|
|
1088
|
+
await seedTokens(prisma, run1.id, { input: 60, output: 30 });
|
|
1089
|
+
|
|
1090
|
+
// Skipped run with a phase — excluded
|
|
1091
|
+
await runService.create(cronId, agentId, {
|
|
1092
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
1093
|
+
skipped: true,
|
|
1094
|
+
phaseId: patchPhaseId,
|
|
1095
|
+
});
|
|
1096
|
+
|
|
1097
|
+
const stats = await statsService.query();
|
|
1098
|
+
|
|
1099
|
+
expect(stats.byPhase).toHaveLength(1);
|
|
1100
|
+
expect(stats.byPhase[0].input).toBe(60);
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
// ─── backfilled legacy rows ──────────────────────────────────────────────────
|
|
1104
|
+
|
|
1105
|
+
it("query(from, to, agentId) scopes every dimension to that agent only", async () => {
|
|
1106
|
+
const agentA = await createAgent(prisma, "Agent A");
|
|
1107
|
+
const agentB = await createAgent(prisma, "Agent B");
|
|
1108
|
+
const cronA = await createCron(cronJobService, agentA, "cron-a");
|
|
1109
|
+
const cronB = await createCron(cronJobService, agentB, "cron-b");
|
|
1110
|
+
const phaseA = await createPhaseCron(cronJobService, agentA, "dev-task");
|
|
1111
|
+
const phaseB = await createPhaseCron(cronJobService, agentB, "review");
|
|
1112
|
+
|
|
1113
|
+
const runA = await runService.create(cronA, agentA, {
|
|
1114
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1115
|
+
skipped: false,
|
|
1116
|
+
phaseId: phaseA,
|
|
1117
|
+
});
|
|
1118
|
+
const runB = await runService.create(cronB, agentB, {
|
|
1119
|
+
startedAt: new Date("2026-01-10T10:00:00Z"),
|
|
1120
|
+
skipped: false,
|
|
1121
|
+
phaseId: phaseB,
|
|
1122
|
+
});
|
|
1123
|
+
await seedTokens(prisma, runA.id, { input: 100, output: 50, model: "m-a" });
|
|
1124
|
+
await seedTokens(prisma, runB.id, {
|
|
1125
|
+
input: 300,
|
|
1126
|
+
output: 150,
|
|
1127
|
+
model: "m-b",
|
|
1128
|
+
});
|
|
1129
|
+
|
|
1130
|
+
const stats = await statsService.query(undefined, undefined, agentA);
|
|
1131
|
+
|
|
1132
|
+
expect(stats.totals.input).toBe(100);
|
|
1133
|
+
expect(stats.byAgent.map((a) => a.key)).toEqual([agentA]);
|
|
1134
|
+
expect(stats.byCron.map((r) => r.key1)).toEqual([agentA]);
|
|
1135
|
+
expect(stats.byModel.map((r) => [r.key1, r.key2])).toEqual([
|
|
1136
|
+
[agentA, "m-a"],
|
|
1137
|
+
]);
|
|
1138
|
+
expect(stats.byCronModel.map((r) => r.key2)).toEqual(["m-a"]);
|
|
1139
|
+
expect(stats.byPhase.map((r) => r.key)).toEqual(["dev-task"]);
|
|
1140
|
+
expect(stats.daily).toHaveLength(1);
|
|
1141
|
+
expect(stats.daily[0].input).toBe(100);
|
|
1142
|
+
|
|
1143
|
+
// Omitted agentId stays fleet-wide.
|
|
1144
|
+
const fleet = await statsService.query();
|
|
1145
|
+
expect(fleet.byAgent).toHaveLength(2);
|
|
1146
|
+
expect(fleet.totals.input).toBe(400);
|
|
1147
|
+
});
|
|
1148
|
+
|
|
1149
|
+
it("surfaces a backfilled legacy run (model=legacy-unattributed, cost=0) in all dimensions", async () => {
|
|
1150
|
+
// Simulates the state after the consolidation migration backfills a
|
|
1151
|
+
// pre-2026-07-01 run whose token counts lived in the now-dropped
|
|
1152
|
+
// AgentCronRun columns. The migration inserts a single breakdown row with
|
|
1153
|
+
// model = 'legacy-unattributed' and costUsd = 0.
|
|
1154
|
+
const agentId = await createAgent(prisma, "Legacy Agent");
|
|
1155
|
+
const cronId = await createCron(cronJobService, agentId, "legacy-cron");
|
|
1156
|
+
|
|
1157
|
+
const run = await runService.create(cronId, agentId, {
|
|
1158
|
+
startedAt: new Date("2026-06-15T09:00:00Z"),
|
|
1159
|
+
skipped: false,
|
|
1160
|
+
});
|
|
1161
|
+
await seedTokens(prisma, run.id, {
|
|
1162
|
+
input: 500,
|
|
1163
|
+
output: 250,
|
|
1164
|
+
cacheRead: 40,
|
|
1165
|
+
cacheCreation: 20,
|
|
1166
|
+
model: "legacy-unattributed",
|
|
1167
|
+
costUsd: 0,
|
|
1168
|
+
});
|
|
1169
|
+
|
|
1170
|
+
const stats = await statsService.query();
|
|
1171
|
+
|
|
1172
|
+
// totals reflect the backfilled token counts
|
|
1173
|
+
expect(stats.totals.input).toBe(500);
|
|
1174
|
+
expect(stats.totals.output).toBe(250);
|
|
1175
|
+
expect(stats.totals.cacheRead).toBe(40);
|
|
1176
|
+
expect(stats.totals.cacheCreation).toBe(20);
|
|
1177
|
+
expect(stats.totals.total).toBe(500 + 250 + 40 + 20);
|
|
1178
|
+
// cost is $0 for backfilled legacy rows
|
|
1179
|
+
expect(stats.totals.costUsd).toBe(0);
|
|
1180
|
+
|
|
1181
|
+
// byModel surfaces the placeholder model name
|
|
1182
|
+
expect(stats.byModel).toHaveLength(1);
|
|
1183
|
+
expect(stats.byModel[0].key1).toBe(agentId);
|
|
1184
|
+
expect(stats.byModel[0].key2).toBe("legacy-unattributed");
|
|
1185
|
+
expect(stats.byModel[0].input).toBe(500);
|
|
1186
|
+
expect(stats.byModel[0].costUsd).toBe(0);
|
|
1187
|
+
|
|
1188
|
+
// byCronModel too
|
|
1189
|
+
expect(stats.byCronModel).toHaveLength(1);
|
|
1190
|
+
expect(stats.byCronModel[0].key1).toBe(`${agentId}:legacy-cron`);
|
|
1191
|
+
expect(stats.byCronModel[0].key2).toBe("legacy-unattributed");
|
|
1192
|
+
expect(stats.byCronModel[0].input).toBe(500);
|
|
1193
|
+
});
|
|
1194
|
+
|
|
1195
|
+
// ─── bySkill / baselines (PAU-1.6) ───────────────────────────────────────────
|
|
1196
|
+
|
|
1197
|
+
it("query() returns empty bySkill and baselines when nothing reported", async () => {
|
|
1198
|
+
const agentId = await createAgent(prisma);
|
|
1199
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
1200
|
+
await runService.create(cronId, agentId, {
|
|
1201
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1202
|
+
skipped: false,
|
|
1203
|
+
});
|
|
1204
|
+
|
|
1205
|
+
const stats = await statsService.query();
|
|
1206
|
+
|
|
1207
|
+
expect(stats.bySkill).toEqual([]);
|
|
1208
|
+
expect(stats.baselines).toEqual([]);
|
|
1209
|
+
});
|
|
1210
|
+
|
|
1211
|
+
it("query() aggregates bySkill per (kind,name) and excludes skipped runs", async () => {
|
|
1212
|
+
const agentId = await createAgent(prisma);
|
|
1213
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
1214
|
+
const run1 = await runService.create(cronId, agentId, {
|
|
1215
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1216
|
+
skipped: false,
|
|
1217
|
+
});
|
|
1218
|
+
const run2 = await runService.create(cronId, agentId, {
|
|
1219
|
+
startedAt: new Date("2026-01-11T09:00:00Z"),
|
|
1220
|
+
skipped: false,
|
|
1221
|
+
});
|
|
1222
|
+
const skippedRun = await runService.create(cronId, agentId, {
|
|
1223
|
+
startedAt: new Date("2026-01-12T09:00:00Z"),
|
|
1224
|
+
skipped: true,
|
|
1225
|
+
});
|
|
1226
|
+
|
|
1227
|
+
const usage = (
|
|
1228
|
+
cronRunId: string,
|
|
1229
|
+
kind: string,
|
|
1230
|
+
name: string,
|
|
1231
|
+
n: number,
|
|
1232
|
+
invokeContextDelta: number | null,
|
|
1233
|
+
) => ({
|
|
1234
|
+
cronRunId,
|
|
1235
|
+
kind,
|
|
1236
|
+
name,
|
|
1237
|
+
invocations: n,
|
|
1238
|
+
turns: n * 2,
|
|
1239
|
+
inputTokens: n * 10,
|
|
1240
|
+
outputTokens: n * 5,
|
|
1241
|
+
cacheReadTokens: n * 3,
|
|
1242
|
+
cacheCreationTokens: n,
|
|
1243
|
+
invokeContextDelta,
|
|
1244
|
+
});
|
|
1245
|
+
await prisma.agentCronRunSkillUsage.createMany({
|
|
1246
|
+
data: [
|
|
1247
|
+
usage(run1.id, "skill", "shipwright:review", 1, 1000),
|
|
1248
|
+
usage(run2.id, "skill", "shipwright:review", 2, 2000),
|
|
1249
|
+
usage(run2.id, "subagent", "general-purpose", 1, null),
|
|
1250
|
+
usage(skippedRun.id, "skill", "shipwright:review", 50, 99999),
|
|
1251
|
+
],
|
|
1252
|
+
});
|
|
1253
|
+
|
|
1254
|
+
const stats = await statsService.query();
|
|
1255
|
+
|
|
1256
|
+
expect(stats.bySkill).toHaveLength(2);
|
|
1257
|
+
const review = stats.bySkill?.find((s) => s.name === "shipwright:review");
|
|
1258
|
+
expect(review).toEqual({
|
|
1259
|
+
kind: "skill",
|
|
1260
|
+
name: "shipwright:review",
|
|
1261
|
+
runs: 2,
|
|
1262
|
+
invocations: 3,
|
|
1263
|
+
turns: 6,
|
|
1264
|
+
input: 30,
|
|
1265
|
+
output: 15,
|
|
1266
|
+
cacheRead: 9,
|
|
1267
|
+
cacheCreation: 3,
|
|
1268
|
+
avgInvokeContextDelta: 1500,
|
|
1269
|
+
});
|
|
1270
|
+
const sub = stats.bySkill?.find((s) => s.kind === "subagent");
|
|
1271
|
+
expect(sub?.runs).toBe(1);
|
|
1272
|
+
expect(sub?.avgInvokeContextDelta).toBeNull();
|
|
1273
|
+
});
|
|
1274
|
+
|
|
1275
|
+
it("query() groups baselines by (fingerprint,model,phase), includes skipped, excludes no-baseline runs, orders by first appearance", async () => {
|
|
1276
|
+
const agentId = await createAgent(prisma);
|
|
1277
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
1278
|
+
const devPhase = await createPhaseCron(cronJobService, agentId, "dev-task");
|
|
1279
|
+
const reviewPhase = await createPhaseCron(
|
|
1280
|
+
cronJobService,
|
|
1281
|
+
agentId,
|
|
1282
|
+
"review",
|
|
1283
|
+
);
|
|
1284
|
+
|
|
1285
|
+
const baseline = (
|
|
1286
|
+
tokens: number,
|
|
1287
|
+
fp: string,
|
|
1288
|
+
turns: number | null,
|
|
1289
|
+
toolCalls: number | null,
|
|
1290
|
+
) => ({
|
|
1291
|
+
baselineModel: "claude-sonnet-5-5",
|
|
1292
|
+
baselineContextTokens: tokens,
|
|
1293
|
+
contextFingerprint: fp,
|
|
1294
|
+
turns,
|
|
1295
|
+
toolCalls,
|
|
1296
|
+
});
|
|
1297
|
+
const mk = async (
|
|
1298
|
+
startedAt: string,
|
|
1299
|
+
extra: Record<string, unknown>,
|
|
1300
|
+
phaseId?: string,
|
|
1301
|
+
) => {
|
|
1302
|
+
const run = await runService.create(cronId, agentId, {
|
|
1303
|
+
startedAt: new Date(startedAt),
|
|
1304
|
+
skipped: (extra.skipped as boolean | undefined) ?? false,
|
|
1305
|
+
phaseId,
|
|
1306
|
+
});
|
|
1307
|
+
const { skipped: _skipped, ...data } = extra;
|
|
1308
|
+
await prisma.agentCronRun.update({ where: { id: run.id }, data });
|
|
1309
|
+
};
|
|
1310
|
+
|
|
1311
|
+
// Second-seen group inserted first to prove ordering is by startedAt.
|
|
1312
|
+
await mk("2026-01-12T09:00:00Z", baseline(500, "fp-b", 4, 8), reviewPhase);
|
|
1313
|
+
await mk("2026-01-10T09:00:00Z", baseline(100, "fp-a", 10, 20), devPhase);
|
|
1314
|
+
await mk("2026-01-11T09:00:00Z", baseline(300, "fp-a", 20, null), devPhase);
|
|
1315
|
+
await mk(
|
|
1316
|
+
"2026-01-13T09:00:00Z",
|
|
1317
|
+
{ ...baseline(200, "fp-a", null, null), skipped: true },
|
|
1318
|
+
devPhase,
|
|
1319
|
+
);
|
|
1320
|
+
// No baseline reported → excluded.
|
|
1321
|
+
await mk("2026-01-09T09:00:00Z", {}, devPhase);
|
|
1322
|
+
|
|
1323
|
+
const stats = await statsService.query();
|
|
1324
|
+
|
|
1325
|
+
expect(stats.baselines).toHaveLength(2);
|
|
1326
|
+
const [first, second] = stats.baselines ?? [];
|
|
1327
|
+
expect(first).toMatchObject({
|
|
1328
|
+
contextFingerprint: "fp-a",
|
|
1329
|
+
baselineModel: "claude-sonnet-5-5",
|
|
1330
|
+
phase: "dev-task",
|
|
1331
|
+
runs: 3,
|
|
1332
|
+
avgContextTokens: 200,
|
|
1333
|
+
minContextTokens: 100,
|
|
1334
|
+
maxContextTokens: 300,
|
|
1335
|
+
avgTurns: 15,
|
|
1336
|
+
avgToolCalls: 20,
|
|
1337
|
+
firstSeen: "2026-01-10T09:00:00.000Z",
|
|
1338
|
+
lastSeen: "2026-01-13T09:00:00.000Z",
|
|
1339
|
+
});
|
|
1340
|
+
expect(second).toMatchObject({
|
|
1341
|
+
contextFingerprint: "fp-b",
|
|
1342
|
+
phase: "review",
|
|
1343
|
+
runs: 1,
|
|
1344
|
+
avgContextTokens: 500,
|
|
1345
|
+
});
|
|
1346
|
+
});
|
|
1347
|
+
|
|
1348
|
+
// ─── outcomes ────────────────────────────────────────────────────────────────
|
|
1349
|
+
|
|
1350
|
+
it("outcomes() groups by (phase, contextFingerprint) with counts, durations, and averages", async () => {
|
|
1351
|
+
const agentId = await createAgent(prisma);
|
|
1352
|
+
const cronId = await createCron(cronJobService, agentId);
|
|
1353
|
+
const devTask = await createPhaseCron(cronJobService, agentId, "dev-task");
|
|
1354
|
+
const review = await createPhaseCron(cronJobService, agentId, "review");
|
|
1355
|
+
|
|
1356
|
+
const mk = (data: {
|
|
1357
|
+
phaseId?: string;
|
|
1358
|
+
fp?: string;
|
|
1359
|
+
minutes?: number;
|
|
1360
|
+
outcome?: string;
|
|
1361
|
+
skipped?: boolean;
|
|
1362
|
+
skipReason?: string;
|
|
1363
|
+
turns?: number;
|
|
1364
|
+
toolCalls?: number;
|
|
1365
|
+
ctx?: number;
|
|
1366
|
+
startedAt?: string;
|
|
1367
|
+
}) => {
|
|
1368
|
+
const startedAt = new Date(data.startedAt ?? "2026-01-10T09:00:00Z");
|
|
1369
|
+
return prisma.agentCronRun.create({
|
|
1370
|
+
data: {
|
|
1371
|
+
cronId,
|
|
1372
|
+
agentId,
|
|
1373
|
+
startedAt,
|
|
1374
|
+
completedAt:
|
|
1375
|
+
data.minutes === undefined
|
|
1376
|
+
? null
|
|
1377
|
+
: new Date(startedAt.getTime() + data.minutes * 60_000),
|
|
1378
|
+
skipped: data.skipped ?? false,
|
|
1379
|
+
skipReason: data.skipReason ?? null,
|
|
1380
|
+
outcome: data.outcome ?? null,
|
|
1381
|
+
phaseId: data.phaseId ?? null,
|
|
1382
|
+
contextFingerprint: data.fp ?? null,
|
|
1383
|
+
turns: data.turns ?? null,
|
|
1384
|
+
toolCalls: data.toolCalls ?? null,
|
|
1385
|
+
baselineContextTokens: data.ctx ?? null,
|
|
1386
|
+
},
|
|
1387
|
+
});
|
|
1388
|
+
};
|
|
1389
|
+
|
|
1390
|
+
// dev-task / fp-a: 3 completed (1, 2, 6 min), 1 failed (4 min), 2 skipped
|
|
1391
|
+
await mk({
|
|
1392
|
+
phaseId: devTask,
|
|
1393
|
+
fp: "fp-a",
|
|
1394
|
+
minutes: 1,
|
|
1395
|
+
outcome: "completed",
|
|
1396
|
+
turns: 10,
|
|
1397
|
+
toolCalls: 20,
|
|
1398
|
+
ctx: 1000,
|
|
1399
|
+
});
|
|
1400
|
+
await mk({
|
|
1401
|
+
phaseId: devTask,
|
|
1402
|
+
fp: "fp-a",
|
|
1403
|
+
minutes: 2,
|
|
1404
|
+
outcome: "completed",
|
|
1405
|
+
turns: 20,
|
|
1406
|
+
toolCalls: 40,
|
|
1407
|
+
ctx: 2000,
|
|
1408
|
+
});
|
|
1409
|
+
await mk({
|
|
1410
|
+
phaseId: devTask,
|
|
1411
|
+
fp: "fp-a",
|
|
1412
|
+
minutes: 6,
|
|
1413
|
+
outcome: "completed",
|
|
1414
|
+
turns: 30,
|
|
1415
|
+
toolCalls: 60,
|
|
1416
|
+
ctx: 3000,
|
|
1417
|
+
});
|
|
1418
|
+
await mk({
|
|
1419
|
+
phaseId: devTask,
|
|
1420
|
+
fp: "fp-a",
|
|
1421
|
+
minutes: 4,
|
|
1422
|
+
outcome: "failed",
|
|
1423
|
+
turns: 40,
|
|
1424
|
+
toolCalls: 80,
|
|
1425
|
+
});
|
|
1426
|
+
await mk({
|
|
1427
|
+
phaseId: devTask,
|
|
1428
|
+
fp: "fp-a",
|
|
1429
|
+
skipped: true,
|
|
1430
|
+
skipReason: "no-ready-task",
|
|
1431
|
+
});
|
|
1432
|
+
await mk({
|
|
1433
|
+
phaseId: devTask,
|
|
1434
|
+
fp: "fp-a",
|
|
1435
|
+
skipped: true,
|
|
1436
|
+
skipReason: "no-ready-task",
|
|
1437
|
+
});
|
|
1438
|
+
// dev-task / fp-b: a different fingerprint is a separate series
|
|
1439
|
+
await mk({
|
|
1440
|
+
phaseId: devTask,
|
|
1441
|
+
fp: "fp-b",
|
|
1442
|
+
minutes: 10,
|
|
1443
|
+
outcome: "completed",
|
|
1444
|
+
turns: 5,
|
|
1445
|
+
});
|
|
1446
|
+
// review / fp-a, and a legacy run with neither phase nor fingerprint
|
|
1447
|
+
await mk({ phaseId: review, fp: "fp-a", minutes: 3, outcome: "completed" });
|
|
1448
|
+
await mk({ skipped: true });
|
|
1449
|
+
// outside the requested range
|
|
1450
|
+
await mk({
|
|
1451
|
+
phaseId: devTask,
|
|
1452
|
+
fp: "fp-a",
|
|
1453
|
+
minutes: 9,
|
|
1454
|
+
outcome: "completed",
|
|
1455
|
+
startedAt: "2025-12-01T09:00:00Z",
|
|
1456
|
+
});
|
|
1457
|
+
|
|
1458
|
+
const { series } = await statsService.outcomes(
|
|
1459
|
+
"2026-01-01T00:00:00Z",
|
|
1460
|
+
"2026-02-01T00:00:00Z",
|
|
1461
|
+
);
|
|
1462
|
+
const find = (phase: string | null, fp: string | null) =>
|
|
1463
|
+
series.find((x) => x.phase === phase && x.contextFingerprint === fp);
|
|
1464
|
+
|
|
1465
|
+
expect(series).toHaveLength(4);
|
|
1466
|
+
|
|
1467
|
+
const a = find("dev-task", "fp-a");
|
|
1468
|
+
expect(a).toEqual({
|
|
1469
|
+
phase: "dev-task",
|
|
1470
|
+
contextFingerprint: "fp-a",
|
|
1471
|
+
runs: 6,
|
|
1472
|
+
completed: 3,
|
|
1473
|
+
failed: 1,
|
|
1474
|
+
skipped: 2,
|
|
1475
|
+
skipReasons: { "no-ready-task": 2 },
|
|
1476
|
+
avgDurationMs: 3.25 * 60_000, // (1+2+6+4)/4 min — skipped runs excluded
|
|
1477
|
+
p50DurationMs: 3 * 60_000, // median of 1, 2, 4, 6
|
|
1478
|
+
avgTurns: 25,
|
|
1479
|
+
avgToolCalls: 50,
|
|
1480
|
+
avgContextTokens: 2000, // null baselines ignored
|
|
1481
|
+
});
|
|
1482
|
+
|
|
1483
|
+
const b = find("dev-task", "fp-b");
|
|
1484
|
+
expect(b?.runs).toBe(1);
|
|
1485
|
+
expect(b?.avgDurationMs).toBe(10 * 60_000);
|
|
1486
|
+
expect(b?.avgToolCalls).toBeNull();
|
|
1487
|
+
expect(b?.avgContextTokens).toBeNull();
|
|
1488
|
+
|
|
1489
|
+
expect(find("review", "fp-a")?.completed).toBe(1);
|
|
1490
|
+
|
|
1491
|
+
const legacy = find(null, null);
|
|
1492
|
+
expect(legacy).toMatchObject({
|
|
1493
|
+
runs: 1,
|
|
1494
|
+
skipped: 1,
|
|
1495
|
+
skipReasons: { unknown: 1 },
|
|
1496
|
+
avgDurationMs: null,
|
|
1497
|
+
p50DurationMs: null,
|
|
1498
|
+
});
|
|
1499
|
+
});
|
|
1500
|
+
|
|
1501
|
+
it("outcomes(from, to, agentId) scopes counts and skip-reason histograms to that agent only", async () => {
|
|
1502
|
+
const agentA = await createAgent(prisma, "Agent A");
|
|
1503
|
+
const agentB = await createAgent(prisma, "Agent B");
|
|
1504
|
+
const cronA = await createCron(cronJobService, agentA, "cron-a");
|
|
1505
|
+
const cronB = await createCron(cronJobService, agentB, "cron-b");
|
|
1506
|
+
const phaseA = await createPhaseCron(cronJobService, agentA, "dev-task");
|
|
1507
|
+
const phaseB = await createPhaseCron(cronJobService, agentB, "dev-task");
|
|
1508
|
+
|
|
1509
|
+
const mk = (cronId: string, agentId: string, phaseId: string, skipped: boolean) =>
|
|
1510
|
+
prisma.agentCronRun.create({
|
|
1511
|
+
data: {
|
|
1512
|
+
cronId,
|
|
1513
|
+
agentId,
|
|
1514
|
+
phaseId,
|
|
1515
|
+
startedAt: new Date("2026-01-10T09:00:00Z"),
|
|
1516
|
+
completedAt: skipped ? null : new Date("2026-01-10T09:02:00Z"),
|
|
1517
|
+
skipped,
|
|
1518
|
+
skipReason: skipped ? "reason-a" : null,
|
|
1519
|
+
outcome: skipped ? null : "completed",
|
|
1520
|
+
contextFingerprint: "fp-a",
|
|
1521
|
+
},
|
|
1522
|
+
});
|
|
1523
|
+
await mk(cronA, agentA, phaseA, false);
|
|
1524
|
+
await mk(cronA, agentA, phaseA, true);
|
|
1525
|
+
await mk(cronB, agentB, phaseB, false);
|
|
1526
|
+
await mk(cronB, agentB, phaseB, false);
|
|
1527
|
+
await mk(cronB, agentB, phaseB, true);
|
|
1528
|
+
await mk(cronB, agentB, phaseB, true);
|
|
1529
|
+
|
|
1530
|
+
const { series } = await statsService.outcomes(undefined, undefined, agentA);
|
|
1531
|
+
expect(series).toHaveLength(1);
|
|
1532
|
+
expect(series[0]).toMatchObject({
|
|
1533
|
+
runs: 2,
|
|
1534
|
+
completed: 1,
|
|
1535
|
+
skipped: 1,
|
|
1536
|
+
skipReasons: { "reason-a": 1 },
|
|
1537
|
+
});
|
|
1538
|
+
|
|
1539
|
+
// Omitted agentId stays fleet-wide.
|
|
1540
|
+
const fleet = await statsService.outcomes();
|
|
1541
|
+
expect(fleet.series).toHaveLength(1);
|
|
1542
|
+
expect(fleet.series[0]).toMatchObject({
|
|
1543
|
+
runs: 6,
|
|
1544
|
+
skipReasons: { "reason-a": 3 },
|
|
1545
|
+
});
|
|
1546
|
+
});
|
|
1547
|
+
|
|
1548
|
+
it("outcomes() returns an empty series when there are no runs", async () => {
|
|
1549
|
+
expect(await statsService.outcomes()).toEqual({ series: [] });
|
|
1550
|
+
});
|
|
1551
|
+
});
|