@pasko70/pibo 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -254
- package/context/codex-base-prompt.md +4 -0
- package/context/compute-worker.md +23 -0
- package/context/pibo-native-tooling.md +18 -0
- package/context/pibo-system-prompt.md +69 -8
- package/dist/api/simple-agent-api.js +136 -0
- package/dist/app-context.js +4 -0
- package/dist/apps/chat/agent-profiles.js +18 -3
- package/dist/apps/chat/agent-store.js +113 -16
- package/dist/apps/chat/chat-api-routes.js +311 -0
- package/dist/apps/chat/chat-capability-routes.js +90 -0
- package/dist/apps/chat/chat-files.js +178 -0
- package/dist/apps/chat/chat-request-normalizers.js +681 -0
- package/dist/apps/chat/chat-settings-routes.js +75 -0
- package/dist/apps/chat/chat-trace-helpers.js +94 -0
- package/dist/apps/chat/chat-user-skill-routes.js +149 -0
- package/dist/apps/chat/cron-api.js +213 -0
- package/dist/apps/chat/data/chat-data-mappers.js +74 -0
- package/dist/apps/chat/data/event-command-service.js +75 -0
- package/dist/apps/chat/data/navigation-query-service.js +11 -0
- package/dist/apps/chat/data/project-service.js +844 -0
- package/dist/apps/chat/data/read-state-service.js +42 -0
- package/dist/apps/chat/data/room-service.js +52 -0
- package/dist/apps/chat/data/session-query-service.js +63 -0
- package/dist/apps/chat/data/timeline-query-service.js +96 -0
- package/dist/apps/chat/event-log.js +1 -307
- package/dist/apps/chat/model-catalog.js +7 -2
- package/dist/apps/chat/output-compactor.js +126 -0
- package/dist/apps/chat/output-event-policy.js +17 -0
- package/dist/apps/chat/project-workflow-human-actions.js +347 -0
- package/dist/apps/chat/project-workflow-sessions.js +260 -0
- package/dist/apps/chat/provider-auth-actions.js +43 -0
- package/dist/apps/chat/ralph-api.js +156 -0
- package/dist/apps/chat/read-model.js +1 -218
- package/dist/apps/chat/rooms.js +1 -321
- package/dist/apps/chat/static-assets.js +996 -0
- package/dist/apps/chat/stream.js +29 -6
- package/dist/apps/chat/trace.js +179 -1003
- package/dist/apps/chat/types/event-store.js +1 -0
- package/dist/apps/chat/types/read-model.js +1 -0
- package/dist/apps/chat/types/rooms.js +36 -0
- package/dist/apps/chat/web-app.js +3131 -1959
- package/dist/apps/chat/workflow-catalog.js +572 -0
- package/dist/apps/chat/workflow-edge-compatibility.js +51 -0
- package/dist/apps/chat/workflow-json-schema-validation.js +335 -0
- package/dist/apps/chat/workflow-persistence-model.js +140 -0
- package/dist/apps/chat/workflow-persistence.js +633 -0
- package/dist/apps/chat/workflow-registered-ref-params.js +130 -0
- package/dist/apps/chat/workflow-registered-ref-pickers.js +179 -0
- package/dist/apps/chat/workflow-registered-ref-validation.js +210 -0
- package/dist/apps/chat/workflow-v2-security-validation.js +93 -0
- package/dist/apps/chat/workflow-validation-helpers.js +114 -0
- package/dist/apps/chat-ui/assets/{dist-CNLWH2qh.js → dist-B4lw9z4F.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-8e4L8ciw.js → dist-BbTaHWON.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CMZOa5ik.js → dist-BcT_c1EJ.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DNcnQRPF.js → dist-CGINd7XU.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CqqUC8Ce.js → dist-CrBzZkUp.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-D2N4-LIc.js → dist-CyyISotB.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CUqpST5w.js → dist-DFbnFNpX.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CspjOeBj.js → dist-DGLWQLX8.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-B7bbI3Qz.js → dist-DfcIkFSN.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-v4pjHlB7.js → dist-sKDHTwaT.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-D9L4MSRF.js → dist-y9D6la48.js} +1 -1
- package/dist/apps/chat-ui/assets/index-C25VYnyb.css +1 -0
- package/dist/apps/chat-ui/assets/index-CUkuvI3v.js +165 -0
- package/dist/apps/chat-ui/assets/pwa-images/android/launchericon-144x144.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/android/launchericon-192x192.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/android/launchericon-48x48.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/android/launchericon-512x512.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/android/launchericon-72x72.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/android/launchericon-96x96.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/1024.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/114.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/120.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/128.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/144.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/152.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/16.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/167.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/180.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/192.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/20.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/256.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/29.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/32.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/40.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/50.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/512.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/57.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/58.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/60.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/64.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/72.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/76.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/80.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/ios/87.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/LargeTile.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/LargeTile.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/LargeTile.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/LargeTile.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/LargeTile.scale-400.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SmallTile.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SmallTile.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SmallTile.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SmallTile.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SmallTile.scale-400.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SplashScreen.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SplashScreen.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SplashScreen.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SplashScreen.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/SplashScreen.scale-400.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square150x150Logo.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square150x150Logo.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square150x150Logo.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square150x150Logo.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square150x150Logo.scale-400.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-16.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-20.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-24.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-256.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-30.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-32.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-36.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-40.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-44.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-48.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-60.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-64.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-72.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-80.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-lightunplated_targetsize-96.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-16.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-20.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-24.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-256.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-30.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-32.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-36.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-40.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-44.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-48.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-60.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-64.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-72.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-80.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.altform-unplated_targetsize-96.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.scale-400.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-16.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-20.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-24.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-256.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-30.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-32.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-36.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-40.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-44.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-48.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-60.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-64.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-72.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-80.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Square44x44Logo.targetsize-96.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/StoreLogo.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/StoreLogo.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/StoreLogo.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/StoreLogo.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/StoreLogo.scale-400.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Wide310x150Logo.scale-100.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Wide310x150Logo.scale-125.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Wide310x150Logo.scale-150.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Wide310x150Logo.scale-200.png +0 -0
- package/dist/apps/chat-ui/assets/pwa-images/windows/Wide310x150Logo.scale-400.png +0 -0
- package/dist/apps/chat-ui/index.html +8 -3
- package/dist/apps/chat-ui/manifest.webmanifest +25 -0
- package/dist/apps/chat-ui/sw.js +41 -0
- package/dist/apps/cli-ui/InkSessionApp.js +1218 -0
- package/dist/apps/cli-ui/InkTerminalLine.js +53 -0
- package/dist/apps/cli-ui/InkTerminalRow.js +304 -0
- package/dist/apps/cli-ui/InkTerminalView.js +25 -0
- package/dist/apps/cli-ui/cliSessionsCommand.js +232 -0
- package/dist/apps/cli-ui/index.js +8 -0
- package/dist/apps/cli-ui/inkColors.js +67 -0
- package/dist/apps/cli-ui/inkJson.js +244 -0
- package/dist/apps/cli-ui/inkMarkdown.js +122 -0
- package/dist/apps/context-files-ui/assets/{dist-DFi2YGux.js → dist-BOiGdrG-.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-DyfSeBMu.js → dist-DK01b7x4.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-CwbH_3RN.js → dist-DPG9jOQ-.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-dTBsFYAl.js → dist-Dp5AH00p.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-CxRLRmMx.js → dist-DqYw0LH7.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-MyZgX9K2.js → dist-TF5iO5ML.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-DQXzt2Ng.js → dist-TbjAhles.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-YwR9dpb-.js → dist-ZsBbt0uY.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-CKQF1zEu.js → dist-kmsvtHkH.js} +1 -1
- package/dist/apps/context-files-ui/assets/{dist-BJW3917N.js → dist-vnd7PMu3.js} +1 -1
- package/dist/apps/context-files-ui/assets/{index-BwRDoHMC.css → index-B92tL3Pp.css} +1 -1
- package/dist/apps/context-files-ui/assets/{index-GwvjXytt.js → index-BhAWD3dv.js} +2 -2
- package/dist/apps/context-files-ui/index.html +2 -2
- package/dist/auth/better-auth.js +2 -1
- package/dist/auth/login-actions.js +251 -0
- package/dist/auth/openai-codex-usage.js +147 -0
- package/dist/bin/pibo.js +0 -0
- package/dist/bin/rg.js +15 -0
- package/dist/cli-session/fakeSessionSource.js +804 -0
- package/dist/cli-session/index.js +3 -0
- package/dist/cli-session/localSessionSource.js +1144 -0
- package/dist/cli-session/sessionSource.js +13 -0
- package/dist/cli.js +142 -14
- package/dist/compute/cli.js +415 -0
- package/dist/compute/docker.js +827 -0
- package/dist/compute/resource-health.js +333 -0
- package/dist/compute/resource-policy.js +98 -0
- package/dist/config/config.js +9 -5
- package/dist/core/context-build.js +677 -0
- package/dist/core/model-defaults.js +51 -9
- package/dist/core/pibo-home.js +8 -0
- package/dist/core/profiles.js +48 -0
- package/dist/core/provider-telemetry.js +222 -0
- package/dist/core/routed-session.js +441 -47
- package/dist/core/runtime-telemetry.js +853 -0
- package/dist/core/runtime.js +103 -15
- package/dist/core/session-errors.js +47 -0
- package/dist/core/session-model.js +39 -0
- package/dist/core/session-router.js +292 -62
- package/dist/core/shared-app.js +17 -0
- package/dist/core/skill-expansion.js +43 -0
- package/dist/core/system-prompt-template.js +9 -9
- package/dist/core/telemetry-staleness.js +128 -0
- package/dist/core/test-output-compaction.js +145 -0
- package/dist/core/user-settings.js +73 -0
- package/dist/cron/channel.js +28 -0
- package/dist/cron/cli.js +230 -0
- package/dist/cron/plugin.js +11 -0
- package/dist/cron/schedule.js +214 -0
- package/dist/cron/service.js +177 -0
- package/dist/cron/store.js +349 -0
- package/dist/cron/types.js +1 -0
- package/dist/data/cli.js +184 -0
- package/dist/data/event-log.js +106 -0
- package/dist/data/final-app-space-cutover-migration.js +728 -0
- package/dist/data/ingest-service.js +389 -0
- package/dist/data/message-store.js +62 -0
- package/dist/data/navigation-store.js +75 -0
- package/dist/data/observation-store.js +46 -0
- package/dist/data/payload-store.js +144 -0
- package/dist/data/pibo-store.js +59 -0
- package/dist/data/schema.js +434 -0
- package/dist/data/session-store.js +63 -0
- package/dist/data/shared-app-migration.js +757 -0
- package/dist/data/sqlite-schema.js +12 -0
- package/dist/data/telemetry-preview.js +65 -0
- package/dist/data/telemetry-queries.js +212 -0
- package/dist/data/telemetry-retention.js +50 -0
- package/dist/data/telemetry-rows.js +158 -0
- package/dist/data/telemetry.js +357 -0
- package/dist/debug/delta-compaction.js +99 -0
- package/dist/debug/detail-format.js +86 -0
- package/dist/debug/events.js +149 -29
- package/dist/debug/failures.js +162 -0
- package/dist/debug/index.js +655 -30
- package/dist/debug/messages.js +185 -0
- package/dist/debug/next-commands.js +6 -0
- package/dist/debug/payloads.js +123 -0
- package/dist/debug/pty.js +924 -0
- package/dist/debug/session.js +45 -15
- package/dist/debug/sql.js +1 -1
- package/dist/debug/stores.js +18 -13
- package/dist/debug/summary.js +72 -0
- package/dist/debug/telemetry.js +649 -0
- package/dist/debug/tools.js +168 -0
- package/dist/debug/trace-render-check.js +151 -0
- package/dist/debug/trace.js +152 -27
- package/dist/debug/web-artifacts.js +63 -0
- package/dist/debug/web-options.js +313 -0
- package/dist/debug/web-render-analysis.js +310 -0
- package/dist/debug/web-snapshot-browser-scripts.js +303 -0
- package/dist/debug/web-streaming-benchmark-analysis.js +605 -0
- package/dist/debug/web-streaming-browser-library.js +928 -0
- package/dist/debug/web-streaming-browser-scripts.js +242 -0
- package/dist/debug/web-streaming-provider-telemetry.js +194 -0
- package/dist/debug/web-streaming-report.js +493 -0
- package/dist/debug/web-streaming-types.js +1 -0
- package/dist/debug/web.js +574 -0
- package/dist/gateway/backup.js +83 -0
- package/dist/gateway/cli.js +511 -0
- package/dist/gateway/fallback.js +155 -0
- package/dist/gateway/pidfile.js +84 -0
- package/dist/gateway/protocol.js +16 -0
- package/dist/gateway/server.js +172 -17
- package/dist/gateway/tool.js +54 -51
- package/dist/gateway/web.js +74 -2
- package/dist/index.js +3 -1
- package/dist/local/client.js +3 -3
- package/dist/local/extension.js +8 -5
- package/dist/plugins/builtin.js +251 -49
- package/dist/plugins/chat-vscode-web.js +28 -0
- package/dist/plugins/codex-compat.js +0 -41
- package/dist/plugins/context-files.js +98 -18
- package/dist/plugins/dev-auth.js +124 -0
- package/dist/plugins/native-tooling.js +15 -0
- package/dist/plugins/registry.js +49 -1
- package/dist/plugins/web-annotations.js +27 -0
- package/dist/providers/minimax.js +70 -0
- package/dist/providers/openai-compat.js +48 -0
- package/dist/ralph/channel.js +8 -0
- package/dist/ralph/cli.js +152 -0
- package/dist/ralph/plugin.js +14 -0
- package/dist/ralph/service.js +271 -0
- package/dist/ralph/stopping.js +151 -0
- package/dist/ralph/store.js +450 -0
- package/dist/ralph/templates.js +200 -0
- package/dist/ralph/types.js +1 -0
- package/dist/reliability/store.js +42 -26
- package/dist/runs/registry.js +72 -35
- package/dist/session-ui/commandCatalog.js +193 -0
- package/dist/session-ui/commandResults.js +95 -0
- package/dist/session-ui/index.js +7 -0
- package/dist/session-ui/ownerViewModel.js +27 -0
- package/dist/session-ui/roomSessionViewModel.js +69 -0
- package/dist/session-ui/statusViewModel.js +127 -0
- package/dist/session-ui/terminalCards.js +210 -0
- package/dist/session-ui/terminalRows.js +937 -0
- package/dist/session-ui/terminalValue.js +45 -0
- package/dist/sessions/pibo-data-store.js +178 -0
- package/dist/sessions/sqlite-store.js +78 -12
- package/dist/sessions/store.js +11 -4
- package/dist/sessions/workflow-session-kind.js +20 -0
- package/dist/setup/cli.js +567 -0
- package/dist/shared/trace-async-agent-runs.js +127 -0
- package/dist/shared/trace-engine.js +96 -0
- package/dist/shared/trace-event-projection.js +679 -0
- package/dist/shared/trace-live-reducer.js +191 -0
- package/dist/shared/trace-nodes.js +76 -0
- package/dist/shared/trace-order.js +11 -10
- package/dist/shared/trace-patch-nodes.js +83 -0
- package/dist/shared/trace-run-notifications.js +69 -0
- package/dist/shared/trace-subagent-links.js +46 -0
- package/dist/shared/trace-transcript.js +303 -0
- package/dist/shared/trace-types.js +1 -0
- package/dist/shared-app.js +4 -0
- package/dist/signals/aggregate.js +65 -0
- package/dist/signals/events.js +1 -0
- package/dist/signals/projector.js +158 -0
- package/dist/signals/registry.js +513 -0
- package/dist/signals/store.js +1 -0
- package/dist/signals/types.js +1 -0
- package/dist/skills/cli.js +117 -0
- package/dist/tools/agent-browser-leases.js +219 -0
- package/dist/tools/agent-browser-wrapper.js +99 -0
- package/dist/tools/browser-pool.js +623 -0
- package/dist/tools/browser-use-cdp.js +56 -4
- package/dist/tools/browser-use-leases.js +215 -37
- package/dist/tools/browser-use-wrapper.js +508 -65
- package/dist/tools/cdp-client.js +189 -0
- package/dist/tools/guides.js +203 -4
- package/dist/tools/index.js +666 -16
- package/dist/tools/npm-runtime.js +136 -0
- package/dist/tools/registry.js +107 -8
- package/dist/tools/runtime/index.js +3 -0
- package/dist/tools/runtime/node-backend.js +225 -0
- package/dist/tools/runtime/node-worker-source.js +206 -0
- package/dist/tools/runtime/python-backend.js +227 -0
- package/dist/tools/runtime/python-worker-source.js +178 -0
- package/dist/tools/runtime/registry.js +255 -0
- package/dist/tools/runtime/tool.js +168 -0
- package/dist/tools/runtime/types.js +1 -0
- package/dist/tools/web-search.js +2 -1
- package/dist/user-skills/installer.js +39 -36
- package/dist/user-skills/store.js +50 -11
- package/dist/web/auth.js +5 -1
- package/dist/web/channel.js +86 -0
- package/dist/web/http.js +79 -14
- package/dist/web-annotations/api.js +348 -0
- package/dist/web-annotations/attachments.js +201 -0
- package/dist/web-annotations/cdp.js +1107 -0
- package/dist/web-annotations/index.js +7 -0
- package/dist/web-annotations/store.js +407 -0
- package/dist/web-annotations/tools.js +350 -0
- package/dist/web-annotations/types.js +34 -0
- package/dist/web-annotations/validation.js +288 -0
- package/docs/README.md +23 -0
- package/docs/ops/install-developer-host.md +112 -0
- package/docs/ops/install-user-host.md +76 -0
- package/docs/ops/upgrade-user-to-developer-host.md +69 -0
- package/package.json +24 -10
- package/skills/builtin/pi-agent-harness/SKILL.md +24 -24
- package/skills/builtin/pibo-docker-system/SKILL.md +170 -0
- package/skills/builtin/pibo-spec-writing/SKILL.md +330 -0
- package/skills/builtin/prd/SKILL.md +143 -0
- package/skills/builtin/ralph-loop/SKILL.md +359 -0
- package/skills/builtin/ralph-prd-json/SKILL.md +123 -0
- package/skills/builtin/skill-creator/LICENSE.txt +202 -0
- package/skills/builtin/skill-creator/SKILL.md +513 -0
- package/skills/builtin/skill-creator/agents/analyzer.md +274 -0
- package/skills/builtin/skill-creator/agents/comparator.md +202 -0
- package/skills/builtin/skill-creator/agents/grader.md +223 -0
- package/skills/builtin/skill-creator/assets/eval_review.html +146 -0
- package/skills/builtin/skill-creator/eval-viewer/generate_review.py +471 -0
- package/skills/builtin/skill-creator/eval-viewer/viewer.html +1325 -0
- package/skills/builtin/skill-creator/references/schemas.md +430 -0
- package/skills/builtin/skill-creator/scripts/__init__.py +0 -0
- package/skills/builtin/skill-creator/scripts/aggregate_benchmark.py +401 -0
- package/skills/builtin/skill-creator/scripts/generate_report.py +326 -0
- package/skills/builtin/skill-creator/scripts/improve_description.py +247 -0
- package/skills/builtin/skill-creator/scripts/package_skill.py +136 -0
- package/skills/builtin/skill-creator/scripts/quick_validate.py +103 -0
- package/skills/builtin/skill-creator/scripts/run_eval.py +310 -0
- package/skills/builtin/skill-creator/scripts/run_loop.py +328 -0
- package/skills/builtin/skill-creator/scripts/utils.py +47 -0
- package/skills/builtin/web-annotations/SKILL.md +93 -0
- package/dist/apps/chat-ui/assets/index-D2Qxambd.js +0 -151
- package/dist/apps/chat-ui/assets/index-DXx368XK.css +0 -1
- package/dist/env.js +0 -32
- package/dist/plugins/core-tools.js +0 -94
- package/dist/plugins/example.js +0 -67
- package/dist/plugins/remote-agent.js +0 -9
- package/dist/remote/channel.js +0 -176
- package/dist/remote/client.js +0 -92
- package/dist/remote/examples/tui-controller.js +0 -263
- package/dist/remote/protocol.js +0 -55
- package/dist/remote/session-client.js +0 -189
- package/dist/sessions/bindings.js +0 -41
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { summarizeStreamingSelectedLiveEventSource } from "./web-streaming-report.js";
|
|
3
|
+
const LIVE_PIPELINE_MIN_PRESERVATION_RATIO = 0.95;
|
|
4
|
+
const LIVE_PIPELINE_MIN_FLUSH_RATIO = 0.75;
|
|
5
|
+
const URL_COMPARISON_MAX_SMOOTHNESS_DROP = 15;
|
|
6
|
+
const URL_COMPARISON_MAX_DOM_LAG_DELTA_MS = 150;
|
|
7
|
+
const URL_COMPARISON_MAX_SSE_LAG_DELTA_MS = 100;
|
|
8
|
+
const URL_COMPARISON_MAX_SSE_CHUNK_GAP_DELTA_MS = 100;
|
|
9
|
+
const URL_COMPARISON_MAX_FIRST_TRANSPORT_LATENCY_DELTA_MS = 250;
|
|
10
|
+
const URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS = 250;
|
|
11
|
+
const URL_COMPARISON_MAX_FIRST_VISIBLE_LATENCY_DELTA_MS = 300;
|
|
12
|
+
const PROVIDER_MIN_TEXT_PRESERVATION_RATIO = 0.95;
|
|
13
|
+
const PROVIDER_MIN_REASONING_PRESERVATION_RATIO = 0.95;
|
|
14
|
+
export function scoreStreamingBenchmark(benchmark) {
|
|
15
|
+
const debugDelta = benchmark.debug.delta ?? {};
|
|
16
|
+
const textDeltaCount = numberField(debugDelta, "textDeltaCount");
|
|
17
|
+
const domGapP50Ms = finiteNumber(benchmark.dom.gapsMs.p50);
|
|
18
|
+
const domGapP90Ms = finiteNumber(benchmark.dom.gapsMs.p90);
|
|
19
|
+
const domJumpP90Chars = finiteNumber(benchmark.dom.positiveCharJumps.p90);
|
|
20
|
+
const firstVisibleMs = finiteNumber(benchmark.dom.firstPositiveUpdateMs);
|
|
21
|
+
const providerToDomRatio = textDeltaCount > 0 ? Math.min(1, benchmark.dom.positiveUpdateCount / textDeltaCount) : 0;
|
|
22
|
+
const smoothness = 0.30 * clampScore(100 - (domGapP50Ms ?? 100))
|
|
23
|
+
+ 0.25 * clampScore(((300 - (domGapP90Ms ?? 300)) / 3))
|
|
24
|
+
+ 0.20 * clampScore(((120 - (domJumpP90Chars ?? 120)) / 1.2))
|
|
25
|
+
+ 0.15 * clampScore(providerToDomRatio * 100)
|
|
26
|
+
+ 0.10 * clampScore(100 - ((firstVisibleMs ?? 500) / 5));
|
|
27
|
+
return {
|
|
28
|
+
smoothness: round3(smoothness),
|
|
29
|
+
domGapP50Ms,
|
|
30
|
+
domGapP90Ms,
|
|
31
|
+
domJumpP90Chars,
|
|
32
|
+
textDeltaCount,
|
|
33
|
+
domPositiveUpdateCount: benchmark.dom.positiveUpdateCount,
|
|
34
|
+
firstVisibleMs,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export function summarizeStreamingCadence(benchmark) {
|
|
38
|
+
const fixtureScheduleGapP90Ms = finiteNumber(benchmark.fixture?.scheduleGapsMs?.p90);
|
|
39
|
+
if (fixtureScheduleGapP90Ms === undefined)
|
|
40
|
+
return undefined;
|
|
41
|
+
const domGapP90Ms = finiteNumber(benchmark.dom.gapsMs.p90);
|
|
42
|
+
const sseTextGapP90Ms = finiteNumber(benchmark.sse?.textEventGapsMs.p90);
|
|
43
|
+
return {
|
|
44
|
+
fixtureScheduleGapP90Ms,
|
|
45
|
+
domGapP90Ms,
|
|
46
|
+
sseTextGapP90Ms,
|
|
47
|
+
domLagOverScheduleP90Ms: domGapP90Ms === undefined ? undefined : round3(domGapP90Ms - fixtureScheduleGapP90Ms),
|
|
48
|
+
sseTextLagOverScheduleP90Ms: sseTextGapP90Ms === undefined ? undefined : round3(sseTextGapP90Ms - fixtureScheduleGapP90Ms),
|
|
49
|
+
domToScheduleP90Ratio: domGapP90Ms === undefined || fixtureScheduleGapP90Ms <= 0 ? undefined : round3(domGapP90Ms / fixtureScheduleGapP90Ms),
|
|
50
|
+
sseTextToScheduleP90Ratio: sseTextGapP90Ms === undefined || fixtureScheduleGapP90Ms <= 0 ? undefined : round3(sseTextGapP90Ms / fixtureScheduleGapP90Ms),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
export function summarizeStreamingProviderPreservation(benchmark) {
|
|
54
|
+
const provider = benchmark.provider;
|
|
55
|
+
if (!provider?.available)
|
|
56
|
+
return undefined;
|
|
57
|
+
const selectedLive = summarizeStreamingSelectedLiveEventSource(benchmark);
|
|
58
|
+
return {
|
|
59
|
+
providerTextDeltaCount: provider.textDeltaCount,
|
|
60
|
+
providerReasoningDeltaCount: provider.reasoningDeltaCount,
|
|
61
|
+
sseTextEventCount: benchmark.sse?.textEventCount,
|
|
62
|
+
sseReasoningEventCount: benchmark.sse?.reasoningEventCount,
|
|
63
|
+
selectedLiveTextEventCountAfterStart: selectedLive?.textEventCountAfterStart,
|
|
64
|
+
selectedLiveReasoningEventCountAfterStart: selectedLive?.reasoningEventCountAfterStart,
|
|
65
|
+
domPositiveUpdateCount: benchmark.dom?.positiveUpdateCount,
|
|
66
|
+
sseTextToProviderRatio: ratioToProvider(benchmark.sse?.textEventCount, provider.textDeltaCount),
|
|
67
|
+
sseReasoningToProviderRatio: ratioToProvider(benchmark.sse?.reasoningEventCount, provider.reasoningDeltaCount),
|
|
68
|
+
selectedLiveTextToProviderRatio: ratioToProvider(selectedLive?.textEventCountAfterStart, provider.textDeltaCount),
|
|
69
|
+
selectedLiveReasoningToProviderRatio: ratioToProvider(selectedLive?.reasoningEventCountAfterStart, provider.reasoningDeltaCount),
|
|
70
|
+
domPositiveToProviderTextRatio: ratioToProvider(benchmark.dom?.positiveUpdateCount, provider.textDeltaCount),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
export function summarizeStreamingLivePipeline(benchmark) {
|
|
74
|
+
const debugDelta = benchmark.debug?.delta ?? {};
|
|
75
|
+
const debugStateBeforeReset = benchmark.debug?.stateBeforeReset ?? {};
|
|
76
|
+
const debugAfter = benchmark.debug?.after ?? {};
|
|
77
|
+
const fixtureTextCount = finiteNumber(benchmark.fixture?.deltaCount);
|
|
78
|
+
const fixtureReasoningCount = finiteNumber(benchmark.fixture?.reasoningDeltaCount);
|
|
79
|
+
const provider = benchmark.provider?.available ? benchmark.provider : undefined;
|
|
80
|
+
const debugTextCount = finiteNumber(debugDelta.textDeltaCount);
|
|
81
|
+
const debugReasoningCount = finiteNumber(debugDelta.reasoningDeltaCount);
|
|
82
|
+
const expectedSource = fixtureTextCount !== undefined || fixtureReasoningCount !== undefined ? "fixture" : provider ? "provider" : "debug";
|
|
83
|
+
const expectedTextDeltaCount = expectedSource === "fixture" ? (fixtureTextCount ?? 0) : expectedSource === "provider" ? provider?.textDeltaCount ?? 0 : debugTextCount ?? 0;
|
|
84
|
+
const expectedReasoningDeltaCount = expectedSource === "fixture" ? (fixtureReasoningCount ?? 0) : expectedSource === "provider" ? provider?.reasoningDeltaCount ?? 0 : debugReasoningCount ?? 0;
|
|
85
|
+
const expectedInputEventCount = expectedTextDeltaCount + expectedReasoningDeltaCount;
|
|
86
|
+
if (expectedInputEventCount <= 0)
|
|
87
|
+
return undefined;
|
|
88
|
+
const expectedPipelineEventCount = expectedStreamingPipelineEventCount(benchmark, expectedInputEventCount);
|
|
89
|
+
const enqueueCount = finiteNumber(debugDelta.enqueueCount);
|
|
90
|
+
const flushCount = finiteNumber(debugDelta.flushCount);
|
|
91
|
+
const flushedEventCount = finiteNumber(debugDelta.flushedEventCount);
|
|
92
|
+
const overlayUpdateCount = finiteNumber(debugDelta.overlayUpdateCount);
|
|
93
|
+
const overlayEventCount = stateWindowNumber(debugAfter.overlayEventCount, debugStateBeforeReset.overlayEventCount);
|
|
94
|
+
const currentOutputLength = stateWindowNumber(debugAfter.currentOutputLength, debugStateBeforeReset.currentOutputLength);
|
|
95
|
+
const expectedTextBytes = finiteNumber(benchmark.fixture?.textBytes);
|
|
96
|
+
const debugStartedAt = typeof debugAfter.startedAt === "string" ? debugAfter.startedAt : undefined;
|
|
97
|
+
return {
|
|
98
|
+
expectedSource,
|
|
99
|
+
expectedTextDeltaCount,
|
|
100
|
+
expectedReasoningDeltaCount,
|
|
101
|
+
expectedInputEventCount,
|
|
102
|
+
expectedPipelineEventCount,
|
|
103
|
+
enqueueCount,
|
|
104
|
+
flushCount,
|
|
105
|
+
flushedEventCount,
|
|
106
|
+
overlayUpdateCount,
|
|
107
|
+
overlayEventCount,
|
|
108
|
+
currentOutputLength,
|
|
109
|
+
expectedTextBytes,
|
|
110
|
+
enqueueToExpectedRatio: ratioToProvider(enqueueCount, expectedPipelineEventCount),
|
|
111
|
+
flushedEventsToExpectedRatio: ratioToProvider(flushedEventCount, expectedPipelineEventCount),
|
|
112
|
+
overlayEventsToExpectedRatio: ratioToProvider(overlayEventCount, expectedInputEventCount),
|
|
113
|
+
currentOutputToExpectedTextBytesRatio: ratioToProvider(currentOutputLength, expectedTextBytes),
|
|
114
|
+
flushToEnqueueRatio: ratioToProvider(flushCount, enqueueCount),
|
|
115
|
+
overlayUpdatesToFlushedEventsRatio: ratioToProvider(overlayUpdateCount, flushedEventCount),
|
|
116
|
+
firstEventMs: elapsedIsoMs(debugStartedAt, debugAfter.firstEventAt),
|
|
117
|
+
firstTextDeltaMs: elapsedIsoMs(debugStartedAt, debugAfter.firstTextDeltaAt),
|
|
118
|
+
firstReasoningDeltaMs: elapsedIsoMs(debugStartedAt, debugAfter.firstReasoningDeltaAt),
|
|
119
|
+
firstEnqueueMs: elapsedIsoMs(debugStartedAt, debugAfter.firstEnqueueAt),
|
|
120
|
+
firstFlushMs: elapsedIsoMs(debugStartedAt, debugAfter.firstFlushAt),
|
|
121
|
+
firstOverlayUpdateMs: elapsedIsoMs(debugStartedAt, debugAfter.firstOverlayUpdateAt),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
export function attachStreamingProviderTelemetryToBenchmark(benchmark, providerTelemetry) {
|
|
125
|
+
if (!providerTelemetry)
|
|
126
|
+
return benchmark;
|
|
127
|
+
const baseRegressions = benchmark.regressions.filter((regression) => !regression.startsWith("provider ") && !regression.startsWith("live pipeline "));
|
|
128
|
+
const withProvider = { ...benchmark, provider: providerTelemetry };
|
|
129
|
+
const providerPreservation = summarizeStreamingProviderPreservation(withProvider);
|
|
130
|
+
const withProviderMetrics = { ...withProvider, providerPreservation, livePipeline: summarizeStreamingLivePipeline(withProvider) };
|
|
131
|
+
return {
|
|
132
|
+
...withProviderMetrics,
|
|
133
|
+
regressions: [...baseRegressions, ...evaluateStreamingLivePipelineRegressions(withProviderMetrics), ...evaluateStreamingProviderRegressions(withProviderMetrics)],
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
export function attachStreamingProviderTelemetryToBenchmarks(benchmarks, providerTelemetry) {
|
|
137
|
+
return providerTelemetry ? benchmarks.map((benchmark) => attachStreamingProviderTelemetryToBenchmark(benchmark, providerTelemetry)) : benchmarks;
|
|
138
|
+
}
|
|
139
|
+
function expectedStreamingPipelineEventCount(_benchmark, fallbackInputEventCount) {
|
|
140
|
+
return fallbackInputEventCount;
|
|
141
|
+
}
|
|
142
|
+
export function evaluateStreamingLivePipelineRegressions(benchmark) {
|
|
143
|
+
const pipeline = benchmark.livePipeline;
|
|
144
|
+
if (!pipeline || pipeline.expectedSource === "debug" || pipeline.expectedInputEventCount <= 0 || benchmark.fixture?.simulation === "trace-catchup")
|
|
145
|
+
return [];
|
|
146
|
+
const hasLiveCounters = [pipeline.enqueueCount, pipeline.flushCount, pipeline.flushedEventCount, pipeline.overlayUpdateCount, pipeline.overlayEventCount].some((value) => value !== undefined);
|
|
147
|
+
if (!hasLiveCounters)
|
|
148
|
+
return [];
|
|
149
|
+
const regressions = [];
|
|
150
|
+
pushMinimumRatioRegression(regressions, pipeline.flushedEventsToExpectedRatio, LIVE_PIPELINE_MIN_PRESERVATION_RATIO, "live pipeline flushed events/overlay expected ratio");
|
|
151
|
+
pushMinimumRatioRegression(regressions, pipeline.overlayEventsToExpectedRatio, LIVE_PIPELINE_MIN_PRESERVATION_RATIO, "live pipeline overlay events/input expected ratio");
|
|
152
|
+
if (pipeline.expectedTextDeltaCount > 0 && pipeline.expectedTextBytes !== undefined) {
|
|
153
|
+
pushMinimumRatioRegression(regressions, pipeline.currentOutputToExpectedTextBytesRatio, LIVE_PIPELINE_MIN_PRESERVATION_RATIO, "live pipeline current text/expected bytes ratio");
|
|
154
|
+
}
|
|
155
|
+
pushMinimumRatioRegression(regressions, pipeline.flushToEnqueueRatio, LIVE_PIPELINE_MIN_FLUSH_RATIO, "live pipeline flush/enqueue ratio");
|
|
156
|
+
pushMinimumRatioRegression(regressions, pipeline.overlayUpdatesToFlushedEventsRatio, LIVE_PIPELINE_MIN_FLUSH_RATIO, "live pipeline overlay updates/flushed ratio");
|
|
157
|
+
return regressions;
|
|
158
|
+
}
|
|
159
|
+
export function evaluateStreamingProviderRegressions(benchmark) {
|
|
160
|
+
const provider = benchmark.provider;
|
|
161
|
+
if (!provider?.available)
|
|
162
|
+
return [];
|
|
163
|
+
const regressions = [];
|
|
164
|
+
if ((provider.parseErrorCount ?? 0) > 0)
|
|
165
|
+
regressions.push(`provider parse errors ${provider.parseErrorCount} > 0`);
|
|
166
|
+
if ((provider.unknownEventCount ?? 0) > 0)
|
|
167
|
+
regressions.push(`provider unknown events ${provider.unknownEventCount} > 0`);
|
|
168
|
+
if (provider.truncated)
|
|
169
|
+
regressions.push("provider telemetry events were truncated");
|
|
170
|
+
const preservation = benchmark.providerPreservation;
|
|
171
|
+
if (provider.textDeltaCount > 0) {
|
|
172
|
+
if (benchmark.sse?.requested)
|
|
173
|
+
pushProviderRatioRegression(regressions, preservation?.sseTextToProviderRatio, PROVIDER_MIN_TEXT_PRESERVATION_RATIO, "provider SSE text preservation ratio");
|
|
174
|
+
if (benchmark.eventSource?.requested)
|
|
175
|
+
pushProviderRatioRegression(regressions, preservation?.selectedLiveTextToProviderRatio, PROVIDER_MIN_TEXT_PRESERVATION_RATIO, "provider selected-live text preservation ratio");
|
|
176
|
+
}
|
|
177
|
+
if (provider.reasoningDeltaCount > 0) {
|
|
178
|
+
if (benchmark.sse?.requested)
|
|
179
|
+
pushProviderRatioRegression(regressions, preservation?.sseReasoningToProviderRatio, PROVIDER_MIN_REASONING_PRESERVATION_RATIO, "provider SSE reasoning preservation ratio");
|
|
180
|
+
if (benchmark.eventSource?.requested)
|
|
181
|
+
pushProviderRatioRegression(regressions, preservation?.selectedLiveReasoningToProviderRatio, PROVIDER_MIN_REASONING_PRESERVATION_RATIO, "provider selected-live reasoning preservation ratio");
|
|
182
|
+
}
|
|
183
|
+
return regressions;
|
|
184
|
+
}
|
|
185
|
+
function pushProviderRatioRegression(regressions, ratio, minimum, label) {
|
|
186
|
+
pushMinimumRatioRegression(regressions, ratio, minimum, label);
|
|
187
|
+
}
|
|
188
|
+
function pushMinimumRatioRegression(regressions, ratio, minimum, label) {
|
|
189
|
+
if (ratio === undefined)
|
|
190
|
+
regressions.push(`${label} unavailable`);
|
|
191
|
+
else if (ratio < minimum)
|
|
192
|
+
regressions.push(`${label} ${ratio} < ${minimum}`);
|
|
193
|
+
}
|
|
194
|
+
function ratioToProvider(numerator, denominator) {
|
|
195
|
+
if (numerator === undefined || denominator === undefined || denominator <= 0)
|
|
196
|
+
return undefined;
|
|
197
|
+
return round3(numerator / denominator);
|
|
198
|
+
}
|
|
199
|
+
function commonStreamingNegativeProfile(runs) {
|
|
200
|
+
const profile = runs[0]?.negativeProfile;
|
|
201
|
+
return profile && runs.every((run) => run.negativeProfile === profile) ? profile : undefined;
|
|
202
|
+
}
|
|
203
|
+
export function summarizeStreamingBenchmarkGroup(runs, baselineRuns) {
|
|
204
|
+
const summary = summarizeStreamingBenchmarks(runs);
|
|
205
|
+
return {
|
|
206
|
+
kind: "streaming-benchmark-runs",
|
|
207
|
+
createdAt: new Date().toISOString(),
|
|
208
|
+
durationMs: runs[0]?.durationMs ?? 0,
|
|
209
|
+
runs,
|
|
210
|
+
negativeProfile: commonStreamingNegativeProfile(runs),
|
|
211
|
+
summary,
|
|
212
|
+
comparison: baselineRuns?.length ? compareStreamingBenchmarkSummaries(summarizeStreamingBenchmarks(baselineRuns), summary) : undefined,
|
|
213
|
+
regressions: runs.flatMap((run, index) => run.regressions.map((regression) => `run ${index + 1}: ${regression}`)),
|
|
214
|
+
warnings: runs.flatMap((run, index) => run.warnings.map((warning) => `run ${index + 1}: ${warning}`)),
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
export function summarizeStreamingBenchmarkUrlComparison(primaryUrl, compareUrl, primary, compare) {
|
|
218
|
+
const summaryComparison = compareStreamingBenchmarkSummaries(primary.summary, compare.summary);
|
|
219
|
+
const negativeProfile = primary.negativeProfile && primary.negativeProfile === compare.negativeProfile ? primary.negativeProfile : undefined;
|
|
220
|
+
return {
|
|
221
|
+
kind: "streaming-benchmark-url-comparison",
|
|
222
|
+
createdAt: new Date().toISOString(),
|
|
223
|
+
durationMs: primary.durationMs,
|
|
224
|
+
primaryUrl,
|
|
225
|
+
compareUrl,
|
|
226
|
+
primary,
|
|
227
|
+
compare,
|
|
228
|
+
negativeProfile,
|
|
229
|
+
comparison: summaryComparison,
|
|
230
|
+
regressions: [
|
|
231
|
+
...primary.regressions.map((regression) => `primary: ${regression}`),
|
|
232
|
+
...compare.regressions.map((regression) => `compare: ${regression}`),
|
|
233
|
+
...evaluateStreamingBenchmarkUrlComparisonRegressions(summaryComparison).map((regression) => `comparison: ${regression}`),
|
|
234
|
+
],
|
|
235
|
+
warnings: [
|
|
236
|
+
...primary.warnings.map((warning) => `primary: ${warning}`),
|
|
237
|
+
...compare.warnings.map((warning) => `compare: ${warning}`),
|
|
238
|
+
],
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
export function evaluateStreamingBenchmarkUrlComparisonRegressions(comparison) {
|
|
242
|
+
const regressions = [];
|
|
243
|
+
if (comparison.smoothnessDelta !== undefined && comparison.smoothnessDelta < -URL_COMPARISON_MAX_SMOOTHNESS_DROP) {
|
|
244
|
+
regressions.push(`compare smoothness delta ${comparison.smoothnessDelta} below -${URL_COMPARISON_MAX_SMOOTHNESS_DROP}`);
|
|
245
|
+
}
|
|
246
|
+
if (comparison.domLagOverFixtureScheduleP90DeltaMs !== undefined && comparison.domLagOverFixtureScheduleP90DeltaMs > URL_COMPARISON_MAX_DOM_LAG_DELTA_MS) {
|
|
247
|
+
regressions.push(`compare DOM lag over schedule delta ${comparison.domLagOverFixtureScheduleP90DeltaMs}ms exceeds ${URL_COMPARISON_MAX_DOM_LAG_DELTA_MS}ms`);
|
|
248
|
+
}
|
|
249
|
+
if (comparison.sseTextLagOverFixtureScheduleP90DeltaMs !== undefined && comparison.sseTextLagOverFixtureScheduleP90DeltaMs > URL_COMPARISON_MAX_SSE_LAG_DELTA_MS) {
|
|
250
|
+
regressions.push(`compare SSE text lag over schedule delta ${comparison.sseTextLagOverFixtureScheduleP90DeltaMs}ms exceeds ${URL_COMPARISON_MAX_SSE_LAG_DELTA_MS}ms`);
|
|
251
|
+
}
|
|
252
|
+
if (comparison.sseChunkGapP90DeltaMs !== undefined && comparison.sseChunkGapP90DeltaMs > URL_COMPARISON_MAX_SSE_CHUNK_GAP_DELTA_MS) {
|
|
253
|
+
regressions.push(`compare SSE chunk p90 gap delta ${comparison.sseChunkGapP90DeltaMs}ms exceeds ${URL_COMPARISON_MAX_SSE_CHUNK_GAP_DELTA_MS}ms`);
|
|
254
|
+
}
|
|
255
|
+
if (comparison.selectedLiveFirstTextEventDeltaMs !== undefined && comparison.selectedLiveFirstTextEventDeltaMs > URL_COMPARISON_MAX_FIRST_TRANSPORT_LATENCY_DELTA_MS) {
|
|
256
|
+
regressions.push(`compare selected-live first text latency delta ${comparison.selectedLiveFirstTextEventDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_TRANSPORT_LATENCY_DELTA_MS}ms`);
|
|
257
|
+
}
|
|
258
|
+
if (comparison.sseFirstTextEventDeltaMs !== undefined && comparison.sseFirstTextEventDeltaMs > URL_COMPARISON_MAX_FIRST_TRANSPORT_LATENCY_DELTA_MS) {
|
|
259
|
+
regressions.push(`compare SSE first text latency delta ${comparison.sseFirstTextEventDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_TRANSPORT_LATENCY_DELTA_MS}ms`);
|
|
260
|
+
}
|
|
261
|
+
if (comparison.liveFirstTextDeltaDeltaMs !== undefined && comparison.liveFirstTextDeltaDeltaMs > URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS) {
|
|
262
|
+
regressions.push(`compare live first text latency delta ${comparison.liveFirstTextDeltaDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS}ms`);
|
|
263
|
+
}
|
|
264
|
+
if (comparison.liveFirstEnqueueDeltaMs !== undefined && comparison.liveFirstEnqueueDeltaMs > URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS) {
|
|
265
|
+
regressions.push(`compare live first enqueue latency delta ${comparison.liveFirstEnqueueDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS}ms`);
|
|
266
|
+
}
|
|
267
|
+
if (comparison.liveFirstFlushDeltaMs !== undefined && comparison.liveFirstFlushDeltaMs > URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS) {
|
|
268
|
+
regressions.push(`compare live first flush latency delta ${comparison.liveFirstFlushDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS}ms`);
|
|
269
|
+
}
|
|
270
|
+
if (comparison.liveFirstOverlayUpdateDeltaMs !== undefined && comparison.liveFirstOverlayUpdateDeltaMs > URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS) {
|
|
271
|
+
regressions.push(`compare live first overlay latency delta ${comparison.liveFirstOverlayUpdateDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_LIVE_LATENCY_DELTA_MS}ms`);
|
|
272
|
+
}
|
|
273
|
+
if (comparison.firstVisibleDeltaMs !== undefined && comparison.firstVisibleDeltaMs > URL_COMPARISON_MAX_FIRST_VISIBLE_LATENCY_DELTA_MS) {
|
|
274
|
+
regressions.push(`compare DOM first visible latency delta ${comparison.firstVisibleDeltaMs}ms exceeds ${URL_COMPARISON_MAX_FIRST_VISIBLE_LATENCY_DELTA_MS}ms`);
|
|
275
|
+
}
|
|
276
|
+
if (comparison.sseTextEventDelta !== undefined && comparison.sseTextEventDelta < 0)
|
|
277
|
+
regressions.push(`compare SSE text events delta ${comparison.sseTextEventDelta} below 0`);
|
|
278
|
+
if (comparison.selectedLiveTextEventDelta !== undefined && comparison.selectedLiveTextEventDelta < 0)
|
|
279
|
+
regressions.push(`compare selected-live text events delta ${comparison.selectedLiveTextEventDelta} below 0`);
|
|
280
|
+
if (comparison.selectedLiveReasoningEventDelta !== undefined && comparison.selectedLiveReasoningEventDelta < 0)
|
|
281
|
+
regressions.push(`compare selected-live reasoning events delta ${comparison.selectedLiveReasoningEventDelta} below 0`);
|
|
282
|
+
return regressions;
|
|
283
|
+
}
|
|
284
|
+
export function applyExpectedStreamingRegressions(benchmark, expectedPatterns) {
|
|
285
|
+
const assertion = evaluateStreamingBenchmarkAssertion(benchmark.regressions, expectedPatterns);
|
|
286
|
+
if (expectedPatterns.length > 0)
|
|
287
|
+
benchmark.assertion = assertion;
|
|
288
|
+
return assertion;
|
|
289
|
+
}
|
|
290
|
+
export function evaluateStreamingBenchmarkAssertion(regressions, expectedPatterns) {
|
|
291
|
+
const expectedRegressionPatterns = expectedPatterns.filter((pattern) => pattern.length > 0);
|
|
292
|
+
const expectedRegressions = [];
|
|
293
|
+
const unexpectedRegressions = [];
|
|
294
|
+
const matchedPatterns = new Set();
|
|
295
|
+
for (const regression of regressions) {
|
|
296
|
+
const matches = expectedRegressionPatterns.filter((pattern) => regression.includes(pattern));
|
|
297
|
+
if (matches.length > 0) {
|
|
298
|
+
expectedRegressions.push(regression);
|
|
299
|
+
for (const pattern of matches)
|
|
300
|
+
matchedPatterns.add(pattern);
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
unexpectedRegressions.push(regression);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const missingExpectedRegressionPatterns = expectedRegressionPatterns.filter((pattern) => !matchedPatterns.has(pattern));
|
|
307
|
+
return {
|
|
308
|
+
expectedRegressionPatterns,
|
|
309
|
+
expectedRegressions,
|
|
310
|
+
unexpectedRegressions,
|
|
311
|
+
missingExpectedRegressionPatterns,
|
|
312
|
+
passed: unexpectedRegressions.length === 0 && missingExpectedRegressionPatterns.length === 0,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
export function summarizeStreamingBenchmarks(runs) {
|
|
316
|
+
return {
|
|
317
|
+
runs: runs.length,
|
|
318
|
+
smoothness: numericStats(runs.map((run) => run.score.smoothness)),
|
|
319
|
+
textDeltaCount: numericStats(runs.map((run) => run.score.textDeltaCount)),
|
|
320
|
+
reasoningDeltaCount: numericStats(runs.map((run) => numberField(run.debug.delta ?? {}, "reasoningDeltaCount"))),
|
|
321
|
+
domPositiveUpdateCount: numericStats(runs.map((run) => run.score.domPositiveUpdateCount)),
|
|
322
|
+
domGapP50Ms: numericStats(runs.map((run) => run.dom.gapsMs.p50)),
|
|
323
|
+
domGapP90Ms: numericStats(runs.map((run) => run.dom.gapsMs.p90)),
|
|
324
|
+
domGapMaxMs: numericStats(runs.map((run) => run.dom.gapsMs.max)),
|
|
325
|
+
domJumpP90Chars: numericStats(runs.map((run) => run.dom.positiveCharJumps.p90)),
|
|
326
|
+
domJumpMaxChars: numericStats(runs.map((run) => run.dom.positiveCharJumps.max)),
|
|
327
|
+
firstVisibleMs: numericStats(runs.map((run) => run.dom.firstPositiveUpdateMs)),
|
|
328
|
+
longTaskMaxMs: numericStats(runs.map((run) => run.longTasks.maxMs)),
|
|
329
|
+
regressionCount: numericStats(runs.map((run) => run.regressions.length)),
|
|
330
|
+
debugEnqueueCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "enqueueCount"))),
|
|
331
|
+
debugFlushCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "flushCount"))),
|
|
332
|
+
debugFlushedEventCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "flushedEventCount"))),
|
|
333
|
+
debugOverlayUpdateCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "overlayUpdateCount"))),
|
|
334
|
+
debugOverlayEventCount: numericStats(runs.map((run) => streamingDebugStateWindowNumber(run, "overlayEventCount"))),
|
|
335
|
+
debugLiveTraceComputeCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "liveTraceComputeCount"))),
|
|
336
|
+
debugLiveTraceComputeDurationTotalMs: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "liveTraceComputeDurationMsTotal"))),
|
|
337
|
+
debugLiveTraceComputeDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "liveTraceComputeDurationMsMax"))),
|
|
338
|
+
debugMarkdownRenderCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderCount"))),
|
|
339
|
+
debugMarkdownRenderPlainCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderPlainCount"))),
|
|
340
|
+
debugMarkdownRenderFullCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderFullCount"))),
|
|
341
|
+
debugMarkdownRenderCommonMarkCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderCommonMarkCount"))),
|
|
342
|
+
debugMarkdownRenderGfmCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderGfmCount"))),
|
|
343
|
+
debugMarkdownRenderGfmFastCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderGfmFastCount"))),
|
|
344
|
+
debugMarkdownRenderDurationTotalMs: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "markdownRenderDurationMsTotal"))),
|
|
345
|
+
debugMarkdownRenderDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "markdownRenderDurationMsMax"))),
|
|
346
|
+
debugMarkdownRenderPlainDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "markdownRenderPlainDurationMsMax"))),
|
|
347
|
+
debugMarkdownRenderFullDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "markdownRenderFullDurationMsMax"))),
|
|
348
|
+
debugMarkdownRenderCommonMarkDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "markdownRenderCommonMarkDurationMsMax"))),
|
|
349
|
+
debugMarkdownRenderGfmDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "markdownRenderGfmDurationMsMax"))),
|
|
350
|
+
debugMarkdownRenderGfmFastDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "markdownRenderGfmFastDurationMsMax"))),
|
|
351
|
+
debugTraceRefreshScheduledCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "traceRefreshScheduledCount"))),
|
|
352
|
+
debugTraceRefreshCompletedCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "traceRefreshCompletedCount"))),
|
|
353
|
+
debugTraceRefreshFailedCount: numericStats(runs.map((run) => streamingDebugDeltaNumber(run, "traceRefreshFailedCount"))),
|
|
354
|
+
debugTraceRefreshDurationMaxMs: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "traceRefreshDurationMsMax"))),
|
|
355
|
+
debugCurrentOutputLength: numericStats(runs.map((run) => streamingDebugStateWindowNumber(run, "currentOutputLength"))),
|
|
356
|
+
debugTraceBaseOutputLength: numericStats(runs.map((run) => streamingDebugAfterNumber(run, "traceBaseOutputLength"))),
|
|
357
|
+
liveExpectedInputEventCount: numericStats(runs.map((run) => streamingLivePipeline(run)?.expectedInputEventCount)),
|
|
358
|
+
liveExpectedPipelineEventCount: numericStats(runs.map((run) => streamingLivePipeline(run)?.expectedPipelineEventCount)),
|
|
359
|
+
liveEnqueueToExpectedRatio: numericStats(runs.map((run) => streamingLivePipeline(run)?.enqueueToExpectedRatio)),
|
|
360
|
+
liveFlushedEventsToExpectedRatio: numericStats(runs.map((run) => streamingLivePipeline(run)?.flushedEventsToExpectedRatio)),
|
|
361
|
+
liveOverlayEventsToExpectedRatio: numericStats(runs.map((run) => streamingLivePipeline(run)?.overlayEventsToExpectedRatio)),
|
|
362
|
+
liveCurrentOutputToExpectedTextBytesRatio: numericStats(runs.map((run) => streamingLivePipeline(run)?.currentOutputToExpectedTextBytesRatio)),
|
|
363
|
+
liveFlushToEnqueueRatio: numericStats(runs.map((run) => streamingLivePipeline(run)?.flushToEnqueueRatio)),
|
|
364
|
+
liveOverlayUpdatesToFlushedEventsRatio: numericStats(runs.map((run) => streamingLivePipeline(run)?.overlayUpdatesToFlushedEventsRatio)),
|
|
365
|
+
liveFirstEventMs: numericStats(runs.map((run) => streamingLivePipeline(run)?.firstEventMs)),
|
|
366
|
+
liveFirstTextDeltaMs: numericStats(runs.map((run) => streamingLivePipeline(run)?.firstTextDeltaMs)),
|
|
367
|
+
liveFirstReasoningDeltaMs: numericStats(runs.map((run) => streamingLivePipeline(run)?.firstReasoningDeltaMs)),
|
|
368
|
+
liveFirstEnqueueMs: numericStats(runs.map((run) => streamingLivePipeline(run)?.firstEnqueueMs)),
|
|
369
|
+
liveFirstFlushMs: numericStats(runs.map((run) => streamingLivePipeline(run)?.firstFlushMs)),
|
|
370
|
+
liveFirstOverlayUpdateMs: numericStats(runs.map((run) => streamingLivePipeline(run)?.firstOverlayUpdateMs)),
|
|
371
|
+
traceSampleCount: numericStats(runs.map((run) => run.trace?.sampleCount)),
|
|
372
|
+
traceLiveVersionCount: numericStats(runs.map((run) => run.trace?.liveVersionCount)),
|
|
373
|
+
traceFirstLiveVersionMs: numericStats(runs.map((run) => run.trace?.firstLiveVersionMs)),
|
|
374
|
+
traceMaxAssistantOutputLength: numericStats(runs.map((run) => run.trace?.maxAssistantOutputLength)),
|
|
375
|
+
traceFinalAssistantOutputLength: numericStats(runs.map((run) => run.trace?.finalAssistantOutputLength)),
|
|
376
|
+
traceDurableEventDelta: numericStats(runs.map((run) => traceDurableEventDelta(run.trace))),
|
|
377
|
+
fixtureScheduleGapP90Ms: numericStats(runs.map((run) => run.fixture?.scheduleGapsMs?.p90)),
|
|
378
|
+
domLagOverFixtureScheduleP90Ms: numericStats(runs.map((run) => run.cadence?.domLagOverScheduleP90Ms)),
|
|
379
|
+
sseTextLagOverFixtureScheduleP90Ms: numericStats(runs.map((run) => run.cadence?.sseTextLagOverScheduleP90Ms)),
|
|
380
|
+
domToFixtureScheduleP90Ratio: numericStats(runs.map((run) => run.cadence?.domToScheduleP90Ratio)),
|
|
381
|
+
sseTextToFixtureScheduleP90Ratio: numericStats(runs.map((run) => run.cadence?.sseTextToScheduleP90Ratio)),
|
|
382
|
+
eventSourceTextEventCountAfterStart: numericStats(runs.map((run) => run.eventSource?.textEventCountAfterStart)),
|
|
383
|
+
eventSourceReasoningEventCountAfterStart: numericStats(runs.map((run) => run.eventSource?.reasoningEventCountAfterStart)),
|
|
384
|
+
eventSourceForcedCloseCountAfterStart: numericStats(runs.map((run) => run.eventSource?.forcedCloseCountAfterStart)),
|
|
385
|
+
eventSourceReconnectOpenCountAfterStart: numericStats(runs.map((run) => run.eventSource?.openCountAfterStart)),
|
|
386
|
+
eventSourceTransientIdCountAfterStart: numericStats(runs.map((run) => run.eventSource?.transientIdCountAfterStart)),
|
|
387
|
+
sseTextEventCount: numericStats(runs.map((run) => run.sse?.textEventCount)),
|
|
388
|
+
sseReasoningEventCount: numericStats(runs.map((run) => run.sse?.reasoningEventCount)),
|
|
389
|
+
sseChunkBytesP50: numericStats(runs.map((run) => run.sse?.chunkBytes.p50)),
|
|
390
|
+
sseChunkGapP90Ms: numericStats(runs.map((run) => run.sse?.chunkGapsMs.p90)),
|
|
391
|
+
sseTextEventsPerChunkP90: numericStats(runs.map((run) => run.sse?.textEventsPerChunk.p90)),
|
|
392
|
+
sseTextEventGapP90Ms: numericStats(runs.map((run) => run.sse?.textEventGapsMs.p90)),
|
|
393
|
+
sseFirstTextEventMs: numericStats(runs.map((run) => run.sse?.firstTextEventMs)),
|
|
394
|
+
selectedLiveTextEventCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.textEventCountAfterStart)),
|
|
395
|
+
selectedLiveReasoningEventCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.reasoningEventCountAfterStart)),
|
|
396
|
+
selectedLiveEventCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.eventCountAfterStart)),
|
|
397
|
+
selectedLiveForcedCloseCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.forcedCloseCountAfterStart)),
|
|
398
|
+
selectedLiveReconnectOpenCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.openCountAfterStart)),
|
|
399
|
+
selectedLiveTransientIdCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.transientIdCountAfterStart)),
|
|
400
|
+
selectedLiveLiveSinceCount: numericStats(runs.map((run) => selectedLiveStream(run)?.liveSinceValues?.length)),
|
|
401
|
+
selectedLiveReplayEventCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.liveReplayEventCountAfterStart)),
|
|
402
|
+
selectedLiveReplayCursorLagMaxAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.liveReplayCursorLagMaxAfterStart)),
|
|
403
|
+
selectedLiveReplayDuplicateCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.liveReplayDuplicateCountAfterStart)),
|
|
404
|
+
selectedLiveReplayMissedCountAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.liveReplayMissedCountAfterStart)),
|
|
405
|
+
selectedLiveFirstTextEventMsAfterStart: numericStats(runs.map((run) => selectedLiveStream(run)?.firstTextEventMsAfterStart)),
|
|
406
|
+
roomSummaryEventCountAfterStart: numericStats(runs.map((run) => roomSummaryStream(run)?.eventCountAfterStart)),
|
|
407
|
+
roomSummaryTextEventCountAfterStart: numericStats(runs.map((run) => roomSummaryStream(run)?.textEventCountAfterStart)),
|
|
408
|
+
roomSummaryReasoningEventCountAfterStart: numericStats(runs.map((run) => roomSummaryStream(run)?.reasoningEventCountAfterStart)),
|
|
409
|
+
providerTextDeltaCount: numericStats(runs.map((run) => run.provider?.textDeltaCount)),
|
|
410
|
+
providerReasoningDeltaCount: numericStats(runs.map((run) => run.provider?.reasoningDeltaCount)),
|
|
411
|
+
providerTextDeltaBytesP50: numericStats(runs.map((run) => run.provider?.textDeltaBytes.p50)),
|
|
412
|
+
providerTextDeltaGapP90Ms: numericStats(runs.map((run) => run.provider?.textDeltaGapsMs.p90)),
|
|
413
|
+
providerFirstTextLatencyMs: numericStats(runs.map((run) => run.provider?.firstTextLatencyMs)),
|
|
414
|
+
providerParseErrorCount: numericStats(runs.map((run) => run.provider?.parseErrorCount)),
|
|
415
|
+
providerUnknownEventCount: numericStats(runs.map((run) => run.provider?.unknownEventCount)),
|
|
416
|
+
providerSseTextRatio: numericStats(runs.map((run) => run.providerPreservation?.sseTextToProviderRatio)),
|
|
417
|
+
providerSseReasoningRatio: numericStats(runs.map((run) => run.providerPreservation?.sseReasoningToProviderRatio)),
|
|
418
|
+
providerSelectedLiveTextRatio: numericStats(runs.map((run) => run.providerPreservation?.selectedLiveTextToProviderRatio)),
|
|
419
|
+
providerSelectedLiveReasoningRatio: numericStats(runs.map((run) => run.providerPreservation?.selectedLiveReasoningToProviderRatio)),
|
|
420
|
+
providerDomPositiveTextRatio: numericStats(runs.map((run) => run.providerPreservation?.domPositiveToProviderTextRatio)),
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
function compareStreamingBenchmarkSummaries(baseline, current) {
|
|
424
|
+
return {
|
|
425
|
+
baselineRuns: baseline.runs,
|
|
426
|
+
currentRuns: current.runs,
|
|
427
|
+
smoothnessDelta: statDelta(current.smoothness, baseline.smoothness),
|
|
428
|
+
domGapP90DeltaMs: statDelta(current.domGapP90Ms, baseline.domGapP90Ms),
|
|
429
|
+
domPositiveUpdateDelta: statDelta(current.domPositiveUpdateCount, baseline.domPositiveUpdateCount),
|
|
430
|
+
domJumpMaxDeltaChars: statDelta(current.domJumpMaxChars, baseline.domJumpMaxChars),
|
|
431
|
+
longTaskMaxDeltaMs: statDelta(current.longTaskMaxMs, baseline.longTaskMaxMs),
|
|
432
|
+
firstVisibleDeltaMs: statDelta(current.firstVisibleMs, baseline.firstVisibleMs),
|
|
433
|
+
liveFirstTextDeltaDeltaMs: statDelta(current.liveFirstTextDeltaMs, baseline.liveFirstTextDeltaMs),
|
|
434
|
+
liveFirstEnqueueDeltaMs: statDelta(current.liveFirstEnqueueMs, baseline.liveFirstEnqueueMs),
|
|
435
|
+
liveFirstFlushDeltaMs: statDelta(current.liveFirstFlushMs, baseline.liveFirstFlushMs),
|
|
436
|
+
liveFirstOverlayUpdateDeltaMs: statDelta(current.liveFirstOverlayUpdateMs, baseline.liveFirstOverlayUpdateMs),
|
|
437
|
+
sseFirstTextEventDeltaMs: statDelta(current.sseFirstTextEventMs, baseline.sseFirstTextEventMs),
|
|
438
|
+
selectedLiveFirstTextEventDeltaMs: statDelta(current.selectedLiveFirstTextEventMsAfterStart, baseline.selectedLiveFirstTextEventMsAfterStart),
|
|
439
|
+
providerFirstTextLatencyDeltaMs: statDelta(current.providerFirstTextLatencyMs, baseline.providerFirstTextLatencyMs),
|
|
440
|
+
debugEnqueueCountDelta: statDelta(current.debugEnqueueCount, baseline.debugEnqueueCount),
|
|
441
|
+
debugFlushCountDelta: statDelta(current.debugFlushCount, baseline.debugFlushCount),
|
|
442
|
+
debugOverlayUpdateCountDelta: statDelta(current.debugOverlayUpdateCount, baseline.debugOverlayUpdateCount),
|
|
443
|
+
debugTraceRefreshCompletedCountDelta: statDelta(current.debugTraceRefreshCompletedCount, baseline.debugTraceRefreshCompletedCount),
|
|
444
|
+
liveEnqueueToExpectedRatioDelta: statDelta(current.liveEnqueueToExpectedRatio, baseline.liveEnqueueToExpectedRatio),
|
|
445
|
+
liveFlushedEventsToExpectedRatioDelta: statDelta(current.liveFlushedEventsToExpectedRatio, baseline.liveFlushedEventsToExpectedRatio),
|
|
446
|
+
liveOverlayEventsToExpectedRatioDelta: statDelta(current.liveOverlayEventsToExpectedRatio, baseline.liveOverlayEventsToExpectedRatio),
|
|
447
|
+
liveFlushToEnqueueRatioDelta: statDelta(current.liveFlushToEnqueueRatio, baseline.liveFlushToEnqueueRatio),
|
|
448
|
+
liveOverlayUpdatesToFlushedEventsRatioDelta: statDelta(current.liveOverlayUpdatesToFlushedEventsRatio, baseline.liveOverlayUpdatesToFlushedEventsRatio),
|
|
449
|
+
traceLiveVersionCountDelta: statDelta(current.traceLiveVersionCount, baseline.traceLiveVersionCount),
|
|
450
|
+
traceMaxAssistantOutputDelta: statDelta(current.traceMaxAssistantOutputLength, baseline.traceMaxAssistantOutputLength),
|
|
451
|
+
traceDurableEventDeltaDelta: statDelta(current.traceDurableEventDelta, baseline.traceDurableEventDelta),
|
|
452
|
+
fixtureScheduleGapP90DeltaMs: statDelta(current.fixtureScheduleGapP90Ms, baseline.fixtureScheduleGapP90Ms),
|
|
453
|
+
domLagOverFixtureScheduleP90DeltaMs: statDelta(current.domLagOverFixtureScheduleP90Ms, baseline.domLagOverFixtureScheduleP90Ms),
|
|
454
|
+
sseTextLagOverFixtureScheduleP90DeltaMs: statDelta(current.sseTextLagOverFixtureScheduleP90Ms, baseline.sseTextLagOverFixtureScheduleP90Ms),
|
|
455
|
+
eventSourceTextEventDelta: statDelta(current.eventSourceTextEventCountAfterStart, baseline.eventSourceTextEventCountAfterStart),
|
|
456
|
+
eventSourceReasoningEventDelta: statDelta(current.eventSourceReasoningEventCountAfterStart, baseline.eventSourceReasoningEventCountAfterStart),
|
|
457
|
+
sseTextEventDelta: statDelta(current.sseTextEventCount, baseline.sseTextEventCount),
|
|
458
|
+
sseChunkGapP90DeltaMs: statDelta(current.sseChunkGapP90Ms, baseline.sseChunkGapP90Ms),
|
|
459
|
+
selectedLiveTextEventDelta: statDelta(current.selectedLiveTextEventCountAfterStart, baseline.selectedLiveTextEventCountAfterStart),
|
|
460
|
+
selectedLiveReasoningEventDelta: statDelta(current.selectedLiveReasoningEventCountAfterStart, baseline.selectedLiveReasoningEventCountAfterStart),
|
|
461
|
+
selectedLiveEventDelta: statDelta(current.selectedLiveEventCountAfterStart, baseline.selectedLiveEventCountAfterStart),
|
|
462
|
+
providerSseTextRatioDelta: statDelta(current.providerSseTextRatio, baseline.providerSseTextRatio),
|
|
463
|
+
providerSelectedLiveTextRatioDelta: statDelta(current.providerSelectedLiveTextRatio, baseline.providerSelectedLiveTextRatio),
|
|
464
|
+
providerSseReasoningRatioDelta: statDelta(current.providerSseReasoningRatio, baseline.providerSseReasoningRatio),
|
|
465
|
+
providerSelectedLiveReasoningRatioDelta: statDelta(current.providerSelectedLiveReasoningRatio, baseline.providerSelectedLiveReasoningRatio),
|
|
466
|
+
providerDomPositiveTextRatioDelta: statDelta(current.providerDomPositiveTextRatio, baseline.providerDomPositiveTextRatio),
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
function streamingLivePipeline(run) {
|
|
470
|
+
return run.livePipeline ?? summarizeStreamingLivePipeline(run);
|
|
471
|
+
}
|
|
472
|
+
function streamingDebugDeltaNumber(run, key) {
|
|
473
|
+
const value = run.debug.delta?.[key];
|
|
474
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
475
|
+
}
|
|
476
|
+
function streamingDebugAfterNumber(run, key) {
|
|
477
|
+
const value = run.debug.after?.[key];
|
|
478
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
479
|
+
}
|
|
480
|
+
function streamingDebugStateWindowNumber(run, key) {
|
|
481
|
+
return stateWindowNumber(run.debug.after?.[key], run.debug.stateBeforeReset?.[key]);
|
|
482
|
+
}
|
|
483
|
+
function selectedLiveStream(run) {
|
|
484
|
+
return summarizeStreamingSelectedLiveEventSource(run);
|
|
485
|
+
}
|
|
486
|
+
function roomSummaryStream(run) {
|
|
487
|
+
return run.eventSource?.streams.find((stream) => stream.role === "room-summary");
|
|
488
|
+
}
|
|
489
|
+
function traceDurableEventDelta(trace) {
|
|
490
|
+
if (!trace || trace.durableEventCountStart === undefined || trace.durableEventCountEnd === undefined)
|
|
491
|
+
return undefined;
|
|
492
|
+
return trace.durableEventCountEnd - trace.durableEventCountStart;
|
|
493
|
+
}
|
|
494
|
+
export async function readStreamingBenchmarkArtifact(file) {
|
|
495
|
+
const parsed = JSON.parse(await readFile(file, "utf8"));
|
|
496
|
+
const value = parsed.benchmark ?? parsed;
|
|
497
|
+
if (value?.kind === "streaming-benchmark")
|
|
498
|
+
return normalizeStreamingBenchmarkRun(value);
|
|
499
|
+
if (value?.kind === "streaming-benchmark-runs") {
|
|
500
|
+
const runs = Array.isArray(value.runs) ? value.runs.map((run) => normalizeStreamingBenchmarkRun(run)) : [];
|
|
501
|
+
return normalizeStreamingBenchmarkGroupArtifact(value, runs);
|
|
502
|
+
}
|
|
503
|
+
if (value?.kind === "streaming-benchmark-url-comparison") {
|
|
504
|
+
const primaryRuns = Array.isArray(value.primary?.runs) ? value.primary.runs.map((run) => normalizeStreamingBenchmarkRun(run)) : [];
|
|
505
|
+
const compareRuns = Array.isArray(value.compare?.runs) ? value.compare.runs.map((run) => normalizeStreamingBenchmarkRun(run)) : [];
|
|
506
|
+
const primary = normalizeStreamingBenchmarkGroupArtifact(value.primary, primaryRuns);
|
|
507
|
+
const compare = normalizeStreamingBenchmarkGroupArtifact(value.compare, compareRuns);
|
|
508
|
+
return {
|
|
509
|
+
...value,
|
|
510
|
+
primary,
|
|
511
|
+
compare,
|
|
512
|
+
comparison: compareStreamingBenchmarkSummaries(primary.summary, compare.summary),
|
|
513
|
+
regressions: Array.isArray(value.regressions) ? value.regressions : [],
|
|
514
|
+
warnings: Array.isArray(value.warnings) ? value.warnings : [],
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
throw new Error(`File is not a streaming benchmark artifact: ${file}`);
|
|
518
|
+
}
|
|
519
|
+
function normalizeStreamingBenchmarkGroupArtifact(value, runs) {
|
|
520
|
+
return {
|
|
521
|
+
...value,
|
|
522
|
+
kind: "streaming-benchmark-runs",
|
|
523
|
+
createdAt: value?.createdAt ?? new Date().toISOString(),
|
|
524
|
+
durationMs: value?.durationMs ?? runs[0]?.durationMs ?? 0,
|
|
525
|
+
runs,
|
|
526
|
+
summary: summarizeStreamingBenchmarks(runs),
|
|
527
|
+
regressions: Array.isArray(value?.regressions) ? value.regressions : [],
|
|
528
|
+
warnings: Array.isArray(value?.warnings) ? value.warnings : [],
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
function normalizeStreamingBenchmarkRun(run) {
|
|
532
|
+
const scored = { ...run, regressions: Array.isArray(run.regressions) ? run.regressions : [], warnings: Array.isArray(run.warnings) ? run.warnings : [], score: run.score ?? scoreStreamingBenchmark(run) };
|
|
533
|
+
const withProviderPreservation = { ...scored, providerPreservation: run.providerPreservation ?? summarizeStreamingProviderPreservation(scored) };
|
|
534
|
+
const withLivePipeline = { ...withProviderPreservation, livePipeline: run.livePipeline ?? summarizeStreamingLivePipeline(withProviderPreservation) };
|
|
535
|
+
return { ...withLivePipeline, cadence: run.cadence ?? summarizeStreamingCadence(withLivePipeline) };
|
|
536
|
+
}
|
|
537
|
+
export function streamingBenchmarkReportTarget(benchmark) {
|
|
538
|
+
const url = benchmark.kind === "streaming-benchmark-url-comparison"
|
|
539
|
+
? benchmark.primaryUrl
|
|
540
|
+
: benchmark.kind === "streaming-benchmark-runs"
|
|
541
|
+
? benchmark.runs[0]?.url ?? ""
|
|
542
|
+
: benchmark.url;
|
|
543
|
+
return { id: "artifact", url, title: "streaming benchmark artifact" };
|
|
544
|
+
}
|
|
545
|
+
export async function readStreamingBenchmarkRuns(file) {
|
|
546
|
+
const value = await readStreamingBenchmarkArtifact(file);
|
|
547
|
+
const runs = value.kind === "streaming-benchmark-runs"
|
|
548
|
+
? value.runs
|
|
549
|
+
: value.kind === "streaming-benchmark-url-comparison"
|
|
550
|
+
? [...value.primary.runs, ...value.compare.runs]
|
|
551
|
+
: [value];
|
|
552
|
+
return runs.filter((run) => run.kind === "streaming-benchmark");
|
|
553
|
+
}
|
|
554
|
+
function numericStats(values) {
|
|
555
|
+
const nums = values.filter((value) => typeof value === "number" && Number.isFinite(value)).slice().sort((a, b) => a - b);
|
|
556
|
+
if (!nums.length)
|
|
557
|
+
return { count: 0 };
|
|
558
|
+
const pick = (q) => nums[Math.min(nums.length - 1, Math.max(0, Math.floor((nums.length - 1) * q)))];
|
|
559
|
+
const avg = nums.reduce((sum, value) => sum + value, 0) / nums.length;
|
|
560
|
+
return {
|
|
561
|
+
count: nums.length,
|
|
562
|
+
min: round3(nums[0]),
|
|
563
|
+
p50: round3(pick(0.50)),
|
|
564
|
+
p90: round3(pick(0.90)),
|
|
565
|
+
p99: round3(pick(0.99)),
|
|
566
|
+
max: round3(nums[nums.length - 1]),
|
|
567
|
+
avg: round3(avg),
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
function statDelta(current, baseline) {
|
|
571
|
+
if (current.p50 === undefined || baseline.p50 === undefined)
|
|
572
|
+
return undefined;
|
|
573
|
+
return round3(current.p50 - baseline.p50);
|
|
574
|
+
}
|
|
575
|
+
function clampScore(value) {
|
|
576
|
+
return Math.min(100, Math.max(0, value));
|
|
577
|
+
}
|
|
578
|
+
function finiteNumber(value) {
|
|
579
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
580
|
+
}
|
|
581
|
+
function numberField(record, key) {
|
|
582
|
+
const value = record[key];
|
|
583
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
584
|
+
}
|
|
585
|
+
function stateWindowNumber(afterValue, beforeValue) {
|
|
586
|
+
const after = finiteNumber(afterValue);
|
|
587
|
+
if (after === undefined)
|
|
588
|
+
return undefined;
|
|
589
|
+
const before = finiteNumber(beforeValue);
|
|
590
|
+
if (before === undefined || before <= 0 || after < before)
|
|
591
|
+
return after;
|
|
592
|
+
return round3(after - before);
|
|
593
|
+
}
|
|
594
|
+
function elapsedIsoMs(startedAt, value) {
|
|
595
|
+
if (typeof startedAt !== "string" || typeof value !== "string")
|
|
596
|
+
return undefined;
|
|
597
|
+
const start = Date.parse(startedAt);
|
|
598
|
+
const end = Date.parse(value);
|
|
599
|
+
if (!Number.isFinite(start) || !Number.isFinite(end))
|
|
600
|
+
return undefined;
|
|
601
|
+
return round3(end - start);
|
|
602
|
+
}
|
|
603
|
+
function round3(value) {
|
|
604
|
+
return Math.round(value * 1000) / 1000;
|
|
605
|
+
}
|