@superblocksteam/vite-plugin-file-sync 2.0.150-next.0 → 2.0.150-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-service/index.d.ts.map +1 -1
- package/dist/ai-service/index.js +30 -2
- package/dist/ai-service/index.js.map +1 -1
- package/dist/ai-service/llm/client.d.ts +2 -9
- package/dist/ai-service/llm/client.d.ts.map +1 -1
- package/dist/ai-service/llm/client.js +2 -81
- package/dist/ai-service/llm/client.js.map +1 -1
- package/dist/ai-service/llm/context-v2/compaction/client-side.d.ts.map +1 -1
- package/dist/ai-service/llm/context-v2/compaction/client-side.js +15 -4
- package/dist/ai-service/llm/context-v2/compaction/client-side.js.map +1 -1
- package/dist/ai-service/llm/context-v2/config.d.ts +8 -0
- package/dist/ai-service/llm/context-v2/config.d.ts.map +1 -1
- package/dist/ai-service/llm/context-v2/config.js +13 -0
- package/dist/ai-service/llm/context-v2/config.js.map +1 -1
- package/dist/ai-service/llm/context-v2/context-metrics.d.ts +18 -0
- package/dist/ai-service/llm/context-v2/context-metrics.d.ts.map +1 -1
- package/dist/ai-service/llm/context-v2/context-metrics.js +94 -2
- package/dist/ai-service/llm/context-v2/context-metrics.js.map +1 -1
- package/dist/ai-service/llm/context-v2/context.d.ts +3 -0
- package/dist/ai-service/llm/context-v2/context.d.ts.map +1 -1
- package/dist/ai-service/llm/context-v2/context.js +13 -1
- package/dist/ai-service/llm/context-v2/context.js.map +1 -1
- package/dist/ai-service/llm/stream/observers/workflow-usage.d.ts +35 -0
- package/dist/ai-service/llm/stream/observers/workflow-usage.d.ts.map +1 -0
- package/dist/ai-service/llm/stream/observers/workflow-usage.js +108 -0
- package/dist/ai-service/llm/stream/observers/workflow-usage.js.map +1 -0
- package/dist/ai-service/llm/stream/orchestrator.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/orchestrator.js +1 -0
- package/dist/ai-service/llm/stream/orchestrator.js.map +1 -1
- package/dist/ai-service/llm/stream/session.d.ts +2 -0
- package/dist/ai-service/llm/stream/session.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/session.js +2 -0
- package/dist/ai-service/llm/stream/session.js.map +1 -1
- package/dist/ai-service/state-machine/clark-fsm.d.ts +3 -0
- package/dist/ai-service/state-machine/clark-fsm.d.ts.map +1 -1
- package/dist/ai-service/state-machine/clark-fsm.js +19 -0
- package/dist/ai-service/state-machine/clark-fsm.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/awaiting-user.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/awaiting-user.js +17 -28
- package/dist/ai-service/state-machine/handlers/awaiting-user.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/idle.d.ts +1 -1
- package/dist/ai-service/state-machine/handlers/idle.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/idle.js +28 -2
- package/dist/ai-service/state-machine/handlers/idle.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/play-dead.d.ts +2 -2
- package/dist/ai-service/state-machine/handlers/play-dead.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/play-dead.js +18 -4
- package/dist/ai-service/state-machine/handlers/play-dead.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/prompt-billing-metrics.d.ts +26 -0
- package/dist/ai-service/state-machine/helpers/prompt-billing-metrics.d.ts.map +1 -0
- package/dist/ai-service/state-machine/helpers/prompt-billing-metrics.js +127 -0
- package/dist/ai-service/state-machine/helpers/prompt-billing-metrics.js.map +1 -0
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.d.ts +32 -10
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.js +79 -18
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.js.map +1 -1
- package/dist/policy-gate-callback-mapper.d.ts.map +1 -1
- package/dist/policy-gate-callback-mapper.js +5 -6
- package/dist/policy-gate-callback-mapper.js.map +1 -1
- package/dist/policy-gate-runner.d.ts.map +1 -1
- package/dist/policy-gate-runner.js +22 -4
- package/dist/policy-gate-runner.js.map +1 -1
- package/package.json +8 -8
|
@@ -2,8 +2,9 @@ import { AiGenerationState } from "@superblocksteam/library-shared/types";
|
|
|
2
2
|
import { getLogger } from "../../../util/logger.js";
|
|
3
3
|
import { APP_RUNTIME_UPDATED_WITHOUT_EDITS, USER_ACCEPTED_DRAFT, USER_CANCELED, AGENT_CANCELED, } from "../clark-fsm.js";
|
|
4
4
|
import { sendUserGenerationStateChannel, sendUserMessageChannel, } from "../helpers/peer.js";
|
|
5
|
-
import {
|
|
6
|
-
|
|
5
|
+
import { finalizePromptBillingMetrics } from "../helpers/prompt-billing-metrics.js";
|
|
6
|
+
import { recordPromptOutcome, promptFailureUserCanceled, resolvePromptMetricLabels, resolvePromptOutcomeDedupeKey, } from "../helpers/prompt-request-metrics.js";
|
|
7
|
+
export const doIdle = (clark, { chatSessionStore, clarkProfiler, features }) => {
|
|
7
8
|
const sendUserMessage = sendUserMessageChannel(clark, chatSessionStore);
|
|
8
9
|
const sendUserGenerationState = sendUserGenerationStateChannel(clark);
|
|
9
10
|
return async ({ event }) => {
|
|
@@ -29,6 +30,7 @@ export const doIdle = (clark, { chatSessionStore, clarkProfiler }) => {
|
|
|
29
30
|
recordPromptOutcome({
|
|
30
31
|
dedupeKey: resolvePromptOutcomeDedupeKey(clark.context),
|
|
31
32
|
outcome: "success",
|
|
33
|
+
...resolvePromptMetricLabels(clark.context),
|
|
32
34
|
});
|
|
33
35
|
await clarkProfiler?.endSession();
|
|
34
36
|
void sendUserGenerationState({
|
|
@@ -38,10 +40,22 @@ export const doIdle = (clark, { chatSessionStore, clarkProfiler }) => {
|
|
|
38
40
|
break;
|
|
39
41
|
case AGENT_CANCELED: {
|
|
40
42
|
const { advice, paywallReason, failure } = event;
|
|
43
|
+
const { promptId, workflowMetrics } = clark.context;
|
|
44
|
+
// Do not await: cancel teardown must not wait on an unbounded billing
|
|
45
|
+
// RPC. finalizePromptBillingMetrics already swallows and logs errors.
|
|
46
|
+
void finalizePromptBillingMetrics({
|
|
47
|
+
chatSessionStore,
|
|
48
|
+
labels: resolvePromptMetricLabels(clark.context),
|
|
49
|
+
logger: getLogger(),
|
|
50
|
+
promptId,
|
|
51
|
+
promptUsageAlarmThreshold: features.promptUsageAlarmThreshold,
|
|
52
|
+
workflowMetrics,
|
|
53
|
+
});
|
|
41
54
|
recordPromptOutcome({
|
|
42
55
|
dedupeKey: resolvePromptOutcomeDedupeKey(clark.context),
|
|
43
56
|
outcome: "failure",
|
|
44
57
|
failure,
|
|
58
|
+
...resolvePromptMetricLabels(clark.context),
|
|
45
59
|
});
|
|
46
60
|
getLogger().info(`[ai-service] Agent canceled with advice: ${advice}`);
|
|
47
61
|
await clarkProfiler.endSession();
|
|
@@ -94,10 +108,22 @@ export const doIdle = (clark, { chatSessionStore, clarkProfiler }) => {
|
|
|
94
108
|
}
|
|
95
109
|
case USER_CANCELED: {
|
|
96
110
|
const isInterrupt = clark.context.isInterruptAbort === true;
|
|
111
|
+
const { promptId, workflowMetrics } = clark.context;
|
|
112
|
+
// Do not await: cancel teardown must not wait on an unbounded billing
|
|
113
|
+
// RPC. finalizePromptBillingMetrics already swallows and logs errors.
|
|
114
|
+
void finalizePromptBillingMetrics({
|
|
115
|
+
chatSessionStore,
|
|
116
|
+
labels: resolvePromptMetricLabels(clark.context),
|
|
117
|
+
logger: getLogger(),
|
|
118
|
+
promptId,
|
|
119
|
+
promptUsageAlarmThreshold: features.promptUsageAlarmThreshold,
|
|
120
|
+
workflowMetrics,
|
|
121
|
+
});
|
|
97
122
|
recordPromptOutcome({
|
|
98
123
|
dedupeKey: resolvePromptOutcomeDedupeKey(clark.context),
|
|
99
124
|
outcome: "failure",
|
|
100
125
|
failure: event.failure ?? promptFailureUserCanceled(),
|
|
126
|
+
...resolvePromptMetricLabels(clark.context),
|
|
101
127
|
});
|
|
102
128
|
getLogger().info(`[ai-service] User canceled (isInterrupt: ${isInterrupt})`);
|
|
103
129
|
await clarkProfiler.endSession();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"idle.js","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/handlers/idle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAE1E,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EACL,iCAAiC,EACjC,mBAAmB,EACnB,aAAa,EACb,cAAc,GAIf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,mBAAmB,EACnB,yBAAyB,EACzB,6BAA6B,GAC9B,MAAM,sCAAsC,CAAC;AAE9C,MAAM,CAAC,MAAM,MAAM,GAAG,CACpB,KAAY,EACZ,EAAE,gBAAgB,EAAE,aAAa,EAA2B,
|
|
1
|
+
{"version":3,"file":"idle.js","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/handlers/idle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAE1E,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EACL,iCAAiC,EACjC,mBAAmB,EACnB,aAAa,EACb,cAAc,GAIf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,4BAA4B,EAAE,MAAM,sCAAsC,CAAC;AACpF,OAAO,EACL,mBAAmB,EACnB,yBAAyB,EACzB,yBAAyB,EACzB,6BAA6B,GAC9B,MAAM,sCAAsC,CAAC;AAE9C,MAAM,CAAC,MAAM,MAAM,GAAG,CACpB,KAAY,EACZ,EAAE,gBAAgB,EAAE,aAAa,EAAE,QAAQ,EAA2B,EACtE,EAAE;IACF,MAAM,eAAe,GAAG,sBAAsB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IACxE,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,KAAK,CAAC,CAAC;IAEtE,OAAO,KAAK,EAAE,EAAE,KAAK,EAAmB,EAAiB,EAAE;QACzD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,mBAAmB;gBACtB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACpB,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,oBAAoB,EAAE,CAAC;oBACpE,IAAI,aAAa,EAAE,CAAC;wBAClB,KAAK,CAAC,aAAa,CAAC;4BAClB,aAAa;yBACd,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,KAAK,uBAAuB,CAAC;oBAC3B,KAAK,EAAE,iBAAiB,CAAC,IAAI;oBAC7B,QAAQ,EAAE,KAAK;iBAChB,CAAC,CAAC;gBACH,MAAM;YAER,KAAK,iCAAiC;gBACpC,SAAS,EAAE,CAAC,IAAI,CACd,oFAAoF,EACpF;oBACE,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,cAAc;iBAC7C,CACF,CAAC;gBACF,mBAAmB,CAAC;oBAClB,SAAS,EAAE,6BAA6B,CAAC,KAAK,CAAC,OAAO,CAAC;oBACvD,OAAO,EAAE,SAAS;oBAClB,GAAG,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC;iBAC5C,CAAC,CAAC;gBACH,MAAM,aAAa,EAAE,UAAU,EAAE,CAAC;gBAElC,KAAK,uBAAuB,CAAC;oBAC3B,KAAK,EAAE,iBAAiB,CAAC,IAAI;oBAC7B,QAAQ,EAAE,KAAK;iBAChB,CAAC,CAAC;gBACH,MAAM;YAER,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;gBACjD,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC;gBACpD,sEAAsE;gBACtE,sEAAsE;gBACtE,KAAK,4BAA4B,CAAC;oBAChC,gBAAgB;oBAChB,MAAM,EAAE,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC;oBAChD,MAAM,EAAE,SAAS,EAAE;oBACnB,QAAQ;oBACR,yBAAyB,EAAE,QAAQ,CAAC,yBAAyB;oBAC7D,eAAe;iBAChB,CAAC,CAAC;gBACH,mBAAmB,CAAC;oBAClB,SAAS,EAAE,6BAA6B,CAAC,KAAK,CAAC,OAAO,CAAC;oBACvD,OAAO,EAAE,SAAS;oBAClB,OAAO;oBACP,GAAG,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC;iBAC5C,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC,IAAI,CAAC,4CAA4C,MAAM,EAAE,CAAC,CAAC;gBAEvE,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;gBAEjC,IAAI,aAAa,EAAE,CAAC;oBAClB,8DAA8D;oBAC9D,+DAA+D;oBAC/D,8CAA8C;oBAC9C,qDAAqD;oBACrD,yDAAyD;oBACzD,+CAA+C;oBAC/C,8DAA8D;oBAC9D,sDAAsD;oBACtD,4DAA4D;oBAC5D,+DAA+D;oBAC/D,2CAA2C;oBAC3C,KAAK,CAAC,aAAa,CAAC;wBAClB,mBAAmB,EAAE;4BACnB,KAAK,EAAE,iBAAiB,CAAC,SAAS;4BAClC,QAAQ,EAAE,IAAI;4BACd,aAAa;yBACd;qBACF,CAAC,CAAC;oBAEH,kEAAkE;oBAClE,oEAAoE;oBACpE,iEAAiE;oBACjE,mEAAmE;oBACnE,+DAA+D;oBAC/D,gEAAgE;oBAChE,mEAAmE;oBACnE,qEAAqE;oBACrE,MAAM,eAAe,CACnB;wBACE,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,MAAM,IAAI,EAAE;wBACrB,IAAI,EAAE,OAAO;qBACd,EACD,CAAC,EACD,IAAI,EACJ,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAC/B,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,KAAK,eAAe,CAClB;wBACE,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,MAAM,IAAI,EAAE;wBACrB,IAAI,EAAE,OAAO;qBACd,EACD,CAAC,EACD,IAAI,CACL,CAAC;gBACJ,CAAC;gBAED,KAAK,uBAAuB,CAAC;oBAC3B,KAAK,EAAE,iBAAiB,CAAC,SAAS;oBAClC,QAAQ,EAAE,IAAI;oBACd,aAAa;iBACd,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,KAAK,aAAa,CAAC,CAAC,CAAC;gBACnB,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB,KAAK,IAAI,CAAC;gBAC5D,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC;gBACpD,sEAAsE;gBACtE,sEAAsE;gBACtE,KAAK,4BAA4B,CAAC;oBAChC,gBAAgB;oBAChB,MAAM,EAAE,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC;oBAChD,MAAM,EAAE,SAAS,EAAE;oBACnB,QAAQ;oBACR,yBAAyB,EAAE,QAAQ,CAAC,yBAAyB;oBAC7D,eAAe;iBAChB,CAAC,CAAC;gBACH,mBAAmB,CAAC;oBAClB,SAAS,EAAE,6BAA6B,CAAC,KAAK,CAAC,OAAO,CAAC;oBACvD,OAAO,EAAE,SAAS;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,yBAAyB,EAAE;oBACrD,GAAG,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC;iBAC5C,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC,IAAI,CACd,4CAA4C,WAAW,GAAG,CAC3D,CAAC;gBAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;gBAEjC,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,MAAM,eAAe,CACnB;wBACE,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,WAAW;wBACpB,IAAI,EAAE,uBAAuB;qBAC9B,EACD,CAAC,EACD,IAAI,CACL,CAAC;oBAEF,KAAK,uBAAuB,CAAC;wBAC3B,KAAK,EAAE,iBAAiB,CAAC,SAAS;wBAClC,QAAQ,EAAE,KAAK;qBAChB,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM;YACR,CAAC;YAED;gBACE,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QAED,iCAAiC;QACjC,KAAK,CAAC,YAAY,EAAE,CAAC;QACrB,KAAK,CAAC,gBAAgB,EAAE,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { type ClarkTransition } from "../clark-fsm.js";
|
|
2
|
-
export declare const playDead: () => ({ event }: ClarkTransition) => Promise<void>;
|
|
1
|
+
import { type Clark, type ClarkStateHandlerParams, type ClarkTransition } from "../clark-fsm.js";
|
|
2
|
+
export declare const playDead: (clark: Clark, { chatSessionStore, features }: ClarkStateHandlerParams) => ({ event }: ClarkTransition) => Promise<void>;
|
|
3
3
|
//# sourceMappingURL=play-dead.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"play-dead.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/handlers/play-dead.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"play-dead.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/handlers/play-dead.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,KAAK,EACV,KAAK,uBAAuB,EAC5B,KAAK,eAAe,EACrB,MAAM,iBAAiB,CAAC;AAIzB,eAAO,MAAM,QAAQ,GAClB,OAAO,KAAK,EAAE,gCAAgC,uBAAuB,MAC/D,WAAW,eAAe,KAAG,OAAO,CAAC,IAAI,CAoB/C,CAAC"}
|
|
@@ -1,9 +1,23 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { getLogger } from "../../../util/logger.js";
|
|
2
|
+
import { AGENT_GAVE_UP, } from "../clark-fsm.js";
|
|
3
|
+
import { finalizePromptBillingMetrics } from "../helpers/prompt-billing-metrics.js";
|
|
4
|
+
import { resolvePromptMetricLabels } from "../helpers/prompt-request-metrics.js";
|
|
5
|
+
export const playDead = (clark, { chatSessionStore, features }) => async ({ event }) => {
|
|
3
6
|
switch (event.type) {
|
|
4
|
-
case AGENT_GAVE_UP:
|
|
5
|
-
|
|
7
|
+
case AGENT_GAVE_UP: {
|
|
8
|
+
const { promptId, workflowMetrics } = clark.context;
|
|
9
|
+
// Do not await: give-up teardown must not wait on an unbounded billing
|
|
10
|
+
// RPC. finalizePromptBillingMetrics already swallows and logs errors.
|
|
11
|
+
void finalizePromptBillingMetrics({
|
|
12
|
+
chatSessionStore,
|
|
13
|
+
labels: resolvePromptMetricLabels(clark.context),
|
|
14
|
+
logger: getLogger(),
|
|
15
|
+
promptId,
|
|
16
|
+
promptUsageAlarmThreshold: features.promptUsageAlarmThreshold,
|
|
17
|
+
workflowMetrics,
|
|
18
|
+
});
|
|
6
19
|
break;
|
|
20
|
+
}
|
|
7
21
|
default:
|
|
8
22
|
throw new Error(`Unhandled event: ${event}`);
|
|
9
23
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"play-dead.js","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/handlers/play-dead.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"play-dead.js","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/handlers/play-dead.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EACL,aAAa,GAId,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,4BAA4B,EAAE,MAAM,sCAAsC,CAAC;AACpF,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAEjF,MAAM,CAAC,MAAM,QAAQ,GACnB,CAAC,KAAY,EAAE,EAAE,gBAAgB,EAAE,QAAQ,EAA2B,EAAE,EAAE,CAC1E,KAAK,EAAE,EAAE,KAAK,EAAmB,EAAiB,EAAE;IAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC;YACpD,uEAAuE;YACvE,sEAAsE;YACtE,KAAK,4BAA4B,CAAC;gBAChC,gBAAgB;gBAChB,MAAM,EAAE,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC;gBAChD,MAAM,EAAE,SAAS,EAAE;gBACnB,QAAQ;gBACR,yBAAyB,EAAE,QAAQ,CAAC,yBAAyB;gBAC7D,eAAe;aAChB,CAAC,CAAC;YACH,MAAM;QACR,CAAC;QAED;YACE,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ChatSessionStore } from "../../chat/chat-session-store.js";
|
|
2
|
+
import { type PromptMetricLabels } from "./prompt-request-metrics.js";
|
|
3
|
+
export declare function _resetPromptBillingMetricsForTesting(): void;
|
|
4
|
+
/**
|
|
5
|
+
* Flush per-prompt billing and context headroom metrics at any terminal
|
|
6
|
+
* outcome — success, cancel, or give-up — so credits-per-prompt is not
|
|
7
|
+
* survivorship-biased toward completed prompts.
|
|
8
|
+
*/
|
|
9
|
+
export declare function finalizePromptBillingMetrics(params: {
|
|
10
|
+
chatSessionStore: ChatSessionStore;
|
|
11
|
+
labels: PromptMetricLabels;
|
|
12
|
+
logger: {
|
|
13
|
+
info: (...args: unknown[]) => void;
|
|
14
|
+
warn: (...args: unknown[]) => void;
|
|
15
|
+
};
|
|
16
|
+
promptId: string | undefined;
|
|
17
|
+
promptUsageAlarmThreshold?: number;
|
|
18
|
+
workflowMetrics?: {
|
|
19
|
+
hadProviderFallback?: boolean;
|
|
20
|
+
llmUsage: Parameters<ChatSessionStore["calculatePromptCost"]>[1];
|
|
21
|
+
llmCallCount: number;
|
|
22
|
+
minHeadroomTokens?: number;
|
|
23
|
+
peakContextUsed?: number;
|
|
24
|
+
};
|
|
25
|
+
}): Promise<void>;
|
|
26
|
+
//# sourceMappingURL=prompt-billing-metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompt-billing-metrics.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/helpers/prompt-billing-metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,OAAO,EAEL,KAAK,kBAAkB,EACxB,MAAM,6BAA6B,CAAC;AAgFrC,wBAAgB,oCAAoC,IAAI,IAAI,CAG3D;AAED;;;;GAIG;AACH,wBAAsB,4BAA4B,CAAC,MAAM,EAAE;IACzD,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,MAAM,EAAE;QACN,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;QACnC,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,eAAe,CAAC,EAAE;QAChB,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,QAAQ,EAAE,UAAU,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH,GAAG,OAAO,CAAC,IAAI,CAAC,CAiEhB"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { recordPromptContextHeadroom } from "../../llm/context-v2/context-metrics.js";
|
|
2
|
+
import { recordPromptCredits, } from "./prompt-request-metrics.js";
|
|
3
|
+
/** Cost round-trips currently in flight, keyed by prompt. */
|
|
4
|
+
const inflightCostRpcs = new Map();
|
|
5
|
+
/**
|
|
6
|
+
* Prompts already priced by a successful round-trip, keyed by promptId with
|
|
7
|
+
* the expiry timestamp (ms). Entries only need to outlive concurrent terminal
|
|
8
|
+
* paths for the same prompt; after that they are pruned so the map cannot grow
|
|
9
|
+
* without bound under sustained traffic.
|
|
10
|
+
*/
|
|
11
|
+
const pricedPrompts = new Map();
|
|
12
|
+
/** How long a successful pricing claim is remembered for race dedupe. */
|
|
13
|
+
const PRICED_PROMPT_TTL_MS = 60_000;
|
|
14
|
+
function pruneExpiredPricedPrompts(now) {
|
|
15
|
+
for (const [promptId, expiresAt] of pricedPrompts) {
|
|
16
|
+
if (expiresAt <= now) {
|
|
17
|
+
pricedPrompts.delete(promptId);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function isPromptPriced(promptId, now = Date.now()) {
|
|
22
|
+
pruneExpiredPricedPrompts(now);
|
|
23
|
+
return pricedPrompts.has(promptId);
|
|
24
|
+
}
|
|
25
|
+
function markPromptPriced(promptId, now = Date.now()) {
|
|
26
|
+
pruneExpiredPricedPrompts(now);
|
|
27
|
+
pricedPrompts.set(promptId, now + PRICED_PROMPT_TTL_MS);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Price a prompt exactly once across terminal paths, returning `undefined`
|
|
31
|
+
* when someone else has already accounted for it.
|
|
32
|
+
*
|
|
33
|
+
* Terminal paths both overlap and repeat. Aborting the stream drives the FSM
|
|
34
|
+
* into its own cancel handler while `handleUserCanceled` is still awaiting the
|
|
35
|
+
* cost call, so a concurrent caller joins the in-flight round-trip; a caller
|
|
36
|
+
* arriving after one succeeded is turned away by `pricedPrompts`.
|
|
37
|
+
*
|
|
38
|
+
* A joiner waits on the result rather than returning immediately, because
|
|
39
|
+
* returning loses the prompt outright when the round-trip fails — it would
|
|
40
|
+
* already be gone by the time the failure surfaced, leaving nobody to retry.
|
|
41
|
+
* Cancel is when the call is most likely to fail, so a joiner that sees a
|
|
42
|
+
* rejection retries instead. Only a success marks the prompt priced, which is
|
|
43
|
+
* what lets a later terminal path pick up after a failed attempt.
|
|
44
|
+
*/
|
|
45
|
+
async function calculatePromptCostOnce(chatSessionStore, promptId, llmUsage) {
|
|
46
|
+
if (isPromptPriced(promptId)) {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
const inflight = inflightCostRpcs.get(promptId);
|
|
50
|
+
if (inflight) {
|
|
51
|
+
try {
|
|
52
|
+
await inflight;
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// Fall through and retry; the owner has already dropped the slot.
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const attempt = chatSessionStore.calculatePromptCost(promptId, llmUsage);
|
|
60
|
+
inflightCostRpcs.set(promptId, attempt);
|
|
61
|
+
try {
|
|
62
|
+
const credits = await attempt;
|
|
63
|
+
markPromptPriced(promptId);
|
|
64
|
+
return credits;
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
inflightCostRpcs.delete(promptId);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export function _resetPromptBillingMetricsForTesting() {
|
|
71
|
+
inflightCostRpcs.clear();
|
|
72
|
+
pricedPrompts.clear();
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Flush per-prompt billing and context headroom metrics at any terminal
|
|
76
|
+
* outcome — success, cancel, or give-up — so credits-per-prompt is not
|
|
77
|
+
* survivorship-biased toward completed prompts.
|
|
78
|
+
*/
|
|
79
|
+
export async function finalizePromptBillingMetrics(params) {
|
|
80
|
+
const { promptId, workflowMetrics } = params;
|
|
81
|
+
if (!promptId || !workflowMetrics) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
// Local measurement off workflowMetrics, so it stays above the RPC: a
|
|
85
|
+
// transient socket failure must not punch holes in the context signal.
|
|
86
|
+
recordPromptContextHeadroom({
|
|
87
|
+
dedupeKey: promptId,
|
|
88
|
+
minHeadroomTokens: workflowMetrics.minHeadroomTokens,
|
|
89
|
+
peakContextUsed: workflowMetrics.peakContextUsed,
|
|
90
|
+
});
|
|
91
|
+
let totalCreditsUsed;
|
|
92
|
+
try {
|
|
93
|
+
totalCreditsUsed = await calculatePromptCostOnce(params.chatSessionStore, promptId, workflowMetrics.llmUsage);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
params.logger.warn("[billing:prompt-cost] Failed to calculate prompt credits", {
|
|
97
|
+
promptId,
|
|
98
|
+
error: error instanceof Error ? error.message : String(error),
|
|
99
|
+
});
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (totalCreditsUsed === undefined) {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
recordPromptCredits({
|
|
106
|
+
...params.labels,
|
|
107
|
+
credits: totalCreditsUsed,
|
|
108
|
+
dedupeKey: promptId,
|
|
109
|
+
});
|
|
110
|
+
const threshold = params.promptUsageAlarmThreshold;
|
|
111
|
+
if (threshold === undefined) {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const costSummary = {
|
|
115
|
+
llmCallCount: workflowMetrics.llmCallCount,
|
|
116
|
+
promptId,
|
|
117
|
+
threshold,
|
|
118
|
+
totalCreditsUsed,
|
|
119
|
+
};
|
|
120
|
+
if (totalCreditsUsed > threshold) {
|
|
121
|
+
params.logger.warn(`[billing:prompt-cost] Prompt credit usage ${totalCreditsUsed} exceeds threshold of ${threshold}`, costSummary);
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
params.logger.info(`[billing:prompt-cost] Prompt cost summary: ${totalCreditsUsed} credits across ${workflowMetrics.llmCallCount} LLM calls`, costSummary);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=prompt-billing-metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompt-billing-metrics.js","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/helpers/prompt-billing-metrics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,MAAM,yCAAyC,CAAC;AACtF,OAAO,EACL,mBAAmB,GAEpB,MAAM,6BAA6B,CAAC;AAErC,6DAA6D;AAC7D,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;AAE5D;;;;;GAKG;AACH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;AAEhD,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAEpC,SAAS,yBAAyB,CAAC,GAAW;IAC5C,KAAK,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,aAAa,EAAE,CAAC;QAClD,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;YACrB,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IACxD,yBAAyB,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IAC1D,yBAAyB,CAAC,GAAG,CAAC,CAAC;IAC/B,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,oBAAoB,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,KAAK,UAAU,uBAAuB,CACpC,gBAAkC,EAClC,QAAgB,EAChB,QAAgE;IAEhE,IAAI,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC;YACf,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzE,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC;QAC9B,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oCAAoC;IAClD,gBAAgB,CAAC,KAAK,EAAE,CAAC;IACzB,aAAa,CAAC,KAAK,EAAE,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,MAgBlD;IACC,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAC7C,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,EAAE,CAAC;QAClC,OAAO;IACT,CAAC;IAED,sEAAsE;IACtE,uEAAuE;IACvE,2BAA2B,CAAC;QAC1B,SAAS,EAAE,QAAQ;QACnB,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;QACpD,eAAe,EAAE,eAAe,CAAC,eAAe;KACjD,CAAC,CAAC;IAEH,IAAI,gBAAoC,CAAC;IACzC,IAAI,CAAC;QACH,gBAAgB,GAAG,MAAM,uBAAuB,CAC9C,MAAM,CAAC,gBAAgB,EACvB,QAAQ,EACR,eAAe,CAAC,QAAQ,CACzB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,0DAA0D,EAC1D;YACE,QAAQ;YACR,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO;IACT,CAAC;IAED,mBAAmB,CAAC;QAClB,GAAG,MAAM,CAAC,MAAM;QAChB,OAAO,EAAE,gBAAgB;QACzB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,CAAC,yBAAyB,CAAC;IACnD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,MAAM,WAAW,GAAG;QAClB,YAAY,EAAE,eAAe,CAAC,YAAY;QAC1C,QAAQ;QACR,SAAS;QACT,gBAAgB;KACjB,CAAC;IAEF,IAAI,gBAAgB,GAAG,SAAS,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,6CAA6C,gBAAgB,yBAAyB,SAAS,EAAE,EACjG,WAAW,CACZ,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,8CAA8C,gBAAgB,mBAAmB,eAAe,CAAC,YAAY,YAAY,EACzH,WAAW,CACZ,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -4,6 +4,13 @@ export type PromptOutcome = (typeof PROMPT_OUTCOMES)[number];
|
|
|
4
4
|
export { PROMPT_OUTCOME_TAG, promptOutcomeTagsFromFailure, type PromptFailure, } from "./prompt-failure.js";
|
|
5
5
|
export { failureDetailFromPromptFailure, PLATFORM_PROMPT_FAILURE_REASONS, PROMPT_FAILURE_ORIGINS, TELEMETRY_FAILURE_ORIGINS, telemetryFailureOrigin, isUserInitiatedPromptAbort, promptFailureFromLlmError, promptFailureFromStreamAbort, promptFailureLlmProviderUnknown, promptFailurePaywall, promptFailurePlatform, promptFailureUserCanceled, type PlatformPromptFailureReason, type PromptFailureOrigin, type TelemetryFailureOrigin, type UserPromptFailureReason, USER_PROMPT_FAILURE_REASONS, } from "./prompt-failure.js";
|
|
6
6
|
export declare const METRIC_AWAITING_USER_DISMISSED = "superblocks.clark.prompt.awaiting_user_dismissed_total";
|
|
7
|
+
export declare const AWAITING_USER_DISMISS_REASONS: readonly ["cancel", "new_prompt"];
|
|
8
|
+
export type AwaitingUserDismissReason = (typeof AWAITING_USER_DISMISS_REASONS)[number];
|
|
9
|
+
export declare const AWAITING_USER_DISMISS_REASON_TAG = "dismiss_reason";
|
|
10
|
+
export declare const PROMPT_APPLICATION_ID_TAG = "application_id";
|
|
11
|
+
export declare const PROMPT_CORE_MODEL_TAG = "core_model";
|
|
12
|
+
export declare const PROMPT_HAD_FALLBACK_TAG = "had_fallback";
|
|
13
|
+
export declare const PROMPT_ORGANIZATION_ID_TAG = "organization_id";
|
|
7
14
|
export declare const METRIC_CREDITS = "superblocks.clark.prompt.credits";
|
|
8
15
|
/**
|
|
9
16
|
* Credit-scale bucket boundaries for credits-per-prompt. Centered on the
|
|
@@ -11,30 +18,42 @@ export declare const METRIC_CREDITS = "superblocks.clark.prompt.credits";
|
|
|
11
18
|
* near the golden-metric alert edge.
|
|
12
19
|
*/
|
|
13
20
|
export declare const PROMPT_CREDITS_BUCKETS: readonly [0.5, 1, 2, 5, 7.5, 10, 15, 25, 50, 100, 250];
|
|
14
|
-
export
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
export declare
|
|
21
|
+
export interface PromptMetricLabels {
|
|
22
|
+
coreModel?: string;
|
|
23
|
+
hadFallback?: boolean;
|
|
24
|
+
}
|
|
25
|
+
export declare function resolvePromptMetricLabels(context: {
|
|
26
|
+
llmConfig?: {
|
|
27
|
+
coreModel?: string;
|
|
28
|
+
};
|
|
29
|
+
workflowMetrics?: {
|
|
30
|
+
hadProviderFallback?: boolean;
|
|
31
|
+
};
|
|
32
|
+
}): PromptMetricLabels;
|
|
19
33
|
export interface PromptRequestMetricIdentity {
|
|
20
34
|
applicationId?: string;
|
|
21
35
|
organizationId?: string;
|
|
22
36
|
}
|
|
23
37
|
export declare function setPromptRequestMetricIdentity(identity: PromptRequestMetricIdentity): void;
|
|
24
|
-
export declare function recordPromptAttempt(
|
|
38
|
+
export declare function recordPromptAttempt(params?: {
|
|
39
|
+
coreModel?: string;
|
|
40
|
+
outcomeDedupeKey?: string;
|
|
41
|
+
}): void;
|
|
25
42
|
export declare function resolvePromptOutcomeDedupeKey(context: {
|
|
26
43
|
promptId?: string;
|
|
27
44
|
promptOutcomeDedupeKey?: string;
|
|
28
45
|
}): string | undefined;
|
|
29
46
|
export declare function recordPromptOutcome(params: {
|
|
47
|
+
coreModel?: string;
|
|
30
48
|
dedupeKey: string | undefined;
|
|
31
|
-
outcome: PromptOutcome;
|
|
32
49
|
failure?: PromptFailure;
|
|
50
|
+
hadFallback?: boolean;
|
|
51
|
+
outcome: PromptOutcome;
|
|
33
52
|
}): void;
|
|
34
53
|
export declare function recordPromptCredits(params: {
|
|
35
|
-
dedupeKey: string | undefined;
|
|
36
54
|
credits: number;
|
|
37
|
-
|
|
55
|
+
dedupeKey: string | undefined;
|
|
56
|
+
} & PromptMetricLabels): void;
|
|
38
57
|
export declare function resolvePromptRequestDedupeKey(request: {
|
|
39
58
|
requestId?: string;
|
|
40
59
|
prompt: string;
|
|
@@ -75,12 +94,15 @@ export declare function recordAwaitingUserHandoffDismissed(params: {
|
|
|
75
94
|
reason: AwaitingUserDismissReason;
|
|
76
95
|
}): void;
|
|
77
96
|
export declare function recordAwaitingUserPromptOutcome(params: {
|
|
78
|
-
dedupeKey: string | undefined;
|
|
79
97
|
advice?: string;
|
|
98
|
+
coreModel?: string;
|
|
80
99
|
countsAsPromptSuccess?: boolean;
|
|
100
|
+
dedupeKey: string | undefined;
|
|
101
|
+
hadFallback?: boolean;
|
|
81
102
|
}): boolean;
|
|
82
103
|
/** Record attempt + immediate failure for pre-FSM entry rejections (no promptId yet). */
|
|
83
104
|
export declare function recordPromptEntryRejection(params: {
|
|
105
|
+
coreModel?: string;
|
|
84
106
|
dedupeKey: string | undefined;
|
|
85
107
|
failure: PromptFailure;
|
|
86
108
|
}): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-request-metrics.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/helpers/prompt-request-metrics.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"prompt-request-metrics.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/state-machine/helpers/prompt-request-metrics.ts"],"names":[],"mappings":"AAYA,OAAO,EAML,KAAK,aAAa,EACnB,MAAM,qBAAqB,CAAC;AAE7B,eAAO,MAAM,eAAe,iCAAkC,CAAC;AAC/D,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,4BAA4B,EAC5B,KAAK,aAAa,GACnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,8BAA8B,EAC9B,+BAA+B,EAC/B,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,0BAA0B,EAC1B,yBAAyB,EACzB,4BAA4B,EAC5B,+BAA+B,EAC/B,oBAAoB,EACpB,qBAAqB,EACrB,yBAAyB,EACzB,KAAK,2BAA2B,EAChC,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,2BAA2B,GAC5B,MAAM,qBAAqB,CAAC;AAK7B,eAAO,MAAM,8BAA8B,2DACe,CAAC;AAC3D,eAAO,MAAM,6BAA6B,mCAAoC,CAAC;AAC/E,MAAM,MAAM,yBAAyB,GACnC,CAAC,OAAO,6BAA6B,CAAC,CAAC,MAAM,CAAC,CAAC;AACjD,eAAO,MAAM,gCAAgC,mBAAmB,CAAC;AACjE,eAAO,MAAM,yBAAyB,mBAAmB,CAAC;AAC1D,eAAO,MAAM,qBAAqB,eAAe,CAAC;AAClD,eAAO,MAAM,uBAAuB,iBAAiB,CAAC;AACtD,eAAO,MAAM,0BAA0B,oBAAoB,CAAC;AAC5D,eAAO,MAAM,cAAc,qCAAqC,CAAC;AACjE;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,wDAEzB,CAAC;AAEX,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,SAAS,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACnC,eAAe,CAAC,EAAE;QAAE,mBAAmB,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;CACrD,GAAG,kBAAkB,CAKrB;AAwBD,MAAM,WAAW,2BAA2B;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AA0MD,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,2BAA2B,GACpC,IAAI,CAEN;AAID,wBAAgB,mBAAmB,CAAC,MAAM,CAAC,EAAE;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,GAAG,IAAI,CAKP;AAED,wBAAgB,6BAA6B,CAAC,OAAO,EAAE;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC,GAAG,MAAM,GAAG,SAAS,CAErB;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,aAAa,CAAC;CACxB,GAAG,IAAI,CAaP;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE;IACN,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/B,GAAG,kBAAkB,GACrB,IAAI,CAYN;AAED,wBAAgB,6BAA6B,CAAC,OAAO,EAAE;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,MAAM,CAET;AAED;;;GAGG;AACH,wBAAgB,kCAAkC,CAAC,KAAK,EAAE;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAmBzB;AAED,wBAAgB,wCAAwC,CAAC,KAAK,EAAE;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAQzB;AAED,wBAAgB,2BAA2B,CAAC,MAAM,EAAE;IAClD,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,yBAAyB,CAAC;CACnC,GAAG,IAAI,CAQP;AAED,wBAAgB,wCAAwC,CAAC,MAAM,EAAE;IAC/D,2BAA2B,CAAC,EAAE,OAAO,CAAC;IACtC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GAAG,OAAO,CAKV;AAED,wBAAgB,4BAA4B,CAAC,OAAO,EAAE;IACpD,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC,GAAG,MAAM,GAAG,SAAS,CAKrB;AAED,wBAAgB,kCAAkC,CAAC,MAAM,EAAE;IACzD,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,2BAA2B,CAAC,EAAE,OAAO,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,MAAM,EAAE,yBAAyB,CAAC;CACnC,GAAG,IAAI,CAQP;AAED,wBAAgB,+BAA+B,CAAC,MAAM,EAAE;IACtD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,GAAG,OAAO,CAyBV;AAED,yFAAyF;AACzF,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,aAAa,CAAC;CACxB,GAAG,IAAI,CASP;AAED,wBAAgB,oCAAoC,IAAI,IAAI,CAE3D"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { metrics } from "@opentelemetry/api";
|
|
2
|
+
import { coreModelLabel } from "@superblocksteam/library-shared/types";
|
|
2
3
|
import { AGENT_CANCELED, AGENT_NEEDS_USER_INPUT, APP_RUNTIME_UPDATED_WITHOUT_EDITS, RUNTIME_REVIEW_FINISHED, USER_CANCELED, V3_AGENT_FINISHED, } from "../clark-fsm.js";
|
|
3
4
|
import { promptFailureAttributes, promptFailurePlatform, promptFailureUserCanceled, promptOutcomeTagsFromFailure, PROMPT_OUTCOME_TAG, } from "./prompt-failure.js";
|
|
4
5
|
export const PROMPT_OUTCOMES = ["failure", "success"];
|
|
@@ -8,6 +9,12 @@ const METER_NAME = "superblocks.clark.prompt";
|
|
|
8
9
|
const METRIC_ATTEMPTS = "superblocks.clark.prompt.attempts_total";
|
|
9
10
|
const METRIC_OUTCOMES = "superblocks.clark.prompt.outcomes_total";
|
|
10
11
|
export const METRIC_AWAITING_USER_DISMISSED = "superblocks.clark.prompt.awaiting_user_dismissed_total";
|
|
12
|
+
export const AWAITING_USER_DISMISS_REASONS = ["cancel", "new_prompt"];
|
|
13
|
+
export const AWAITING_USER_DISMISS_REASON_TAG = "dismiss_reason";
|
|
14
|
+
export const PROMPT_APPLICATION_ID_TAG = "application_id";
|
|
15
|
+
export const PROMPT_CORE_MODEL_TAG = "core_model";
|
|
16
|
+
export const PROMPT_HAD_FALLBACK_TAG = "had_fallback";
|
|
17
|
+
export const PROMPT_ORGANIZATION_ID_TAG = "organization_id";
|
|
11
18
|
export const METRIC_CREDITS = "superblocks.clark.prompt.credits";
|
|
12
19
|
/**
|
|
13
20
|
* Credit-scale bucket boundaries for credits-per-prompt. Centered on the
|
|
@@ -17,10 +24,28 @@ export const METRIC_CREDITS = "superblocks.clark.prompt.credits";
|
|
|
17
24
|
export const PROMPT_CREDITS_BUCKETS = [
|
|
18
25
|
0.5, 1, 2, 5, 7.5, 10, 15, 25, 50, 100, 250,
|
|
19
26
|
];
|
|
20
|
-
export
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
27
|
+
export function resolvePromptMetricLabels(context) {
|
|
28
|
+
return {
|
|
29
|
+
coreModel: context.llmConfig?.coreModel,
|
|
30
|
+
hadFallback: context.workflowMetrics?.hadProviderFallback ?? false,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function promptCoreModelAttributes(labels) {
|
|
34
|
+
const attributes = {};
|
|
35
|
+
// Absent stays absent: a prompt that reported no model and one that reported
|
|
36
|
+
// a model we don't recognize are different facts, and only the latter needs
|
|
37
|
+
// bounding.
|
|
38
|
+
if (labels.coreModel) {
|
|
39
|
+
attributes[PROMPT_CORE_MODEL_TAG] = coreModelLabel(labels.coreModel);
|
|
40
|
+
}
|
|
41
|
+
return attributes;
|
|
42
|
+
}
|
|
43
|
+
function promptOutcomeModelAttributes(labels) {
|
|
44
|
+
return {
|
|
45
|
+
...promptCoreModelAttributes(labels),
|
|
46
|
+
[PROMPT_HAD_FALLBACK_TAG]: labels.hadFallback ? "true" : "false",
|
|
47
|
+
};
|
|
48
|
+
}
|
|
24
49
|
class PromptRequestMetrics {
|
|
25
50
|
cachedMeter;
|
|
26
51
|
cachedInstruments;
|
|
@@ -43,8 +68,18 @@ class PromptRequestMetrics {
|
|
|
43
68
|
}
|
|
44
69
|
return attributes;
|
|
45
70
|
}
|
|
46
|
-
/**
|
|
47
|
-
|
|
71
|
+
/**
|
|
72
|
+
* Org + core_model + had_fallback for the credits histogram. Credits is
|
|
73
|
+
* cardinality-sensitive, so application_id stays off it; core_model and
|
|
74
|
+
* had_fallback are bounded (a fixed model set and a boolean) and are what
|
|
75
|
+
* separate per-model cost from fallback-contaminated traffic.
|
|
76
|
+
*/
|
|
77
|
+
creditsAttributes(labels) {
|
|
78
|
+
const attributes = this.creditsAttributesBase();
|
|
79
|
+
Object.assign(attributes, promptOutcomeModelAttributes(labels));
|
|
80
|
+
return attributes;
|
|
81
|
+
}
|
|
82
|
+
creditsAttributesBase() {
|
|
48
83
|
const attributes = {};
|
|
49
84
|
if (this.organizationId) {
|
|
50
85
|
attributes[PROMPT_ORGANIZATION_ID_TAG] = this.organizationId;
|
|
@@ -67,7 +102,7 @@ class PromptRequestMetrics {
|
|
|
67
102
|
unit: "1",
|
|
68
103
|
}),
|
|
69
104
|
creditsHistogram: meter.createHistogram(METRIC_CREDITS, {
|
|
70
|
-
description: "AI credits consumed by a
|
|
105
|
+
description: "AI credits consumed by a Clark user prompt at any terminal outcome (golden cost metric). Labeled by organization_id, core_model, and had_fallback.",
|
|
71
106
|
unit: "{credit}",
|
|
72
107
|
advice: {
|
|
73
108
|
explicitBucketBoundaries: [...PROMPT_CREDITS_BUCKETS],
|
|
@@ -80,8 +115,11 @@ class PromptRequestMetrics {
|
|
|
80
115
|
};
|
|
81
116
|
return this.cachedInstruments;
|
|
82
117
|
}
|
|
83
|
-
recordAttempt() {
|
|
84
|
-
this.getInstruments().attemptCounter.add(1,
|
|
118
|
+
recordAttempt(labels = {}) {
|
|
119
|
+
this.getInstruments().attemptCounter.add(1, {
|
|
120
|
+
...this.baseAttributes(),
|
|
121
|
+
...promptCoreModelAttributes(labels),
|
|
122
|
+
});
|
|
85
123
|
}
|
|
86
124
|
/** Allow a new terminal outcome when the same request is retried. */
|
|
87
125
|
forgetOutcomeDedupeKey(dedupeKey) {
|
|
@@ -98,16 +136,17 @@ class PromptRequestMetrics {
|
|
|
98
136
|
});
|
|
99
137
|
}
|
|
100
138
|
/**
|
|
101
|
-
* Record credits for one
|
|
102
|
-
*
|
|
103
|
-
*
|
|
139
|
+
* Record credits for one prompt at its terminal outcome, whether it
|
|
140
|
+
* completed, was canceled, or gave up. Dedupe key is `promptId` (a fresh
|
|
141
|
+
* UUID per generate), so retries do not need a clear path — unlike
|
|
142
|
+
* outcomes, which reuse requestId-based keys across attempts.
|
|
104
143
|
*/
|
|
105
144
|
recordCredits(params) {
|
|
106
145
|
if (this.recordedCreditsKeys.has(params.dedupeKey)) {
|
|
107
146
|
return;
|
|
108
147
|
}
|
|
109
148
|
this.recordedCreditsKeys.add(params.dedupeKey);
|
|
110
|
-
this.getInstruments().creditsHistogram.record(params.credits, this.creditsAttributes());
|
|
149
|
+
this.getInstruments().creditsHistogram.record(params.credits, this.creditsAttributes(params.labels));
|
|
111
150
|
}
|
|
112
151
|
recordOutcome(params) {
|
|
113
152
|
if (this.recordedOutcomeKeys.has(params.dedupeKey)) {
|
|
@@ -116,6 +155,7 @@ class PromptRequestMetrics {
|
|
|
116
155
|
this.recordedOutcomeKeys.add(params.dedupeKey);
|
|
117
156
|
const attributes = {
|
|
118
157
|
...this.baseAttributes(),
|
|
158
|
+
...promptOutcomeModelAttributes(params.labels ?? {}),
|
|
119
159
|
outcome: params.outcome,
|
|
120
160
|
};
|
|
121
161
|
if (params.outcome === "failure" && params.failure) {
|
|
@@ -129,9 +169,15 @@ class PromptRequestMetrics {
|
|
|
129
169
|
return;
|
|
130
170
|
}
|
|
131
171
|
this.recordedOutcomeKeys.add(params.dedupeKey);
|
|
132
|
-
|
|
172
|
+
const coreModelAttributes = promptCoreModelAttributes(params.labels ?? {});
|
|
173
|
+
const outcomeModelAttributes = promptOutcomeModelAttributes(params.labels ?? {});
|
|
174
|
+
this.getInstruments().attemptCounter.add(1, {
|
|
175
|
+
...this.baseAttributes(),
|
|
176
|
+
...coreModelAttributes,
|
|
177
|
+
});
|
|
133
178
|
this.getInstruments().outcomeCounter.add(1, {
|
|
134
179
|
...this.baseAttributes(),
|
|
180
|
+
...outcomeModelAttributes,
|
|
135
181
|
outcome: "failure",
|
|
136
182
|
...promptFailureAttributes(params.failure),
|
|
137
183
|
});
|
|
@@ -150,11 +196,11 @@ export function setPromptRequestMetricIdentity(identity) {
|
|
|
150
196
|
promptRequestMetrics.setIdentity(identity);
|
|
151
197
|
}
|
|
152
198
|
const promptRequestMetrics = new PromptRequestMetrics();
|
|
153
|
-
export function recordPromptAttempt(
|
|
154
|
-
if (outcomeDedupeKey) {
|
|
155
|
-
promptRequestMetrics.forgetOutcomeDedupeKey(outcomeDedupeKey);
|
|
199
|
+
export function recordPromptAttempt(params) {
|
|
200
|
+
if (params?.outcomeDedupeKey) {
|
|
201
|
+
promptRequestMetrics.forgetOutcomeDedupeKey(params.outcomeDedupeKey);
|
|
156
202
|
}
|
|
157
|
-
promptRequestMetrics.recordAttempt();
|
|
203
|
+
promptRequestMetrics.recordAttempt({ coreModel: params?.coreModel });
|
|
158
204
|
}
|
|
159
205
|
export function resolvePromptOutcomeDedupeKey(context) {
|
|
160
206
|
return context.promptOutcomeDedupeKey ?? context.promptId;
|
|
@@ -167,6 +213,10 @@ export function recordPromptOutcome(params) {
|
|
|
167
213
|
dedupeKey: params.dedupeKey,
|
|
168
214
|
outcome: params.outcome,
|
|
169
215
|
failure: params.failure,
|
|
216
|
+
labels: {
|
|
217
|
+
coreModel: params.coreModel,
|
|
218
|
+
hadFallback: params.hadFallback ?? false,
|
|
219
|
+
},
|
|
170
220
|
});
|
|
171
221
|
}
|
|
172
222
|
export function recordPromptCredits(params) {
|
|
@@ -176,6 +226,10 @@ export function recordPromptCredits(params) {
|
|
|
176
226
|
promptRequestMetrics.recordCredits({
|
|
177
227
|
dedupeKey: params.dedupeKey,
|
|
178
228
|
credits: params.credits,
|
|
229
|
+
labels: {
|
|
230
|
+
coreModel: params.coreModel,
|
|
231
|
+
hadFallback: params.hadFallback ?? false,
|
|
232
|
+
},
|
|
179
233
|
});
|
|
180
234
|
}
|
|
181
235
|
export function resolvePromptRequestDedupeKey(request) {
|
|
@@ -244,10 +298,15 @@ export function recordAwaitingUserPromptOutcome(params) {
|
|
|
244
298
|
if (!params.dedupeKey) {
|
|
245
299
|
return false;
|
|
246
300
|
}
|
|
301
|
+
const outcomeLabels = {
|
|
302
|
+
coreModel: params.coreModel,
|
|
303
|
+
hadFallback: params.hadFallback ?? false,
|
|
304
|
+
};
|
|
247
305
|
if (params.countsAsPromptSuccess) {
|
|
248
306
|
recordPromptOutcome({
|
|
249
307
|
dedupeKey: params.dedupeKey,
|
|
250
308
|
outcome: "success",
|
|
309
|
+
...outcomeLabels,
|
|
251
310
|
});
|
|
252
311
|
return true;
|
|
253
312
|
}
|
|
@@ -256,6 +315,7 @@ export function recordAwaitingUserPromptOutcome(params) {
|
|
|
256
315
|
dedupeKey: params.dedupeKey,
|
|
257
316
|
outcome: "failure",
|
|
258
317
|
failure: promptFailurePlatform("unexpected"),
|
|
318
|
+
...outcomeLabels,
|
|
259
319
|
});
|
|
260
320
|
}
|
|
261
321
|
return false;
|
|
@@ -268,6 +328,7 @@ export function recordPromptEntryRejection(params) {
|
|
|
268
328
|
promptRequestMetrics.recordEntryRejection({
|
|
269
329
|
dedupeKey: params.dedupeKey,
|
|
270
330
|
failure: params.failure,
|
|
331
|
+
labels: { coreModel: params.coreModel },
|
|
271
332
|
});
|
|
272
333
|
}
|
|
273
334
|
export function _resetPromptRequestMetricsForTesting() {
|