@llm4ts/flow 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"EstimatedUsage.d.ts","sourceRoot":"","sources":["../src/EstimatedUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"EstimatedUsage.d.ts","sourceRoot":"","sources":["../src/EstimatedUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAKvC,OAAO,KAAK,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAA;AAC7E,OAAO,EAA+B,UAAU,EAAgB,MAAM,qBAAqB,CAAA;AAmB3F,MAAM,WAAW,qBAAqB;IACpC,uEAAuE;IACvE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,wDAAwD;IACxD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAChC;AAED,eAAO,MAAM,qBAAqB,oBAAoB,CAAA;AAEtD,eAAO,MAAM,4BAA4B,GACvC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,KACxD,qBAMF,CAAA;AAED,6EAA6E;AAC7E,eAAO,MAAM,mBAAmB,GAAI,gBAAgB,MAAM,KAAG,MAAuC,CAAA;AAEpG,eAAO,MAAM,gBAAgB,GAAI,OAAO,MAAM,GAAG,SAAS,KAAG,OACN,CAAA;AAKvD,eAAO,MAAM,aAAa,GACxB,aAAa,MAAM,EACnB,iBAAiB,MAAM,EACvB,SAAS,qBAAqB,KAC7B,UAOF,CAAA;AA6BD,MAAM,WAAW,mBAAmB;IAClC,sEAAsE;IACtE,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAA;IACjC,yFAAyF;IACzF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,CAAA;CACvD;AA8CD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,gHAqGlC,CAAA"}
|
package/dist/EstimatedUsage.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect";
|
|
2
2
|
import * as Ref from "effect/Ref";
|
|
3
|
+
import * as Result from "effect/Result";
|
|
3
4
|
import * as Stream from "effect/Stream";
|
|
4
|
-
import { LlmChunk, TokenUsage } from "@llm4ts/core/Models";
|
|
5
|
+
import { LabelDistribution, LlmChunk, TokenUsage } from "@llm4ts/core/Models";
|
|
5
6
|
import { estimateCostUsd } from "./PriceList.js";
|
|
6
7
|
export const defaultReferenceModel = "claude-sonnet-4";
|
|
7
8
|
export const estimatedUsageOptionsFromEnv = (environment) => {
|
|
@@ -24,6 +25,8 @@ export const estimateUsage = (promptChars, completionChars, options) => {
|
|
|
24
25
|
return costUsd === undefined ? base : TokenUsage.make({ ...base, costUsd });
|
|
25
26
|
};
|
|
26
27
|
const messagesChars = (messages) => messages.reduce((sum, message) => sum + message.content.length, 0);
|
|
28
|
+
/** Characters the backend produced for a batch: the readable answers only. */
|
|
29
|
+
const sequenceCompletionChars = (sequence) => sequence.entries.reduce((sum, entry) => sum + (Result.isSuccess(entry) ? JSON.stringify(entry.success.probabilities).length : 0), 0);
|
|
27
30
|
const mergeTotals = (previous, usage) => {
|
|
28
31
|
const cached = [previous?.cached, usage.cached].flatMap((value) => value === undefined ? [] : [value]);
|
|
29
32
|
const cost = [previous?.costUsd, usage.costUsd].flatMap((value) => value === undefined ? [] : [value]);
|
|
@@ -70,6 +73,22 @@ const estimatedStream = (stream, promptChars, options, record) => Stream.unwrap(
|
|
|
70
73
|
export const makeEstimatedUsageMeter = Effect.fn("@llm4ts/flow/EstimatedUsage.make")(function* (service, options) {
|
|
71
74
|
const totals = yield* Ref.make(undefined);
|
|
72
75
|
const record = (usage) => Ref.update(totals, (previous) => mergeTotals(previous, usage));
|
|
76
|
+
const estimatedLabel = estimatedModelLabel(options.referenceModel);
|
|
77
|
+
const withEstimate = (usage, carry) => record(usage).pipe(Effect.as(carry(usage)));
|
|
78
|
+
// `scoreLabelSequence` is optional on the shape and the judgment layer reads
|
|
79
|
+
// its absence as "this backend cannot batch" (falling back to a verbalized
|
|
80
|
+
// sequence). So it is forwarded only when the seat really has it: defining it
|
|
81
|
+
// unconditionally would advertise a capability the backend lacks, and leaving
|
|
82
|
+
// it out of the decorator would hide one it has.
|
|
83
|
+
const batched = service.scoreLabelSequence;
|
|
84
|
+
const scoreLabelSequence = batched === undefined
|
|
85
|
+
? undefined
|
|
86
|
+
: (prompt, labelSets) => batched(prompt, labelSets).pipe(Effect.flatMap((sequence) => sequence.usage === undefined
|
|
87
|
+
? withEstimate(
|
|
88
|
+
// One call, so one estimate for the whole batch: the
|
|
89
|
+
// per-question entries of a sequence carry no usage.
|
|
90
|
+
estimateUsage(prompt.length, sequenceCompletionChars(sequence), options), (usage) => ({ ...sequence, usage, model: estimatedLabel }))
|
|
91
|
+
: record(sequence.usage).pipe(Effect.as(sequence))));
|
|
73
92
|
const decorated = {
|
|
74
93
|
executeStream: (prompt) => estimatedStream(service.executeStream(prompt), prompt.length, options, record),
|
|
75
94
|
executeStreamWithHistory: (messages) => estimatedStream(service.executeStreamWithHistory(messages), messagesChars(messages), options, record),
|
|
@@ -84,10 +103,19 @@ export const makeEstimatedUsageMeter = Effect.fn("@llm4ts/flow/EstimatedUsage.ma
|
|
|
84
103
|
const estimated = estimateUsage(prompt.length, (JSON.stringify(value) ?? "").length, options);
|
|
85
104
|
return record(estimated).pipe(Effect.as([value, estimated, estimatedModelLabel(options.referenceModel)]));
|
|
86
105
|
})),
|
|
106
|
+
// A connector builds its verbalized `scoreLabels` from its own raw
|
|
107
|
+
// `executeStructuredWithUsage`, so the substitution below is the only one
|
|
108
|
+
// that reaches label scoring. Relabelling the model is the point of it:
|
|
109
|
+
// published under the backend's real name, an estimate would land in a
|
|
110
|
+
// measured column. The judgment origin then records `estimated:<model>`
|
|
111
|
+
// rather than the backend's name for such a seat, which is the accepted
|
|
112
|
+
// cost of never mixing the two.
|
|
87
113
|
scoreLabels: (prompt, labels) => service
|
|
88
114
|
.scoreLabels(prompt, labels)
|
|
89
|
-
.pipe(Effect.
|
|
90
|
-
estimateUsage(prompt.length, JSON.stringify(distribution.probabilities).length, options))))
|
|
115
|
+
.pipe(Effect.flatMap((distribution) => distribution.usage === undefined
|
|
116
|
+
? withEstimate(estimateUsage(prompt.length, JSON.stringify(distribution.probabilities).length, options), (usage) => LabelDistribution.make({ ...distribution, usage, model: estimatedLabel }))
|
|
117
|
+
: record(distribution.usage).pipe(Effect.as(distribution)))),
|
|
118
|
+
...(scoreLabelSequence === undefined ? {} : { scoreLabelSequence }),
|
|
91
119
|
isAvailable: service.isAvailable
|
|
92
120
|
};
|
|
93
121
|
return { service: decorated, totals: Ref.get(totals) };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"EstimatedUsage.js","sourceRoot":"","sources":["../src/EstimatedUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,GAAG,MAAM,YAAY,CAAA;AACjC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAgB,MAAM,qBAAqB,CAAA;
|
|
1
|
+
{"version":3,"file":"EstimatedUsage.js","sourceRoot":"","sources":["../src/EstimatedUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,GAAG,MAAM,YAAY,CAAA;AACjC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,UAAU,EAAgB,MAAM,qBAAqB,CAAA;AAC3F,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAyBhD,MAAM,CAAC,MAAM,qBAAqB,GAAG,iBAAiB,CAAA;AAEtD,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAC1C,WAAyD,EAClC,EAAE;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;IACrF,OAAO;QACL,cAAc,EAAE,WAAW,CAAC,qBAAqB,EAAE,IAAI,EAAE,IAAI,qBAAqB;QAClF,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAA;AACH,CAAC,CAAA;AAED,6EAA6E;AAC7E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,cAAsB,EAAU,EAAE,CAAC,aAAa,cAAc,EAAE,CAAA;AAEpG,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,KAAyB,EAAW,EAAE,CACrE,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;AAEvD,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,aAAqB,EAAU,EAAE,CACjE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,aAAa,CAAC,CAAA;AAE/C,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,WAAmB,EACnB,eAAuB,EACvB,OAA8B,EAClB,EAAE;IACd,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,aAAa,CAAC,CAAA;IACpD,MAAM,UAAU,GAAG,SAAS,CAAC,eAAe,EAAE,aAAa,CAAC,CAAA;IAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU,EAAE,CAAC,CAAA;IAChF,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAA;IAC7D,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;AAC7E,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,CAAC,QAAgC,EAAU,EAAE,CACjE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;AAEpE,8EAA8E;AAC9E,MAAM,uBAAuB,GAAG,CAAC,QAAuB,EAAU,EAAE,CAClE,QAAQ,CAAC,OAAO,CAAC,MAAM,CACrB,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CACb,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAC1F,CAAC,CACF,CAAA;AAEH,MAAM,WAAW,GAAG,CAAC,QAAgC,EAAE,KAAiB,EAAc,EAAE;IACtF,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAChE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CACnC,CAAA;IACD,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAChE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CACnC,CAAA;IACD,OAAO,UAAU,CAAC,IAAI,CAAC;QACrB,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM;QAC9C,UAAU,EAAE,CAAC,QAAQ,EAAE,UAAU,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU;QAC1D,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK;QAC3C,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC;QACzF,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC;KACvF,CAAC,CAAA;AACJ,CAAC,CAAA;AASD,MAAM,eAAe,GAAG,CACtB,MAAyC,EACzC,WAAmB,EACnB,OAA8B,EAC9B,MAAkD,EACf,EAAE,CACrC,MAAM,CAAC,MAAM,CACX,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACvC,MAAM,eAAe,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACzE,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;YAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC,CAAC,CACH,CACF,CAAA;IACD,mEAAmE;IACnE,qEAAqE;IACrE,iCAAiC;IACjC,MAAM,IAAI,GAAsC,MAAM,CAAC,MAAM,CAC3D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,OAAO,MAAM,CAAC,KAAK,CAAA;QACrB,CAAC;QACD,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,CAAA;QAClF,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,OAAO,MAAM,CAAC,OAAO,CACnB,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,EAAE;YACT,KAAK;YACL,QAAQ,EAAE,EAAE,KAAK,EAAE,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;SACjE,CAAC,CACH,CAAA;IACH,CAAC,CAAC,CACH,CAAA;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;AACpC,CAAC,CAAC,CACH,CAAA;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC,EAAE,CAAC,kCAAkC,CAAC,CAAC,QAAQ,CAAC,EAC5F,OAAwB,EACxB,OAA8B;IAE9B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAyB,SAAS,CAAC,CAAA;IACjE,MAAM,MAAM,GAAG,CAAC,KAAiB,EAAuB,EAAE,CACxD,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;IAChE,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,CAAI,KAAiB,EAAE,KAA+B,EAAoB,EAAE,CAC/F,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE7C,6EAA6E;IAC7E,2EAA2E;IAC3E,8EAA8E;IAC9E,8EAA8E;IAC9E,iDAAiD;IACjD,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAA;IAC1C,MAAM,kBAAkB,GACtB,OAAO,KAAK,SAAS;QACnB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,CACE,MAAc,EACd,SAA+C,EACP,EAAE,CAC1C,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,IAAI,CAC7B,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC1B,QAAQ,CAAC,KAAK,KAAK,SAAS;YAC1B,CAAC,CAAC,YAAY;YACV,qDAAqD;YACrD,qDAAqD;YACrD,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,EACxE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAC3D;YACH,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CACrD,CACF,CAAA;IAET,MAAM,SAAS,GAAoB;QACjC,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CACxB,eAAe,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;QAChF,wBAAwB,EAAE,CAAC,QAAQ,EAAE,EAAE,CACrC,eAAe,CACb,OAAO,CAAC,wBAAwB,CAAC,QAAQ,CAAC,EAC1C,aAAa,CAAC,QAAQ,CAAC,EACvB,OAAO,EACP,MAAM,CACP;QACH,gBAAgB,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC;QAC5E,iBAAiB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CAChD,OAAO;aACJ,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC;aAC7C,IAAI,CACH,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CACpF,CACF;QACL,0BAA0B,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CACzD,OAAO,CAAC,0BAA0B,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,IAAI,CACjE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE;YACvC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC,CAAC,CAAA;YACtE,CAAC;YACD,MAAM,SAAS,GAAG,aAAa,CAC7B,MAAM,CAAC,MAAM,EACb,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EACpC,OAAO,CACR,CAAA;YACD,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAC3B,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAU,CAAC,CACpF,CAAA;QACH,CAAC,CAAC,CACH;QACH,mEAAmE;QACnE,0EAA0E;QAC1E,wEAAwE;QACxE,uEAAuE;QACvE,wEAAwE;QACxE,wEAAwE;QACxE,gCAAgC;QAChC,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAC9B,OAAO;aACJ,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC;aAC3B,IAAI,CACH,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,EAAE,CAC9B,YAAY,CAAC,KAAK,KAAK,SAAS;YAC9B,CAAC,CAAC,YAAY,CACV,aAAa,CACX,MAAM,CAAC,MAAM,EACb,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,MAAM,EACjD,OAAO,CACR,EACD,CAAC,KAAK,EAAE,EAAE,CACR,iBAAiB,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAC5E;YACH,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAC7D,CACF;QACL,GAAG,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC;QACnE,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAA;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAA;AACxD,CAAC,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@llm4ts/flow",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "Effect-native LLM workflow, persistence, review, and repository automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -98,7 +98,7 @@
|
|
|
98
98
|
"typescript"
|
|
99
99
|
],
|
|
100
100
|
"dependencies": {
|
|
101
|
-
"@llm4ts/core": "2.
|
|
101
|
+
"@llm4ts/core": "2.7.0"
|
|
102
102
|
},
|
|
103
103
|
"peerDependencies": {
|
|
104
104
|
"effect": "4.0.0-rc.115"
|
package/src/EstimatedUsage.ts
CHANGED
|
@@ -1,19 +1,27 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect"
|
|
2
2
|
import * as Ref from "effect/Ref"
|
|
3
|
+
import * as Result from "effect/Result"
|
|
3
4
|
import * as Stream from "effect/Stream"
|
|
4
5
|
import type { LlmError } from "@llm4ts/core/Errors"
|
|
5
|
-
import type { LlmServiceShape } from "@llm4ts/core/LlmService"
|
|
6
|
-
import { LlmChunk, TokenUsage, type Message } from "@llm4ts/core/Models"
|
|
6
|
+
import type { LabelSequence, LlmServiceShape } from "@llm4ts/core/LlmService"
|
|
7
|
+
import { LabelDistribution, LlmChunk, TokenUsage, type Message } from "@llm4ts/core/Models"
|
|
7
8
|
import { estimateCostUsd } from "./PriceList.ts"
|
|
8
9
|
|
|
9
|
-
// ESTIMATED token accounting for connectors that report none (
|
|
10
|
-
//
|
|
11
|
-
// counts characters on both sides
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
10
|
+
// ESTIMATED token accounting for connectors that report none (Antigravity,
|
|
11
|
+
// Copilot, and Cursor on the capability matrix; every backend whose
|
|
12
|
+
// `usageReporting` is false). The decorator counts characters on both sides
|
|
13
|
+
// of each request and, when the backend reported nothing, carries the
|
|
14
|
+
// estimate back on the value the caller already reads — a synthetic final
|
|
15
|
+
// chunk for streams, the returned usage for structured calls and label
|
|
16
|
+
// scoring — so `collect`, `Chat`, `LlmJudgment`'s usage hook, `CostTracker`,
|
|
17
|
+
// and the cost summary all see it through the normal event path, labelled
|
|
18
|
+
// `estimated:<model>` so no report can mistake it for measured usage
|
|
19
|
+
// (ADR 0012; the PoC decision of 2026-08-30 is estimates-only, no CLI usage
|
|
20
|
+
// parsing).
|
|
21
|
+
//
|
|
22
|
+
// The runner meters every seat with this by default, so a flow inherits it
|
|
23
|
+
// without wrapping anything; wrapping again is safe, because an estimate is
|
|
24
|
+
// only ever filled in where no usage was reported at all.
|
|
17
25
|
|
|
18
26
|
export interface EstimatedUsageOptions {
|
|
19
27
|
/** Pricing reference for the estimate — a `PriceList` model prefix. */
|
|
@@ -59,6 +67,14 @@ export const estimateUsage = (
|
|
|
59
67
|
const messagesChars = (messages: ReadonlyArray<Message>): number =>
|
|
60
68
|
messages.reduce((sum, message) => sum + message.content.length, 0)
|
|
61
69
|
|
|
70
|
+
/** Characters the backend produced for a batch: the readable answers only. */
|
|
71
|
+
const sequenceCompletionChars = (sequence: LabelSequence): number =>
|
|
72
|
+
sequence.entries.reduce(
|
|
73
|
+
(sum, entry) =>
|
|
74
|
+
sum + (Result.isSuccess(entry) ? JSON.stringify(entry.success.probabilities).length : 0),
|
|
75
|
+
0
|
|
76
|
+
)
|
|
77
|
+
|
|
62
78
|
const mergeTotals = (previous: TokenUsage | undefined, usage: TokenUsage): TokenUsage => {
|
|
63
79
|
const cached = [previous?.cached, usage.cached].flatMap((value) =>
|
|
64
80
|
value === undefined ? [] : [value]
|
|
@@ -138,6 +154,35 @@ export const makeEstimatedUsageMeter = Effect.fn("@llm4ts/flow/EstimatedUsage.ma
|
|
|
138
154
|
const totals = yield* Ref.make<TokenUsage | undefined>(undefined)
|
|
139
155
|
const record = (usage: TokenUsage): Effect.Effect<void> =>
|
|
140
156
|
Ref.update(totals, (previous) => mergeTotals(previous, usage))
|
|
157
|
+
const estimatedLabel = estimatedModelLabel(options.referenceModel)
|
|
158
|
+
const withEstimate = <A>(usage: TokenUsage, carry: (usage: TokenUsage) => A): Effect.Effect<A> =>
|
|
159
|
+
record(usage).pipe(Effect.as(carry(usage)))
|
|
160
|
+
|
|
161
|
+
// `scoreLabelSequence` is optional on the shape and the judgment layer reads
|
|
162
|
+
// its absence as "this backend cannot batch" (falling back to a verbalized
|
|
163
|
+
// sequence). So it is forwarded only when the seat really has it: defining it
|
|
164
|
+
// unconditionally would advertise a capability the backend lacks, and leaving
|
|
165
|
+
// it out of the decorator would hide one it has.
|
|
166
|
+
const batched = service.scoreLabelSequence
|
|
167
|
+
const scoreLabelSequence =
|
|
168
|
+
batched === undefined
|
|
169
|
+
? undefined
|
|
170
|
+
: (
|
|
171
|
+
prompt: string,
|
|
172
|
+
labelSets: ReadonlyArray<ReadonlyArray<string>>
|
|
173
|
+
): Effect.Effect<LabelSequence, LlmError> =>
|
|
174
|
+
batched(prompt, labelSets).pipe(
|
|
175
|
+
Effect.flatMap((sequence) =>
|
|
176
|
+
sequence.usage === undefined
|
|
177
|
+
? withEstimate(
|
|
178
|
+
// One call, so one estimate for the whole batch: the
|
|
179
|
+
// per-question entries of a sequence carry no usage.
|
|
180
|
+
estimateUsage(prompt.length, sequenceCompletionChars(sequence), options),
|
|
181
|
+
(usage) => ({ ...sequence, usage, model: estimatedLabel })
|
|
182
|
+
)
|
|
183
|
+
: record(sequence.usage).pipe(Effect.as(sequence))
|
|
184
|
+
)
|
|
185
|
+
)
|
|
141
186
|
|
|
142
187
|
const decorated: LlmServiceShape = {
|
|
143
188
|
executeStream: (prompt) =>
|
|
@@ -174,21 +219,32 @@ export const makeEstimatedUsageMeter = Effect.fn("@llm4ts/flow/EstimatedUsage.ma
|
|
|
174
219
|
)
|
|
175
220
|
})
|
|
176
221
|
),
|
|
222
|
+
// A connector builds its verbalized `scoreLabels` from its own raw
|
|
223
|
+
// `executeStructuredWithUsage`, so the substitution below is the only one
|
|
224
|
+
// that reaches label scoring. Relabelling the model is the point of it:
|
|
225
|
+
// published under the backend's real name, an estimate would land in a
|
|
226
|
+
// measured column. The judgment origin then records `estimated:<model>`
|
|
227
|
+
// rather than the backend's name for such a seat, which is the accepted
|
|
228
|
+
// cost of never mixing the two.
|
|
177
229
|
scoreLabels: (prompt, labels) =>
|
|
178
230
|
service
|
|
179
231
|
.scoreLabels(prompt, labels)
|
|
180
232
|
.pipe(
|
|
181
|
-
Effect.
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
233
|
+
Effect.flatMap((distribution) =>
|
|
234
|
+
distribution.usage === undefined
|
|
235
|
+
? withEstimate(
|
|
236
|
+
estimateUsage(
|
|
237
|
+
prompt.length,
|
|
238
|
+
JSON.stringify(distribution.probabilities).length,
|
|
239
|
+
options
|
|
240
|
+
),
|
|
241
|
+
(usage) =>
|
|
242
|
+
LabelDistribution.make({ ...distribution, usage, model: estimatedLabel })
|
|
188
243
|
)
|
|
189
|
-
|
|
244
|
+
: record(distribution.usage).pipe(Effect.as(distribution))
|
|
190
245
|
)
|
|
191
246
|
),
|
|
247
|
+
...(scoreLabelSequence === undefined ? {} : { scoreLabelSequence }),
|
|
192
248
|
isAvailable: service.isAvailable
|
|
193
249
|
}
|
|
194
250
|
return { service: decorated, totals: Ref.get(totals) }
|