@modelprofile.com/flexharness-models 17.0.0 → 18.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts_models/index.d.ts +1 -0
- package/dist_ts_models/index.js +2 -1
- package/dist_ts_models/smartai.failures.d.ts +2 -1
- package/dist_ts_models/smartai.failures.js +4 -1
- package/dist_ts_models/smartai.terminalevent.d.ts +56 -0
- package/dist_ts_models/smartai.terminalevent.js +221 -0
- package/package.json +1 -1
- package/readme.md +22 -1
- package/ts_models/index.ts +1 -0
- package/ts_models/readme.md +22 -1
- package/ts_models/smartai.failures.ts +4 -1
- package/ts_models/smartai.terminalevent.ts +275 -0
|
@@ -3,6 +3,7 @@ export * from './classes.modelregistry.js';
|
|
|
3
3
|
export * from './smartai.cache.js';
|
|
4
4
|
export * from './smartai.limits.js';
|
|
5
5
|
export * from './smartai.failures.js';
|
|
6
|
+
export * from './smartai.terminalevent.js';
|
|
6
7
|
export * from './smartai.textfiles.js';
|
|
7
8
|
export * from './smartai.pricing.js';
|
|
8
9
|
export { generateText, streamText, tool, jsonSchema } from './plugins.js';
|
package/dist_ts_models/index.js
CHANGED
|
@@ -3,7 +3,8 @@ export * from './classes.modelregistry.js';
|
|
|
3
3
|
export * from './smartai.cache.js';
|
|
4
4
|
export * from './smartai.limits.js';
|
|
5
5
|
export * from './smartai.failures.js';
|
|
6
|
+
export * from './smartai.terminalevent.js';
|
|
6
7
|
export * from './smartai.textfiles.js';
|
|
7
8
|
export * from './smartai.pricing.js';
|
|
8
9
|
export { generateText, streamText, tool, jsonSchema } from './plugins.js';
|
|
9
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
10
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90c19tb2RlbHMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FBYyxpQkFBaUIsQ0FBQztBQUNoQyxjQUFjLDRCQUE0QixDQUFDO0FBQzNDLGNBQWMsb0JBQW9CLENBQUM7QUFDbkMsY0FBYyxxQkFBcUIsQ0FBQztBQUNwQyxjQUFjLHVCQUF1QixDQUFDO0FBQ3RDLGNBQWMsNEJBQTRCLENBQUM7QUFDM0MsY0FBYyx3QkFBd0IsQ0FBQztBQUN2QyxjQUFjLHNCQUFzQixDQUFDO0FBQ3JDLE9BQU8sRUFBRSxZQUFZLEVBQUUsVUFBVSxFQUFFLElBQUksRUFBRSxVQUFVLEVBQUUsTUFBTSxjQUFjLENBQUMifQ==
|
|
@@ -4,7 +4,8 @@ import { type IModelLimitInfo } from './smartai.limits.js';
|
|
|
4
4
|
* `http`: the provider answered with an HTTP error status.
|
|
5
5
|
* `stream`: the provider answered with a response stream and reported an error inside it, before or
|
|
6
6
|
* after output began.
|
|
7
|
-
* `response`: the provider answered, but its response could not be read or processed
|
|
7
|
+
* `response`: the provider answered, but its response could not be read or processed, or its
|
|
8
|
+
* stream ended before its terminal event (`ModelStreamIncompleteError`).
|
|
8
9
|
* `network`: no response arrived because the connection failed.
|
|
9
10
|
* `aborted`: an abort signal ended the call, a timeout signal included.
|
|
10
11
|
* `unknown`: the failure is none of these.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import { isModelLimitError, readJsonRecord } from './smartai.limits.js';
|
|
3
|
+
import { isModelStreamIncompleteError } from './smartai.terminalevent.js';
|
|
3
4
|
/** How many causes are followed before a failure counts as unknown. */
|
|
4
5
|
const MAX_CAUSE_DEPTH = 8;
|
|
5
6
|
/** The longest provider code that is kept. */
|
|
@@ -86,6 +87,8 @@ function describeKnownFailure(error) {
|
|
|
86
87
|
return { kind: 'limit', limit: error.limit };
|
|
87
88
|
if (plugins.isAbortError(error))
|
|
88
89
|
return { kind: 'aborted' };
|
|
90
|
+
if (isModelStreamIncompleteError(error))
|
|
91
|
+
return { kind: 'response' };
|
|
89
92
|
if (plugins.StreamProviderError.isInstance(error) || plugins.isProviderStreamError(error)) {
|
|
90
93
|
return describeStreamFailure(error.code ?? readFrameCode(error.data));
|
|
91
94
|
}
|
|
@@ -124,4 +127,4 @@ export function describeModelFailure(error) {
|
|
|
124
127
|
}
|
|
125
128
|
return { kind: 'unknown' };
|
|
126
129
|
}
|
|
127
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
130
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRhaS5mYWlsdXJlcy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzX21vZGVscy9zbWFydGFpLmZhaWx1cmVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxPQUFPLE1BQU0sY0FBYyxDQUFDO0FBQ3hDLE9BQU8sRUFBRSxpQkFBaUIsRUFBRSxjQUFjLEVBQXdCLE1BQU0scUJBQXFCLENBQUM7QUFDOUYsT0FBTyxFQUFFLDRCQUE0QixFQUFFLE1BQU0sNEJBQTRCLENBQUM7QUFnQzFFLHVFQUF1RTtBQUN2RSxNQUFNLGVBQWUsR0FBRyxDQUFDLENBQUM7QUFFMUIsOENBQThDO0FBQzlDLE1BQU0sd0JBQXdCLEdBQUcsRUFBRSxDQUFDO0FBRXBDOzs7O0dBSUc7QUFDSCxNQUFNLG1CQUFtQixHQUFHLGlGQUFpRixDQUFDO0FBRTlHLG9EQUFvRDtBQUNwRCxNQUFNLHVCQUF1QixHQUFHLG1GQUFtRixDQUFDO0FBRXBILDhFQUE4RTtBQUM5RSxNQUFNLG9CQUFvQixHQUFHLEVBQUUsQ0FBQztBQUVoQyxxRkFBcUY7QUFDckYsTUFBTSw0QkFBNEIsR0FBRyxDQUFDLENBQUM7QUFFdkMsdUdBQXVHO0FBQ3ZHLFNBQVMsVUFBVSxDQUFDLElBQVk7SUFDOUIsSUFBSSxJQUFJLENBQUMsTUFBTSxHQUFHLG9CQUFvQjtRQUFFLE9BQU8sS0FBSyxDQUFDO0lBQ3JELE9BQU8sSUFBSSxDQUFDLE1BQU0sR0FBRyw0QkFBNEIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0FBQ3JHLENBQUM7QUFFRCx1R0FBdUc7QUFDdkcsU0FBUyxjQUFjLENBQUMsSUFBWTtJQUNsQyxPQUFPLElBQUksQ0FBQyxNQUFNLElBQUksd0JBQXdCO1dBQ3pDLG1CQUFtQixDQUFDLElBQUksQ0FBQyxJQUFJLENBQUM7V0FDOUIsQ0FBQyx1QkFBdUIsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDO1dBQ25DLElBQUksQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLENBQUMsS0FBSyxDQUFDLFVBQVUsQ0FBQyxDQUFDO0FBQzVDLENBQUM7QUFFRCxTQUFTLFVBQVUsQ0FBQyxLQUF5QjtJQUMzQyxPQUFPLE9BQU8sS0FBSyxLQUFLLFFBQVEsSUFBSSxNQUFNLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQyxJQUFJLEtBQUssSUFBSSxHQUFHLElBQUksS0FBSyxJQUFJLEdBQUcsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUM7QUFDbEgsQ0FBQztBQUVELFNBQVMsZ0JBQWdCLENBQUMsS0FBYztJQUN0QyxNQUFNLElBQUksR0FBRyxPQUFPLEtBQUssS0FBSyxRQUFRLElBQUksTUFBTSxDQUFDLGFBQWEsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUM7SUFDOUYsT0FBTyxPQUFPLElBQUksS0FBSyxRQUFRLElBQUksY0FBYyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztBQUM3RSxDQUFDO0FBRUQsMkNBQTJDO0FBQzNDLE1BQU0sdUJBQXVCLEdBQUcsbUJBQW1CLENBQUM7QUFFcEQ7Ozs7R0FJRztBQUNILFNBQVMsa0JBQWtCLENBQUMsS0FBMkI7SUFDckQsTUFBTSxXQUFXLEdBQUcsTUFBTSxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsZUFBZSxJQUFJLEVBQUUsQ0FBQztTQUM1RCxJQUFJLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsV0FBVyxFQUFFLEtBQUssY0FBYyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUNoRSxPQUFPLFdBQVcsRUFBRSxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUUsQ0FBQyxVQUFVLENBQUMsdUJBQXVCLENBQUMsS0FBSyxJQUFJLENBQUM7QUFDeEYsQ0FBQztBQUVELFNBQVMsU0FBUyxDQUFDLElBQXdCO0lBQ3pDLElBQUksSUFBSSxLQUFLLFNBQVM7UUFBRSxPQUFPLFNBQVMsQ0FBQztJQUN6QyxJQUFJLENBQUM7UUFDSCxPQUFPLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDMUIsQ0FBQztJQUFDLE1BQU0sQ0FBQztRQUNQLE9BQU8sU0FBUyxDQUFDO0lBQ25CLENBQUM7QUFDSCxDQUFDO0FBRUQsOEdBQThHO0FBQzlHLE1BQU0sZ0JBQWdCLEdBQUcsT0FBTyxDQUFDO0FBRWpDOzs7Ozs7R0FNRztBQUNILFNBQVMsYUFBYSxDQUFDLEtBQWM7SUFDbkMsTUFBTSxLQUFLLEdBQUcsY0FBYyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3BDLElBQUksS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLFNBQVMsQ0FBQztJQUMxQyxNQUFNLE1BQU0sR0FBRyxjQUFjLENBQUMsY0FBYyxDQUFDLEtBQUssQ0FBQyxRQUFRLENBQUMsRUFBRSxLQUFLLENBQUMsSUFBSSxjQUFjLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFJLEtBQUssQ0FBQztJQUM3RyxNQUFNLElBQUksR0FBRyxNQUFNLENBQUMsSUFBSSxJQUFJLEtBQUssQ0FBQyxJQUFJLENBQUM7SUFDdkMsSUFBSSxJQUFJLEtBQUssU0FBUyxJQUFJLElBQUksS0FBSyxJQUFJO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDckQsT0FBTyxNQUFNLENBQUMsSUFBSSxLQUFLLGdCQUFnQixDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUM7QUFDcEUsQ0FBQztBQUVELFNBQVMscUJBQXFCLENBQUMsSUFBYTtJQUMxQyxNQUFNLFlBQVksR0FBRyxnQkFBZ0IsQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUM1QyxPQUFPLFlBQVksS0FBSyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsWUFBWSxFQUFFLENBQUM7QUFDNUYsQ0FBQztBQUVELFNBQVMsb0JBQW9CLENBQUMsS0FBYztJQUMxQyxJQUFJLGlCQUFpQixDQUFDLEtBQUssQ0FBQztRQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxLQUFLLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDM0UsSUFBSSxPQUFPLENBQUMsWUFBWSxDQUFDLEtBQUssQ0FBQztRQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLENBQUM7SUFDNUQsSUFBSSw0QkFBNEIsQ0FBQyxLQUFLLENBQUM7UUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLFVBQVUsRUFBRSxDQUFDO0lBQ3JFLElBQUksT0FBTyxDQUFDLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxLQUFLLENBQUMsSUFBSSxPQUFPLENBQUMscUJBQXFCLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUMxRixPQUFPLHFCQUFxQixDQUFDLEtBQUssQ0FBQyxJQUFJLElBQUksYUFBYSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO0lBQ3hFLENBQUM7SUFDRCxJQUFJLE9BQU8sQ0FBQyxZQUFZLENBQUMsVUFBVSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDM0MsSUFBSSxrQkFBa0IsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDO1lBQzlCLDRGQUE0RjtZQUM1RixxRkFBcUY7WUFDckYsMkNBQTJDO1lBQzNDLElBQUksS0FBSyxDQUFDLFVBQVUsS0FBSyxTQUFTO2dCQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsVUFBVSxFQUFFLENBQUM7WUFDaEUsT0FBTyxxQkFBcUIsQ0FBQyxhQUFhLENBQUMsS0FBSyxDQUFDLElBQUksSUFBSSxTQUFTLENBQUMsS0FBSyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUMzRixDQUFDO1FBQ0QsTUFBTSxNQUFNLEdBQUcsVUFBVSxDQUFDLEtBQUssQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUM1QyxJQUFJLE1BQU0sS0FBSyxTQUFTO1lBQUUsT0FBTyxFQUFFLElBQUksRUFBRSxTQUFTLEVBQUUsQ0FBQztRQUNyRCxPQUFPLEVBQUUsSUFBSSxFQUFFLE1BQU0sSUFBSSxHQUFHLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsVUFBVSxFQUFFLE1BQU0sRUFBRSxDQUFDO0lBQy9ELENBQUM7SUFDRCxPQUFPLFNBQVMsQ0FBQztBQUNuQixDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sVUFBVSxvQkFBb0IsQ0FBQyxLQUFjO0lBQ2pELE1BQU0sT0FBTyxHQUFHLElBQUksR0FBRyxFQUFXLENBQUM7SUFDbkMsSUFBSSxPQUFPLEdBQUcsS0FBSyxDQUFDO0lBQ3BCLEtBQUssSUFBSSxLQUFLLEdBQUcsQ0FBQyxFQUFFLEtBQUssSUFBSSxlQUFlLElBQUksT0FBTyxLQUFLLFNBQVMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLEVBQUUsS0FBSyxJQUFJLENBQUMsRUFBRSxDQUFDO1FBQzNHLE9BQU8sQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDckIsTUFBTSxXQUFXLEdBQUcsb0JBQW9CLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDbEQsSUFBSSxXQUFXO1lBQUUsT0FBTyxXQUFXLENBQUM7UUFDcEMsT0FBTyxHQUFHLE9BQU8sQ0FBQyxVQUFVLENBQUMsVUFBVSxDQUFDLE9BQU8sQ0FBQztZQUM5QyxDQUFDLENBQUMsT0FBTyxDQUFDLFNBQVM7WUFDbkIsQ0FBQyxDQUFDLE9BQU8sWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztJQUMzRCxDQUFDO0lBQ0QsT0FBTyxFQUFFLElBQUksRUFBRSxTQUFTLEVBQUUsQ0FBQztBQUM3QixDQUFDIn0=
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a model's streamed answer ends: at its stream's terminal event, never at the end of the
|
|
3
|
+
* connection (ruling of 2026-10-27, AGL-1130). A stream that ends without its terminal event is an
|
|
4
|
+
* incomplete answer and fails; a stream that has it is the answer, and nothing after it is read, so
|
|
5
|
+
* neither a connection that breaks afterwards nor one a proxy keeps open changes or holds it.
|
|
6
|
+
*/
|
|
7
|
+
import type * as plugins from './plugins.js';
|
|
8
|
+
/** How a dialect's response stream ends. */
|
|
9
|
+
export interface IModelStreamTerminal {
|
|
10
|
+
/** The event that ends a finished answer, as `ModelStreamIncompleteError` names it, such as `message_stop`. */
|
|
11
|
+
readonly event: string;
|
|
12
|
+
/**
|
|
13
|
+
* Whether a server-sent event ends the stream, by its data (its `data` lines joined with a line
|
|
14
|
+
* feed): the event that ends a finished answer, or a failure the dialect ends its stream with.
|
|
15
|
+
*/
|
|
16
|
+
ends(data: string): boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* A model's response stream ended before its terminal event: the server closed it early, so the
|
|
20
|
+
* answer is incomplete. The call fails like one whose connection broke mid-stream, which the
|
|
21
|
+
* engine does not retry.
|
|
22
|
+
*/
|
|
23
|
+
export declare class ModelStreamIncompleteError extends Error {
|
|
24
|
+
/** The terminal event the stream ended without. */
|
|
25
|
+
readonly terminalEvent: string;
|
|
26
|
+
constructor(terminalEvent: string);
|
|
27
|
+
}
|
|
28
|
+
/** Recognizes a ModelStreamIncompleteError, also one created by another copy of this package. */
|
|
29
|
+
export declare function isModelStreamIncompleteError(error: unknown): error is ModelStreamIncompleteError;
|
|
30
|
+
/** The two halves of `endModelStreamsAtTerminalEvent`: the model's fetch and its innermost middleware. */
|
|
31
|
+
export interface IModelStreamTerminalEnding {
|
|
32
|
+
/**
|
|
33
|
+
* The fetch to give the provider's model: a successful event-stream answer ends right after its
|
|
34
|
+
* terminal event, which ends the model's answer there. Every other answer, an error status or a
|
|
35
|
+
* JSON body among them, passes unchanged.
|
|
36
|
+
*/
|
|
37
|
+
fetch: typeof globalThis.fetch;
|
|
38
|
+
/**
|
|
39
|
+
* The middleware that fails a streamed call whose stream ended before its terminal event: an
|
|
40
|
+
* error part with a `ModelStreamIncompleteError` follows what the stream carried, in place of the
|
|
41
|
+
* finish part the model may still give it, unless the stream reported an error of its own, with
|
|
42
|
+
* which the call fails then. It must be the innermost middleware around the model,
|
|
43
|
+
* since it gives each streamed call an abort signal of its own (following the call's), by which
|
|
44
|
+
* the fetch reports how the call's stream ended.
|
|
45
|
+
*/
|
|
46
|
+
middleware: plugins.LanguageModelV4Middleware;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Ends a provider model's answers at its dialect's terminal event (see `IModelStreamTerminal`).
|
|
50
|
+
* Give the model `fetch` (which calls `transport` for every request) and wrap it in `middleware`,
|
|
51
|
+
* innermost: an answer is then what its stream carried up to its terminal event, whatever the
|
|
52
|
+
* connection does afterwards, and an answer whose stream ends without it fails with a
|
|
53
|
+
* `ModelStreamIncompleteError` after the parts it streamed. The provider modules do this for
|
|
54
|
+
* their models.
|
|
55
|
+
*/
|
|
56
|
+
export declare function endModelStreamsAtTerminalEvent(terminal: IModelStreamTerminal, transport: typeof globalThis.fetch): IModelStreamTerminalEnding;
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
const modelStreamIncompleteErrorMarker = Symbol.for('@modelprofile.com/flexharness-models/ModelStreamIncompleteError');
|
|
2
|
+
/**
|
|
3
|
+
* A model's response stream ended before its terminal event: the server closed it early, so the
|
|
4
|
+
* answer is incomplete. The call fails like one whose connection broke mid-stream, which the
|
|
5
|
+
* engine does not retry.
|
|
6
|
+
*/
|
|
7
|
+
export class ModelStreamIncompleteError extends Error {
|
|
8
|
+
/** The terminal event the stream ended without. */
|
|
9
|
+
terminalEvent;
|
|
10
|
+
constructor(terminalEvent) {
|
|
11
|
+
super(`The model's response stream ended before its terminal event (${terminalEvent}), so its answer is incomplete.`);
|
|
12
|
+
this.name = 'ModelStreamIncompleteError';
|
|
13
|
+
this.terminalEvent = terminalEvent;
|
|
14
|
+
Object.defineProperty(this, modelStreamIncompleteErrorMarker, { value: true });
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/** Recognizes a ModelStreamIncompleteError, also one created by another copy of this package. */
|
|
18
|
+
export function isModelStreamIncompleteError(error) {
|
|
19
|
+
return error instanceof Error
|
|
20
|
+
&& error[modelStreamIncompleteErrorMarker] === true;
|
|
21
|
+
}
|
|
22
|
+
const streamEndings = new WeakMap();
|
|
23
|
+
const LF = 0x0a;
|
|
24
|
+
const CR = 0x0d;
|
|
25
|
+
const COLON = 0x3a;
|
|
26
|
+
const UTF8_BOM = [0xef, 0xbb, 0xbf];
|
|
27
|
+
const concatBytes = (parts, length) => {
|
|
28
|
+
if (parts.length === 1)
|
|
29
|
+
return parts[0];
|
|
30
|
+
const bytes = new Uint8Array(length);
|
|
31
|
+
let offset = 0;
|
|
32
|
+
for (const part of parts) {
|
|
33
|
+
bytes.set(part, offset);
|
|
34
|
+
offset += part.length;
|
|
35
|
+
}
|
|
36
|
+
return bytes;
|
|
37
|
+
};
|
|
38
|
+
/** Whether a line, from `start`, is a `data` field: `data`, then a colon or the end of the line. */
|
|
39
|
+
const isDataLine = (line, start) => line.length - start >= 4
|
|
40
|
+
&& line[start] === 0x64 && line[start + 1] === 0x61 && line[start + 2] === 0x74 && line[start + 3] === 0x61
|
|
41
|
+
&& (line.length - start === 4 || line[start + 4] === COLON);
|
|
42
|
+
/**
|
|
43
|
+
* A server-sent event stream that ends right after its terminal event: the bytes up to the blank
|
|
44
|
+
* line that ends that event, then the end of the stream, and the source is cancelled, so nothing
|
|
45
|
+
* after the event is read. A source that fails passes its error on as it is. A source that ends
|
|
46
|
+
* before the terminal event ends the stream too, with all it carried, and marks `ending`
|
|
47
|
+
* incomplete; without an `ending` it fails the stream with a `ModelStreamIncompleteError`.
|
|
48
|
+
*
|
|
49
|
+
* Lines end with LF, CR or CR LF, as in the SSE format. The bytes the stream passes on are the
|
|
50
|
+
* source's, with one exception: a terminal event whose blank line ends with a CR the source had not
|
|
51
|
+
* followed with its LF yet gets that LF, since a parser waits for it before it ends the line.
|
|
52
|
+
*/
|
|
53
|
+
function endEventStreamAtTerminalEvent(source, terminal, ending) {
|
|
54
|
+
const reader = source.getReader();
|
|
55
|
+
const decoder = new TextDecoder();
|
|
56
|
+
/** The bytes of the line being read, in the pieces of the chunks it spans. */
|
|
57
|
+
let line = [];
|
|
58
|
+
let lineLength = 0;
|
|
59
|
+
/** The data of the event being read: its `data` lines joined with LF, or `undefined` before the first. */
|
|
60
|
+
let data;
|
|
61
|
+
/** The last byte ended a line with a CR: an LF right after it belongs to that line end. */
|
|
62
|
+
let afterCr = false;
|
|
63
|
+
let firstLine = true;
|
|
64
|
+
/** Ends the line being read; whether it was the blank line that ends the terminal event. */
|
|
65
|
+
const endLine = () => {
|
|
66
|
+
const bytes = concatBytes(line, lineLength);
|
|
67
|
+
line = [];
|
|
68
|
+
lineLength = 0;
|
|
69
|
+
const start = firstLine && UTF8_BOM.every((byte, index) => bytes[index] === byte) ? UTF8_BOM.length : 0;
|
|
70
|
+
firstLine = false;
|
|
71
|
+
if (bytes.length === start) {
|
|
72
|
+
const event = data;
|
|
73
|
+
data = undefined;
|
|
74
|
+
return event !== undefined && terminal.ends(event);
|
|
75
|
+
}
|
|
76
|
+
if (!isDataLine(bytes, start))
|
|
77
|
+
return false;
|
|
78
|
+
// `data:` takes one space after the colon off its value; `data` alone has an empty value.
|
|
79
|
+
const valueStart = bytes.length - start === 4 ? bytes.length : start + (bytes[start + 5] === 0x20 ? 6 : 5);
|
|
80
|
+
const value = decoder.decode(bytes.subarray(valueStart));
|
|
81
|
+
data = data === undefined ? value : `${data}\n${value}`;
|
|
82
|
+
return false;
|
|
83
|
+
};
|
|
84
|
+
/** Reads a chunk's lines: where the terminal event ends in it, or `undefined`. */
|
|
85
|
+
const scan = (chunk) => {
|
|
86
|
+
let start = 0;
|
|
87
|
+
for (let index = 0; index < chunk.length; index += 1) {
|
|
88
|
+
const byte = chunk[index];
|
|
89
|
+
if (afterCr) {
|
|
90
|
+
afterCr = false;
|
|
91
|
+
if (byte === LF) {
|
|
92
|
+
start = index + 1;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (byte !== LF && byte !== CR)
|
|
97
|
+
continue;
|
|
98
|
+
if (index > start) {
|
|
99
|
+
line.push(chunk.subarray(start, index));
|
|
100
|
+
lineLength += index - start;
|
|
101
|
+
}
|
|
102
|
+
afterCr = byte === CR;
|
|
103
|
+
start = index + 1;
|
|
104
|
+
if (endLine()) {
|
|
105
|
+
if (byte === LF)
|
|
106
|
+
return { end: index + 1, addLf: false };
|
|
107
|
+
return chunk[index + 1] === LF ? { end: index + 2, addLf: false } : { end: index + 1, addLf: true };
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
if (start < chunk.length) {
|
|
111
|
+
line.push(chunk.slice(start));
|
|
112
|
+
lineLength += chunk.length - start;
|
|
113
|
+
}
|
|
114
|
+
return undefined;
|
|
115
|
+
};
|
|
116
|
+
return new ReadableStream({
|
|
117
|
+
async pull(controller) {
|
|
118
|
+
for (;;) {
|
|
119
|
+
// A source that fails, such as a connection that breaks, fails the stream with its own error.
|
|
120
|
+
const next = await reader.read();
|
|
121
|
+
if (next.done) {
|
|
122
|
+
if (!ending) {
|
|
123
|
+
controller.error(new ModelStreamIncompleteError(terminal.event));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
// The stream ends, so the provider's model reads all it carried; the middleware then
|
|
127
|
+
// fails the call. A failed byte stream would drop what its readers had not read yet.
|
|
128
|
+
ending.incomplete = true;
|
|
129
|
+
controller.close();
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const chunk = next.value;
|
|
133
|
+
const terminalEnd = scan(chunk);
|
|
134
|
+
if (!terminalEnd) {
|
|
135
|
+
if (chunk.length === 0)
|
|
136
|
+
continue;
|
|
137
|
+
controller.enqueue(chunk);
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
controller.enqueue(chunk.subarray(0, terminalEnd.end));
|
|
141
|
+
if (terminalEnd.addLf)
|
|
142
|
+
controller.enqueue(Uint8Array.of(LF));
|
|
143
|
+
controller.close();
|
|
144
|
+
// Nothing after the terminal event is read: a connection kept open is let go now.
|
|
145
|
+
void reader.cancel().catch(() => undefined);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
},
|
|
149
|
+
cancel(reason) {
|
|
150
|
+
return reader.cancel(reason);
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
/** Whether a response is a successful server-sent event stream. */
|
|
155
|
+
const isEventStream = (response) => response.ok
|
|
156
|
+
&& response.body !== null
|
|
157
|
+
&& (response.headers.get('content-type') ?? '').trim().toLowerCase().startsWith('text/event-stream');
|
|
158
|
+
/**
|
|
159
|
+
* Ends a provider model's answers at its dialect's terminal event (see `IModelStreamTerminal`).
|
|
160
|
+
* Give the model `fetch` (which calls `transport` for every request) and wrap it in `middleware`,
|
|
161
|
+
* innermost: an answer is then what its stream carried up to its terminal event, whatever the
|
|
162
|
+
* connection does afterwards, and an answer whose stream ends without it fails with a
|
|
163
|
+
* `ModelStreamIncompleteError` after the parts it streamed. The provider modules do this for
|
|
164
|
+
* their models.
|
|
165
|
+
*/
|
|
166
|
+
export function endModelStreamsAtTerminalEvent(terminal, transport) {
|
|
167
|
+
const fetch = async (input, init) => {
|
|
168
|
+
const response = await transport(input, init);
|
|
169
|
+
if (!isEventStream(response))
|
|
170
|
+
return response;
|
|
171
|
+
const ending = init?.signal ? streamEndings.get(init.signal) : undefined;
|
|
172
|
+
return new Response(endEventStreamAtTerminalEvent(response.body, terminal, ending), {
|
|
173
|
+
status: response.status,
|
|
174
|
+
statusText: response.statusText,
|
|
175
|
+
headers: response.headers,
|
|
176
|
+
});
|
|
177
|
+
};
|
|
178
|
+
const middleware = {
|
|
179
|
+
specificationVersion: 'v4',
|
|
180
|
+
transformParams: async ({ params, type }) => {
|
|
181
|
+
if (type !== 'stream')
|
|
182
|
+
return params;
|
|
183
|
+
const abortSignal = params.abortSignal ? AbortSignal.any([params.abortSignal]) : new AbortController().signal;
|
|
184
|
+
streamEndings.set(abortSignal, { incomplete: false });
|
|
185
|
+
return { ...params, abortSignal };
|
|
186
|
+
},
|
|
187
|
+
wrapStream: async ({ doStream, params }) => {
|
|
188
|
+
const ending = params.abortSignal ? streamEndings.get(params.abortSignal) : undefined;
|
|
189
|
+
const result = await doStream();
|
|
190
|
+
if (!ending)
|
|
191
|
+
return result;
|
|
192
|
+
// A stream that reported an error has failed with it; an early end changes nothing then.
|
|
193
|
+
let failed = false;
|
|
194
|
+
const fail = (controller) => {
|
|
195
|
+
failed = true;
|
|
196
|
+
controller.enqueue({ type: 'error', error: new ModelStreamIncompleteError(terminal.event) });
|
|
197
|
+
};
|
|
198
|
+
return {
|
|
199
|
+
...result,
|
|
200
|
+
stream: result.stream.pipeThrough(new TransformStream({
|
|
201
|
+
transform: (part, controller) => {
|
|
202
|
+
if (part.type === 'error')
|
|
203
|
+
failed = true;
|
|
204
|
+
// The model finishes an answer whose stream ended early as if complete; it fails instead.
|
|
205
|
+
if (part.type === 'finish' && ending.incomplete && !failed) {
|
|
206
|
+
fail(controller);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
controller.enqueue(part);
|
|
210
|
+
},
|
|
211
|
+
flush: (controller) => {
|
|
212
|
+
if (ending.incomplete && !failed)
|
|
213
|
+
fail(controller);
|
|
214
|
+
},
|
|
215
|
+
})),
|
|
216
|
+
};
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
return { fetch, middleware };
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRhaS50ZXJtaW5hbGV2ZW50LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHNfbW9kZWxzL3NtYXJ0YWkudGVybWluYWxldmVudC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFtQkEsTUFBTSxnQ0FBZ0MsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDLGlFQUFpRSxDQUFDLENBQUM7QUFFdkg7Ozs7R0FJRztBQUNILE1BQU0sT0FBTywwQkFBMkIsU0FBUSxLQUFLO0lBQ25ELG1EQUFtRDtJQUNuQyxhQUFhLENBQVM7SUFFdEMsWUFBWSxhQUFxQjtRQUMvQixLQUFLLENBQUMsZ0VBQWdFLGFBQWEsaUNBQWlDLENBQUMsQ0FBQztRQUN0SCxJQUFJLENBQUMsSUFBSSxHQUFHLDRCQUE0QixDQUFDO1FBQ3pDLElBQUksQ0FBQyxhQUFhLEdBQUcsYUFBYSxDQUFDO1FBQ25DLE1BQU0sQ0FBQyxjQUFjLENBQUMsSUFBSSxFQUFFLGdDQUFnQyxFQUFFLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDakYsQ0FBQztDQUNGO0FBRUQsaUdBQWlHO0FBQ2pHLE1BQU0sVUFBVSw0QkFBNEIsQ0FBQyxLQUFjO0lBQ3pELE9BQU8sS0FBSyxZQUFZLEtBQUs7V0FDdkIsS0FBeUMsQ0FBQyxnQ0FBZ0MsQ0FBQyxLQUFLLElBQUksQ0FBQztBQUM3RixDQUFDO0FBUUQsTUFBTSxhQUFhLEdBQUcsSUFBSSxPQUFPLEVBQThCLENBQUM7QUFFaEUsTUFBTSxFQUFFLEdBQUcsSUFBSSxDQUFDO0FBQ2hCLE1BQU0sRUFBRSxHQUFHLElBQUksQ0FBQztBQUNoQixNQUFNLEtBQUssR0FBRyxJQUFJLENBQUM7QUFDbkIsTUFBTSxRQUFRLEdBQUcsQ0FBQyxJQUFJLEVBQUUsSUFBSSxFQUFFLElBQUksQ0FBVSxDQUFDO0FBRTdDLE1BQU0sV0FBVyxHQUFHLENBQUMsS0FBbUIsRUFBRSxNQUFjLEVBQWMsRUFBRTtJQUN0RSxJQUFJLEtBQUssQ0FBQyxNQUFNLEtBQUssQ0FBQztRQUFFLE9BQU8sS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQ3hDLE1BQU0sS0FBSyxHQUFHLElBQUksVUFBVSxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQ3JDLElBQUksTUFBTSxHQUFHLENBQUMsQ0FBQztJQUNmLEtBQUssTUFBTSxJQUFJLElBQUksS0FBSyxFQUFFLENBQUM7UUFDekIsS0FBSyxDQUFDLEdBQUcsQ0FBQyxJQUFJLEVBQUUsTUFBTSxDQUFDLENBQUM7UUFDeEIsTUFBTSxJQUFJLElBQUksQ0FBQyxNQUFNLENBQUM7SUFDeEIsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQyxDQUFDO0FBRUYsb0dBQW9HO0FBQ3BHLE1BQU0sVUFBVSxHQUFHLENBQUMsSUFBZ0IsRUFBRSxLQUFhLEVBQVcsRUFBRSxDQUM5RCxJQUFJLENBQUMsTUFBTSxHQUFHLEtBQUssSUFBSSxDQUFDO09BQ3JCLElBQUksQ0FBQyxLQUFLLENBQUMsS0FBSyxJQUFJLElBQUksSUFBSSxDQUFDLEtBQUssR0FBRyxDQUFDLENBQUMsS0FBSyxJQUFJLElBQUksSUFBSSxDQUFDLEtBQUssR0FBRyxDQUFDLENBQUMsS0FBSyxJQUFJLElBQUksSUFBSSxDQUFDLEtBQUssR0FBRyxDQUFDLENBQUMsS0FBSyxJQUFJO09BQ3hHLENBQUMsSUFBSSxDQUFDLE1BQU0sR0FBRyxLQUFLLEtBQUssQ0FBQyxJQUFJLElBQUksQ0FBQyxLQUFLLEdBQUcsQ0FBQyxDQUFDLEtBQUssS0FBSyxDQUFDLENBQUM7QUFFOUQ7Ozs7Ozs7Ozs7R0FVRztBQUNILFNBQVMsNkJBQTZCLENBQ3BDLE1BQWtDLEVBQ2xDLFFBQThCLEVBQzlCLE1BQWlDO0lBRWpDLE1BQU0sTUFBTSxHQUFHLE1BQU0sQ0FBQyxTQUFTLEVBQUUsQ0FBQztJQUNsQyxNQUFNLE9BQU8sR0FBRyxJQUFJLFdBQVcsRUFBRSxDQUFDO0lBQ2xDLDhFQUE4RTtJQUM5RSxJQUFJLElBQUksR0FBaUIsRUFBRSxDQUFDO0lBQzVCLElBQUksVUFBVSxHQUFHLENBQUMsQ0FBQztJQUNuQiwwR0FBMEc7SUFDMUcsSUFBSSxJQUF3QixDQUFDO0lBQzdCLDJGQUEyRjtJQUMzRixJQUFJLE9BQU8sR0FBRyxLQUFLLENBQUM7SUFDcEIsSUFBSSxTQUFTLEdBQUcsSUFBSSxDQUFDO0lBRXJCLDRGQUE0RjtJQUM1RixNQUFNLE9BQU8sR0FBRyxHQUFZLEVBQUU7UUFDNUIsTUFBTSxLQUFLLEdBQUcsV0FBVyxDQUFDLElBQUksRUFBRSxVQUFVLENBQUMsQ0FBQztRQUM1QyxJQUFJLEdBQUcsRUFBRSxDQUFDO1FBQ1YsVUFBVSxHQUFHLENBQUMsQ0FBQztRQUNmLE1BQU0sS0FBSyxHQUFHLFNBQVMsSUFBSSxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUMsSUFBSSxFQUFFLEtBQUssRUFBRSxFQUFFLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxLQUFLLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDeEcsU0FBUyxHQUFHLEtBQUssQ0FBQztRQUNsQixJQUFJLEtBQUssQ0FBQyxNQUFNLEtBQUssS0FBSyxFQUFFLENBQUM7WUFDM0IsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDO1lBQ25CLElBQUksR0FBRyxTQUFTLENBQUM7WUFDakIsT0FBTyxLQUFLLEtBQUssU0FBUyxJQUFJLFFBQVEsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDckQsQ0FBQztRQUNELElBQUksQ0FBQyxVQUFVLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQztZQUFFLE9BQU8sS0FBSyxDQUFDO1FBQzVDLDBGQUEwRjtRQUMxRixNQUFNLFVBQVUsR0FBRyxLQUFLLENBQUMsTUFBTSxHQUFHLEtBQUssS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEtBQUssR0FBRyxDQUFDLEtBQUssQ0FBQyxLQUFLLEdBQUcsQ0FBQyxDQUFDLEtBQUssSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQzNHLE1BQU0sS0FBSyxHQUFHLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDO1FBQ3pELElBQUksR0FBRyxJQUFJLEtBQUssU0FBUyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsSUFBSSxLQUFLLEtBQUssRUFBRSxDQUFDO1FBQ3hELE9BQU8sS0FBSyxDQUFDO0lBQ2YsQ0FBQyxDQUFDO0lBRUYsa0ZBQWtGO0lBQ2xGLE1BQU0sSUFBSSxHQUFHLENBQUMsS0FBaUIsRUFBK0MsRUFBRTtRQUM5RSxJQUFJLEtBQUssR0FBRyxDQUFDLENBQUM7UUFDZCxLQUFLLElBQUksS0FBSyxHQUFHLENBQUMsRUFBRSxLQUFLLEdBQUcsS0FBSyxDQUFDLE1BQU0sRUFBRSxLQUFLLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDckQsTUFBTSxJQUFJLEdBQUcsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFDO1lBQzFCLElBQUksT0FBTyxFQUFFLENBQUM7Z0JBQ1osT0FBTyxHQUFHLEtBQUssQ0FBQztnQkFDaEIsSUFBSSxJQUFJLEtBQUssRUFBRSxFQUFFLENBQUM7b0JBQ2hCLEtBQUssR0FBRyxLQUFLLEdBQUcsQ0FBQyxDQUFDO29CQUNsQixTQUFTO2dCQUNYLENBQUM7WUFDSCxDQUFDO1lBQ0QsSUFBSSxJQUFJLEtBQUssRUFBRSxJQUFJLElBQUksS0FBSyxFQUFFO2dCQUFFLFNBQVM7WUFDekMsSUFBSSxLQUFLLEdBQUcsS0FBSyxFQUFFLENBQUM7Z0JBQ2xCLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxLQUFLLEVBQUUsS0FBSyxDQUFDLENBQUMsQ0FBQztnQkFDeEMsVUFBVSxJQUFJLEtBQUssR0FBRyxLQUFLLENBQUM7WUFDOUIsQ0FBQztZQUNELE9BQU8sR0FBRyxJQUFJLEtBQUssRUFBRSxDQUFDO1lBQ3RCLEtBQUssR0FBRyxLQUFLLEdBQUcsQ0FBQyxDQUFDO1lBQ2xCLElBQUksT0FBTyxFQUFFLEVBQUUsQ0FBQztnQkFDZCxJQUFJLElBQUksS0FBSyxFQUFFO29CQUFFLE9BQU8sRUFBRSxHQUFHLEVBQUUsS0FBSyxHQUFHLENBQUMsRUFBRSxLQUFLLEVBQUUsS0FBSyxFQUFFLENBQUM7Z0JBQ3pELE9BQU8sS0FBSyxDQUFDLEtBQUssR0FBRyxDQUFDLENBQUMsS0FBSyxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsR0FBRyxFQUFFLEtBQUssR0FBRyxDQUFDLEVBQUUsS0FBSyxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEdBQUcsRUFBRSxLQUFLLEdBQUcsQ0FBQyxFQUFFLEtBQUssRUFBRSxJQUFJLEVBQUUsQ0FBQztZQUN0RyxDQUFDO1FBQ0gsQ0FBQztRQUNELElBQUksS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUUsQ0FBQztZQUN6QixJQUFJLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQztZQUM5QixVQUFVLElBQUksS0FBSyxDQUFDLE1BQU0sR0FBRyxLQUFLLENBQUM7UUFDckMsQ0FBQztRQUNELE9BQU8sU0FBUyxDQUFDO0lBQ25CLENBQUMsQ0FBQztJQUVGLE9BQU8sSUFBSSxjQUFjLENBQWE7UUFDcEMsS0FBSyxDQUFDLElBQUksQ0FBQyxVQUFVO1lBQ25CLFNBQVMsQ0FBQztnQkFDUiw4RkFBOEY7Z0JBQzlGLE1BQU0sSUFBSSxHQUFHLE1BQU0sTUFBTSxDQUFDLElBQUksRUFBRSxDQUFDO2dCQUNqQyxJQUFJLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztvQkFDZCxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7d0JBQ1osVUFBVSxDQUFDLEtBQUssQ0FBQyxJQUFJLDBCQUEwQixDQUFDLFFBQVEsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDO3dCQUNqRSxPQUFPO29CQUNULENBQUM7b0JBQ0QscUZBQXFGO29CQUNyRixxRkFBcUY7b0JBQ3JGLE1BQU0sQ0FBQyxVQUFVLEdBQUcsSUFBSSxDQUFDO29CQUN6QixVQUFVLENBQUMsS0FBSyxFQUFFLENBQUM7b0JBQ25CLE9BQU87Z0JBQ1QsQ0FBQztnQkFDRCxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDO2dCQUN6QixNQUFNLFdBQVcsR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUM7Z0JBQ2hDLElBQUksQ0FBQyxXQUFXLEVBQUUsQ0FBQztvQkFDakIsSUFBSSxLQUFLLENBQUMsTUFBTSxLQUFLLENBQUM7d0JBQUUsU0FBUztvQkFDakMsVUFBVSxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQztvQkFDMUIsT0FBTztnQkFDVCxDQUFDO2dCQUNELFVBQVUsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxDQUFDLEVBQUUsV0FBVyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7Z0JBQ3ZELElBQUksV0FBVyxDQUFDLEtBQUs7b0JBQUUsVUFBVSxDQUFDLE9BQU8sQ0FBQyxVQUFVLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUM7Z0JBQzdELFVBQVUsQ0FBQyxLQUFLLEVBQUUsQ0FBQztnQkFDbkIsa0ZBQWtGO2dCQUNsRixLQUFLLE1BQU0sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxLQUFLLENBQUMsR0FBRyxFQUFFLENBQUMsU0FBUyxDQUFDLENBQUM7Z0JBQzVDLE9BQU87WUFDVCxDQUFDO1FBQ0gsQ0FBQztRQUNELE1BQU0sQ0FBQyxNQUFNO1lBQ1gsT0FBTyxNQUFNLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQy9CLENBQUM7S0FDRixDQUFDLENBQUM7QUFDTCxDQUFDO0FBRUQsbUVBQW1FO0FBQ25FLE1BQU0sYUFBYSxHQUFHLENBQUMsUUFBa0IsRUFBVyxFQUFFLENBQ3BELFFBQVEsQ0FBQyxFQUFFO09BQ1IsUUFBUSxDQUFDLElBQUksS0FBSyxJQUFJO09BQ3RCLENBQUMsUUFBUSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsY0FBYyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsV0FBVyxFQUFFLENBQUMsVUFBVSxDQUFDLG1CQUFtQixDQUFDLENBQUM7QUFxQnZHOzs7Ozs7O0dBT0c7QUFDSCxNQUFNLFVBQVUsOEJBQThCLENBQzVDLFFBQThCLEVBQzlCLFNBQWtDO0lBRWxDLE1BQU0sS0FBSyxHQUE0QixLQUFLLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO1FBQzNELE1BQU0sUUFBUSxHQUFHLE1BQU0sU0FBUyxDQUFDLEtBQUssRUFBRSxJQUFJLENBQUMsQ0FBQztRQUM5QyxJQUFJLENBQUMsYUFBYSxDQUFDLFFBQVEsQ0FBQztZQUFFLE9BQU8sUUFBUSxDQUFDO1FBQzlDLE1BQU0sTUFBTSxHQUFHLElBQUksRUFBRSxNQUFNLENBQUMsQ0FBQyxDQUFDLGFBQWEsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUM7UUFDekUsT0FBTyxJQUFJLFFBQVEsQ0FBQyw2QkFBNkIsQ0FBQyxRQUFRLENBQUMsSUFBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLENBQUMsRUFBRTtZQUNuRixNQUFNLEVBQUUsUUFBUSxDQUFDLE1BQU07WUFDdkIsVUFBVSxFQUFFLFFBQVEsQ0FBQyxVQUFVO1lBQy9CLE9BQU8sRUFBRSxRQUFRLENBQUMsT0FBTztTQUMxQixDQUFDLENBQUM7SUFDTCxDQUFDLENBQUM7SUFDRixNQUFNLFVBQVUsR0FBc0M7UUFDcEQsb0JBQW9CLEVBQUUsSUFBSTtRQUMxQixlQUFlLEVBQUUsS0FBSyxFQUFFLEVBQUUsTUFBTSxFQUFFLElBQUksRUFBRSxFQUFFLEVBQUU7WUFDMUMsSUFBSSxJQUFJLEtBQUssUUFBUTtnQkFBRSxPQUFPLE1BQU0sQ0FBQztZQUNyQyxNQUFNLFdBQVcsR0FBRyxNQUFNLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxXQUFXLENBQUMsR0FBRyxDQUFDLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksZUFBZSxFQUFFLENBQUMsTUFBTSxDQUFDO1lBQzlHLGFBQWEsQ0FBQyxHQUFHLENBQUMsV0FBVyxFQUFFLEVBQUUsVUFBVSxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7WUFDdEQsT0FBTyxFQUFFLEdBQUcsTUFBTSxFQUFFLFdBQVcsRUFBRSxDQUFDO1FBQ3BDLENBQUM7UUFDRCxVQUFVLEVBQUUsS0FBSyxFQUFFLEVBQUUsUUFBUSxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUU7WUFDekMsTUFBTSxNQUFNLEdBQUcsTUFBTSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztZQUN0RixNQUFNLE1BQU0sR0FBRyxNQUFNLFFBQVEsRUFBRSxDQUFDO1lBQ2hDLElBQUksQ0FBQyxNQUFNO2dCQUFFLE9BQU8sTUFBTSxDQUFDO1lBQzNCLHlGQUF5RjtZQUN6RixJQUFJLE1BQU0sR0FBRyxLQUFLLENBQUM7WUFDbkIsTUFBTSxJQUFJLEdBQUcsQ0FBQyxVQUErRSxFQUFRLEVBQUU7Z0JBQ3JHLE1BQU0sR0FBRyxJQUFJLENBQUM7Z0JBQ2QsVUFBVSxDQUFDLE9BQU8sQ0FBQyxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsS0FBSyxFQUFFLElBQUksMEJBQTBCLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUMsQ0FBQztZQUMvRixDQUFDLENBQUM7WUFDRixPQUFPO2dCQUNMLEdBQUcsTUFBTTtnQkFDVCxNQUFNLEVBQUUsTUFBTSxDQUFDLE1BQU0sQ0FBQyxXQUFXLENBQUMsSUFBSSxlQUFlLENBQXVFO29CQUMxSCxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsVUFBVSxFQUFFLEVBQUU7d0JBQzlCLElBQUksSUFBSSxDQUFDLElBQUksS0FBSyxPQUFPOzRCQUFFLE1BQU0sR0FBRyxJQUFJLENBQUM7d0JBQ3pDLDBGQUEwRjt3QkFDMUYsSUFBSSxJQUFJLENBQUMsSUFBSSxLQUFLLFFBQVEsSUFBSSxNQUFNLENBQUMsVUFBVSxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7NEJBQzNELElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQzs0QkFDakIsT0FBTzt3QkFDVCxDQUFDO3dCQUNELFVBQVUsQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7b0JBQzNCLENBQUM7b0JBQ0QsS0FBSyxFQUFFLENBQUMsVUFBVSxFQUFFLEVBQUU7d0JBQ3BCLElBQUksTUFBTSxDQUFDLFVBQVUsSUFBSSxDQUFDLE1BQU07NEJBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO29CQUNyRCxDQUFDO2lCQUNGLENBQUMsQ0FBQzthQUNKLENBQUM7UUFDSixDQUFDO0tBQ0YsQ0FBQztJQUNGLE9BQU8sRUFBRSxLQUFLLEVBQUUsVUFBVSxFQUFFLENBQUM7QUFDL0IsQ0FBQyJ9
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"author": "Task Venture Capital GmbH",
|
|
3
3
|
"license": "MIT",
|
|
4
4
|
"name": "@modelprofile.com/flexharness-models",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "18.0.0",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"description": "Provider-independent model contracts, prompt caching and explicit model registration.",
|
|
8
8
|
"main": "./dist_ts_models/index.js",
|
package/readme.md
CHANGED
|
@@ -32,7 +32,8 @@ failed and never holds provider response text, headers or credentials. Its `kind
|
|
|
32
32
|
`providerCode` the provider states, its error `code` or, for an error with none such as
|
|
33
33
|
Anthropic's, its error `type` (`overloaded_error`), and no `status`: the answer succeeded, and
|
|
34
34
|
the HTTP-equivalent status the AI SDK derives from the code, such as 500 for `server_error`, is
|
|
35
|
-
no HTTP answer), `response` (an answer that could not be processed, with its `status
|
|
35
|
+
no HTTP answer), `response` (an answer that could not be processed, with its `status`, or a
|
|
36
|
+
stream that ended before its terminal event, a `ModelStreamIncompleteError`),
|
|
36
37
|
`network` (no answer arrived), `aborted` (an abort or timeout signal) or `unknown`. A provider code
|
|
37
38
|
is kept only in a shape providers name errors in (snake_case words, optionally dotted, such as
|
|
38
39
|
`insufficient_quota`; upper-case status names such as `RESOURCE_EXHAUSTED`; numeric codes) of at
|
|
@@ -40,6 +41,24 @@ most 64 characters, and never when it looks like a credential: a key or token pr
|
|
|
40
41
|
`sk_`, `pk_`, `ghp_` or `eyJ`, or a long run of letters and digits. The description follows `cause`
|
|
41
42
|
and the last attempt of a retried call.
|
|
42
43
|
|
|
44
|
+
A streamed answer ends at its stream's terminal event, never at the end of the connection.
|
|
45
|
+
`endModelStreamsAtTerminalEvent(terminal, transport)` gives a provider's model what it needs for
|
|
46
|
+
that: a `fetch` to create the model with, which calls `transport`, and a `middleware` to wrap the
|
|
47
|
+
model in, innermost. `terminal` (`IModelStreamTerminal`) names the dialect's terminal event
|
|
48
|
+
(`event`, such as `message_stop`) and tells by an event's data whether it ends the stream (`ends`):
|
|
49
|
+
the event that ends a finished answer, or the failure frame a dialect ends a failed one with. A
|
|
50
|
+
successful event-stream answer is then passed on up to the blank line that ends its terminal
|
|
51
|
+
event and cancelled after it, so a connection that breaks or stays open afterwards never changes
|
|
52
|
+
or holds the answer. A stream that ends before its terminal event fails the call with a
|
|
53
|
+
`ModelStreamIncompleteError` (`terminalEvent`, `isModelStreamIncompleteError()`) as an error part
|
|
54
|
+
after what it streamed, in place of the finish part, unless the stream reported an error of its
|
|
55
|
+
own, such as a chunk the model could not read; a stream that breaks fails with the connection's
|
|
56
|
+
own error, as before. Other answers, an error status or a JSON body, pass
|
|
57
|
+
unchanged. The middleware gives each streamed call an abort signal of its own (which follows the
|
|
58
|
+
call's) and learns from the fetch how that call's stream ended. The provider modules of
|
|
59
|
+
`flexharness-providers` do this for Anthropic Messages, OpenAI Responses and Chat Completions
|
|
60
|
+
(AGL-1130).
|
|
61
|
+
|
|
43
62
|
`createTextFilePartMiddleware()` sends the text files of user messages as text parts, for
|
|
44
63
|
providers that read only images and PDFs as files. A file part with inline text, or whose media
|
|
45
64
|
type names text (`isTextMediaType`: `text/*`, JSON, XML, YAML, CSV, script types, `+json`, `+xml`
|
|
@@ -90,6 +109,8 @@ incomplete provider usage with `ModelUsageReadError`.
|
|
|
90
109
|
costs nothing, so the planner places breakpoints without counting tokens it cannot count before
|
|
91
110
|
the request is sent.
|
|
92
111
|
- **OpenAI** (Responses and the ChatGPT Codex backend): `store: false` and a `prompt_cache_key`.
|
|
112
|
+
Chat Completions sends them only to OpenAI's own API, or to a server the openai provider's
|
|
113
|
+
`openAiFields` says takes them.
|
|
93
114
|
`deriveSmartAiPromptCacheKey({ system, tools, scope })` derives the key from the instructions and
|
|
94
115
|
tool definitions every turn repeats, so sessions with one prefix reach the machine that caches it;
|
|
95
116
|
any change to the prefix changes the key. `keyScope` keeps tenants apart, and `key` replaces the
|
package/ts_models/index.ts
CHANGED
|
@@ -3,6 +3,7 @@ export * from './classes.modelregistry.js';
|
|
|
3
3
|
export * from './smartai.cache.js';
|
|
4
4
|
export * from './smartai.limits.js';
|
|
5
5
|
export * from './smartai.failures.js';
|
|
6
|
+
export * from './smartai.terminalevent.js';
|
|
6
7
|
export * from './smartai.textfiles.js';
|
|
7
8
|
export * from './smartai.pricing.js';
|
|
8
9
|
export { generateText, streamText, tool, jsonSchema } from './plugins.js';
|
package/ts_models/readme.md
CHANGED
|
@@ -32,7 +32,8 @@ failed and never holds provider response text, headers or credentials. Its `kind
|
|
|
32
32
|
`providerCode` the provider states, its error `code` or, for an error with none such as
|
|
33
33
|
Anthropic's, its error `type` (`overloaded_error`), and no `status`: the answer succeeded, and
|
|
34
34
|
the HTTP-equivalent status the AI SDK derives from the code, such as 500 for `server_error`, is
|
|
35
|
-
no HTTP answer), `response` (an answer that could not be processed, with its `status
|
|
35
|
+
no HTTP answer), `response` (an answer that could not be processed, with its `status`, or a
|
|
36
|
+
stream that ended before its terminal event, a `ModelStreamIncompleteError`),
|
|
36
37
|
`network` (no answer arrived), `aborted` (an abort or timeout signal) or `unknown`. A provider code
|
|
37
38
|
is kept only in a shape providers name errors in (snake_case words, optionally dotted, such as
|
|
38
39
|
`insufficient_quota`; upper-case status names such as `RESOURCE_EXHAUSTED`; numeric codes) of at
|
|
@@ -40,6 +41,24 @@ most 64 characters, and never when it looks like a credential: a key or token pr
|
|
|
40
41
|
`sk_`, `pk_`, `ghp_` or `eyJ`, or a long run of letters and digits. The description follows `cause`
|
|
41
42
|
and the last attempt of a retried call.
|
|
42
43
|
|
|
44
|
+
A streamed answer ends at its stream's terminal event, never at the end of the connection.
|
|
45
|
+
`endModelStreamsAtTerminalEvent(terminal, transport)` gives a provider's model what it needs for
|
|
46
|
+
that: a `fetch` to create the model with, which calls `transport`, and a `middleware` to wrap the
|
|
47
|
+
model in, innermost. `terminal` (`IModelStreamTerminal`) names the dialect's terminal event
|
|
48
|
+
(`event`, such as `message_stop`) and tells by an event's data whether it ends the stream (`ends`):
|
|
49
|
+
the event that ends a finished answer, or the failure frame a dialect ends a failed one with. A
|
|
50
|
+
successful event-stream answer is then passed on up to the blank line that ends its terminal
|
|
51
|
+
event and cancelled after it, so a connection that breaks or stays open afterwards never changes
|
|
52
|
+
or holds the answer. A stream that ends before its terminal event fails the call with a
|
|
53
|
+
`ModelStreamIncompleteError` (`terminalEvent`, `isModelStreamIncompleteError()`) as an error part
|
|
54
|
+
after what it streamed, in place of the finish part, unless the stream reported an error of its
|
|
55
|
+
own, such as a chunk the model could not read; a stream that breaks fails with the connection's
|
|
56
|
+
own error, as before. Other answers, an error status or a JSON body, pass
|
|
57
|
+
unchanged. The middleware gives each streamed call an abort signal of its own (which follows the
|
|
58
|
+
call's) and learns from the fetch how that call's stream ended. The provider modules of
|
|
59
|
+
`flexharness-providers` do this for Anthropic Messages, OpenAI Responses and Chat Completions
|
|
60
|
+
(AGL-1130).
|
|
61
|
+
|
|
43
62
|
`createTextFilePartMiddleware()` sends the text files of user messages as text parts, for
|
|
44
63
|
providers that read only images and PDFs as files. A file part with inline text, or whose media
|
|
45
64
|
type names text (`isTextMediaType`: `text/*`, JSON, XML, YAML, CSV, script types, `+json`, `+xml`
|
|
@@ -90,6 +109,8 @@ incomplete provider usage with `ModelUsageReadError`.
|
|
|
90
109
|
costs nothing, so the planner places breakpoints without counting tokens it cannot count before
|
|
91
110
|
the request is sent.
|
|
92
111
|
- **OpenAI** (Responses and the ChatGPT Codex backend): `store: false` and a `prompt_cache_key`.
|
|
112
|
+
Chat Completions sends them only to OpenAI's own API, or to a server the openai provider's
|
|
113
|
+
`openAiFields` says takes them.
|
|
93
114
|
`deriveSmartAiPromptCacheKey({ system, tools, scope })` derives the key from the instructions and
|
|
94
115
|
tool definitions every turn repeats, so sessions with one prefix reach the machine that caches it;
|
|
95
116
|
any change to the prefix changes the key. `keyScope` keeps tenants apart, and `key` replaces the
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import { isModelLimitError, readJsonRecord, type IModelLimitInfo } from './smartai.limits.js';
|
|
3
|
+
import { isModelStreamIncompleteError } from './smartai.terminalevent.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* `limit`: a usage or rate limit refused the call; `limit` describes it.
|
|
6
7
|
* `http`: the provider answered with an HTTP error status.
|
|
7
8
|
* `stream`: the provider answered with a response stream and reported an error inside it, before or
|
|
8
9
|
* after output began.
|
|
9
|
-
* `response`: the provider answered, but its response could not be read or processed
|
|
10
|
+
* `response`: the provider answered, but its response could not be read or processed, or its
|
|
11
|
+
* stream ended before its terminal event (`ModelStreamIncompleteError`).
|
|
10
12
|
* `network`: no response arrived because the connection failed.
|
|
11
13
|
* `aborted`: an abort signal ended the call, a timeout signal included.
|
|
12
14
|
* `unknown`: the failure is none of these.
|
|
@@ -125,6 +127,7 @@ function describeStreamFailure(code: unknown): IModelFailureDescription {
|
|
|
125
127
|
function describeKnownFailure(error: unknown): IModelFailureDescription | undefined {
|
|
126
128
|
if (isModelLimitError(error)) return { kind: 'limit', limit: error.limit };
|
|
127
129
|
if (plugins.isAbortError(error)) return { kind: 'aborted' };
|
|
130
|
+
if (isModelStreamIncompleteError(error)) return { kind: 'response' };
|
|
128
131
|
if (plugins.StreamProviderError.isInstance(error) || plugins.isProviderStreamError(error)) {
|
|
129
132
|
return describeStreamFailure(error.code ?? readFrameCode(error.data));
|
|
130
133
|
}
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a model's streamed answer ends: at its stream's terminal event, never at the end of the
|
|
3
|
+
* connection (ruling of 2026-10-27, AGL-1130). A stream that ends without its terminal event is an
|
|
4
|
+
* incomplete answer and fails; a stream that has it is the answer, and nothing after it is read, so
|
|
5
|
+
* neither a connection that breaks afterwards nor one a proxy keeps open changes or holds it.
|
|
6
|
+
*/
|
|
7
|
+
import type * as plugins from './plugins.js';
|
|
8
|
+
|
|
9
|
+
/** How a dialect's response stream ends. */
|
|
10
|
+
export interface IModelStreamTerminal {
|
|
11
|
+
/** The event that ends a finished answer, as `ModelStreamIncompleteError` names it, such as `message_stop`. */
|
|
12
|
+
readonly event: string;
|
|
13
|
+
/**
|
|
14
|
+
* Whether a server-sent event ends the stream, by its data (its `data` lines joined with a line
|
|
15
|
+
* feed): the event that ends a finished answer, or a failure the dialect ends its stream with.
|
|
16
|
+
*/
|
|
17
|
+
ends(data: string): boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const modelStreamIncompleteErrorMarker = Symbol.for('@modelprofile.com/flexharness-models/ModelStreamIncompleteError');
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* A model's response stream ended before its terminal event: the server closed it early, so the
|
|
24
|
+
* answer is incomplete. The call fails like one whose connection broke mid-stream, which the
|
|
25
|
+
* engine does not retry.
|
|
26
|
+
*/
|
|
27
|
+
export class ModelStreamIncompleteError extends Error {
|
|
28
|
+
/** The terminal event the stream ended without. */
|
|
29
|
+
public readonly terminalEvent: string;
|
|
30
|
+
|
|
31
|
+
constructor(terminalEvent: string) {
|
|
32
|
+
super(`The model's response stream ended before its terminal event (${terminalEvent}), so its answer is incomplete.`);
|
|
33
|
+
this.name = 'ModelStreamIncompleteError';
|
|
34
|
+
this.terminalEvent = terminalEvent;
|
|
35
|
+
Object.defineProperty(this, modelStreamIncompleteErrorMarker, { value: true });
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Recognizes a ModelStreamIncompleteError, also one created by another copy of this package. */
|
|
40
|
+
export function isModelStreamIncompleteError(error: unknown): error is ModelStreamIncompleteError {
|
|
41
|
+
return error instanceof Error
|
|
42
|
+
&& (error as Error & Record<symbol, unknown>)[modelStreamIncompleteErrorMarker] === true;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** How a streamed model call's response stream ended, by the abort signal its request carries. */
|
|
46
|
+
interface IStreamEnding {
|
|
47
|
+
/** The stream ended before its terminal event. */
|
|
48
|
+
incomplete: boolean;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const streamEndings = new WeakMap<AbortSignal, IStreamEnding>();
|
|
52
|
+
|
|
53
|
+
const LF = 0x0a;
|
|
54
|
+
const CR = 0x0d;
|
|
55
|
+
const COLON = 0x3a;
|
|
56
|
+
const UTF8_BOM = [0xef, 0xbb, 0xbf] as const;
|
|
57
|
+
|
|
58
|
+
const concatBytes = (parts: Uint8Array[], length: number): Uint8Array => {
|
|
59
|
+
if (parts.length === 1) return parts[0];
|
|
60
|
+
const bytes = new Uint8Array(length);
|
|
61
|
+
let offset = 0;
|
|
62
|
+
for (const part of parts) {
|
|
63
|
+
bytes.set(part, offset);
|
|
64
|
+
offset += part.length;
|
|
65
|
+
}
|
|
66
|
+
return bytes;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
/** Whether a line, from `start`, is a `data` field: `data`, then a colon or the end of the line. */
|
|
70
|
+
const isDataLine = (line: Uint8Array, start: number): boolean =>
|
|
71
|
+
line.length - start >= 4
|
|
72
|
+
&& line[start] === 0x64 && line[start + 1] === 0x61 && line[start + 2] === 0x74 && line[start + 3] === 0x61
|
|
73
|
+
&& (line.length - start === 4 || line[start + 4] === COLON);
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A server-sent event stream that ends right after its terminal event: the bytes up to the blank
|
|
77
|
+
* line that ends that event, then the end of the stream, and the source is cancelled, so nothing
|
|
78
|
+
* after the event is read. A source that fails passes its error on as it is. A source that ends
|
|
79
|
+
* before the terminal event ends the stream too, with all it carried, and marks `ending`
|
|
80
|
+
* incomplete; without an `ending` it fails the stream with a `ModelStreamIncompleteError`.
|
|
81
|
+
*
|
|
82
|
+
* Lines end with LF, CR or CR LF, as in the SSE format. The bytes the stream passes on are the
|
|
83
|
+
* source's, with one exception: a terminal event whose blank line ends with a CR the source had not
|
|
84
|
+
* followed with its LF yet gets that LF, since a parser waits for it before it ends the line.
|
|
85
|
+
*/
|
|
86
|
+
function endEventStreamAtTerminalEvent(
|
|
87
|
+
source: ReadableStream<Uint8Array>,
|
|
88
|
+
terminal: IModelStreamTerminal,
|
|
89
|
+
ending: IStreamEnding | undefined,
|
|
90
|
+
): ReadableStream<Uint8Array> {
|
|
91
|
+
const reader = source.getReader();
|
|
92
|
+
const decoder = new TextDecoder();
|
|
93
|
+
/** The bytes of the line being read, in the pieces of the chunks it spans. */
|
|
94
|
+
let line: Uint8Array[] = [];
|
|
95
|
+
let lineLength = 0;
|
|
96
|
+
/** The data of the event being read: its `data` lines joined with LF, or `undefined` before the first. */
|
|
97
|
+
let data: string | undefined;
|
|
98
|
+
/** The last byte ended a line with a CR: an LF right after it belongs to that line end. */
|
|
99
|
+
let afterCr = false;
|
|
100
|
+
let firstLine = true;
|
|
101
|
+
|
|
102
|
+
/** Ends the line being read; whether it was the blank line that ends the terminal event. */
|
|
103
|
+
const endLine = (): boolean => {
|
|
104
|
+
const bytes = concatBytes(line, lineLength);
|
|
105
|
+
line = [];
|
|
106
|
+
lineLength = 0;
|
|
107
|
+
const start = firstLine && UTF8_BOM.every((byte, index) => bytes[index] === byte) ? UTF8_BOM.length : 0;
|
|
108
|
+
firstLine = false;
|
|
109
|
+
if (bytes.length === start) {
|
|
110
|
+
const event = data;
|
|
111
|
+
data = undefined;
|
|
112
|
+
return event !== undefined && terminal.ends(event);
|
|
113
|
+
}
|
|
114
|
+
if (!isDataLine(bytes, start)) return false;
|
|
115
|
+
// `data:` takes one space after the colon off its value; `data` alone has an empty value.
|
|
116
|
+
const valueStart = bytes.length - start === 4 ? bytes.length : start + (bytes[start + 5] === 0x20 ? 6 : 5);
|
|
117
|
+
const value = decoder.decode(bytes.subarray(valueStart));
|
|
118
|
+
data = data === undefined ? value : `${data}\n${value}`;
|
|
119
|
+
return false;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/** Reads a chunk's lines: where the terminal event ends in it, or `undefined`. */
|
|
123
|
+
const scan = (chunk: Uint8Array): { end: number; addLf: boolean } | undefined => {
|
|
124
|
+
let start = 0;
|
|
125
|
+
for (let index = 0; index < chunk.length; index += 1) {
|
|
126
|
+
const byte = chunk[index];
|
|
127
|
+
if (afterCr) {
|
|
128
|
+
afterCr = false;
|
|
129
|
+
if (byte === LF) {
|
|
130
|
+
start = index + 1;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (byte !== LF && byte !== CR) continue;
|
|
135
|
+
if (index > start) {
|
|
136
|
+
line.push(chunk.subarray(start, index));
|
|
137
|
+
lineLength += index - start;
|
|
138
|
+
}
|
|
139
|
+
afterCr = byte === CR;
|
|
140
|
+
start = index + 1;
|
|
141
|
+
if (endLine()) {
|
|
142
|
+
if (byte === LF) return { end: index + 1, addLf: false };
|
|
143
|
+
return chunk[index + 1] === LF ? { end: index + 2, addLf: false } : { end: index + 1, addLf: true };
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (start < chunk.length) {
|
|
147
|
+
line.push(chunk.slice(start));
|
|
148
|
+
lineLength += chunk.length - start;
|
|
149
|
+
}
|
|
150
|
+
return undefined;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
return new ReadableStream<Uint8Array>({
|
|
154
|
+
async pull(controller) {
|
|
155
|
+
for (;;) {
|
|
156
|
+
// A source that fails, such as a connection that breaks, fails the stream with its own error.
|
|
157
|
+
const next = await reader.read();
|
|
158
|
+
if (next.done) {
|
|
159
|
+
if (!ending) {
|
|
160
|
+
controller.error(new ModelStreamIncompleteError(terminal.event));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
// The stream ends, so the provider's model reads all it carried; the middleware then
|
|
164
|
+
// fails the call. A failed byte stream would drop what its readers had not read yet.
|
|
165
|
+
ending.incomplete = true;
|
|
166
|
+
controller.close();
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
const chunk = next.value;
|
|
170
|
+
const terminalEnd = scan(chunk);
|
|
171
|
+
if (!terminalEnd) {
|
|
172
|
+
if (chunk.length === 0) continue;
|
|
173
|
+
controller.enqueue(chunk);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
controller.enqueue(chunk.subarray(0, terminalEnd.end));
|
|
177
|
+
if (terminalEnd.addLf) controller.enqueue(Uint8Array.of(LF));
|
|
178
|
+
controller.close();
|
|
179
|
+
// Nothing after the terminal event is read: a connection kept open is let go now.
|
|
180
|
+
void reader.cancel().catch(() => undefined);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
cancel(reason) {
|
|
185
|
+
return reader.cancel(reason);
|
|
186
|
+
},
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** Whether a response is a successful server-sent event stream. */
|
|
191
|
+
const isEventStream = (response: Response): boolean =>
|
|
192
|
+
response.ok
|
|
193
|
+
&& response.body !== null
|
|
194
|
+
&& (response.headers.get('content-type') ?? '').trim().toLowerCase().startsWith('text/event-stream');
|
|
195
|
+
|
|
196
|
+
/** The two halves of `endModelStreamsAtTerminalEvent`: the model's fetch and its innermost middleware. */
|
|
197
|
+
export interface IModelStreamTerminalEnding {
|
|
198
|
+
/**
|
|
199
|
+
* The fetch to give the provider's model: a successful event-stream answer ends right after its
|
|
200
|
+
* terminal event, which ends the model's answer there. Every other answer, an error status or a
|
|
201
|
+
* JSON body among them, passes unchanged.
|
|
202
|
+
*/
|
|
203
|
+
fetch: typeof globalThis.fetch;
|
|
204
|
+
/**
|
|
205
|
+
* The middleware that fails a streamed call whose stream ended before its terminal event: an
|
|
206
|
+
* error part with a `ModelStreamIncompleteError` follows what the stream carried, in place of the
|
|
207
|
+
* finish part the model may still give it, unless the stream reported an error of its own, with
|
|
208
|
+
* which the call fails then. It must be the innermost middleware around the model,
|
|
209
|
+
* since it gives each streamed call an abort signal of its own (following the call's), by which
|
|
210
|
+
* the fetch reports how the call's stream ended.
|
|
211
|
+
*/
|
|
212
|
+
middleware: plugins.LanguageModelV4Middleware;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Ends a provider model's answers at its dialect's terminal event (see `IModelStreamTerminal`).
|
|
217
|
+
* Give the model `fetch` (which calls `transport` for every request) and wrap it in `middleware`,
|
|
218
|
+
* innermost: an answer is then what its stream carried up to its terminal event, whatever the
|
|
219
|
+
* connection does afterwards, and an answer whose stream ends without it fails with a
|
|
220
|
+
* `ModelStreamIncompleteError` after the parts it streamed. The provider modules do this for
|
|
221
|
+
* their models.
|
|
222
|
+
*/
|
|
223
|
+
export function endModelStreamsAtTerminalEvent(
|
|
224
|
+
terminal: IModelStreamTerminal,
|
|
225
|
+
transport: typeof globalThis.fetch,
|
|
226
|
+
): IModelStreamTerminalEnding {
|
|
227
|
+
const fetch: typeof globalThis.fetch = async (input, init) => {
|
|
228
|
+
const response = await transport(input, init);
|
|
229
|
+
if (!isEventStream(response)) return response;
|
|
230
|
+
const ending = init?.signal ? streamEndings.get(init.signal) : undefined;
|
|
231
|
+
return new Response(endEventStreamAtTerminalEvent(response.body!, terminal, ending), {
|
|
232
|
+
status: response.status,
|
|
233
|
+
statusText: response.statusText,
|
|
234
|
+
headers: response.headers,
|
|
235
|
+
});
|
|
236
|
+
};
|
|
237
|
+
const middleware: plugins.LanguageModelV4Middleware = {
|
|
238
|
+
specificationVersion: 'v4',
|
|
239
|
+
transformParams: async ({ params, type }) => {
|
|
240
|
+
if (type !== 'stream') return params;
|
|
241
|
+
const abortSignal = params.abortSignal ? AbortSignal.any([params.abortSignal]) : new AbortController().signal;
|
|
242
|
+
streamEndings.set(abortSignal, { incomplete: false });
|
|
243
|
+
return { ...params, abortSignal };
|
|
244
|
+
},
|
|
245
|
+
wrapStream: async ({ doStream, params }) => {
|
|
246
|
+
const ending = params.abortSignal ? streamEndings.get(params.abortSignal) : undefined;
|
|
247
|
+
const result = await doStream();
|
|
248
|
+
if (!ending) return result;
|
|
249
|
+
// A stream that reported an error has failed with it; an early end changes nothing then.
|
|
250
|
+
let failed = false;
|
|
251
|
+
const fail = (controller: TransformStreamDefaultController<plugins.LanguageModelV4StreamPart>): void => {
|
|
252
|
+
failed = true;
|
|
253
|
+
controller.enqueue({ type: 'error', error: new ModelStreamIncompleteError(terminal.event) });
|
|
254
|
+
};
|
|
255
|
+
return {
|
|
256
|
+
...result,
|
|
257
|
+
stream: result.stream.pipeThrough(new TransformStream<plugins.LanguageModelV4StreamPart, plugins.LanguageModelV4StreamPart>({
|
|
258
|
+
transform: (part, controller) => {
|
|
259
|
+
if (part.type === 'error') failed = true;
|
|
260
|
+
// The model finishes an answer whose stream ended early as if complete; it fails instead.
|
|
261
|
+
if (part.type === 'finish' && ending.incomplete && !failed) {
|
|
262
|
+
fail(controller);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
controller.enqueue(part);
|
|
266
|
+
},
|
|
267
|
+
flush: (controller) => {
|
|
268
|
+
if (ending.incomplete && !failed) fail(controller);
|
|
269
|
+
},
|
|
270
|
+
})),
|
|
271
|
+
};
|
|
272
|
+
},
|
|
273
|
+
};
|
|
274
|
+
return { fetch, middleware };
|
|
275
|
+
}
|