@pipedog/openai 0.0.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/_constants.d.ts +4 -0
- package/dist/_constants.js +9 -0
- package/dist/base_llm.d.ts +272 -0
- package/dist/base_llm.js +619 -0
- package/dist/image.d.ts +116 -0
- package/dist/image.js +179 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +12 -0
- package/dist/live/events.d.ts +768 -0
- package/dist/live/events.js +1020 -0
- package/dist/live/index.d.ts +2 -0
- package/dist/live/index.js +8 -0
- package/dist/live/llm.d.ts +566 -0
- package/dist/live/llm.js +1652 -0
- package/dist/llm.d.ts +30 -0
- package/dist/llm.js +50 -0
- package/dist/open_ai_live_adapter.d.ts +94 -0
- package/dist/open_ai_live_adapter.js +192 -0
- package/dist/open_ai_realtime_adapter.d.ts +95 -0
- package/dist/open_ai_realtime_adapter.js +221 -0
- package/dist/open_ai_responses_adapter.d.ts +95 -0
- package/dist/open_ai_responses_adapter.js +358 -0
- package/dist/realtime/events.d.ts +1401 -0
- package/dist/realtime/events.js +1639 -0
- package/dist/realtime/index.d.ts +2 -0
- package/dist/realtime/index.js +8 -0
- package/dist/realtime/llm.d.ts +457 -0
- package/dist/realtime/llm.js +1550 -0
- package/dist/responses/index.d.ts +2 -0
- package/dist/responses/index.js +8 -0
- package/dist/responses/llm.d.ts +448 -0
- package/dist/responses/llm.js +1517 -0
- package/dist/stt.d.ts +409 -0
- package/dist/stt.js +741 -0
- package/dist/tts.d.ts +86 -0
- package/dist/tts.js +238 -0
- package/package.json +49 -3
- package/README.md +0 -5
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/adapters/services/open_ai_realtime_adapter.py
|
|
6
|
+
/** OpenAI Realtime LLM adapter for Pipecat. */
|
|
7
|
+
import { BaseLLMAdapter } from "@pipedog/core/adapters/base_llm_adapter";
|
|
8
|
+
import { AdapterType } from "@pipedog/core/adapters/schemas/tools_schema";
|
|
9
|
+
import { LLMSpecificMessage, } from "@pipedog/core/processors/aggregators/llm_context";
|
|
10
|
+
import { logger } from "@pipedog/core/utils/logger";
|
|
11
|
+
import * as events from "./realtime/events.js";
|
|
12
|
+
/** Container for OpenAI-formatted messages converted from universal context. */
|
|
13
|
+
export class ConvertedMessages {
|
|
14
|
+
messages;
|
|
15
|
+
systemInstruction;
|
|
16
|
+
constructor(options) {
|
|
17
|
+
this.messages = options.messages;
|
|
18
|
+
this.systemInstruction = options.systemInstruction ?? null;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* LLM adapter for OpenAI Realtime API function calling.
|
|
23
|
+
*
|
|
24
|
+
* Converts Pipecat's tool schemas into the specific format required by
|
|
25
|
+
* OpenAI's Realtime API for function calling capabilities.
|
|
26
|
+
*/
|
|
27
|
+
export class OpenAIRealtimeLLMAdapter extends BaseLLMAdapter {
|
|
28
|
+
/** Container for OpenAI-formatted messages converted from universal context. */
|
|
29
|
+
static ConvertedMessages = ConvertedMessages;
|
|
30
|
+
/** The identifier used in LLMSpecificMessage instances for OpenAI Realtime. */
|
|
31
|
+
get idForLlmSpecificMessages() {
|
|
32
|
+
return "openai-realtime";
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Get OpenAI Realtime-specific LLM invocation parameters from a universal
|
|
36
|
+
* LLM context.
|
|
37
|
+
*
|
|
38
|
+
* The `tools` returned are the set the session should carry: the context's
|
|
39
|
+
* own when it has any, else the init-provided ones. Built-in tools ride
|
|
40
|
+
* along with whichever set is in use, and are the set when there is
|
|
41
|
+
* neither.
|
|
42
|
+
*
|
|
43
|
+
* @param context The LLM context containing messages, tools, etc.
|
|
44
|
+
* @param options.systemInstruction Optional system instruction from service settings.
|
|
45
|
+
* @param options.serviceTools The service's init-provided tools, as
|
|
46
|
+
* `_serviceTools()` returns them: a `ToolsSchema` or a provider-native
|
|
47
|
+
* tool list, or `null`.
|
|
48
|
+
* @returns Dictionary of parameters for invoking OpenAI Realtime's API.
|
|
49
|
+
*/
|
|
50
|
+
async getLlmInvocationParams(context, options = {}) {
|
|
51
|
+
const messages = this._fromUniversalContextMessages(this.getMessages(context));
|
|
52
|
+
const effectiveSystem = this._resolveSystemInstruction(messages.systemInstruction, options.systemInstruction ?? null, { discardContextSystem: true });
|
|
53
|
+
return {
|
|
54
|
+
system_instruction: effectiveSystem,
|
|
55
|
+
messages: messages.messages,
|
|
56
|
+
tools: (this._realtimeSessionTools(context.tools, options.serviceTools ?? null) ??
|
|
57
|
+
[]),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Get messages from a universal LLM context in a format ready for logging
|
|
62
|
+
* about OpenAI Realtime.
|
|
63
|
+
*
|
|
64
|
+
* Binary data (images, audio) is replaced with short placeholders.
|
|
65
|
+
*
|
|
66
|
+
* This is a placeholder until support for universal LLMContext machinery is
|
|
67
|
+
* added for OpenAI Realtime.
|
|
68
|
+
*
|
|
69
|
+
* @param context The LLM context containing messages.
|
|
70
|
+
* @returns List of messages in a format ready for logging about OpenAI Realtime.
|
|
71
|
+
*/
|
|
72
|
+
getMessagesForLogging(context) {
|
|
73
|
+
return this.getMessages(context, { truncateLargeValues: true });
|
|
74
|
+
}
|
|
75
|
+
_fromUniversalContextMessages(universalContextMessages) {
|
|
76
|
+
// We can't load a long conversation history into the openai realtime api yet. (The API/model
|
|
77
|
+
// forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So
|
|
78
|
+
// our general strategy until this is fixed is just to put everything into a first "user"
|
|
79
|
+
// message as a single input.
|
|
80
|
+
if (universalContextMessages.length === 0) {
|
|
81
|
+
return new ConvertedMessages({ messages: [] });
|
|
82
|
+
}
|
|
83
|
+
// NOTE: This adapter does not yet handle `LLMSpecificMessage` — those
|
|
84
|
+
// are filtered out below. Other adapters (e.g. Anthropic) dispatch
|
|
85
|
+
// LLMSpecific items through a per-provider passthrough. For OpenAI
|
|
86
|
+
// Realtime, the strategy here packs a multi-message history into a
|
|
87
|
+
// single text message (see comment further down), which doesn't
|
|
88
|
+
// compose with opaque per-provider payloads. If/when this adapter
|
|
89
|
+
// adopts the per-message strategy, LLMSpecific items can flow
|
|
90
|
+
// through `_fromUniversalContextMessage` like in other adapters.
|
|
91
|
+
const messages = universalContextMessages
|
|
92
|
+
.filter((m) => !(m instanceof LLMSpecificMessage))
|
|
93
|
+
.map((m) => structuredClone(m));
|
|
94
|
+
let systemInstruction = null;
|
|
95
|
+
// If we have a "system" message as our first message,
|
|
96
|
+
// pull that out into session "instructions"
|
|
97
|
+
if (messages.length > 0 && messages[0].role === "system") {
|
|
98
|
+
this._warnContextSystemMessage();
|
|
99
|
+
const system = messages.shift();
|
|
100
|
+
const content = system.content;
|
|
101
|
+
if (typeof content === "string") {
|
|
102
|
+
systemInstruction = content;
|
|
103
|
+
}
|
|
104
|
+
else if (Array.isArray(content)) {
|
|
105
|
+
systemInstruction = content[0].text ?? null;
|
|
106
|
+
}
|
|
107
|
+
if (messages.length === 0) {
|
|
108
|
+
return new ConvertedMessages({ messages: [], systemInstruction });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
// Convert any remaining "system"/"developer" messages to "user"
|
|
112
|
+
for (const msg of messages) {
|
|
113
|
+
if (msg.role === "system" || msg.role === "developer") {
|
|
114
|
+
msg.role = "user";
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// If we have just a single "user" item, we can just send it normally
|
|
118
|
+
if (messages.length === 1 && messages[0].role === "user") {
|
|
119
|
+
return new ConvertedMessages({
|
|
120
|
+
messages: [this._fromUniversalContextMessage(messages[0])],
|
|
121
|
+
systemInstruction,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
// Otherwise, let's pack everything into a single "user" message with a bit of
|
|
125
|
+
// explanation for the LLM
|
|
126
|
+
const introText = `
|
|
127
|
+
This is a previously saved conversation. Please treat this conversation history as a
|
|
128
|
+
starting point for the current conversation.`;
|
|
129
|
+
const trailingText = `
|
|
130
|
+
This is the end of the previously saved conversation. Please continue the conversation
|
|
131
|
+
from here. If the last message is a user instruction or question, act on that instruction
|
|
132
|
+
or answer the question. If the last message is an assistant response, simple say that you
|
|
133
|
+
are ready to continue the conversation.`;
|
|
134
|
+
return new ConvertedMessages({
|
|
135
|
+
messages: [
|
|
136
|
+
new events.ConversationItem({
|
|
137
|
+
role: "user",
|
|
138
|
+
type: "message",
|
|
139
|
+
content: [
|
|
140
|
+
new events.ItemContent({
|
|
141
|
+
type: "input_text",
|
|
142
|
+
// `json.dumps(messages, indent=2, ensure_ascii=False)`.
|
|
143
|
+
// `pyJsonDumps` has no `indent`; `JSON.stringify` matches it for
|
|
144
|
+
// context messages except for integral floats (`1.0` → `1`).
|
|
145
|
+
text: [introText, JSON.stringify(messages, null, 2), trailingText].join("\n\n"),
|
|
146
|
+
}),
|
|
147
|
+
],
|
|
148
|
+
}),
|
|
149
|
+
],
|
|
150
|
+
systemInstruction,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
_fromUniversalContextMessage(message) {
|
|
154
|
+
// NOTE: `LLMSpecificMessage` is not yet handled here — see the
|
|
155
|
+
// corresponding note in `_fromUniversalContextMessages`.
|
|
156
|
+
const msg = message;
|
|
157
|
+
if (msg.role === "user") {
|
|
158
|
+
let content = msg.content;
|
|
159
|
+
if (Array.isArray(content)) {
|
|
160
|
+
content = "";
|
|
161
|
+
for (const c of (msg.content ?? [])) {
|
|
162
|
+
if (c.type === "text") {
|
|
163
|
+
content += ` ${c.text}`;
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
logger.error(`Unhandled content type in context message: ${c.type} - ${JSON.stringify(msg)}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return new events.ConversationItem({
|
|
171
|
+
role: "user",
|
|
172
|
+
type: "message",
|
|
173
|
+
content: [new events.ItemContent({ type: "input_text", text: content })],
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
if (msg.role === "assistant" && msg.tool_calls && msg.tool_calls.length > 0) {
|
|
177
|
+
const tc = msg.tool_calls[0];
|
|
178
|
+
return new events.ConversationItem({
|
|
179
|
+
type: "function_call",
|
|
180
|
+
callId: tc.id,
|
|
181
|
+
name: tc.function.name,
|
|
182
|
+
arguments: tc.function.arguments,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
throw new Error(`Unhandled message type in _fromUniversalContextMessage: ${JSON.stringify(msg)}`);
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Convert a function schema to OpenAI Realtime format.
|
|
189
|
+
*
|
|
190
|
+
* @param func The function schema to convert.
|
|
191
|
+
* @returns Dictionary in OpenAI Realtime function format.
|
|
192
|
+
*/
|
|
193
|
+
static _toOpenaiRealtimeFunctionFormat(func) {
|
|
194
|
+
return {
|
|
195
|
+
type: "function",
|
|
196
|
+
name: func.name,
|
|
197
|
+
description: func.description,
|
|
198
|
+
parameters: {
|
|
199
|
+
type: "object",
|
|
200
|
+
properties: func.properties,
|
|
201
|
+
required: func.required,
|
|
202
|
+
},
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Convert tool schemas to OpenAI Realtime function-calling format.
|
|
207
|
+
*
|
|
208
|
+
* @param toolsSchema The tools schema containing functions to convert.
|
|
209
|
+
* @returns List of function definitions in OpenAI Realtime format.
|
|
210
|
+
*/
|
|
211
|
+
toProviderToolsFormat(toolsSchema) {
|
|
212
|
+
const functionsSchema = toolsSchema.standardTools;
|
|
213
|
+
const formattedStandardTools = functionsSchema.map((func) => OpenAIRealtimeLLMAdapter._toOpenaiRealtimeFunctionFormat(func));
|
|
214
|
+
let customOpenaiTools = [];
|
|
215
|
+
if (toolsSchema.customTools) {
|
|
216
|
+
customOpenaiTools = toolsSchema.customTools[AdapterType.OPENAI] ?? [];
|
|
217
|
+
}
|
|
218
|
+
return [...formattedStandardTools, ...customOpenaiTools];
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=open_ai_realtime_adapter.js.map
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { BaseLLMAdapter } from "@pipedog/core/adapters/base_llm_adapter";
|
|
2
|
+
import { type ToolsSchema } from "@pipedog/core/adapters/schemas/tools_schema";
|
|
3
|
+
import { type OpenAINotGiven } from "@pipedog/core/adapters/services/open_ai_adapter";
|
|
4
|
+
import { type LLMContext, type LLMContextMessage, LLMSpecificMessage } from "@pipedog/core/processors/aggregators/llm_context";
|
|
5
|
+
type Dict = Record<string, any>;
|
|
6
|
+
/** A Responses API input item (`ResponseInputItemParam`). */
|
|
7
|
+
export type ResponseInputItemParam = Dict;
|
|
8
|
+
/** A Responses API tool (`ToolParam`). */
|
|
9
|
+
export type ToolParam = Dict;
|
|
10
|
+
/** A Responses API function tool (`FunctionToolParam`). */
|
|
11
|
+
export interface FunctionToolParam {
|
|
12
|
+
type: "function";
|
|
13
|
+
name: string;
|
|
14
|
+
parameters: Dict | null;
|
|
15
|
+
strict: boolean | null;
|
|
16
|
+
description?: string | null;
|
|
17
|
+
}
|
|
18
|
+
/** Context-based parameters for invoking OpenAI Responses API. */
|
|
19
|
+
export interface OpenAIResponsesLLMInvocationParams {
|
|
20
|
+
input: ResponseInputItemParam[];
|
|
21
|
+
tools: ToolParam[] | OpenAINotGiven;
|
|
22
|
+
instructions?: string;
|
|
23
|
+
}
|
|
24
|
+
/** Options for `OpenAIResponsesLLMAdapter.getLlmInvocationParams`. */
|
|
25
|
+
export interface OpenAIResponsesLLMInvocationParamsOptions {
|
|
26
|
+
/** Optional system instruction from service settings. */
|
|
27
|
+
systemInstruction?: string | null;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* OpenAI Responses API adapter for Pipecat.
|
|
31
|
+
*
|
|
32
|
+
* Handles:
|
|
33
|
+
*
|
|
34
|
+
* - Converting LLMContext messages to Responses API input items
|
|
35
|
+
* - Converting Pipecat's standardized tools schema to Responses API function tool format
|
|
36
|
+
* - Extracting and sanitizing messages from the LLM context for logging
|
|
37
|
+
*/
|
|
38
|
+
export declare class OpenAIResponsesLLMAdapter extends BaseLLMAdapter<OpenAIResponsesLLMInvocationParams> {
|
|
39
|
+
/** The identifier used in LLMSpecificMessage instances. */
|
|
40
|
+
get idForLlmSpecificMessages(): string;
|
|
41
|
+
/**
|
|
42
|
+
* Get Responses API invocation parameters from a universal LLM context.
|
|
43
|
+
*
|
|
44
|
+
* @param context The LLM context containing messages, tools, etc.
|
|
45
|
+
* @param options.systemInstruction Optional system instruction from service settings.
|
|
46
|
+
* @returns Dictionary of parameters for the Responses API.
|
|
47
|
+
*/
|
|
48
|
+
getLlmInvocationParams(context: LLMContext, options?: OpenAIResponsesLLMInvocationParamsOptions): Promise<OpenAIResponsesLLMInvocationParams>;
|
|
49
|
+
/**
|
|
50
|
+
* Convert function schemas to Responses API function tool format.
|
|
51
|
+
*
|
|
52
|
+
* @param toolsSchema The Pipecat tools schema to convert.
|
|
53
|
+
* @returns List of Responses API function tool definitions.
|
|
54
|
+
*/
|
|
55
|
+
toProviderToolsFormat(toolsSchema: ToolsSchema): ToolParam[];
|
|
56
|
+
/**
|
|
57
|
+
* Get messages from context in a format ready for logging.
|
|
58
|
+
*
|
|
59
|
+
* Binary data (images, audio) is replaced with short placeholders, and
|
|
60
|
+
* reasoning messages' encrypted payloads are elided.
|
|
61
|
+
*
|
|
62
|
+
* @param context The LLM context containing messages.
|
|
63
|
+
* @returns List of messages in a format ready for logging.
|
|
64
|
+
*/
|
|
65
|
+
getMessagesForLogging(context: LLMContext): Record<string, unknown>[];
|
|
66
|
+
/**
|
|
67
|
+
* Convert LLMContext messages to Responses API input items.
|
|
68
|
+
*
|
|
69
|
+
* @param messages Messages from the LLMContext.
|
|
70
|
+
* @returns List of Responses API input items.
|
|
71
|
+
*/
|
|
72
|
+
protected _convertMessagesToInput(messages: LLMContextMessage[]): ResponseInputItemParam[];
|
|
73
|
+
/**
|
|
74
|
+
* Convert an OpenAI-Responses-specific message to an input item.
|
|
75
|
+
*
|
|
76
|
+
* Reasoning messages — persisted so the model's prior reasoning round-trips
|
|
77
|
+
* on later turns — become Responses `reasoning` input items. Anything
|
|
78
|
+
* else is assumed to already be in Responses input shape.
|
|
79
|
+
*
|
|
80
|
+
* @param message The LLM-specific message from the context.
|
|
81
|
+
* @returns A Responses API input item.
|
|
82
|
+
*/
|
|
83
|
+
protected _fromSpecificMessage(message: LLMSpecificMessage): ResponseInputItemParam;
|
|
84
|
+
/** The Responses API fetches image and file URLs itself. */
|
|
85
|
+
supportsFileUrl(url: string, _mimeType: string): boolean;
|
|
86
|
+
/**
|
|
87
|
+
* Convert multimodal content parts to Responses API format.
|
|
88
|
+
*
|
|
89
|
+
* @param content List of content parts from the LLMContext message.
|
|
90
|
+
* @returns List of content parts in Responses API format.
|
|
91
|
+
*/
|
|
92
|
+
protected _convertMultimodalContent(content: Dict[]): Dict[];
|
|
93
|
+
}
|
|
94
|
+
export {};
|
|
95
|
+
//# sourceMappingURL=open_ai_responses_adapter.d.ts.map
|
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/adapters/services/open_ai_responses_adapter.py
|
|
6
|
+
/**
|
|
7
|
+
* OpenAI Responses API adapter for Pipecat.
|
|
8
|
+
*
|
|
9
|
+
* Porting note: like `open_ai_adapter.ts` in core, this module does not
|
|
10
|
+
* depend on the `openai` SDK. The Responses API types are declared
|
|
11
|
+
* structurally, and the SDK's `NOT_GIVEN` sentinel is `undefined`.
|
|
12
|
+
*/
|
|
13
|
+
import { rethrowIfCancelled } from "@pipedog/asyncio";
|
|
14
|
+
import { BaseLLMAdapter, LLMContextConversionError } from "@pipedog/core/adapters/base_llm_adapter";
|
|
15
|
+
import { AdapterType } from "@pipedog/core/adapters/schemas/tools_schema";
|
|
16
|
+
import { openaiFromLlmContextTools, } from "@pipedog/core/adapters/services/open_ai_adapter";
|
|
17
|
+
import { LLMSpecificMessage, } from "@pipedog/core/processors/aggregators/llm_context";
|
|
18
|
+
import { pyJsonDumps } from "@pipedog/core/utils/py_json";
|
|
19
|
+
/**
|
|
20
|
+
* OpenAI Responses API adapter for Pipecat.
|
|
21
|
+
*
|
|
22
|
+
* Handles:
|
|
23
|
+
*
|
|
24
|
+
* - Converting LLMContext messages to Responses API input items
|
|
25
|
+
* - Converting Pipecat's standardized tools schema to Responses API function tool format
|
|
26
|
+
* - Extracting and sanitizing messages from the LLM context for logging
|
|
27
|
+
*/
|
|
28
|
+
export class OpenAIResponsesLLMAdapter extends BaseLLMAdapter {
|
|
29
|
+
/** The identifier used in LLMSpecificMessage instances. */
|
|
30
|
+
get idForLlmSpecificMessages() {
|
|
31
|
+
return "openai_responses";
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Get Responses API invocation parameters from a universal LLM context.
|
|
35
|
+
*
|
|
36
|
+
* @param context The LLM context containing messages, tools, etc.
|
|
37
|
+
* @param options.systemInstruction Optional system instruction from service settings.
|
|
38
|
+
* @returns Dictionary of parameters for the Responses API.
|
|
39
|
+
*/
|
|
40
|
+
async getLlmInvocationParams(context, options = {}) {
|
|
41
|
+
const systemInstruction = options.systemInstruction ?? null;
|
|
42
|
+
await this.prepareFileContent(context);
|
|
43
|
+
const messages = this.getMessages(context);
|
|
44
|
+
if (messages.length > 0) {
|
|
45
|
+
const firstMsg = messages[0] instanceof LLMSpecificMessage ? null : messages[0];
|
|
46
|
+
if (firstMsg && firstMsg.role === "system") {
|
|
47
|
+
this._warnContextSystemMessage();
|
|
48
|
+
// Check for conflict: systemInstruction + initial system message.
|
|
49
|
+
// `content` is `string | Iterable<...>`; we only forward it for
|
|
50
|
+
// warning purposes. Coerce non-strings to null.
|
|
51
|
+
const firstContent = firstMsg.content ?? "";
|
|
52
|
+
if (systemInstruction) {
|
|
53
|
+
this._resolveSystemInstruction(typeof firstContent === "string" ? firstContent : null, systemInstruction, { discardContextSystem: false });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const inputItems = this._convertMessagesToInput(messages);
|
|
58
|
+
const params = {
|
|
59
|
+
input: inputItems,
|
|
60
|
+
// NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
|
|
61
|
+
tools: openaiFromLlmContextTools(this.fromStandardTools(context.tools)),
|
|
62
|
+
};
|
|
63
|
+
if (systemInstruction) {
|
|
64
|
+
// Compatibility: The Responses API requires at least one input
|
|
65
|
+
// message when instructions are provided. Contexts that worked with
|
|
66
|
+
// OpenAILLMService (systemInstruction + empty messages) need the
|
|
67
|
+
// instructions converted to an initial developer message.
|
|
68
|
+
//
|
|
69
|
+
// NOTE: The service layer (OpenAIResponsesLLMService) internally
|
|
70
|
+
// manages `previous_response_id` for incremental context delivery
|
|
71
|
+
// over WebSocket. This runs post-adapter — the adapter always
|
|
72
|
+
// produces the full input list and the service determines what
|
|
73
|
+
// subset to send. This empty-input fallback is therefore only
|
|
74
|
+
// relevant for one-shot or initial calls.
|
|
75
|
+
//
|
|
76
|
+
// If we added support for user-provided explicit
|
|
77
|
+
// `previous_response_id` and/or `conversation_id` (overriding
|
|
78
|
+
// internal management), we'd need to revisit this logic, as it'd
|
|
79
|
+
// be legit to provide instructions without input items. Note that
|
|
80
|
+
// over HTTP, `previous_response_id` requires `store=True` (30-day
|
|
81
|
+
// OpenAI-side storage), which is why the HTTP variant doesn't use
|
|
82
|
+
// it. The WebSocket variant avoids this via a connection-local
|
|
83
|
+
// in-memory cache — see the class docstrings in llm.ts.
|
|
84
|
+
if (inputItems.length === 0) {
|
|
85
|
+
params.input = [{ role: "developer", content: systemInstruction }];
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
params.instructions = systemInstruction;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return params;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Convert function schemas to Responses API function tool format.
|
|
95
|
+
*
|
|
96
|
+
* @param toolsSchema The Pipecat tools schema to convert.
|
|
97
|
+
* @returns List of Responses API function tool definitions.
|
|
98
|
+
*/
|
|
99
|
+
toProviderToolsFormat(toolsSchema) {
|
|
100
|
+
const functionsSchema = toolsSchema.standardTools;
|
|
101
|
+
const result = [];
|
|
102
|
+
for (const func of functionsSchema) {
|
|
103
|
+
const d = func.toDefaultDict();
|
|
104
|
+
const tool = {
|
|
105
|
+
type: "function",
|
|
106
|
+
name: d.name,
|
|
107
|
+
parameters: pyGet(d, "parameters", {}),
|
|
108
|
+
strict: d.strict ?? null,
|
|
109
|
+
};
|
|
110
|
+
if ("description" in d) {
|
|
111
|
+
tool.description = d.description;
|
|
112
|
+
}
|
|
113
|
+
result.push(tool);
|
|
114
|
+
}
|
|
115
|
+
let customOpenaiTools = [];
|
|
116
|
+
if (toolsSchema.customTools) {
|
|
117
|
+
customOpenaiTools = toolsSchema.customTools[AdapterType.OPENAI] ?? [];
|
|
118
|
+
}
|
|
119
|
+
return [...result, ...customOpenaiTools];
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Get messages from context in a format ready for logging.
|
|
123
|
+
*
|
|
124
|
+
* Binary data (images, audio) is replaced with short placeholders, and
|
|
125
|
+
* reasoning messages' encrypted payloads are elided.
|
|
126
|
+
*
|
|
127
|
+
* @param context The LLM context containing messages.
|
|
128
|
+
* @returns List of messages in a format ready for logging.
|
|
129
|
+
*/
|
|
130
|
+
getMessagesForLogging(context) {
|
|
131
|
+
// Sanitize messages for logging
|
|
132
|
+
const messagesForLogging = [];
|
|
133
|
+
for (const message of this.getMessages(context, { truncateLargeValues: true })) {
|
|
134
|
+
if (message instanceof LLMSpecificMessage) {
|
|
135
|
+
// Responses-specific messages are reasoning items (see
|
|
136
|
+
// _BaseOpenAIResponsesLLMService._appendReasoningMessage).
|
|
137
|
+
// Elide the encrypted payload, which is opaque noise in logs.
|
|
138
|
+
const msg = structuredClone(message.message);
|
|
139
|
+
if (isDict(msg) && msg.encrypted_content) {
|
|
140
|
+
msg.encrypted_content = "...";
|
|
141
|
+
}
|
|
142
|
+
messagesForLogging.push(msg);
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
messagesForLogging.push(message);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return messagesForLogging;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Convert LLMContext messages to Responses API input items.
|
|
152
|
+
*
|
|
153
|
+
* @param messages Messages from the LLMContext.
|
|
154
|
+
* @returns List of Responses API input items.
|
|
155
|
+
*/
|
|
156
|
+
_convertMessagesToInput(messages) {
|
|
157
|
+
const result = [];
|
|
158
|
+
for (const message of messages) {
|
|
159
|
+
if (message instanceof LLMSpecificMessage) {
|
|
160
|
+
result.push(this._fromSpecificMessage(message));
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
const role = message.role;
|
|
164
|
+
if (role === "system" || role === "developer") {
|
|
165
|
+
let content = pyGet(message, "content", "");
|
|
166
|
+
if (Array.isArray(content)) {
|
|
167
|
+
content = this._convertMultimodalContent(content);
|
|
168
|
+
}
|
|
169
|
+
result.push({ role: "developer", content });
|
|
170
|
+
}
|
|
171
|
+
else if (role === "user") {
|
|
172
|
+
let content = pyGet(message, "content", "");
|
|
173
|
+
if (Array.isArray(content)) {
|
|
174
|
+
content = this._convertMultimodalContent(content);
|
|
175
|
+
}
|
|
176
|
+
result.push({ role: "user", content });
|
|
177
|
+
}
|
|
178
|
+
else if (role === "assistant") {
|
|
179
|
+
const toolCalls = message.tool_calls;
|
|
180
|
+
if (toolCalls && toolCalls.length > 0) {
|
|
181
|
+
for (const tc of toolCalls) {
|
|
182
|
+
const func = pyGet(tc, "function", {});
|
|
183
|
+
result.push({
|
|
184
|
+
type: "function_call",
|
|
185
|
+
call_id: pyGet(tc, "id", ""),
|
|
186
|
+
name: pyGet(func, "name", ""),
|
|
187
|
+
arguments: pyGet(func, "arguments", ""),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
let content = pyGet(message, "content", "");
|
|
193
|
+
if (Array.isArray(content)) {
|
|
194
|
+
content = this._convertMultimodalContent(content);
|
|
195
|
+
}
|
|
196
|
+
result.push({ role: "assistant", content });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
else if (role === "tool") {
|
|
200
|
+
let content = pyGet(message, "content", "");
|
|
201
|
+
if (typeof content !== "string") {
|
|
202
|
+
// DIVERGENCE: upstream uses Python's `str(content)` (a Python repr
|
|
203
|
+
// for lists/dicts); we write JSON for objects instead.
|
|
204
|
+
content =
|
|
205
|
+
content === null
|
|
206
|
+
? "None"
|
|
207
|
+
: typeof content === "boolean"
|
|
208
|
+
? content
|
|
209
|
+
? "True"
|
|
210
|
+
: "False"
|
|
211
|
+
: typeof content === "object"
|
|
212
|
+
? pyJsonDumps(content)
|
|
213
|
+
: String(content);
|
|
214
|
+
}
|
|
215
|
+
result.push({
|
|
216
|
+
type: "function_call_output",
|
|
217
|
+
call_id: pyGet(message, "tool_call_id", ""),
|
|
218
|
+
output: content,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return result;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Convert an OpenAI-Responses-specific message to an input item.
|
|
226
|
+
*
|
|
227
|
+
* Reasoning messages — persisted so the model's prior reasoning round-trips
|
|
228
|
+
* on later turns — become Responses `reasoning` input items. Anything
|
|
229
|
+
* else is assumed to already be in Responses input shape.
|
|
230
|
+
*
|
|
231
|
+
* @param message The LLM-specific message from the context.
|
|
232
|
+
* @returns A Responses API input item.
|
|
233
|
+
*/
|
|
234
|
+
_fromSpecificMessage(message) {
|
|
235
|
+
const payload = message.message;
|
|
236
|
+
if (isDict(payload) && payload.type === "reasoning") {
|
|
237
|
+
const item = {
|
|
238
|
+
type: "reasoning",
|
|
239
|
+
id: payload.id ?? null,
|
|
240
|
+
summary: pyGet(payload, "summary", []),
|
|
241
|
+
};
|
|
242
|
+
const encrypted = payload.encrypted_content;
|
|
243
|
+
if (encrypted) {
|
|
244
|
+
item.encrypted_content = encrypted;
|
|
245
|
+
}
|
|
246
|
+
return item;
|
|
247
|
+
}
|
|
248
|
+
return payload;
|
|
249
|
+
}
|
|
250
|
+
/** The Responses API fetches image and file URLs itself. */
|
|
251
|
+
supportsFileUrl(url, _mimeType) {
|
|
252
|
+
return url.startsWith("http://") || url.startsWith("https://");
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Convert multimodal content parts to Responses API format.
|
|
256
|
+
*
|
|
257
|
+
* @param content List of content parts from the LLMContext message.
|
|
258
|
+
* @returns List of content parts in Responses API format.
|
|
259
|
+
*/
|
|
260
|
+
_convertMultimodalContent(content) {
|
|
261
|
+
const result = [];
|
|
262
|
+
for (const part of content) {
|
|
263
|
+
const partType = part.type;
|
|
264
|
+
if (partType === "text") {
|
|
265
|
+
result.push({ type: "input_text", text: pyGet(part, "text", "") });
|
|
266
|
+
}
|
|
267
|
+
else if (partType === "image_url") {
|
|
268
|
+
const imageUrlObj = pyGet(part, "image_url", {});
|
|
269
|
+
result.push({
|
|
270
|
+
type: "input_image",
|
|
271
|
+
image_url: pyGet(imageUrlObj, "url", ""),
|
|
272
|
+
detail: pyGet(imageUrlObj, "detail", "auto"),
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
else if (partType === "file_base64") {
|
|
276
|
+
const fData = part.file;
|
|
277
|
+
if (fData.mime_type.startsWith("image/")) {
|
|
278
|
+
result.push({
|
|
279
|
+
type: "input_image",
|
|
280
|
+
image_url: fData.file_data,
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
result.push({
|
|
285
|
+
type: "input_file",
|
|
286
|
+
filename: fData.filename,
|
|
287
|
+
file_data: fData.file_data,
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
else if (partType === "file_url") {
|
|
292
|
+
const fData = part.file;
|
|
293
|
+
// inlinedFileContent throws a plain Error so adapters with a
|
|
294
|
+
// message-level wrapper don't double-wrap; this adapter has no such
|
|
295
|
+
// wrapper, so wrap here.
|
|
296
|
+
let resolved;
|
|
297
|
+
try {
|
|
298
|
+
resolved = this.inlinedFileContent(fData);
|
|
299
|
+
}
|
|
300
|
+
catch (e) {
|
|
301
|
+
rethrowIfCancelled(e);
|
|
302
|
+
throw new LLMContextConversionError(e);
|
|
303
|
+
}
|
|
304
|
+
if (resolved === null) {
|
|
305
|
+
if (fData.mime_type.startsWith("image/")) {
|
|
306
|
+
result.push({
|
|
307
|
+
type: "input_image",
|
|
308
|
+
image_url: fData.url,
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
else {
|
|
312
|
+
result.push({
|
|
313
|
+
type: "input_file",
|
|
314
|
+
file_url: fData.url,
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
else {
|
|
319
|
+
// Non-raw adapters always cache the data-URL form.
|
|
320
|
+
const dataUrl = resolved;
|
|
321
|
+
if (fData.mime_type.startsWith("image/")) {
|
|
322
|
+
result.push({
|
|
323
|
+
type: "input_image",
|
|
324
|
+
image_url: dataUrl,
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
result.push({
|
|
329
|
+
type: "input_file",
|
|
330
|
+
filename: pyGet(fData, "filename", ""),
|
|
331
|
+
file_data: dataUrl,
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
// Pass through other types as-is. Note: "input_audio" is not
|
|
338
|
+
// yet supported by the Responses API (coming soon per OpenAI
|
|
339
|
+
// docs) but the LLMContext format already matches the expected
|
|
340
|
+
// shape, so it should work once support is enabled.
|
|
341
|
+
result.push(part);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return result;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Python's `dict.get(key, default)`: the default only replaces a missing key
|
|
349
|
+
* (`undefined`); an explicit `null` (None) is kept.
|
|
350
|
+
*/
|
|
351
|
+
function pyGet(d, key, dflt) {
|
|
352
|
+
const v = d[key];
|
|
353
|
+
return v === undefined ? dflt : v;
|
|
354
|
+
}
|
|
355
|
+
function isDict(value) {
|
|
356
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
357
|
+
}
|
|
358
|
+
//# sourceMappingURL=open_ai_responses_adapter.js.map
|