@pure01fx/dsh-openai-codex-auth 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/CODEX-COMPATIBILITY.md +93 -0
- package/README.md +60 -1
- package/client.js +95 -12
- package/lib/catalog.d.ts +6 -0
- package/lib/catalog.js +24 -0
- package/lib/cloud-context.d.ts +20 -0
- package/lib/cloud-context.js +89 -0
- package/lib/cloud-http.d.ts +28 -0
- package/lib/cloud-http.js +170 -0
- package/lib/cloud-images.d.ts +51 -0
- package/lib/cloud-images.js +122 -0
- package/lib/cloud-media.d.ts +52 -0
- package/lib/cloud-media.js +112 -0
- package/lib/cloud-search.d.ts +31 -0
- package/lib/cloud-search.js +172 -0
- package/lib/cloud-tools.d.ts +25 -0
- package/lib/cloud-tools.js +129 -0
- package/lib/cloud-vision.d.ts +34 -0
- package/lib/cloud-vision.js +108 -0
- package/lib/cloud-web-tool.d.ts +12 -0
- package/lib/cloud-web-tool.js +241 -0
- package/lib/index.d.ts +21 -1
- package/lib/index.js +601 -149
- package/lib/native-adapter.d.ts +5 -0
- package/lib/native-adapter.js +22 -10
- package/lib/native-http.d.ts +2 -0
- package/lib/native-http.js +3 -0
- package/lib/native-websocket.js +11 -5
- package/lib/replay.d.ts +1 -0
- package/lib/replay.js +8 -2
- package/lib/response-usage.d.ts +4 -2
- package/lib/response-usage.js +26 -6
- package/lib/responses.d.ts +10 -1
- package/lib/responses.js +86 -10
- package/lib/upstream.d.ts +6 -4
- package/lib/upstream.js +6 -4
- package/lib/usage.d.ts +1 -0
- package/lib/usage.js +10 -3
- package/package.json +62 -9
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/** Wire contract: Codex b348fc26674189f758d5941cdab3f78f258b2aa7, codex-api/src/search.rs. */
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { LlmError } from '@deepseek-ai/dsh-llm';
|
|
4
|
+
const MAX_REQUEST_BYTES = 256 * 1024;
|
|
5
|
+
const MAX_OUTPUT_BYTES = 128 * 1024;
|
|
6
|
+
const MAX_RESULTS_BYTES = 256 * 1024;
|
|
7
|
+
const invalid = () => new LlmError('Invalid codex_web arguments; check command fields and limits', 'INVALID_ARGS');
|
|
8
|
+
function object(value) {
|
|
9
|
+
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.getPrototypeOf(value) !== Object.prototype)
|
|
10
|
+
throw invalid();
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
function text(value, empty = false) {
|
|
14
|
+
if (typeof value !== 'string' || (!empty && !value.trim()) || value.length > MAX_REQUEST_BYTES)
|
|
15
|
+
throw invalid();
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
const string = value => text(value);
|
|
19
|
+
const uint = value => {
|
|
20
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0)
|
|
21
|
+
throw invalid();
|
|
22
|
+
return value;
|
|
23
|
+
};
|
|
24
|
+
const strings = value => {
|
|
25
|
+
if (!Array.isArray(value) || value.length > 100)
|
|
26
|
+
throw invalid();
|
|
27
|
+
return value.map(item => text(item));
|
|
28
|
+
};
|
|
29
|
+
const bool = value => { if (typeof value !== 'boolean')
|
|
30
|
+
throw invalid(); return value; };
|
|
31
|
+
const choice = (...values) => value => { if (typeof value !== 'string' || !values.includes(value))
|
|
32
|
+
throw invalid(); return value; };
|
|
33
|
+
const date = value => {
|
|
34
|
+
const s = text(value);
|
|
35
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(s) || !Number.isFinite(Date.parse(s)) || new Date(s).toISOString().slice(0, 10) !== s)
|
|
36
|
+
throw invalid();
|
|
37
|
+
return s;
|
|
38
|
+
};
|
|
39
|
+
function fields(value, rules, required = []) {
|
|
40
|
+
const input = object(value);
|
|
41
|
+
if (required.some(key => input[key] === undefined))
|
|
42
|
+
throw invalid();
|
|
43
|
+
const result = {};
|
|
44
|
+
for (const [key, item] of Object.entries(input)) {
|
|
45
|
+
if (!Object.hasOwn(rules, key))
|
|
46
|
+
throw invalid();
|
|
47
|
+
if (item !== undefined)
|
|
48
|
+
result[key] = rules[key](item);
|
|
49
|
+
}
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
const query = { q: string, recency: uint, domains: strings };
|
|
53
|
+
const operations = {
|
|
54
|
+
search_query: { rules: query, required: ['q'] },
|
|
55
|
+
image_query: { rules: query, required: ['q'] },
|
|
56
|
+
open: { rules: { ref_id: string, lineno: uint }, required: ['ref_id'] },
|
|
57
|
+
click: { rules: { ref_id: string, id: uint }, required: ['ref_id', 'id'] },
|
|
58
|
+
find: { rules: { ref_id: string, pattern: string }, required: ['ref_id', 'pattern'] },
|
|
59
|
+
screenshot: { rules: { ref_id: string, pageno: uint }, required: ['ref_id', 'pageno'] },
|
|
60
|
+
finance: { rules: { ticker: string, type: choice('equity', 'fund', 'crypto', 'index'), market: value => text(value, true) }, required: ['ticker', 'type'] },
|
|
61
|
+
weather: { rules: { location: string, start: date, duration: uint }, required: ['location'] },
|
|
62
|
+
sports: { rules: { tool: choice('sports'), fn: choice('schedule', 'standings'), league: choice('nba', 'wnba', 'nfl', 'nhl', 'mlb', 'epl', 'ncaamb', 'ncaawb', 'ipl'), team: string, opponent: string, date_from: date, date_to: date, num_games: uint, locale: string }, required: ['fn', 'league'] },
|
|
63
|
+
time: { rules: { utc_offset: value => {
|
|
64
|
+
const s = text(value);
|
|
65
|
+
if (!/^[+-](?:0\d|1\d|2[0-3]):[0-5]\d$/.test(s))
|
|
66
|
+
throw invalid();
|
|
67
|
+
return s;
|
|
68
|
+
} }, required: ['utc_offset'] },
|
|
69
|
+
};
|
|
70
|
+
function boundedRequest(value) {
|
|
71
|
+
try {
|
|
72
|
+
if (Buffer.byteLength(JSON.stringify(value)) > MAX_REQUEST_BYTES)
|
|
73
|
+
throw invalid();
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
throw invalid();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
export function parseCloudSearchArgs(value) {
|
|
80
|
+
const input = object(value);
|
|
81
|
+
const parsed = fields(input, {
|
|
82
|
+
commands: value => {
|
|
83
|
+
const commands = object(value);
|
|
84
|
+
const result = {};
|
|
85
|
+
let count = 0;
|
|
86
|
+
for (const [key, items] of Object.entries(commands)) {
|
|
87
|
+
if (key === 'response_length') {
|
|
88
|
+
result[key] = choice('short', 'medium', 'long')(items);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (!Object.hasOwn(operations, key) || !Array.isArray(items) || items.length === 0 || items.length > 100)
|
|
92
|
+
throw invalid();
|
|
93
|
+
const operation = operations[key];
|
|
94
|
+
count += items.length;
|
|
95
|
+
result[key] = items.map(item => fields(item, operation.rules, operation.required));
|
|
96
|
+
}
|
|
97
|
+
if (count === 0 || count > 100)
|
|
98
|
+
throw invalid();
|
|
99
|
+
return result;
|
|
100
|
+
},
|
|
101
|
+
model: string, context: value => text(value, true), mode: choice('cached', 'indexed', 'live'),
|
|
102
|
+
search_context_size: choice('low', 'medium', 'high'),
|
|
103
|
+
user_location: value => fields(value, { country: string, region: string, city: string, timezone: string }),
|
|
104
|
+
filters: value => fields(value, { allowed_domains: strings, blocked_domains: strings }),
|
|
105
|
+
image_settings: value => fields(value, { max_results: uint, caption: bool }),
|
|
106
|
+
}, ['commands']);
|
|
107
|
+
boundedRequest(parsed);
|
|
108
|
+
return parsed;
|
|
109
|
+
}
|
|
110
|
+
/** The caller selects the model and visible context using the same request credential. */
|
|
111
|
+
export function buildCloudSearchRequest(args, context) {
|
|
112
|
+
const parsed = parseCloudSearchArgs(args);
|
|
113
|
+
const id = createHash('sha256').update(JSON.stringify(['dsh-codex-search-v1', text(context.sessionId), text(context.accountId)])).digest('hex');
|
|
114
|
+
const settings = { allowed_callers: ['direct'], external_web_access: parsed.mode === 'cached' ? false : parsed.mode === 'indexed' ? 'indexed' : true };
|
|
115
|
+
for (const key of ['search_context_size', 'filters', 'image_settings'])
|
|
116
|
+
if (parsed[key] !== undefined)
|
|
117
|
+
settings[key] = parsed[key];
|
|
118
|
+
if (parsed.user_location !== undefined)
|
|
119
|
+
settings.user_location = { type: 'approximate', ...parsed.user_location };
|
|
120
|
+
const selected = parsed.context ?? context.input;
|
|
121
|
+
const input = typeof selected === 'string' ? [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: selected }] }] : selected;
|
|
122
|
+
if (input !== undefined && typeof input !== 'string' && !Array.isArray(input))
|
|
123
|
+
throw invalid();
|
|
124
|
+
const result = { id, model: text(context.model), commands: parsed.commands, settings, ...(input === undefined ? {} : { input }) };
|
|
125
|
+
boundedRequest(result);
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
/** Keep whole opaque result entries and identify any omitted data explicitly. */
|
|
129
|
+
export function parseCloudSearchResponse(body) {
|
|
130
|
+
let data;
|
|
131
|
+
try {
|
|
132
|
+
data = object(body);
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
throw new LlmError('Invalid Codex search response', 'CODEX_CLOUD_FAILED');
|
|
136
|
+
}
|
|
137
|
+
if (typeof data.output !== 'string' || (data.results != null && !Array.isArray(data.results)))
|
|
138
|
+
throw new LlmError('Invalid Codex search response', 'CODEX_CLOUD_FAILED');
|
|
139
|
+
let output = data.output;
|
|
140
|
+
let truncated = false;
|
|
141
|
+
if (Buffer.byteLength(output) > MAX_OUTPUT_BYTES) {
|
|
142
|
+
// Truncate only at complete UTF-8 boundaries; the notice warns that references may be omitted.
|
|
143
|
+
output = new TextDecoder().decode(Buffer.from(output).subarray(0, MAX_OUTPUT_BYTES)).replace(/\uFFFD$/, '');
|
|
144
|
+
truncated = true;
|
|
145
|
+
}
|
|
146
|
+
let results;
|
|
147
|
+
if (Array.isArray(data.results)) {
|
|
148
|
+
results = [];
|
|
149
|
+
let bytes = 2;
|
|
150
|
+
for (const item of data.results) {
|
|
151
|
+
let serialized;
|
|
152
|
+
try {
|
|
153
|
+
serialized = JSON.stringify(item);
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
throw new LlmError('Invalid Codex search results', 'CODEX_CLOUD_FAILED');
|
|
157
|
+
}
|
|
158
|
+
if (serialized === undefined)
|
|
159
|
+
throw new LlmError('Invalid Codex search results', 'CODEX_CLOUD_FAILED');
|
|
160
|
+
const size = Buffer.byteLength(serialized) + 1;
|
|
161
|
+
if (results.length >= 100 || bytes + size > MAX_RESULTS_BYTES) {
|
|
162
|
+
truncated = true;
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
results.push(JSON.parse(serialized));
|
|
166
|
+
bytes += size;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (truncated)
|
|
170
|
+
output += '\n[Search response truncated; some content or references were omitted. Narrow the request to retrieve them.]';
|
|
171
|
+
return { output, ...(results === undefined ? {} : { results }), truncated };
|
|
172
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Ordinary DSH image tools; cloud credentials never enter tool arguments. */
|
|
2
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
3
|
+
import { type NativeCodexModelCatalog } from './catalog.js';
|
|
4
|
+
import { NativeCodexCloudClient } from './cloud-http.js';
|
|
5
|
+
import type { NativeCodexHttpOptions } from './native-http.js';
|
|
6
|
+
import type { CodexImageDetail } from './responses.js';
|
|
7
|
+
export interface CloudToolsConfig {
|
|
8
|
+
enabled?: boolean;
|
|
9
|
+
search?: boolean;
|
|
10
|
+
imageGeneration?: boolean;
|
|
11
|
+
imageEditing?: boolean;
|
|
12
|
+
imageInspection?: boolean;
|
|
13
|
+
searchModel?: string;
|
|
14
|
+
visionModel?: string;
|
|
15
|
+
imageModel?: string;
|
|
16
|
+
searchMode?: 'cached' | 'indexed' | 'live';
|
|
17
|
+
visionDetail?: CodexImageDetail;
|
|
18
|
+
}
|
|
19
|
+
export interface CloudToolsDependencies {
|
|
20
|
+
client: NativeCodexCloudClient;
|
|
21
|
+
catalog: NativeCodexModelCatalog;
|
|
22
|
+
transport: NativeCodexHttpOptions;
|
|
23
|
+
config: CloudToolsConfig;
|
|
24
|
+
}
|
|
25
|
+
export declare function registerCodexImageTools(ctx: Context, deps: CloudToolsDependencies): void;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { nativeCodexAuthorityHash } from './catalog.js';
|
|
2
|
+
import { buildImageGenerationRequest, buildImageEditRequest, parseImageResponse, validateImageEditArgs } from './cloud-images.js';
|
|
3
|
+
import { resolveImageSources, saveImageOutputs } from './cloud-media.js';
|
|
4
|
+
import { inspectCloudImages, parseCloudVisionArgs } from './cloud-vision.js';
|
|
5
|
+
import { chooseCloudModel, visibleCloudImages } from './cloud-context.js';
|
|
6
|
+
const imageRefs = { type: 'array', minItems: 1, maxItems: 5, description: 'Images visible in this session, or absolute file paths read under your file policy. Preserve order.', items: {
|
|
7
|
+
oneOf: [
|
|
8
|
+
{ type: 'object', properties: { attachment_id: { type: 'string' } }, required: ['attachment_id'], additionalProperties: false },
|
|
9
|
+
{ type: 'object', properties: { path: { type: 'string' } }, required: ['path'], additionalProperties: false },
|
|
10
|
+
],
|
|
11
|
+
} };
|
|
12
|
+
const common = {
|
|
13
|
+
prompt: { type: 'string', description: 'Describe the image to generate or the edits to perform.', minLength: 1, maxLength: 32000 },
|
|
14
|
+
model: { type: 'string', description: 'Optional Codex image model; defaults to configured imageModel or gpt-image-2.' },
|
|
15
|
+
quality: { type: 'string', enum: ['auto', 'low', 'medium', 'high'] },
|
|
16
|
+
background: { type: 'string', enum: ['auto', 'transparent', 'opaque'] },
|
|
17
|
+
size: { type: 'string', description: 'auto or a service-supported WIDTHxHEIGHT, such as 1024x1024.' },
|
|
18
|
+
n: { type: 'integer', minimum: 1, maximum: 4, description: 'Optional number of images; omitted by default.' },
|
|
19
|
+
};
|
|
20
|
+
const resultSchema = { type: 'object', additionalProperties: true };
|
|
21
|
+
function render(_args, value) {
|
|
22
|
+
const row = value;
|
|
23
|
+
const content = [{ type: 'text', text: JSON.stringify(value) }];
|
|
24
|
+
if (row.image_output)
|
|
25
|
+
for (const image of row.images ?? [])
|
|
26
|
+
if (image.attachment)
|
|
27
|
+
content.push({ type: 'image', attachment: image.attachment });
|
|
28
|
+
return content;
|
|
29
|
+
}
|
|
30
|
+
function requireAgent(exec) {
|
|
31
|
+
if (!exec.agent)
|
|
32
|
+
throw new Error('Codex cloud tools require an agent session');
|
|
33
|
+
}
|
|
34
|
+
const recentImages = new WeakMap();
|
|
35
|
+
function mediaContext(exec) {
|
|
36
|
+
requireAgent(exec);
|
|
37
|
+
const ctx = exec.agent.ctx;
|
|
38
|
+
const workspace = exec.agent.session.header.cwd;
|
|
39
|
+
if (!workspace || !ctx.fs || !ctx.attachments)
|
|
40
|
+
throw new Error('Codex image tools require the caller workspace, filesystem and attachment services');
|
|
41
|
+
const messages = exec.agent.session.deriveMessages();
|
|
42
|
+
const cached = recentImages.get(exec.agent.session);
|
|
43
|
+
const visible = visibleCloudImages(messages);
|
|
44
|
+
if (cached?.historyLength === messages.length) {
|
|
45
|
+
for (const ref of cached.refs)
|
|
46
|
+
if (!visible.some(value => value.attachmentId === ref.attachmentId))
|
|
47
|
+
visible.push(ref);
|
|
48
|
+
}
|
|
49
|
+
else
|
|
50
|
+
recentImages.delete(exec.agent.session);
|
|
51
|
+
return { fs: ctx.fs, attachments: ctx.attachments, workspace, signal: exec.signal, visibleImages: visible };
|
|
52
|
+
}
|
|
53
|
+
async function imageOutputSupported(exec) {
|
|
54
|
+
requireAgent(exec);
|
|
55
|
+
const { provider, model } = exec.agent.options;
|
|
56
|
+
if (!provider || !model || !exec.agent.ctx.llm)
|
|
57
|
+
return false;
|
|
58
|
+
try {
|
|
59
|
+
return (await exec.agent.ctx.llm.resolveModelInfo(provider, model, exec.signal)).inputModalities?.includes('image') ?? false;
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
exec.signal.throwIfAborted();
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
export function registerCodexImageTools(ctx, deps) {
|
|
67
|
+
const config = deps.config;
|
|
68
|
+
if (config.enabled === false)
|
|
69
|
+
return;
|
|
70
|
+
const output = { schema: resultSchema, render };
|
|
71
|
+
for (const kind of ['generate', 'edit']) {
|
|
72
|
+
if (kind === 'generate' ? config.imageGeneration === false : config.imageEditing === false)
|
|
73
|
+
continue;
|
|
74
|
+
const tool = {
|
|
75
|
+
name: 'codex_image_' + kind,
|
|
76
|
+
description: kind === 'generate' ? 'Generate images using the current Codex account. Returns original file paths and image attachments for display or further editing.'
|
|
77
|
+
: 'Edit 1–5 images with the current Codex account. Use explicit images OR num_last_images_to_include. Returns originals and preview attachments. Does not support mask/seed parameters.',
|
|
78
|
+
parameters: { type: 'object', additionalProperties: false, properties: { ...common, ...(kind === 'edit' ? {
|
|
79
|
+
images: imageRefs, num_last_images_to_include: { type: 'integer', minimum: 1, maximum: 5, description: 'Use the most recent visible session images; mutually exclusive with images.' },
|
|
80
|
+
} : {}) }, required: ['prompt'] },
|
|
81
|
+
output, timeoutMs: 180_000, isConcurrencySafe: () => false,
|
|
82
|
+
async execute(value, exec) {
|
|
83
|
+
requireAgent(exec);
|
|
84
|
+
const media = mediaContext(exec);
|
|
85
|
+
const scope = exec.agent.ctx;
|
|
86
|
+
if (!scope.shell?.sandboxMode || !scope.sandboxPolicy)
|
|
87
|
+
throw new Error('Codex image output requires the policy-enforcing shell and sandboxPolicy services');
|
|
88
|
+
const policy = scope.sandboxPolicy.resolve({ session: exec.agent.session });
|
|
89
|
+
const credential = await deps.client.resolveCredential(exec.signal);
|
|
90
|
+
const args = kind === 'edit' ? validateImageEditArgs(value, config.imageModel) : buildImageGenerationRequest(value, config.imageModel);
|
|
91
|
+
const request = kind === 'edit' ? buildImageEditRequest(value, await resolveImageSources(validateImageEditArgs(value, config.imageModel), media), config.imageModel) : { ...args };
|
|
92
|
+
const result = await deps.client.post(kind === 'edit' ? 'images/edits' : 'images/generations', request, { credential, signal: exec.signal });
|
|
93
|
+
const saved = await saveImageOutputs(parseImageResponse(result.body, result.requestId), { ...media, shell: scope.shell, policy });
|
|
94
|
+
const image_output = await imageOutputSupported(exec);
|
|
95
|
+
const old = recentImages.get(exec.agent.session)?.refs ?? [];
|
|
96
|
+
recentImages.set(exec.agent.session, { historyLength: exec.agent.session.deriveMessages().length,
|
|
97
|
+
refs: [...old, ...saved.images.map(image => image.attachment)].slice(-20) });
|
|
98
|
+
return { ...saved, image_output };
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
ctx.tools.register(tool);
|
|
102
|
+
}
|
|
103
|
+
if (config.imageInspection !== false)
|
|
104
|
+
ctx.tools.register({
|
|
105
|
+
name: 'codex_image_inspect',
|
|
106
|
+
description: 'Ask a Codex vision model to analyze images, OCR, screenshots or charts. This makes one additional cloud inference call, without recursive tools. Use explicit model or configured visionModel for non-Codex callers. Reports effective detail and actual dimensions.',
|
|
107
|
+
parameters: { type: 'object', additionalProperties: false, required: ['prompt', 'images'], properties: {
|
|
108
|
+
prompt: { type: 'string', minLength: 1, maxLength: 32000 }, images: imageRefs,
|
|
109
|
+
model: { type: 'string' }, detail: { type: 'string', enum: ['auto', 'low', 'high', 'original'] }, reasoning_effort: { type: 'string' },
|
|
110
|
+
} },
|
|
111
|
+
output, timeoutMs: 120_000, isConcurrencySafe: () => true,
|
|
112
|
+
async execute(value, exec) {
|
|
113
|
+
requireAgent(exec);
|
|
114
|
+
const args = parseCloudVisionArgs(value);
|
|
115
|
+
const credential = await deps.client.resolveCredential(exec.signal);
|
|
116
|
+
const view = await deps.catalog.listWithAuthority?.(exec.signal) ?? { models: await deps.catalog.list(exec.signal) };
|
|
117
|
+
if (view.authorityHash !== undefined && view.authorityHash !== nativeCodexAuthorityHash(credential.accountId))
|
|
118
|
+
throw new Error('Codex account changed before image inspection; retry with the current account');
|
|
119
|
+
const model = chooseCloudModel({ purpose: 'vision', ...(args.model === undefined ? {} : { requested: args.model }),
|
|
120
|
+
...(config.visionModel === undefined ? {} : { configured: config.visionModel }),
|
|
121
|
+
...exec.agent.options.provider === undefined ? {} : { currentProvider: exec.agent.options.provider },
|
|
122
|
+
...exec.agent.options.model === undefined ? {} : { currentModel: exec.agent.options.model },
|
|
123
|
+
}, view.models);
|
|
124
|
+
const images = await resolveImageSources({ images: args.images }, mediaContext(exec));
|
|
125
|
+
return inspectCloudImages(args, { credential, model, images, signal: exec.signal, transport: deps.transport,
|
|
126
|
+
...(config.visionDetail === undefined ? {} : { defaultDetail: config.visionDetail }) });
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type TokenUsage } from '@deepseek-ai/dsh-llm';
|
|
2
|
+
import type { NativeCodexCredential, NativeCodexModel } from './catalog.js';
|
|
3
|
+
import { type NativeCodexHttpOptions } from './native-http.js';
|
|
4
|
+
import type { CodexImageDetail } from './responses.js';
|
|
5
|
+
import { type ImageSource } from './cloud-images.js';
|
|
6
|
+
import type { ResolvedCloudImage } from './cloud-media.js';
|
|
7
|
+
export interface CloudVisionArgs {
|
|
8
|
+
prompt: string;
|
|
9
|
+
images: ImageSource[];
|
|
10
|
+
model?: string;
|
|
11
|
+
detail?: CodexImageDetail;
|
|
12
|
+
reasoning_effort?: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function parseCloudVisionArgs(value: unknown): CloudVisionArgs;
|
|
15
|
+
export interface CloudVisionDependencies {
|
|
16
|
+
model: NativeCodexModel;
|
|
17
|
+
credential: NativeCodexCredential;
|
|
18
|
+
images: readonly ResolvedCloudImage[];
|
|
19
|
+
signal: AbortSignal;
|
|
20
|
+
defaultDetail?: CodexImageDetail;
|
|
21
|
+
transport: NativeCodexHttpOptions;
|
|
22
|
+
}
|
|
23
|
+
export declare function inspectCloudImages(args: CloudVisionArgs, deps: CloudVisionDependencies): Promise<{
|
|
24
|
+
images: {
|
|
25
|
+
requested_detail: CodexImageDetail;
|
|
26
|
+
effective_detail: string;
|
|
27
|
+
width: number;
|
|
28
|
+
height: number;
|
|
29
|
+
scaled: boolean;
|
|
30
|
+
}[];
|
|
31
|
+
usage?: TokenUsage;
|
|
32
|
+
text: string;
|
|
33
|
+
model: string;
|
|
34
|
+
}>;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/** One explicit, tool-free Codex image understanding request. */
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { createUserMessage, LlmError, ReasoningEffortId } from '@deepseek-ai/dsh-llm';
|
|
4
|
+
import { nativeCodexWireReasoningEffort } from './native-adapter.js';
|
|
5
|
+
import { NativeCodexHttpTransport } from './native-http.js';
|
|
6
|
+
import { validateImageSelection } from './cloud-images.js';
|
|
7
|
+
export function parseCloudVisionArgs(value) {
|
|
8
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
9
|
+
throw new Error('Vision arguments must be an object');
|
|
10
|
+
const row = value;
|
|
11
|
+
if (Object.keys(row).some(key => !['prompt', 'images', 'model', 'detail', 'reasoning_effort'].includes(key)))
|
|
12
|
+
throw new Error('Unknown vision argument');
|
|
13
|
+
if (typeof row.prompt !== 'string' || !row.prompt.trim() || Buffer.byteLength(row.prompt) > 32_000)
|
|
14
|
+
throw new Error('Vision prompt must be a nonempty bounded string');
|
|
15
|
+
const selection = validateImageSelection(row);
|
|
16
|
+
if (!selection.images)
|
|
17
|
+
throw new Error('Vision requires explicit images');
|
|
18
|
+
for (const key of ['model', 'reasoning_effort']) {
|
|
19
|
+
if (row[key] !== undefined && (typeof row[key] !== 'string' || !row[key].trim() || Buffer.byteLength(row[key]) > 256))
|
|
20
|
+
throw new Error('Invalid vision ' + key);
|
|
21
|
+
}
|
|
22
|
+
if (row.detail !== undefined && !['auto', 'low', 'high', 'original'].includes(row.detail))
|
|
23
|
+
throw new Error('Invalid vision detail');
|
|
24
|
+
return { prompt: row.prompt, images: selection.images,
|
|
25
|
+
...(row.model === undefined ? {} : { model: row.model }),
|
|
26
|
+
...(row.detail === undefined ? {} : { detail: row.detail }),
|
|
27
|
+
...(row.reasoning_effort === undefined ? {} : { reasoning_effort: row.reasoning_effort }) };
|
|
28
|
+
}
|
|
29
|
+
export async function inspectCloudImages(args, deps) {
|
|
30
|
+
const { model } = deps;
|
|
31
|
+
deps.signal.throwIfAborted();
|
|
32
|
+
if (!model.inputModalities.includes('image'))
|
|
33
|
+
throw new LlmError('Select a Codex model with image input', 'VISION_UNSUPPORTED');
|
|
34
|
+
if (deps.images.length !== args.images.length)
|
|
35
|
+
throw new Error('Resolved vision images do not match the request');
|
|
36
|
+
const requested = args.detail ?? deps.defaultDetail ?? 'auto';
|
|
37
|
+
const effective = model.useResponsesLite ? undefined
|
|
38
|
+
: requested === 'original' && !model.supportsImageDetailOriginal ? 'high' : requested;
|
|
39
|
+
const images = deps.images.map(image => {
|
|
40
|
+
if (!image.width || !image.height)
|
|
41
|
+
throw new Error('Vision requires verified encoded image dimensions');
|
|
42
|
+
// Ephemeral references are resolved only by this invocation's reader, never persisted.
|
|
43
|
+
const ref = { attachmentId: ('vision-' + randomUUID()),
|
|
44
|
+
mediaType: image.mediaType, bytes: image.data.byteLength, width: image.width, height: image.height };
|
|
45
|
+
return { image, ref };
|
|
46
|
+
});
|
|
47
|
+
const signal = AbortSignal.any([deps.signal, AbortSignal.timeout(120_000)]);
|
|
48
|
+
let recovered = false;
|
|
49
|
+
const transport = new NativeCodexHttpTransport({ ...deps.transport, maxTransientRetries: 0,
|
|
50
|
+
resolveCredential: async (readSignal) => {
|
|
51
|
+
const current = recovered ? await deps.transport.resolveCredential(readSignal) : deps.credential;
|
|
52
|
+
if (current.accountId !== deps.credential.accountId)
|
|
53
|
+
throw new LlmError('Codex account changed during image inspection', 'CODEX_ACCOUNT_CHANGED');
|
|
54
|
+
return current;
|
|
55
|
+
},
|
|
56
|
+
recoverCredential: async (previous, readSignal) => {
|
|
57
|
+
const ok = await deps.transport.recoverCredential?.(previous, readSignal) ?? false;
|
|
58
|
+
recovered = ok;
|
|
59
|
+
return ok;
|
|
60
|
+
},
|
|
61
|
+
readImage: async (ref) => {
|
|
62
|
+
const entry = images.find(image => image.ref.attachmentId === ref.attachmentId);
|
|
63
|
+
if (!entry)
|
|
64
|
+
throw new LlmError('Image is outside this inspection request', 'INVALID_ATTACHMENT');
|
|
65
|
+
return { data: entry.image.data, ...(effective === undefined ? {} : { detail: effective }) };
|
|
66
|
+
} });
|
|
67
|
+
const effort = nativeCodexWireReasoningEffort(args.reasoning_effort ?? (model.useResponsesLite ? model.defaultReasoningLevel : undefined), model);
|
|
68
|
+
const chunks = transport.stream({ provider: 'openai-codex', model: model.slug, signal,
|
|
69
|
+
system: 'Analyze the provided images and answer the user request. State uncertainty where image details are unclear.',
|
|
70
|
+
messages: [createUserMessage({ source: { kind: 'user' }, content: [{ type: 'text', text: args.prompt }, ...images.map(({ ref }) => ({ type: 'image', attachment: ref }))] })],
|
|
71
|
+
tools: [], ...(effort === undefined ? {} : { reasoningEffort: ReasoningEffortId(effort) }) }, { pinnedAccountId: deps.credential.accountId, ...(model.useResponsesLite ? { responsesLite: {
|
|
72
|
+
...(model.defaultVerbosity === undefined ? {} : { defaultVerbosity: model.defaultVerbosity }),
|
|
73
|
+
...(model.instructionsTemplate === undefined ? {} : { instructionsTemplate: model.instructionsTemplate }),
|
|
74
|
+
} } : {}) });
|
|
75
|
+
const texts = new Map();
|
|
76
|
+
let visibleBytes = 0;
|
|
77
|
+
let usage;
|
|
78
|
+
let finished = false;
|
|
79
|
+
for await (const chunk of chunks) {
|
|
80
|
+
if (chunk.type === 'tool-call-delta' || (chunk.type === 'block-start' && chunk.blockType === 'tool-call'))
|
|
81
|
+
throw new LlmError('Image inspection returned an unexpected tool call', 'UNSUPPORTED');
|
|
82
|
+
if (chunk.type === 'text-delta') {
|
|
83
|
+
visibleBytes += Buffer.byteLength(chunk.text);
|
|
84
|
+
if (visibleBytes > 128 * 1024)
|
|
85
|
+
throw new LlmError('Image inspection output exceeded limit', 'RESPONSE_TOO_LARGE');
|
|
86
|
+
}
|
|
87
|
+
if (chunk.type === 'block-end' && chunk.block.type === 'text') {
|
|
88
|
+
texts.set(chunk.index, chunk.block.text);
|
|
89
|
+
if ([...texts.values()].reduce((bytes, text) => bytes + Buffer.byteLength(text), 0) > 128 * 1024)
|
|
90
|
+
throw new LlmError('Image inspection output exceeded limit', 'RESPONSE_TOO_LARGE');
|
|
91
|
+
}
|
|
92
|
+
if (chunk.type === 'usage')
|
|
93
|
+
usage = chunk.usage;
|
|
94
|
+
if (chunk.type === 'finish') {
|
|
95
|
+
if (chunk.reason.kind === 'error')
|
|
96
|
+
throw new LlmError(chunk.reason.failure.message, chunk.reason.failure.code);
|
|
97
|
+
if (chunk.reason.kind !== 'stop')
|
|
98
|
+
throw new LlmError('Image inspection did not complete normally', 'INCOMPLETE_RESPONSE');
|
|
99
|
+
finished = true;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const text = [...texts.entries()].sort(([a], [b]) => a - b).map(([, value]) => value).join('');
|
|
103
|
+
if (!finished || !text.trim())
|
|
104
|
+
throw new LlmError('Image inspection returned no completed text', 'EMPTY_RESPONSE');
|
|
105
|
+
return { text, model: model.slug, ...(usage === undefined ? {} : { usage }), images: images.map(({ image }) => ({
|
|
106
|
+
requested_detail: requested, effective_detail: effective ?? 'omitted', width: image.width, height: image.height, scaled: image.scaled,
|
|
107
|
+
})) };
|
|
108
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Native codex_web registration. No private host APIs or third-party OAuth requests. */
|
|
2
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
3
|
+
import { type NativeCodexModelCatalog } from './catalog.js';
|
|
4
|
+
import type { NativeCodexCloudClient } from './cloud-http.js';
|
|
5
|
+
export declare function registerCodexWeb(ctx: Context, deps: {
|
|
6
|
+
client: NativeCodexCloudClient;
|
|
7
|
+
catalog: NativeCodexModelCatalog;
|
|
8
|
+
config: {
|
|
9
|
+
searchModel?: string;
|
|
10
|
+
searchMode?: 'live' | 'indexed' | 'cached';
|
|
11
|
+
};
|
|
12
|
+
}): void;
|