@onkernel/cua-ai 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +229 -0
- package/dist/api-keys.d.ts +8 -0
- package/dist/api-keys.d.ts.map +1 -0
- package/dist/api-keys.js +48 -0
- package/dist/api-keys.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -0
- package/dist/models.d.ts +33 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +159 -0
- package/dist/models.js.map +1 -0
- package/dist/providers/anthropic/index.d.ts +10 -0
- package/dist/providers/anthropic/index.d.ts.map +1 -0
- package/dist/providers/anthropic/index.js +16 -0
- package/dist/providers/anthropic/index.js.map +1 -0
- package/dist/providers/common.d.ts +111 -0
- package/dist/providers/common.d.ts.map +1 -0
- package/dist/providers/common.js +138 -0
- package/dist/providers/common.js.map +1 -0
- package/dist/providers/gemini/index.d.ts +11 -0
- package/dist/providers/gemini/index.d.ts.map +1 -0
- package/dist/providers/gemini/index.js +14 -0
- package/dist/providers/gemini/index.js.map +1 -0
- package/dist/providers/openai/index.d.ts +8 -0
- package/dist/providers/openai/index.d.ts.map +1 -0
- package/dist/providers/openai/index.js +22 -0
- package/dist/providers/openai/index.js.map +1 -0
- package/dist/providers/tzafon/index.d.ts +12 -0
- package/dist/providers/tzafon/index.d.ts.map +1 -0
- package/dist/providers/tzafon/index.js +18 -0
- package/dist/providers/tzafon/index.js.map +1 -0
- package/dist/providers/tzafon/provider.d.ts +8 -0
- package/dist/providers/tzafon/provider.d.ts.map +1 -0
- package/dist/providers/tzafon/provider.js +234 -0
- package/dist/providers/tzafon/provider.js.map +1 -0
- package/dist/providers/yutori/index.d.ts +12 -0
- package/dist/providers/yutori/index.d.ts.map +1 -0
- package/dist/providers/yutori/index.js +23 -0
- package/dist/providers/yutori/index.js.map +1 -0
- package/dist/providers/yutori/provider.d.ts +9 -0
- package/dist/providers/yutori/provider.d.ts.map +1 -0
- package/dist/providers/yutori/provider.js +307 -0
- package/dist/providers/yutori/provider.js.map +1 -0
- package/dist/providers.d.ts +6 -0
- package/dist/providers.d.ts.map +1 -0
- package/dist/providers.js +26 -0
- package/dist/providers.js.map +1 -0
- package/dist/runtime-spec.d.ts +29 -0
- package/dist/runtime-spec.d.ts.map +1 -0
- package/dist/runtime-spec.js +58 -0
- package/dist/runtime-spec.js.map +1 -0
- package/examples/quickstart.ts +59 -0
- package/examples/screenshot.png +0 -0
- package/package.json +48 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { Type, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai";
|
|
2
|
+
export declare const CUA_ACTION_TYPES: readonly ["click", "double_click", "mouse_down", "mouse_up", "type", "keypress", "scroll", "move", "drag", "wait", "screenshot", "goto", "back", "forward", "url", "cursor_position"];
|
|
3
|
+
export type CuaActionType = (typeof CUA_ACTION_TYPES)[number];
|
|
4
|
+
export interface CuaActionClick {
|
|
5
|
+
type: "click";
|
|
6
|
+
x: number;
|
|
7
|
+
y: number;
|
|
8
|
+
button?: string;
|
|
9
|
+
hold_keys?: string[];
|
|
10
|
+
}
|
|
11
|
+
export interface CuaActionDoubleClick {
|
|
12
|
+
type: "double_click";
|
|
13
|
+
x: number;
|
|
14
|
+
y: number;
|
|
15
|
+
hold_keys?: string[];
|
|
16
|
+
}
|
|
17
|
+
export interface CuaActionMouseDown {
|
|
18
|
+
type: "mouse_down";
|
|
19
|
+
x: number;
|
|
20
|
+
y: number;
|
|
21
|
+
button?: string;
|
|
22
|
+
hold_keys?: string[];
|
|
23
|
+
}
|
|
24
|
+
export interface CuaActionMouseUp {
|
|
25
|
+
type: "mouse_up";
|
|
26
|
+
x: number;
|
|
27
|
+
y: number;
|
|
28
|
+
button?: string;
|
|
29
|
+
hold_keys?: string[];
|
|
30
|
+
}
|
|
31
|
+
export interface CuaActionTypeText {
|
|
32
|
+
type: "type";
|
|
33
|
+
text: string;
|
|
34
|
+
}
|
|
35
|
+
export interface CuaActionKeypress {
|
|
36
|
+
type: "keypress";
|
|
37
|
+
keys: string[];
|
|
38
|
+
}
|
|
39
|
+
export interface CuaActionScroll {
|
|
40
|
+
type: "scroll";
|
|
41
|
+
x?: number;
|
|
42
|
+
y?: number;
|
|
43
|
+
scroll_x?: number;
|
|
44
|
+
scroll_y?: number;
|
|
45
|
+
hold_keys?: string[];
|
|
46
|
+
}
|
|
47
|
+
export interface CuaActionMove {
|
|
48
|
+
type: "move";
|
|
49
|
+
x: number;
|
|
50
|
+
y: number;
|
|
51
|
+
}
|
|
52
|
+
export interface CuaActionDrag {
|
|
53
|
+
type: "drag";
|
|
54
|
+
path: Array<{
|
|
55
|
+
x: number;
|
|
56
|
+
y: number;
|
|
57
|
+
}>;
|
|
58
|
+
button?: string;
|
|
59
|
+
hold_keys?: string[];
|
|
60
|
+
}
|
|
61
|
+
export interface CuaActionWait {
|
|
62
|
+
type: "wait";
|
|
63
|
+
ms?: number;
|
|
64
|
+
}
|
|
65
|
+
export interface CuaActionScreenshot {
|
|
66
|
+
type: "screenshot";
|
|
67
|
+
}
|
|
68
|
+
export interface CuaActionGoto {
|
|
69
|
+
type: "goto";
|
|
70
|
+
url: string;
|
|
71
|
+
}
|
|
72
|
+
export interface CuaActionBack {
|
|
73
|
+
type: "back";
|
|
74
|
+
}
|
|
75
|
+
export interface CuaActionForward {
|
|
76
|
+
type: "forward";
|
|
77
|
+
}
|
|
78
|
+
export interface CuaActionUrl {
|
|
79
|
+
type: "url";
|
|
80
|
+
}
|
|
81
|
+
export interface CuaActionCursorPosition {
|
|
82
|
+
type: "cursor_position";
|
|
83
|
+
}
|
|
84
|
+
export type CuaAction = CuaActionClick | CuaActionDoubleClick | CuaActionMouseDown | CuaActionMouseUp | CuaActionTypeText | CuaActionKeypress | CuaActionScroll | CuaActionMove | CuaActionDrag | CuaActionWait | CuaActionScreenshot | CuaActionGoto | CuaActionBack | CuaActionForward | CuaActionUrl | CuaActionCursorPosition;
|
|
85
|
+
export declare function createCuaActionSchema(actions?: readonly CuaActionType[]): TSchema;
|
|
86
|
+
export declare const CuaActionSchema: Type.TSchema;
|
|
87
|
+
export declare function createCuaBatchSchema(actions?: readonly CuaActionType[]): TSchema;
|
|
88
|
+
export declare const CuaBatchSchema: Type.TSchema;
|
|
89
|
+
export declare const CuaNavigationSchema: Type.TObject<{
|
|
90
|
+
action: Type.TUnion<[Type.TLiteral<"goto">, Type.TLiteral<"back">, Type.TLiteral<"forward">, Type.TLiteral<"url">]>;
|
|
91
|
+
url: Type.TOptional<Type.TString>;
|
|
92
|
+
}>;
|
|
93
|
+
export interface CuaBatchInput {
|
|
94
|
+
actions: CuaAction[];
|
|
95
|
+
}
|
|
96
|
+
export type CuaNavigationInput = Static<typeof CuaNavigationSchema>;
|
|
97
|
+
export declare const CUA_BATCH_TOOL_NAME = "batch_computer_actions";
|
|
98
|
+
export declare const CUA_NAVIGATION_TOOL_NAME = "computer_use_extra";
|
|
99
|
+
export declare const CUA_BATCH_TOOL_DESCRIPTION: string;
|
|
100
|
+
export declare const CUA_NAVIGATION_TOOL_DESCRIPTION = "High-level browser navigation helpers for goto, back, forward, and url.";
|
|
101
|
+
export interface CreateComputerToolDefinitionsOptions {
|
|
102
|
+
actions?: readonly CuaActionType[];
|
|
103
|
+
}
|
|
104
|
+
export type ComputerToolCoordinateSystem = {
|
|
105
|
+
type: "pixel";
|
|
106
|
+
} | {
|
|
107
|
+
type: "normalized";
|
|
108
|
+
range: readonly [number, number];
|
|
109
|
+
};
|
|
110
|
+
export declare function createComputerToolDefinitions(options?: CreateComputerToolDefinitionsOptions): Tool[];
|
|
111
|
+
//# sourceMappingURL=common.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/providers/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,MAAM,EAAE,KAAK,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAEnF,eAAO,MAAM,gBAAgB,uLAiBnB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9D,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACpC,IAAI,EAAE,cAAc,CAAC;IACrB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,YAAY,CAAC;IACnB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,UAAU,CAAC;IACjB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACV;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,mBAAmB;IACnC,IAAI,EAAE,YAAY,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,SAAS,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,KAAK,CAAC;CACZ;AAED,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,iBAAiB,CAAC;CACxB;AAED,MAAM,MAAM,SAAS,GAClB,cAAc,GACd,oBAAoB,GACpB,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,iBAAiB,GACjB,eAAe,GACf,aAAa,GACb,aAAa,GACb,aAAa,GACb,mBAAmB,GACnB,aAAa,GACb,aAAa,GACb,gBAAgB,GAChB,YAAY,GACZ,uBAAuB,CAAC;AAiH3B,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,SAAS,aAAa,EAAqB,GAAG,OAAO,CAInG;AAED,eAAO,MAAM,eAAe,cAA0B,CAAC;AAEvD,wBAAgB,oBAAoB,CAAC,OAAO,CAAC,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAIhF;AAED,eAAO,MAAM,cAAc,cAAyB,CAAC;AAErD,eAAO,MAAM,mBAAmB;;;EAM/B,CAAC;AAEF,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE,SAAS,EAAE,CAAC;CACrB;AACD,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEpE,eAAO,MAAM,mBAAmB,2BAA2B,CAAC;AAC5D,eAAO,MAAM,wBAAwB,uBAAuB,CAAC;AAE7D,eAAO,MAAM,0BAA0B,QAI3B,CAAC;AAEb,eAAO,MAAM,+BAA+B,4EAA4E,CAAC;AAEzH,MAAM,WAAW,oCAAoC;IACpD,OAAO,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;CACnC;AAED,MAAM,MAAM,4BAA4B,GACrC;IACA,IAAI,EAAE,OAAO,CAAC;CACd,GACA;IACA,IAAI,EAAE,YAAY,CAAC;IACnB,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC,CAAC;AAEJ,wBAAgB,6BAA6B,CAAC,OAAO,GAAE,oCAAyC,GAAG,IAAI,EAAE,CAkBxG"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { Type } from "@earendil-works/pi-ai";
|
|
2
|
+
export const CUA_ACTION_TYPES = [
|
|
3
|
+
"click",
|
|
4
|
+
"double_click",
|
|
5
|
+
"mouse_down",
|
|
6
|
+
"mouse_up",
|
|
7
|
+
"type",
|
|
8
|
+
"keypress",
|
|
9
|
+
"scroll",
|
|
10
|
+
"move",
|
|
11
|
+
"drag",
|
|
12
|
+
"wait",
|
|
13
|
+
"screenshot",
|
|
14
|
+
"goto",
|
|
15
|
+
"back",
|
|
16
|
+
"forward",
|
|
17
|
+
"url",
|
|
18
|
+
"cursor_position",
|
|
19
|
+
];
|
|
20
|
+
const PointSchema = Type.Object({
|
|
21
|
+
x: Type.Number(),
|
|
22
|
+
y: Type.Number(),
|
|
23
|
+
}, { additionalProperties: false });
|
|
24
|
+
const CUA_ACTION_SCHEMA_BY_TYPE = {
|
|
25
|
+
click: Type.Object({
|
|
26
|
+
type: Type.Literal("click"),
|
|
27
|
+
x: Type.Number(),
|
|
28
|
+
y: Type.Number(),
|
|
29
|
+
button: Type.Optional(Type.String()),
|
|
30
|
+
hold_keys: Type.Optional(Type.Array(Type.String())),
|
|
31
|
+
}, { additionalProperties: false }),
|
|
32
|
+
double_click: Type.Object({
|
|
33
|
+
type: Type.Literal("double_click"),
|
|
34
|
+
x: Type.Number(),
|
|
35
|
+
y: Type.Number(),
|
|
36
|
+
hold_keys: Type.Optional(Type.Array(Type.String())),
|
|
37
|
+
}, { additionalProperties: false }),
|
|
38
|
+
mouse_down: Type.Object({
|
|
39
|
+
type: Type.Literal("mouse_down"),
|
|
40
|
+
x: Type.Number(),
|
|
41
|
+
y: Type.Number(),
|
|
42
|
+
button: Type.Optional(Type.String()),
|
|
43
|
+
hold_keys: Type.Optional(Type.Array(Type.String())),
|
|
44
|
+
}, { additionalProperties: false }),
|
|
45
|
+
mouse_up: Type.Object({
|
|
46
|
+
type: Type.Literal("mouse_up"),
|
|
47
|
+
x: Type.Number(),
|
|
48
|
+
y: Type.Number(),
|
|
49
|
+
button: Type.Optional(Type.String()),
|
|
50
|
+
hold_keys: Type.Optional(Type.Array(Type.String())),
|
|
51
|
+
}, { additionalProperties: false }),
|
|
52
|
+
type: Type.Object({
|
|
53
|
+
type: Type.Literal("type"),
|
|
54
|
+
text: Type.String(),
|
|
55
|
+
}, { additionalProperties: false }),
|
|
56
|
+
keypress: Type.Object({
|
|
57
|
+
type: Type.Literal("keypress"),
|
|
58
|
+
keys: Type.Array(Type.String()),
|
|
59
|
+
}, { additionalProperties: false }),
|
|
60
|
+
scroll: Type.Object({
|
|
61
|
+
type: Type.Literal("scroll"),
|
|
62
|
+
x: Type.Optional(Type.Number()),
|
|
63
|
+
y: Type.Optional(Type.Number()),
|
|
64
|
+
scroll_x: Type.Optional(Type.Number()),
|
|
65
|
+
scroll_y: Type.Optional(Type.Number()),
|
|
66
|
+
hold_keys: Type.Optional(Type.Array(Type.String())),
|
|
67
|
+
}, { additionalProperties: false }),
|
|
68
|
+
move: Type.Object({
|
|
69
|
+
type: Type.Literal("move"),
|
|
70
|
+
x: Type.Number(),
|
|
71
|
+
y: Type.Number(),
|
|
72
|
+
}, { additionalProperties: false }),
|
|
73
|
+
drag: Type.Object({
|
|
74
|
+
type: Type.Literal("drag"),
|
|
75
|
+
path: Type.Array(PointSchema, { minItems: 2 }),
|
|
76
|
+
button: Type.Optional(Type.String()),
|
|
77
|
+
hold_keys: Type.Optional(Type.Array(Type.String())),
|
|
78
|
+
}, { additionalProperties: false }),
|
|
79
|
+
wait: Type.Object({
|
|
80
|
+
type: Type.Literal("wait"),
|
|
81
|
+
ms: Type.Optional(Type.Number()),
|
|
82
|
+
}, { additionalProperties: false }),
|
|
83
|
+
screenshot: Type.Object({ type: Type.Literal("screenshot") }, { additionalProperties: false }),
|
|
84
|
+
goto: Type.Object({
|
|
85
|
+
type: Type.Literal("goto"),
|
|
86
|
+
url: Type.String(),
|
|
87
|
+
}, { additionalProperties: false }),
|
|
88
|
+
back: Type.Object({ type: Type.Literal("back") }, { additionalProperties: false }),
|
|
89
|
+
forward: Type.Object({ type: Type.Literal("forward") }, { additionalProperties: false }),
|
|
90
|
+
url: Type.Object({ type: Type.Literal("url") }, { additionalProperties: false }),
|
|
91
|
+
cursor_position: Type.Object({ type: Type.Literal("cursor_position") }, { additionalProperties: false }),
|
|
92
|
+
};
|
|
93
|
+
export function createCuaActionSchema(actions = CUA_ACTION_TYPES) {
|
|
94
|
+
if (actions.length === 0)
|
|
95
|
+
throw new Error("actions must include at least one CUA action type");
|
|
96
|
+
if (actions.length === 1)
|
|
97
|
+
return CUA_ACTION_SCHEMA_BY_TYPE[actions[0]];
|
|
98
|
+
return Type.Union(actions.map((action) => CUA_ACTION_SCHEMA_BY_TYPE[action]));
|
|
99
|
+
}
|
|
100
|
+
export const CuaActionSchema = createCuaActionSchema();
|
|
101
|
+
export function createCuaBatchSchema(actions) {
|
|
102
|
+
return Type.Object({
|
|
103
|
+
actions: Type.Array(createCuaActionSchema(actions), { description: "Ordered computer actions to execute." }),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
export const CuaBatchSchema = createCuaBatchSchema();
|
|
107
|
+
export const CuaNavigationSchema = Type.Object({
|
|
108
|
+
action: Type.Union([Type.Literal("goto"), Type.Literal("back"), Type.Literal("forward"), Type.Literal("url")]),
|
|
109
|
+
url: Type.Optional(Type.String()),
|
|
110
|
+
}, { additionalProperties: false });
|
|
111
|
+
export const CUA_BATCH_TOOL_NAME = "batch_computer_actions";
|
|
112
|
+
export const CUA_NAVIGATION_TOOL_NAME = "computer_use_extra";
|
|
113
|
+
export const CUA_BATCH_TOOL_DESCRIPTION = [
|
|
114
|
+
"Execute multiple computer actions in sequence, including ordered read steps like url(), cursor_position(), and screenshot().",
|
|
115
|
+
"Prefer this tool for predictable browser interaction sequences such as click-then-type, typing a URL, keyboard navigation, drag paths, and mixed write/read batches.",
|
|
116
|
+
"If no explicit read step is included, the tool returns one fresh screenshot after execution.",
|
|
117
|
+
].join("\n");
|
|
118
|
+
export const CUA_NAVIGATION_TOOL_DESCRIPTION = "High-level browser navigation helpers for goto, back, forward, and url.";
|
|
119
|
+
export function createComputerToolDefinitions(options = {}) {
|
|
120
|
+
const actions = options.actions;
|
|
121
|
+
return [
|
|
122
|
+
{
|
|
123
|
+
name: CUA_BATCH_TOOL_NAME,
|
|
124
|
+
description: CUA_BATCH_TOOL_DESCRIPTION,
|
|
125
|
+
parameters: createCuaBatchSchema(actions),
|
|
126
|
+
},
|
|
127
|
+
...(actions === undefined
|
|
128
|
+
? [
|
|
129
|
+
{
|
|
130
|
+
name: CUA_NAVIGATION_TOOL_NAME,
|
|
131
|
+
description: CUA_NAVIGATION_TOOL_DESCRIPTION,
|
|
132
|
+
parameters: CuaNavigationSchema,
|
|
133
|
+
},
|
|
134
|
+
]
|
|
135
|
+
: []),
|
|
136
|
+
];
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=common.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/providers/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAwC,MAAM,uBAAuB,CAAC;AAEnF,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,OAAO;IACP,cAAc;IACd,YAAY;IACZ,UAAU;IACV,MAAM;IACN,UAAU;IACV,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,YAAY;IACZ,MAAM;IACN,MAAM;IACN,SAAS;IACT,KAAK;IACL,iBAAiB;CACR,CAAC;AAmHX,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAC9B;IACC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;IAChB,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;CAChB,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B,CAAC;AAEF,MAAM,yBAAyB,GAAG;IACjC,KAAK,EAAE,IAAI,CAAC,MAAM,CACjB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QAC3B,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;KACnD,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,YAAY,EAAE,IAAI,CAAC,MAAM,CACxB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAClC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;KACnD,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,UAAU,EAAE,IAAI,CAAC,MAAM,CACtB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QAChC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;KACnD,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,QAAQ,EAAE,IAAI,CAAC,MAAM,CACpB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;KACnD,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,IAAI,EAAE,IAAI,CAAC,MAAM,CAChB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1B,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;KACnB,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,QAAQ,EAAE,IAAI,CAAC,MAAM,CACpB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;QAC9B,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;KAC/B,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,MAAM,EAAE,IAAI,CAAC,MAAM,CAClB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC5B,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;KACnD,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,IAAI,EAAE,IAAI,CAAC,MAAM,CAChB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;QAChB,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE;KAChB,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,IAAI,EAAE,IAAI,CAAC,MAAM,CAChB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1B,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC9C,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;KACnD,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,IAAI,EAAE,IAAI,CAAC,MAAM,CAChB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1B,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;KAChC,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;IAC9F,IAAI,EAAE,IAAI,CAAC,MAAM,CAChB;QACC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1B,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE;KAClB,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B;IACD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;IAClF,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;IACxF,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;IAChF,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;CAC/D,CAAC;AAE3C,MAAM,UAAU,qBAAqB,CAAC,UAAoC,gBAAgB;IACzF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IAC/F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;IACxE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAC;AAEvD,MAAM,UAAU,oBAAoB,CAAC,OAAkC;IACtE,OAAO,IAAI,CAAC,MAAM,CAAC;QAClB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,sCAAsC,EAAE,CAAC;KAC5G,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,oBAAoB,EAAE,CAAC;AAErD,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAC7C;IACC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9G,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;CACjC,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC/B,CAAC;AAOF,MAAM,CAAC,MAAM,mBAAmB,GAAG,wBAAwB,CAAC;AAC5D,MAAM,CAAC,MAAM,wBAAwB,GAAG,oBAAoB,CAAC;AAE7D,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACzC,8HAA8H;IAC9H,sKAAsK;IACtK,8FAA8F;CAC9F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,CAAC,MAAM,+BAA+B,GAAG,yEAAyE,CAAC;AAezH,MAAM,UAAU,6BAA6B,CAAC,UAAgD,EAAE;IAC/F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,OAAO;QACN;YACC,IAAI,EAAE,mBAAmB;YACzB,WAAW,EAAE,0BAA0B;YACvC,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC;SACzC;QACD,GAAG,CAAC,OAAO,KAAK,SAAS;YACxB,CAAC,CAAC;gBACA;oBACC,IAAI,EAAE,wBAAwB;oBAC9B,WAAW,EAAE,+BAA+B;oBAC5C,UAAU,EAAE,mBAAmB;iBAC/B;aACD;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, CUA_BATCH_TOOL_DESCRIPTION as GEMINI_BATCH_DESCRIPTION, CUA_BATCH_TOOL_NAME as GEMINI_BATCH_TOOL_NAME, createComputerToolDefinitions, createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as GeminiBatchSchema, } from "../common.js";
|
|
2
|
+
export type { CuaAction as GeminiAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as GeminiBatchInput, } from "../common.js";
|
|
3
|
+
export declare const COMPUTER_TOOL_COORDINATES: {
|
|
4
|
+
readonly type: "normalized";
|
|
5
|
+
readonly range: readonly [0, 999];
|
|
6
|
+
};
|
|
7
|
+
export declare const GEMINI_INSTRUCTIONS_RAW = "You control a Kernel cloud browser through computer-use tools. Use pixel coordinates, batch predictable action sequences, and request screenshots or URL reads when state changes.";
|
|
8
|
+
export declare function buildGeminiSystemPrompt(opts?: {
|
|
9
|
+
suffix?: string;
|
|
10
|
+
}): string;
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/providers/gemini/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,gBAAgB,IAAI,uBAAuB,EAC3C,0BAA0B,IAAI,wBAAwB,EACtD,mBAAmB,IAAI,sBAAsB,EAC7C,6BAA6B,EAC7B,qBAAqB,IAAI,kBAAkB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,cAAc,IAAI,iBAAiB,GACnC,MAAM,WAAW,CAAC;AACnB,YAAY,EACX,SAAS,IAAI,YAAY,EACzB,oCAAoC,EACpC,aAAa,IAAI,gBAAgB,GACjC,MAAM,WAAW,CAAC;AASnB,eAAO,MAAM,yBAAyB;;;CAA0F,CAAC;AAEjI,eAAO,MAAM,uBAAuB,uLAAuL,CAAC;AAE5N,wBAAgB,uBAAuB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,MAAM,CAE9E"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, CUA_BATCH_TOOL_DESCRIPTION as GEMINI_BATCH_DESCRIPTION, CUA_BATCH_TOOL_NAME as GEMINI_BATCH_TOOL_NAME, createComputerToolDefinitions, createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as GeminiBatchSchema, } from "../common.js";
|
|
2
|
+
// Provider-native function names emitted on `functionCall.name` (PREDEFINED_COMPUTER_USE_FUNCTIONS):
|
|
3
|
+
// open_web_browser, click_at, hover_at, type_text_at, scroll_document,
|
|
4
|
+
// scroll_at, wait_5_seconds, go_back, go_forward, search, navigate,
|
|
5
|
+
// key_combination, drag_and_drop
|
|
6
|
+
// Coordinates are normalized to 0-999 regardless of input image size.
|
|
7
|
+
// Source: https://github.com/google/computer-use-preview/blob/main/agent.py
|
|
8
|
+
// Docs: https://ai.google.dev/gemini-api/docs/computer-use
|
|
9
|
+
export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] };
|
|
10
|
+
export const GEMINI_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through computer-use tools. Use pixel coordinates, batch predictable action sequences, and request screenshots or URL reads when state changes.`;
|
|
11
|
+
export function buildGeminiSystemPrompt(opts = {}) {
|
|
12
|
+
return [GEMINI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n");
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/providers/gemini/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,gBAAgB,IAAI,uBAAuB,EAC3C,0BAA0B,IAAI,wBAAwB,EACtD,mBAAmB,IAAI,sBAAsB,EAC7C,6BAA6B,EAC7B,qBAAqB,IAAI,kBAAkB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,cAAc,IAAI,iBAAiB,GACnC,MAAM,WAAW,CAAC;AAOnB,qGAAqG;AACrG,yEAAyE;AACzE,sEAAsE;AACtE,mCAAmC;AACnC,sEAAsE;AACtE,4EAA4E;AAC5E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAkD,CAAC;AAEjI,MAAM,CAAC,MAAM,uBAAuB,GAAG,oLAAoL,CAAC;AAE5N,MAAM,UAAU,uBAAuB,CAAC,OAA4B,EAAE;IACrE,OAAO,CAAC,uBAAuB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5E,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, CUA_BATCH_TOOL_DESCRIPTION as OPENAI_BATCH_DESCRIPTION, CUA_BATCH_TOOL_NAME as OPENAI_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_DESCRIPTION as OPENAI_EXTRA_TOOL_DESCRIPTION, CUA_NAVIGATION_TOOL_NAME as OPENAI_EXTRA_TOOL_NAME, createComputerToolDefinitions, createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as OpenAIBatchSchema, CuaNavigationSchema as OpenAIExtraSchema, } from "../common";
|
|
2
|
+
export type { CuaAction as OpenAIAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as OpenAIBatchInput, CuaNavigationInput as OpenAIExtraInput, } from "../common";
|
|
3
|
+
export declare const COMPUTER_TOOL_COORDINATES: {
|
|
4
|
+
readonly type: "pixel";
|
|
5
|
+
};
|
|
6
|
+
export declare const OPENAI_BATCH_INSTRUCTIONS = "You have two browser tools:\n1. batch_computer_actions for click, double_click, mouse_down, mouse_up, type, keypress, scroll, move, drag, wait, goto, back, forward, url, cursor_position, and screenshot.\n2. computer_use_extra for a single high-level goto, back, forward, or url action.\n\nPrefer batch_computer_actions for predictable multi-step browser interaction. Include explicit url(), cursor_position(), or screenshot() read steps when you need intermediate state.";
|
|
7
|
+
export declare function openaiResponsesStoreOnPayload(payload: unknown): unknown | undefined;
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/providers/openai/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,gBAAgB,IAAI,uBAAuB,EAC3C,0BAA0B,IAAI,wBAAwB,EACtD,mBAAmB,IAAI,sBAAsB,EAC7C,+BAA+B,IAAI,6BAA6B,EAChE,wBAAwB,IAAI,sBAAsB,EAClD,6BAA6B,EAC7B,qBAAqB,IAAI,kBAAkB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,cAAc,IAAI,iBAAiB,EACnC,mBAAmB,IAAI,iBAAiB,GACxC,MAAM,WAAW,CAAC;AACnB,YAAY,EACX,SAAS,IAAI,YAAY,EACzB,oCAAoC,EACpC,aAAa,IAAI,gBAAgB,EACjC,kBAAkB,IAAI,gBAAgB,GACtC,MAAM,WAAW,CAAC;AAKnB,eAAO,MAAM,yBAAyB;;CAAoE,CAAC;AAE3G,eAAO,MAAM,yBAAyB,2dAIgJ,CAAC;AAEvL,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,GAAG,SAAS,CAQnF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, CUA_BATCH_TOOL_DESCRIPTION as OPENAI_BATCH_DESCRIPTION, CUA_BATCH_TOOL_NAME as OPENAI_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_DESCRIPTION as OPENAI_EXTRA_TOOL_DESCRIPTION, CUA_NAVIGATION_TOOL_NAME as OPENAI_EXTRA_TOOL_NAME, createComputerToolDefinitions, createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as OpenAIBatchSchema, CuaNavigationSchema as OpenAIExtraSchema, } from "../common";
|
|
2
|
+
// Provider-native action vocabulary emitted on `computer_call.action.type`:
|
|
3
|
+
// click, double_click, drag, move, scroll, type, keypress, wait, screenshot
|
|
4
|
+
// Source: https://github.com/openai/openai-cua-sample-app/blob/main/packages/runner-core/src/responses-loop.ts
|
|
5
|
+
export const COMPUTER_TOOL_COORDINATES = { type: "pixel" };
|
|
6
|
+
export const OPENAI_BATCH_INSTRUCTIONS = `You have two browser tools:
|
|
7
|
+
1. batch_computer_actions for click, double_click, mouse_down, mouse_up, type, keypress, scroll, move, drag, wait, goto, back, forward, url, cursor_position, and screenshot.
|
|
8
|
+
2. computer_use_extra for a single high-level goto, back, forward, or url action.
|
|
9
|
+
|
|
10
|
+
Prefer batch_computer_actions for predictable multi-step browser interaction. Include explicit url(), cursor_position(), or screenshot() read steps when you need intermediate state.`;
|
|
11
|
+
export function openaiResponsesStoreOnPayload(payload) {
|
|
12
|
+
if (!payload || typeof payload !== "object")
|
|
13
|
+
return undefined;
|
|
14
|
+
const current = payload;
|
|
15
|
+
if (current.store === true)
|
|
16
|
+
return undefined;
|
|
17
|
+
return {
|
|
18
|
+
...current,
|
|
19
|
+
store: true,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/providers/openai/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,gBAAgB,IAAI,uBAAuB,EAC3C,0BAA0B,IAAI,wBAAwB,EACtD,mBAAmB,IAAI,sBAAsB,EAC7C,+BAA+B,IAAI,6BAA6B,EAChE,wBAAwB,IAAI,sBAAsB,EAClD,6BAA6B,EAC7B,qBAAqB,IAAI,kBAAkB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,cAAc,IAAI,iBAAiB,EACnC,mBAAmB,IAAI,iBAAiB,GACxC,MAAM,WAAW,CAAC;AAQnB,4EAA4E;AAC5E,8EAA8E;AAC9E,+GAA+G;AAC/G,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,IAAI,EAAE,OAAO,EAAkD,CAAC;AAE3G,MAAM,CAAC,MAAM,yBAAyB,GAAG;;;;sLAI6I,CAAC;AAEvL,MAAM,UAAU,6BAA6B,CAAC,OAAgB;IAC7D,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC9D,MAAM,OAAO,GAAG,OAAkC,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAC7C,OAAO;QACN,GAAG,OAAO;QACV,KAAK,EAAE,IAAI;KACX,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, CUA_BATCH_TOOL_DESCRIPTION as TZAFON_BATCH_DESCRIPTION, CUA_BATCH_TOOL_NAME as TZAFON_BATCH_TOOL_NAME, createComputerToolDefinitions, createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as TzafonBatchSchema, } from "../common.js";
|
|
2
|
+
export type { CuaAction as TzafonAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as TzafonBatchInput, } from "../common.js";
|
|
3
|
+
export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses, } from "./provider.js";
|
|
4
|
+
export declare const COMPUTER_TOOL_COORDINATES: {
|
|
5
|
+
readonly type: "normalized";
|
|
6
|
+
readonly range: readonly [0, 999];
|
|
7
|
+
};
|
|
8
|
+
export declare const TZAFON_INSTRUCTIONS_RAW = "You control a Kernel cloud browser. Prefer batched computer actions for browser interaction and include screenshot or URL reads when you need updated state.";
|
|
9
|
+
export declare function buildTzafonSystemPrompt(opts?: {
|
|
10
|
+
suffix?: string;
|
|
11
|
+
}): string;
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/providers/tzafon/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,gBAAgB,IAAI,mBAAmB,EACvC,0BAA0B,IAAI,wBAAwB,EACtD,mBAAmB,IAAI,sBAAsB,EAC7C,6BAA6B,EAC7B,qBAAqB,IAAI,kBAAkB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,cAAc,IAAI,iBAAiB,GACnC,MAAM,WAAW,CAAC;AACnB,YAAY,EACX,SAAS,IAAI,YAAY,EACzB,oCAAoC,EACpC,aAAa,IAAI,gBAAgB,GACjC,MAAM,WAAW,CAAC;AACnB,OAAO,EACN,oBAAoB,EACpB,2BAA2B,EAC3B,qBAAqB,GACrB,MAAM,YAAY,CAAC;AAYpB,eAAO,MAAM,yBAAyB;;;CAA0F,CAAC;AAEjI,eAAO,MAAM,uBAAuB,iKAAiK,CAAC;AAEtM,wBAAgB,uBAAuB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,MAAM,CAE9E"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, CUA_BATCH_TOOL_DESCRIPTION as TZAFON_BATCH_DESCRIPTION, CUA_BATCH_TOOL_NAME as TZAFON_BATCH_TOOL_NAME, createComputerToolDefinitions, createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as TzafonBatchSchema, } from "../common.js";
|
|
2
|
+
export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses, } from "./provider.js";
|
|
3
|
+
// Provider-native action vocabulary. The model card lists supported actions;
|
|
4
|
+
// the Responses API loop dispatches on `action.type` and adds terminal control
|
|
5
|
+
// types (`answer`, `done`).
|
|
6
|
+
// Model actions: click, double_click, triple_click, right_click, drag, type,
|
|
7
|
+
// key, scroll, hscroll, navigate, wait, terminate
|
|
8
|
+
// Responses API `action.type` also includes: keypress, answer, done
|
|
9
|
+
// Sources:
|
|
10
|
+
// https://huggingface.co/Tzafon/Northstar-CUA-Fast
|
|
11
|
+
// https://docs.lightcone.ai/guides/cua-protocol/
|
|
12
|
+
// https://docs.lightcone.ai/guides/coordinates/
|
|
13
|
+
export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] };
|
|
14
|
+
export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser. Prefer batched computer actions for browser interaction and include screenshot or URL reads when you need updated state.`;
|
|
15
|
+
export function buildTzafonSystemPrompt(opts = {}) {
|
|
16
|
+
return [TZAFON_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n");
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/providers/tzafon/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,gBAAgB,IAAI,mBAAmB,EACvC,0BAA0B,IAAI,wBAAwB,EACtD,mBAAmB,IAAI,sBAAsB,EAC7C,6BAA6B,EAC7B,qBAAqB,IAAI,kBAAkB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,cAAc,IAAI,iBAAiB,GACnC,MAAM,WAAW,CAAC;AAMnB,OAAO,EACN,oBAAoB,EACpB,2BAA2B,EAC3B,qBAAqB,GACrB,MAAM,YAAY,CAAC;AAEpB,6EAA6E;AAC7E,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAC/E,mEAAmE;AACnE,sEAAsE;AACtE,WAAW;AACX,qDAAqD;AACrD,mDAAmD;AACnD,kDAAkD;AAClD,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAkD,CAAC;AAEjI,MAAM,CAAC,MAAM,uBAAuB,GAAG,8JAA8J,CAAC;AAEtM,MAAM,UAAU,uBAAuB,CAAC,OAA4B,EAAE;IACrE,OAAO,CAAC,uBAAuB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5E,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type SimpleStreamOptions, type StreamFunction, type StreamOptions } from "@earendil-works/pi-ai";
|
|
2
|
+
export declare const TZAFON_RESPONSES_API = "tzafon-responses";
|
|
3
|
+
export interface TzafonResponsesOptions extends StreamOptions {
|
|
4
|
+
maxOutputTokens?: number;
|
|
5
|
+
}
|
|
6
|
+
export declare const streamSimpleTzafonResponses: StreamFunction<string, SimpleStreamOptions>;
|
|
7
|
+
export declare const streamTzafonResponses: StreamFunction<string, TzafonResponsesOptions>;
|
|
8
|
+
//# sourceMappingURL=provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/providers/tzafon/provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAON,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,EAIlB,MAAM,uBAAuB,CAAC;AAG/B,eAAO,MAAM,oBAAoB,qBAAqB,CAAC;AAEvD,MAAM,WAAW,sBAAuB,SAAQ,aAAa;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,2BAA2B,EAAE,cAAc,CAAC,MAAM,EAAE,mBAAmB,CAEnF,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,cAAc,CAAC,MAAM,EAAE,sBAAsB,CAsDhF,CAAC"}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { createAssistantMessageEventStream, } from "@earendil-works/pi-ai";
|
|
2
|
+
import Lightcone from "@tzafon/lightcone";
|
|
3
|
+
export const TZAFON_RESPONSES_API = "tzafon-responses";
|
|
4
|
+
export const streamSimpleTzafonResponses = (model, context, options) => {
|
|
5
|
+
return streamTzafonResponses(model, context, options);
|
|
6
|
+
};
|
|
7
|
+
export const streamTzafonResponses = (model, context, options) => {
|
|
8
|
+
const stream = createAssistantMessageEventStream();
|
|
9
|
+
const output = initialAssistantMessage(model);
|
|
10
|
+
void (async () => {
|
|
11
|
+
try {
|
|
12
|
+
const apiKey = options?.apiKey || process.env.TZAFON_API_KEY;
|
|
13
|
+
if (!apiKey)
|
|
14
|
+
throw new Error(`No API key for provider: ${model.provider}`);
|
|
15
|
+
const client = new Lightcone({ apiKey });
|
|
16
|
+
const payload = {
|
|
17
|
+
model: model.id,
|
|
18
|
+
input: convertContextMessages(context),
|
|
19
|
+
tools: convertTools(context.tools ?? []),
|
|
20
|
+
instructions: context.systemPrompt,
|
|
21
|
+
temperature: options?.temperature ?? 0,
|
|
22
|
+
max_output_tokens: options?.maxOutputTokens ?? options?.maxTokens ?? model.maxTokens,
|
|
23
|
+
};
|
|
24
|
+
const nextPayload = await options?.onPayload?.(payload, model);
|
|
25
|
+
if (options?.signal?.aborted)
|
|
26
|
+
throw new Error("Request was aborted");
|
|
27
|
+
const response = await client.responses.create((nextPayload ?? payload), { signal: options?.signal });
|
|
28
|
+
if (options?.signal?.aborted)
|
|
29
|
+
throw new Error("Request was aborted");
|
|
30
|
+
stream.push({ type: "start", partial: output });
|
|
31
|
+
output.responseId = getString(response, "id") || undefined;
|
|
32
|
+
output.usage = usageFromTzafon(getValue(response, "usage"));
|
|
33
|
+
for (const item of getArray(response, "output")) {
|
|
34
|
+
const type = getString(item, "type");
|
|
35
|
+
if (type === "message") {
|
|
36
|
+
const text = extractMessageText(item);
|
|
37
|
+
if (text)
|
|
38
|
+
emitText(stream, output, text);
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (type === "function_call") {
|
|
42
|
+
emitToolCall(stream, output, {
|
|
43
|
+
type: "toolCall",
|
|
44
|
+
id: getString(item, "call_id"),
|
|
45
|
+
name: getString(item, "name"),
|
|
46
|
+
arguments: parseArguments(getValue(item, "arguments")),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
output.stopReason = output.content.some((part) => part.type === "toolCall") ? "toolUse" : "stop";
|
|
51
|
+
stream.push({ type: "done", reason: output.stopReason, message: output });
|
|
52
|
+
stream.end();
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
|
56
|
+
output.errorMessage = err instanceof Error ? err.message : String(err);
|
|
57
|
+
stream.push({ type: "error", reason: output.stopReason, error: output });
|
|
58
|
+
stream.end();
|
|
59
|
+
}
|
|
60
|
+
})();
|
|
61
|
+
return stream;
|
|
62
|
+
};
|
|
63
|
+
function initialAssistantMessage(model) {
|
|
64
|
+
return {
|
|
65
|
+
role: "assistant",
|
|
66
|
+
content: [],
|
|
67
|
+
api: model.api,
|
|
68
|
+
provider: model.provider,
|
|
69
|
+
model: model.id,
|
|
70
|
+
usage: {
|
|
71
|
+
input: 0,
|
|
72
|
+
output: 0,
|
|
73
|
+
cacheRead: 0,
|
|
74
|
+
cacheWrite: 0,
|
|
75
|
+
totalTokens: 0,
|
|
76
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
77
|
+
},
|
|
78
|
+
stopReason: "stop",
|
|
79
|
+
timestamp: Date.now(),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function emitText(stream, output, text) {
|
|
83
|
+
const contentIndex = output.content.length;
|
|
84
|
+
const content = { type: "text", text };
|
|
85
|
+
output.content.push(content);
|
|
86
|
+
stream.push({ type: "text_start", contentIndex, partial: output });
|
|
87
|
+
stream.push({ type: "text_delta", contentIndex, delta: text, partial: output });
|
|
88
|
+
stream.push({ type: "text_end", contentIndex, content: text, partial: output });
|
|
89
|
+
}
|
|
90
|
+
function emitToolCall(stream, output, toolCall) {
|
|
91
|
+
const contentIndex = output.content.length;
|
|
92
|
+
output.content.push(toolCall);
|
|
93
|
+
stream.push({ type: "toolcall_start", contentIndex, partial: output });
|
|
94
|
+
stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output });
|
|
95
|
+
}
|
|
96
|
+
function convertTools(tools) {
|
|
97
|
+
return tools.map((tool) => ({
|
|
98
|
+
type: "function",
|
|
99
|
+
name: tool.name,
|
|
100
|
+
description: tool.description,
|
|
101
|
+
parameters: tool.parameters,
|
|
102
|
+
}));
|
|
103
|
+
}
|
|
104
|
+
function convertContextMessages(context) {
|
|
105
|
+
const items = [];
|
|
106
|
+
for (const message of context.messages) {
|
|
107
|
+
if (message.role === "user") {
|
|
108
|
+
items.push({ role: "user", content: convertUserContent(message.content) });
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
if (message.role === "assistant") {
|
|
112
|
+
const text = message.content
|
|
113
|
+
.filter((part) => part.type === "text")
|
|
114
|
+
.map((part) => part.text)
|
|
115
|
+
.join("\n")
|
|
116
|
+
.trim();
|
|
117
|
+
if (text)
|
|
118
|
+
items.push({ role: "assistant", content: text });
|
|
119
|
+
for (const part of message.content) {
|
|
120
|
+
if (part.type !== "toolCall")
|
|
121
|
+
continue;
|
|
122
|
+
items.push({
|
|
123
|
+
type: "function_call",
|
|
124
|
+
call_id: part.id,
|
|
125
|
+
name: part.name,
|
|
126
|
+
arguments: JSON.stringify(part.arguments ?? {}),
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (message.role === "toolResult") {
|
|
132
|
+
const text = message.content
|
|
133
|
+
.filter((part) => part.type === "text")
|
|
134
|
+
.map((part) => part.text)
|
|
135
|
+
.join("\n")
|
|
136
|
+
.trim();
|
|
137
|
+
items.push({
|
|
138
|
+
type: "function_call_output",
|
|
139
|
+
call_id: message.toolCallId,
|
|
140
|
+
output: message.isError ? `Error: ${text || "tool execution failed"}` : text || "ok",
|
|
141
|
+
});
|
|
142
|
+
const image = [...message.content].reverse().find((part) => part.type === "image");
|
|
143
|
+
if (image) {
|
|
144
|
+
items.push({
|
|
145
|
+
role: "user",
|
|
146
|
+
content: [
|
|
147
|
+
{ type: "input_text", text: "screenshot" },
|
|
148
|
+
{ type: "input_image", image_url: `data:${image.mimeType};base64,${image.data}`, detail: "auto" },
|
|
149
|
+
],
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return items;
|
|
155
|
+
}
|
|
156
|
+
function convertUserContent(content) {
|
|
157
|
+
if (typeof content === "string")
|
|
158
|
+
return [{ type: "input_text", text: content }];
|
|
159
|
+
return content.map((part) => {
|
|
160
|
+
if (part.type === "text")
|
|
161
|
+
return { type: "input_text", text: part.text };
|
|
162
|
+
return { type: "input_image", image_url: `data:${part.mimeType};base64,${part.data}`, detail: "auto" };
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
function extractMessageText(item) {
|
|
166
|
+
return getArray(item, "content")
|
|
167
|
+
.map((block) => getString(block, "text"))
|
|
168
|
+
.filter(Boolean)
|
|
169
|
+
.join("\n")
|
|
170
|
+
.trim();
|
|
171
|
+
}
|
|
172
|
+
function parseArguments(value) {
|
|
173
|
+
const top = typeof value === "string" && value.trim()
|
|
174
|
+
? safeJsonParse(value)
|
|
175
|
+
: value && typeof value === "object"
|
|
176
|
+
? value
|
|
177
|
+
: {};
|
|
178
|
+
if (!top || typeof top !== "object")
|
|
179
|
+
return {};
|
|
180
|
+
// Tzafon sometimes nests JSON-encoded arrays/objects inside the top-level argument object
|
|
181
|
+
// (observed: { "actions": "[{...}]" }). Unwrap one level so consumers get real values.
|
|
182
|
+
const out = {};
|
|
183
|
+
for (const [key, val] of Object.entries(top)) {
|
|
184
|
+
out[key] = typeof val === "string" && looksLikeJson(val) ? safeJsonParse(val) ?? val : val;
|
|
185
|
+
}
|
|
186
|
+
return out;
|
|
187
|
+
}
|
|
188
|
+
function safeJsonParse(value) {
|
|
189
|
+
try {
|
|
190
|
+
const parsed = JSON.parse(value);
|
|
191
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
return null;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
function looksLikeJson(value) {
|
|
198
|
+
const trimmed = value.trim();
|
|
199
|
+
return trimmed.startsWith("[") || trimmed.startsWith("{");
|
|
200
|
+
}
|
|
201
|
+
function usageFromTzafon(usage) {
|
|
202
|
+
const input = readNumber(usage, "input_tokens");
|
|
203
|
+
const output = readNumber(usage, "output_tokens");
|
|
204
|
+
const cacheRead = readNumber(getValue(usage, "input_tokens_details"), "cached_tokens");
|
|
205
|
+
const totalTokens = readNumber(usage, "total_tokens") || input + output;
|
|
206
|
+
return {
|
|
207
|
+
input,
|
|
208
|
+
output,
|
|
209
|
+
cacheRead,
|
|
210
|
+
cacheWrite: 0,
|
|
211
|
+
totalTokens,
|
|
212
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
function readNumber(obj, key) {
|
|
216
|
+
if (!obj || typeof obj !== "object")
|
|
217
|
+
return 0;
|
|
218
|
+
const n = obj[key];
|
|
219
|
+
return typeof n === "number" && Number.isFinite(n) ? n : 0;
|
|
220
|
+
}
|
|
221
|
+
function getArray(obj, key) {
|
|
222
|
+
const value = getValue(obj, key);
|
|
223
|
+
return Array.isArray(value) ? value : [];
|
|
224
|
+
}
|
|
225
|
+
function getString(obj, key) {
|
|
226
|
+
const value = getValue(obj, key);
|
|
227
|
+
return typeof value === "string" ? value : "";
|
|
228
|
+
}
|
|
229
|
+
function getValue(obj, key) {
|
|
230
|
+
if (!obj || typeof obj !== "object")
|
|
231
|
+
return undefined;
|
|
232
|
+
return obj[key];
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=provider.js.map
|