@agents-at-scale/ark 0.1.40 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arkServices.js +12 -0
- package/dist/commands/completion/index.js +11 -1
- package/dist/commands/evaluation/index.d.ts +3 -0
- package/dist/commands/evaluation/index.js +60 -0
- package/dist/commands/evaluation/index.spec.d.ts +1 -0
- package/dist/commands/evaluation/index.spec.js +166 -0
- package/dist/commands/memory/index.d.ts +15 -0
- package/dist/commands/memory/index.js +130 -0
- package/dist/commands/memory/index.spec.d.ts +1 -0
- package/dist/commands/memory/index.spec.js +124 -0
- package/dist/commands/models/create.d.ts +5 -6
- package/dist/commands/models/create.js +14 -119
- package/dist/commands/models/create.spec.js +47 -0
- package/dist/commands/models/kubernetes/manifest-builder.d.ts +11 -0
- package/dist/commands/models/kubernetes/manifest-builder.js +109 -0
- package/dist/commands/models/kubernetes/secret-manager.d.ts +7 -0
- package/dist/commands/models/kubernetes/secret-manager.js +20 -0
- package/dist/commands/models/providers/azure.d.ts +31 -0
- package/dist/commands/models/providers/azure.js +82 -0
- package/dist/commands/models/providers/azure.spec.d.ts +1 -0
- package/dist/commands/models/providers/azure.spec.js +230 -0
- package/dist/commands/models/providers/bedrock.d.ts +37 -0
- package/dist/commands/models/providers/bedrock.js +105 -0
- package/dist/commands/models/providers/bedrock.spec.d.ts +1 -0
- package/dist/commands/models/providers/bedrock.spec.js +241 -0
- package/dist/commands/models/providers/factory.d.ts +18 -0
- package/dist/commands/models/providers/factory.js +31 -0
- package/dist/commands/models/providers/index.d.ts +17 -0
- package/dist/commands/models/providers/index.js +9 -0
- package/dist/commands/models/providers/openai.d.ts +28 -0
- package/dist/commands/models/providers/openai.js +68 -0
- package/dist/commands/models/providers/openai.spec.d.ts +1 -0
- package/dist/commands/models/providers/openai.spec.js +180 -0
- package/dist/commands/models/providers/types.d.ts +51 -0
- package/dist/commands/models/providers/types.js +1 -0
- package/dist/commands/queries/index.d.ts +3 -0
- package/dist/commands/queries/index.js +70 -0
- package/dist/commands/query/index.js +3 -1
- package/dist/commands/query/index.spec.js +24 -0
- package/dist/components/ChatUI.js +82 -16
- package/dist/index.js +6 -0
- package/dist/lib/arkApiClient.d.ts +4 -0
- package/dist/lib/arkApiClient.js +55 -0
- package/dist/lib/errors.d.ts +1 -0
- package/dist/lib/errors.js +1 -0
- package/dist/lib/executeEvaluation.d.ts +16 -0
- package/dist/lib/executeEvaluation.js +155 -0
- package/dist/lib/executeQuery.d.ts +1 -0
- package/dist/lib/executeQuery.js +48 -10
- package/dist/lib/executeQuery.spec.js +3 -3
- package/dist/lib/kubectl.d.ts +8 -0
- package/dist/lib/kubectl.js +20 -0
- package/dist/lib/kubectl.spec.d.ts +1 -0
- package/dist/lib/kubectl.spec.js +88 -0
- package/dist/lib/stdin.d.ts +1 -0
- package/dist/lib/stdin.js +16 -0
- package/dist/lib/stdin.spec.d.ts +1 -0
- package/dist/lib/stdin.spec.js +82 -0
- package/dist/lib/types.d.ts +39 -0
- package/dist/ui/TargetSelector.d.ts +19 -0
- package/dist/ui/TargetSelector.js +48 -0
- package/package.json +2 -1
- package/dist/ui/AgentSelector.d.ts +0 -8
- package/dist/ui/AgentSelector.js +0 -53
- package/dist/ui/ModelSelector.d.ts +0 -8
- package/dist/ui/ModelSelector.js +0 -53
- package/dist/ui/TeamSelector.d.ts +0 -8
- package/dist/ui/TeamSelector.js +0 -55
- package/dist/ui/ToolSelector.d.ts +0 -8
- package/dist/ui/ToolSelector.js +0 -53
|
@@ -7,10 +7,7 @@ import { marked } from 'marked';
|
|
|
7
7
|
// @ts-ignore - no types available
|
|
8
8
|
import TerminalRenderer from 'marked-terminal';
|
|
9
9
|
import { APIError } from 'openai';
|
|
10
|
-
import {
|
|
11
|
-
import { ModelSelector } from '../ui/ModelSelector.js';
|
|
12
|
-
import { TeamSelector } from '../ui/TeamSelector.js';
|
|
13
|
-
import { ToolSelector } from '../ui/ToolSelector.js';
|
|
10
|
+
import { TargetSelector } from '../ui/TargetSelector.js';
|
|
14
11
|
import { useAsyncOperation, AsyncOperationStatus } from './AsyncOperation.js';
|
|
15
12
|
import { createConnectingToArkOperation } from '../ui/asyncOperations/connectingToArk.js';
|
|
16
13
|
// Generate a unique ID for messages
|
|
@@ -21,10 +18,12 @@ const generateMessageId = () => {
|
|
|
21
18
|
// Configure marked with terminal renderer for markdown output
|
|
22
19
|
const configureMarkdown = () => {
|
|
23
20
|
marked.setOptions({
|
|
21
|
+
// @ts-expect-error - TerminalRenderer types are incomplete
|
|
24
22
|
renderer: new TerminalRenderer({
|
|
25
23
|
showSectionPrefix: false,
|
|
26
24
|
width: 80,
|
|
27
25
|
reflowText: true,
|
|
26
|
+
// @ts-expect-error - preserveNewlines exists but not in types
|
|
28
27
|
preserveNewlines: true,
|
|
29
28
|
}),
|
|
30
29
|
});
|
|
@@ -48,6 +47,12 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
48
47
|
const [showModelSelector, setShowModelSelector] = React.useState(false);
|
|
49
48
|
const [showTeamSelector, setShowTeamSelector] = React.useState(false);
|
|
50
49
|
const [showToolSelector, setShowToolSelector] = React.useState(false);
|
|
50
|
+
const [agents, setAgents] = React.useState([]);
|
|
51
|
+
const [models, setModels] = React.useState([]);
|
|
52
|
+
const [teams, setTeams] = React.useState([]);
|
|
53
|
+
const [tools, setTools] = React.useState([]);
|
|
54
|
+
const [selectorLoading, setSelectorLoading] = React.useState(false);
|
|
55
|
+
const [selectorError, setSelectorError] = React.useState(undefined);
|
|
51
56
|
// Message history navigation
|
|
52
57
|
const [messageHistory, setMessageHistory] = React.useState([]);
|
|
53
58
|
const [historyIndex, setHistoryIndex] = React.useState(-1);
|
|
@@ -56,6 +61,50 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
56
61
|
streamingEnabled: config?.chat?.streaming ?? true,
|
|
57
62
|
currentTarget: undefined,
|
|
58
63
|
});
|
|
64
|
+
React.useEffect(() => {
|
|
65
|
+
if (showAgentSelector && agents.length === 0) {
|
|
66
|
+
setSelectorLoading(true);
|
|
67
|
+
setSelectorError(undefined);
|
|
68
|
+
arkApiClient
|
|
69
|
+
.getAgents()
|
|
70
|
+
.then(setAgents)
|
|
71
|
+
.catch((err) => setSelectorError(err.message))
|
|
72
|
+
.finally(() => setSelectorLoading(false));
|
|
73
|
+
}
|
|
74
|
+
}, [showAgentSelector, arkApiClient, agents.length]);
|
|
75
|
+
React.useEffect(() => {
|
|
76
|
+
if (showModelSelector && models.length === 0) {
|
|
77
|
+
setSelectorLoading(true);
|
|
78
|
+
setSelectorError(undefined);
|
|
79
|
+
arkApiClient
|
|
80
|
+
.getModels()
|
|
81
|
+
.then(setModels)
|
|
82
|
+
.catch((err) => setSelectorError(err.message))
|
|
83
|
+
.finally(() => setSelectorLoading(false));
|
|
84
|
+
}
|
|
85
|
+
}, [showModelSelector, arkApiClient, models.length]);
|
|
86
|
+
React.useEffect(() => {
|
|
87
|
+
if (showTeamSelector && teams.length === 0) {
|
|
88
|
+
setSelectorLoading(true);
|
|
89
|
+
setSelectorError(undefined);
|
|
90
|
+
arkApiClient
|
|
91
|
+
.getTeams()
|
|
92
|
+
.then(setTeams)
|
|
93
|
+
.catch((err) => setSelectorError(err.message))
|
|
94
|
+
.finally(() => setSelectorLoading(false));
|
|
95
|
+
}
|
|
96
|
+
}, [showTeamSelector, arkApiClient, teams.length]);
|
|
97
|
+
React.useEffect(() => {
|
|
98
|
+
if (showToolSelector && tools.length === 0) {
|
|
99
|
+
setSelectorLoading(true);
|
|
100
|
+
setSelectorError(undefined);
|
|
101
|
+
arkApiClient
|
|
102
|
+
.getTools()
|
|
103
|
+
.then(setTools)
|
|
104
|
+
.catch((err) => setSelectorError(err.message))
|
|
105
|
+
.finally(() => setSelectorLoading(false));
|
|
106
|
+
}
|
|
107
|
+
}, [showToolSelector, arkApiClient, tools.length]);
|
|
59
108
|
const chatClientRef = React.useRef(undefined);
|
|
60
109
|
// Configure markdown when output format changes
|
|
61
110
|
React.useEffect(() => {
|
|
@@ -608,8 +657,10 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
608
657
|
}
|
|
609
658
|
// Show agent selector if requested
|
|
610
659
|
if (showAgentSelector) {
|
|
611
|
-
return (_jsx(
|
|
612
|
-
|
|
660
|
+
return (_jsx(TargetSelector, { targets: agents, title: "Select Agent", subtitle: "Choose an agent to start a conversation with", loading: selectorLoading, error: selectorError, formatInlineDetail: (t) => t.description, showDetailPanel: true, onSelect: (target) => {
|
|
661
|
+
const agent = agents.find((a) => a.name === target.name);
|
|
662
|
+
if (!agent)
|
|
663
|
+
return;
|
|
613
664
|
const agentTarget = {
|
|
614
665
|
id: `agent/${agent.name}`,
|
|
615
666
|
name: agent.name,
|
|
@@ -620,7 +671,6 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
620
671
|
setChatConfig((prev) => ({ ...prev, currentTarget: agentTarget }));
|
|
621
672
|
setMessages([]);
|
|
622
673
|
setShowAgentSelector(false);
|
|
623
|
-
// Add system message about the selection
|
|
624
674
|
const systemMessage = {
|
|
625
675
|
id: generateMessageId(),
|
|
626
676
|
type: 'system',
|
|
@@ -633,8 +683,18 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
633
683
|
}
|
|
634
684
|
// Show model selector if requested
|
|
635
685
|
if (showModelSelector) {
|
|
636
|
-
return (_jsx(
|
|
637
|
-
|
|
686
|
+
return (_jsx(TargetSelector, { targets: models, title: "Select Model", subtitle: "Choose a model to start a conversation with", loading: selectorLoading, error: selectorError, formatLabel: (t) => {
|
|
687
|
+
const model = models.find((m) => m.name === t.name);
|
|
688
|
+
return model
|
|
689
|
+
? `${model.name}${model.type ? ` (${model.type})` : ''}`
|
|
690
|
+
: t.name;
|
|
691
|
+
}, formatInlineDetail: (t) => {
|
|
692
|
+
const model = models.find((m) => m.name === t.name);
|
|
693
|
+
return model?.model;
|
|
694
|
+
}, showDetailPanel: false, onSelect: (target) => {
|
|
695
|
+
const model = models.find((m) => m.name === target.name);
|
|
696
|
+
if (!model)
|
|
697
|
+
return;
|
|
638
698
|
const modelTarget = {
|
|
639
699
|
id: `model/${model.name}`,
|
|
640
700
|
name: model.name,
|
|
@@ -645,7 +705,6 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
645
705
|
setChatConfig((prev) => ({ ...prev, currentTarget: modelTarget }));
|
|
646
706
|
setMessages([]);
|
|
647
707
|
setShowModelSelector(false);
|
|
648
|
-
// Add system message about the selection
|
|
649
708
|
const systemMessage = {
|
|
650
709
|
id: generateMessageId(),
|
|
651
710
|
type: 'system',
|
|
@@ -658,8 +717,15 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
658
717
|
}
|
|
659
718
|
// Show team selector if requested
|
|
660
719
|
if (showTeamSelector) {
|
|
661
|
-
return (_jsx(
|
|
662
|
-
|
|
720
|
+
return (_jsx(TargetSelector, { targets: teams, title: "Select Team", subtitle: "Choose a team to start a conversation with", loading: selectorLoading, error: selectorError, formatLabel: (t) => {
|
|
721
|
+
const team = teams.find((tm) => tm.name === t.name);
|
|
722
|
+
return team
|
|
723
|
+
? `${team.name}${team.strategy ? ` (${team.strategy})` : ''}`
|
|
724
|
+
: t.name;
|
|
725
|
+
}, formatInlineDetail: (t) => t.description, showDetailPanel: true, onSelect: (target) => {
|
|
726
|
+
const team = teams.find((tm) => tm.name === target.name);
|
|
727
|
+
if (!team)
|
|
728
|
+
return;
|
|
663
729
|
const teamTarget = {
|
|
664
730
|
id: `team/${team.name}`,
|
|
665
731
|
name: team.name,
|
|
@@ -670,7 +736,6 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
670
736
|
setChatConfig((prev) => ({ ...prev, currentTarget: teamTarget }));
|
|
671
737
|
setMessages([]);
|
|
672
738
|
setShowTeamSelector(false);
|
|
673
|
-
// Add system message about the selection
|
|
674
739
|
const systemMessage = {
|
|
675
740
|
id: generateMessageId(),
|
|
676
741
|
type: 'system',
|
|
@@ -683,8 +748,10 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
683
748
|
}
|
|
684
749
|
// Show tool selector if requested
|
|
685
750
|
if (showToolSelector) {
|
|
686
|
-
return (_jsx(
|
|
687
|
-
|
|
751
|
+
return (_jsx(TargetSelector, { targets: tools, title: "Select Tool", subtitle: "Choose a tool to start a conversation with", loading: selectorLoading, error: selectorError, formatInlineDetail: (t) => t.description, showDetailPanel: true, onSelect: (target) => {
|
|
752
|
+
const tool = tools.find((tl) => tl.name === target.name);
|
|
753
|
+
if (!tool)
|
|
754
|
+
return;
|
|
688
755
|
const toolTarget = {
|
|
689
756
|
id: `tool/${tool.name}`,
|
|
690
757
|
name: tool.name,
|
|
@@ -695,7 +762,6 @@ const ChatUI = ({ initialTargetId, arkApiClient, arkApiProxy, config, }) => {
|
|
|
695
762
|
setChatConfig((prev) => ({ ...prev, currentTarget: toolTarget }));
|
|
696
763
|
setMessages([]);
|
|
697
764
|
setShowToolSelector(false);
|
|
698
|
-
// Add system message about the selection
|
|
699
765
|
const systemMessage = {
|
|
700
766
|
id: generateMessageId(),
|
|
701
767
|
type: 'system',
|
package/dist/index.js
CHANGED
|
@@ -13,10 +13,13 @@ import { createClusterCommand } from './commands/cluster/index.js';
|
|
|
13
13
|
import { createCompletionCommand } from './commands/completion/index.js';
|
|
14
14
|
import { createDashboardCommand } from './commands/dashboard/index.js';
|
|
15
15
|
import { createDocsCommand } from './commands/docs/index.js';
|
|
16
|
+
import { createEvaluationCommand } from './commands/evaluation/index.js';
|
|
16
17
|
import { createGenerateCommand } from './commands/generate/index.js';
|
|
17
18
|
import { createInstallCommand } from './commands/install/index.js';
|
|
19
|
+
import { createMemoryCommand } from './commands/memory/index.js';
|
|
18
20
|
import { createModelsCommand } from './commands/models/index.js';
|
|
19
21
|
import { createQueryCommand } from './commands/query/index.js';
|
|
22
|
+
import { createQueriesCommand } from './commands/queries/index.js';
|
|
20
23
|
import { createUninstallCommand } from './commands/uninstall/index.js';
|
|
21
24
|
import { createStatusCommand } from './commands/status/index.js';
|
|
22
25
|
import { createConfigCommand } from './commands/config/index.js';
|
|
@@ -43,10 +46,13 @@ async function main() {
|
|
|
43
46
|
program.addCommand(createCompletionCommand(config));
|
|
44
47
|
program.addCommand(createDashboardCommand(config));
|
|
45
48
|
program.addCommand(createDocsCommand(config));
|
|
49
|
+
program.addCommand(createEvaluationCommand(config));
|
|
46
50
|
program.addCommand(createGenerateCommand(config));
|
|
47
51
|
program.addCommand(createInstallCommand(config));
|
|
52
|
+
program.addCommand(createMemoryCommand(config));
|
|
48
53
|
program.addCommand(createModelsCommand(config));
|
|
49
54
|
program.addCommand(createQueryCommand(config));
|
|
55
|
+
program.addCommand(createQueriesCommand(config));
|
|
50
56
|
program.addCommand(createUninstallCommand(config));
|
|
51
57
|
program.addCommand(createStatusCommand());
|
|
52
58
|
program.addCommand(createConfigCommand(config));
|
|
@@ -47,6 +47,10 @@ export declare class ArkApiClient {
|
|
|
47
47
|
getModels(): Promise<Model[]>;
|
|
48
48
|
getTools(): Promise<Tool[]>;
|
|
49
49
|
getTeams(): Promise<Team[]>;
|
|
50
|
+
getSessions(): Promise<any[]>;
|
|
51
|
+
deleteSession(sessionId: string): Promise<any>;
|
|
52
|
+
deleteQueryMessages(sessionId: string, queryId: string): Promise<any>;
|
|
53
|
+
deleteAllSessions(): Promise<any>;
|
|
50
54
|
createChatCompletion(params: OpenAI.Chat.Completions.ChatCompletionCreateParams): Promise<OpenAI.Chat.Completions.ChatCompletion>;
|
|
51
55
|
createChatCompletionStream(params: OpenAI.Chat.Completions.ChatCompletionCreateParams): AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>;
|
|
52
56
|
}
|
package/dist/lib/arkApiClient.js
CHANGED
|
@@ -84,6 +84,61 @@ export class ArkApiClient {
|
|
|
84
84
|
throw new Error(`Failed to get teams: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
+
async getSessions() {
|
|
88
|
+
try {
|
|
89
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions`);
|
|
90
|
+
if (!response.ok) {
|
|
91
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
92
|
+
}
|
|
93
|
+
const data = (await response.json());
|
|
94
|
+
return data.items || [];
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
throw new Error(`Failed to get sessions: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async deleteSession(sessionId) {
|
|
101
|
+
try {
|
|
102
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions/${sessionId}`, {
|
|
103
|
+
method: 'DELETE',
|
|
104
|
+
});
|
|
105
|
+
if (!response.ok) {
|
|
106
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
107
|
+
}
|
|
108
|
+
return await response.json();
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
throw new Error(`Failed to delete session: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
async deleteQueryMessages(sessionId, queryId) {
|
|
115
|
+
try {
|
|
116
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions/${sessionId}/queries/${queryId}/messages`, {
|
|
117
|
+
method: 'DELETE',
|
|
118
|
+
});
|
|
119
|
+
if (!response.ok) {
|
|
120
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
121
|
+
}
|
|
122
|
+
return await response.json();
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
throw new Error(`Failed to delete query messages: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async deleteAllSessions() {
|
|
129
|
+
try {
|
|
130
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions`, {
|
|
131
|
+
method: 'DELETE',
|
|
132
|
+
});
|
|
133
|
+
if (!response.ok) {
|
|
134
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
135
|
+
}
|
|
136
|
+
return await response.json();
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
throw new Error(`Failed to delete all sessions: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
87
142
|
async createChatCompletion(params) {
|
|
88
143
|
return (await this.openai.chat.completions.create({
|
|
89
144
|
...params,
|
package/dist/lib/errors.d.ts
CHANGED
package/dist/lib/errors.js
CHANGED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface DirectEvaluationOptions {
|
|
2
|
+
evaluatorName: string;
|
|
3
|
+
input: string;
|
|
4
|
+
output: string;
|
|
5
|
+
timeout?: string;
|
|
6
|
+
watchTimeout?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface QueryEvaluationOptions {
|
|
9
|
+
evaluatorName: string;
|
|
10
|
+
queryName: string;
|
|
11
|
+
responseTarget?: string;
|
|
12
|
+
timeout?: string;
|
|
13
|
+
watchTimeout?: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function executeDirectEvaluation(options: DirectEvaluationOptions): Promise<void>;
|
|
16
|
+
export declare function executeQueryEvaluation(options: QueryEvaluationOptions): Promise<void>;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { execa } from 'execa';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
import output from './output.js';
|
|
5
|
+
import { ExitCodes } from './errors.js';
|
|
6
|
+
import { parseDuration } from './duration.js';
|
|
7
|
+
async function waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay) {
|
|
8
|
+
const spinner = ora('Waiting for evaluation completion...').start();
|
|
9
|
+
try {
|
|
10
|
+
await execa('kubectl', [
|
|
11
|
+
'wait',
|
|
12
|
+
'--for=condition=Completed',
|
|
13
|
+
`evaluation/${evaluationName}`,
|
|
14
|
+
`--timeout=${Math.floor(watchTimeoutMs / 1000)}s`,
|
|
15
|
+
], { timeout: watchTimeoutMs });
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
spinner.stop();
|
|
19
|
+
if (error instanceof Error && error.message.includes('timed out waiting')) {
|
|
20
|
+
console.error(chalk.red(`Evaluation did not complete within ${watchTimeoutDisplay}`));
|
|
21
|
+
process.exit(ExitCodes.Timeout);
|
|
22
|
+
}
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
spinner.stop();
|
|
26
|
+
const { stdout } = await execa('kubectl', ['get', 'evaluation', evaluationName, '-o', 'json'], { stdio: 'pipe' });
|
|
27
|
+
const evaluation = JSON.parse(stdout);
|
|
28
|
+
const status = evaluation.status;
|
|
29
|
+
if (status?.phase === 'done') {
|
|
30
|
+
console.log(chalk.green('\nEvaluation completed successfully:'));
|
|
31
|
+
if (status.score !== undefined) {
|
|
32
|
+
console.log(`Score: ${status.score}`);
|
|
33
|
+
}
|
|
34
|
+
if (status.passed !== undefined) {
|
|
35
|
+
console.log(`Result: ${status.passed ? chalk.green('PASSED') : chalk.red('FAILED')}`);
|
|
36
|
+
}
|
|
37
|
+
if (status.message) {
|
|
38
|
+
console.log(`Message: ${status.message}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else if (status?.phase === 'error') {
|
|
42
|
+
console.error(chalk.red(status.message || 'Evaluation failed with unknown error'));
|
|
43
|
+
process.exit(ExitCodes.OperationError);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
output.warning(`Unexpected evaluation phase: ${status?.phase}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export async function executeDirectEvaluation(options) {
|
|
50
|
+
const spinner = ora('Creating evaluation...').start();
|
|
51
|
+
const queryTimeoutMs = options.timeout
|
|
52
|
+
? parseDuration(options.timeout)
|
|
53
|
+
: parseDuration('5m');
|
|
54
|
+
const watchTimeoutMs = options.watchTimeout
|
|
55
|
+
? parseDuration(options.watchTimeout)
|
|
56
|
+
: queryTimeoutMs + 60000;
|
|
57
|
+
const timestamp = Date.now();
|
|
58
|
+
const evaluationName = `cli-eval-${timestamp}`;
|
|
59
|
+
const evaluationManifest = {
|
|
60
|
+
apiVersion: 'ark.mckinsey.com/v1alpha1',
|
|
61
|
+
kind: 'Evaluation',
|
|
62
|
+
metadata: {
|
|
63
|
+
name: evaluationName,
|
|
64
|
+
},
|
|
65
|
+
spec: {
|
|
66
|
+
type: 'direct',
|
|
67
|
+
evaluator: {
|
|
68
|
+
name: options.evaluatorName,
|
|
69
|
+
},
|
|
70
|
+
config: {
|
|
71
|
+
input: options.input,
|
|
72
|
+
output: options.output,
|
|
73
|
+
},
|
|
74
|
+
...(options.timeout && { timeout: options.timeout }),
|
|
75
|
+
ttl: '1h',
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
try {
|
|
79
|
+
spinner.text = 'Submitting evaluation...';
|
|
80
|
+
await execa('kubectl', ['apply', '-f', '-'], {
|
|
81
|
+
input: JSON.stringify(evaluationManifest),
|
|
82
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
83
|
+
});
|
|
84
|
+
spinner.stop();
|
|
85
|
+
const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
|
|
86
|
+
await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
spinner.stop();
|
|
90
|
+
console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
|
|
91
|
+
process.exit(ExitCodes.CliError);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
export async function executeQueryEvaluation(options) {
|
|
95
|
+
const spinner = ora('Creating evaluation...').start();
|
|
96
|
+
const queryTimeoutMs = options.timeout
|
|
97
|
+
? parseDuration(options.timeout)
|
|
98
|
+
: parseDuration('5m');
|
|
99
|
+
const watchTimeoutMs = options.watchTimeout
|
|
100
|
+
? parseDuration(options.watchTimeout)
|
|
101
|
+
: queryTimeoutMs + 60000;
|
|
102
|
+
const timestamp = Date.now();
|
|
103
|
+
const evaluationName = `cli-eval-${timestamp}`;
|
|
104
|
+
let responseTarget;
|
|
105
|
+
if (options.responseTarget) {
|
|
106
|
+
const parts = options.responseTarget.split(':');
|
|
107
|
+
if (parts.length === 2) {
|
|
108
|
+
responseTarget = {
|
|
109
|
+
type: parts[0],
|
|
110
|
+
name: parts[1],
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
spinner.stop();
|
|
115
|
+
console.error(chalk.red('Invalid response-target format. Use: type:name (e.g., agent:my-agent)'));
|
|
116
|
+
process.exit(ExitCodes.CliError);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const evaluationManifest = {
|
|
120
|
+
apiVersion: 'ark.mckinsey.com/v1alpha1',
|
|
121
|
+
kind: 'Evaluation',
|
|
122
|
+
metadata: {
|
|
123
|
+
name: evaluationName,
|
|
124
|
+
},
|
|
125
|
+
spec: {
|
|
126
|
+
type: 'query',
|
|
127
|
+
evaluator: {
|
|
128
|
+
name: options.evaluatorName,
|
|
129
|
+
},
|
|
130
|
+
config: {
|
|
131
|
+
queryRef: {
|
|
132
|
+
name: options.queryName,
|
|
133
|
+
},
|
|
134
|
+
...(responseTarget && { responseTarget }),
|
|
135
|
+
},
|
|
136
|
+
...(options.timeout && { timeout: options.timeout }),
|
|
137
|
+
ttl: '1h',
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
try {
|
|
141
|
+
spinner.text = 'Submitting evaluation...';
|
|
142
|
+
await execa('kubectl', ['apply', '-f', '-'], {
|
|
143
|
+
input: JSON.stringify(evaluationManifest),
|
|
144
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
145
|
+
});
|
|
146
|
+
spinner.stop();
|
|
147
|
+
const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
|
|
148
|
+
await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
spinner.stop();
|
|
152
|
+
console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
|
|
153
|
+
process.exit(ExitCodes.CliError);
|
|
154
|
+
}
|
|
155
|
+
}
|
package/dist/lib/executeQuery.js
CHANGED
|
@@ -7,12 +7,15 @@ import chalk from 'chalk';
|
|
|
7
7
|
import output from './output.js';
|
|
8
8
|
import { ExitCodes } from './errors.js';
|
|
9
9
|
import { parseDuration } from './duration.js';
|
|
10
|
+
import { getResource } from './kubectl.js';
|
|
10
11
|
/**
|
|
11
12
|
* Execute a query against any ARK target (model, agent, team)
|
|
12
13
|
* This is the shared implementation used by all query commands
|
|
13
14
|
*/
|
|
14
15
|
export async function executeQuery(options) {
|
|
15
|
-
const spinner =
|
|
16
|
+
const spinner = options.outputFormat
|
|
17
|
+
? null
|
|
18
|
+
: ora('Creating query...').start();
|
|
16
19
|
const queryTimeoutMs = options.timeout
|
|
17
20
|
? parseDuration(options.timeout)
|
|
18
21
|
: parseDuration('5m');
|
|
@@ -40,13 +43,15 @@ export async function executeQuery(options) {
|
|
|
40
43
|
};
|
|
41
44
|
try {
|
|
42
45
|
// Apply the query
|
|
43
|
-
spinner
|
|
46
|
+
if (spinner)
|
|
47
|
+
spinner.text = 'Submitting query...';
|
|
44
48
|
await execa('kubectl', ['apply', '-f', '-'], {
|
|
45
49
|
input: JSON.stringify(queryManifest),
|
|
46
50
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
47
51
|
});
|
|
48
52
|
// Watch for query completion using kubectl wait
|
|
49
|
-
spinner
|
|
53
|
+
if (spinner)
|
|
54
|
+
spinner.text = 'Waiting for query completion...';
|
|
50
55
|
try {
|
|
51
56
|
await execa('kubectl', [
|
|
52
57
|
'wait',
|
|
@@ -56,7 +61,8 @@ export async function executeQuery(options) {
|
|
|
56
61
|
], { timeout: watchTimeoutMs });
|
|
57
62
|
}
|
|
58
63
|
catch (error) {
|
|
59
|
-
spinner
|
|
64
|
+
if (spinner)
|
|
65
|
+
spinner.stop();
|
|
60
66
|
// Check if it's a timeout or other error
|
|
61
67
|
if (error instanceof Error &&
|
|
62
68
|
error.message.includes('timed out waiting')) {
|
|
@@ -65,11 +71,35 @@ export async function executeQuery(options) {
|
|
|
65
71
|
}
|
|
66
72
|
// For other errors, fetch the query to check status
|
|
67
73
|
}
|
|
68
|
-
spinner
|
|
74
|
+
if (spinner)
|
|
75
|
+
spinner.stop();
|
|
76
|
+
// If output format is specified, output the resource and return
|
|
77
|
+
if (options.outputFormat) {
|
|
78
|
+
try {
|
|
79
|
+
if (options.outputFormat === 'name') {
|
|
80
|
+
console.log(queryName);
|
|
81
|
+
}
|
|
82
|
+
else if (options.outputFormat === 'json' ||
|
|
83
|
+
options.outputFormat === 'yaml') {
|
|
84
|
+
const { stdout } = await execa('kubectl', ['get', 'query', queryName, '-o', options.outputFormat], { stdio: 'pipe' });
|
|
85
|
+
console.log(stdout);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
console.error(chalk.red(`Invalid output format: ${options.outputFormat}. Use: yaml, json, or name`));
|
|
89
|
+
process.exit(ExitCodes.CliError);
|
|
90
|
+
}
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
console.error(chalk.red(error instanceof Error
|
|
95
|
+
? error.message
|
|
96
|
+
: 'Failed to fetch query resource'));
|
|
97
|
+
process.exit(ExitCodes.CliError);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
69
100
|
// Fetch final query state
|
|
70
101
|
try {
|
|
71
|
-
const
|
|
72
|
-
const query = JSON.parse(stdout);
|
|
102
|
+
const query = await getResource('queries', queryName);
|
|
73
103
|
const phase = query.status?.phase;
|
|
74
104
|
// Check if query completed successfully or with error
|
|
75
105
|
if (phase === 'done') {
|
|
@@ -88,7 +118,12 @@ export async function executeQuery(options) {
|
|
|
88
118
|
process.exit(ExitCodes.OperationError);
|
|
89
119
|
}
|
|
90
120
|
else if (phase === 'canceled') {
|
|
91
|
-
spinner
|
|
121
|
+
if (spinner) {
|
|
122
|
+
spinner.warn('Query canceled');
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
output.warning('Query canceled');
|
|
126
|
+
}
|
|
92
127
|
if (query.status?.message) {
|
|
93
128
|
output.warning(query.status.message);
|
|
94
129
|
}
|
|
@@ -96,12 +131,15 @@ export async function executeQuery(options) {
|
|
|
96
131
|
}
|
|
97
132
|
}
|
|
98
133
|
catch (error) {
|
|
99
|
-
console.error(chalk.red(error instanceof Error
|
|
134
|
+
console.error(chalk.red(error instanceof Error
|
|
135
|
+
? error.message
|
|
136
|
+
: 'Failed to fetch query result'));
|
|
100
137
|
process.exit(ExitCodes.CliError);
|
|
101
138
|
}
|
|
102
139
|
}
|
|
103
140
|
catch (error) {
|
|
104
|
-
spinner
|
|
141
|
+
if (spinner)
|
|
142
|
+
spinner.stop();
|
|
105
143
|
console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
|
|
106
144
|
process.exit(ExitCodes.CliError);
|
|
107
145
|
}
|
|
@@ -69,7 +69,7 @@ describe('executeQuery', () => {
|
|
|
69
69
|
if (args.includes('apply')) {
|
|
70
70
|
return { stdout: '', stderr: '', exitCode: 0 };
|
|
71
71
|
}
|
|
72
|
-
if (args.includes('get') && args.includes('
|
|
72
|
+
if (args.includes('get') && args.includes('queries')) {
|
|
73
73
|
return {
|
|
74
74
|
stdout: JSON.stringify(mockQueryResponse),
|
|
75
75
|
stderr: '',
|
|
@@ -98,7 +98,7 @@ describe('executeQuery', () => {
|
|
|
98
98
|
if (args.includes('apply')) {
|
|
99
99
|
return { stdout: '', stderr: '', exitCode: 0 };
|
|
100
100
|
}
|
|
101
|
-
if (args.includes('get') && args.includes('
|
|
101
|
+
if (args.includes('get') && args.includes('queries')) {
|
|
102
102
|
return {
|
|
103
103
|
stdout: JSON.stringify(mockQueryResponse),
|
|
104
104
|
stderr: '',
|
|
@@ -132,7 +132,7 @@ describe('executeQuery', () => {
|
|
|
132
132
|
if (args.includes('apply')) {
|
|
133
133
|
return { stdout: '', stderr: '', exitCode: 0 };
|
|
134
134
|
}
|
|
135
|
-
if (args.includes('get') && args.includes('
|
|
135
|
+
if (args.includes('get') && args.includes('queries')) {
|
|
136
136
|
return {
|
|
137
137
|
stdout: JSON.stringify(mockQueryResponse),
|
|
138
138
|
stderr: '',
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { execa } from 'execa';
|
|
2
|
+
export async function getResource(resourceType, name) {
|
|
3
|
+
if (name === '@latest') {
|
|
4
|
+
const result = await execa('kubectl', [
|
|
5
|
+
'get',
|
|
6
|
+
resourceType,
|
|
7
|
+
'--sort-by=.metadata.creationTimestamp',
|
|
8
|
+
'-o',
|
|
9
|
+
'json',
|
|
10
|
+
], { stdio: 'pipe' });
|
|
11
|
+
const data = JSON.parse(result.stdout);
|
|
12
|
+
const resources = data.items || [];
|
|
13
|
+
if (resources.length === 0) {
|
|
14
|
+
throw new Error(`No ${resourceType} found`);
|
|
15
|
+
}
|
|
16
|
+
return resources[resources.length - 1];
|
|
17
|
+
}
|
|
18
|
+
const result = await execa('kubectl', ['get', resourceType, name, '-o', 'json'], { stdio: 'pipe' });
|
|
19
|
+
return JSON.parse(result.stdout);
|
|
20
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|