@agents-at-scale/ark 0.1.41 → 0.1.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/arkServices.js +0 -9
  2. package/dist/commands/completion/index.js +46 -1
  3. package/dist/commands/evaluation/index.d.ts +3 -0
  4. package/dist/commands/evaluation/index.js +60 -0
  5. package/dist/commands/evaluation/index.spec.d.ts +1 -0
  6. package/dist/commands/evaluation/index.spec.js +161 -0
  7. package/dist/commands/generate/generators/team.js +4 -1
  8. package/dist/commands/install/index.js +27 -0
  9. package/dist/commands/marketplace/index.d.ts +4 -0
  10. package/dist/commands/marketplace/index.js +50 -0
  11. package/dist/commands/memory/index.d.ts +15 -0
  12. package/dist/commands/memory/index.js +130 -0
  13. package/dist/commands/memory/index.spec.d.ts +1 -0
  14. package/dist/commands/memory/index.spec.js +124 -0
  15. package/dist/commands/models/create.d.ts +5 -6
  16. package/dist/commands/models/create.js +14 -119
  17. package/dist/commands/models/create.spec.js +51 -0
  18. package/dist/commands/models/kubernetes/manifest-builder.d.ts +11 -0
  19. package/dist/commands/models/kubernetes/manifest-builder.js +109 -0
  20. package/dist/commands/models/kubernetes/secret-manager.d.ts +7 -0
  21. package/dist/commands/models/kubernetes/secret-manager.js +20 -0
  22. package/dist/commands/models/providers/azure.d.ts +31 -0
  23. package/dist/commands/models/providers/azure.js +82 -0
  24. package/dist/commands/models/providers/azure.spec.d.ts +1 -0
  25. package/dist/commands/models/providers/azure.spec.js +232 -0
  26. package/dist/commands/models/providers/bedrock.d.ts +37 -0
  27. package/dist/commands/models/providers/bedrock.js +105 -0
  28. package/dist/commands/models/providers/bedrock.spec.d.ts +1 -0
  29. package/dist/commands/models/providers/bedrock.spec.js +241 -0
  30. package/dist/commands/models/providers/factory.d.ts +18 -0
  31. package/dist/commands/models/providers/factory.js +31 -0
  32. package/dist/commands/models/providers/index.d.ts +17 -0
  33. package/dist/commands/models/providers/index.js +9 -0
  34. package/dist/commands/models/providers/openai.d.ts +28 -0
  35. package/dist/commands/models/providers/openai.js +68 -0
  36. package/dist/commands/models/providers/openai.spec.d.ts +1 -0
  37. package/dist/commands/models/providers/openai.spec.js +180 -0
  38. package/dist/commands/models/providers/types.d.ts +51 -0
  39. package/dist/commands/models/providers/types.js +1 -0
  40. package/dist/commands/queries/delete.d.ts +7 -0
  41. package/dist/commands/queries/delete.js +24 -0
  42. package/dist/commands/queries/delete.spec.d.ts +1 -0
  43. package/dist/commands/queries/delete.spec.js +74 -0
  44. package/dist/commands/queries/index.d.ts +3 -0
  45. package/dist/commands/queries/index.js +108 -0
  46. package/dist/commands/queries/list.d.ts +6 -0
  47. package/dist/commands/queries/list.js +66 -0
  48. package/dist/commands/queries/list.spec.d.ts +1 -0
  49. package/dist/commands/queries/list.spec.js +170 -0
  50. package/dist/commands/queries/validation.d.ts +2 -0
  51. package/dist/commands/queries/validation.js +10 -0
  52. package/dist/commands/queries/validation.spec.d.ts +1 -0
  53. package/dist/commands/queries/validation.spec.js +27 -0
  54. package/dist/commands/query/index.js +3 -1
  55. package/dist/commands/query/index.spec.js +24 -0
  56. package/dist/commands/uninstall/index.js +27 -0
  57. package/dist/components/ChatUI.js +2 -0
  58. package/dist/index.js +8 -0
  59. package/dist/lib/arkApiClient.d.ts +4 -0
  60. package/dist/lib/arkApiClient.js +57 -0
  61. package/dist/lib/errors.d.ts +1 -0
  62. package/dist/lib/errors.js +1 -0
  63. package/dist/lib/executeEvaluation.d.ts +16 -0
  64. package/dist/lib/executeEvaluation.js +155 -0
  65. package/dist/lib/executeQuery.d.ts +1 -4
  66. package/dist/lib/executeQuery.js +98 -68
  67. package/dist/lib/executeQuery.spec.js +176 -99
  68. package/dist/lib/kubectl.d.ts +15 -0
  69. package/dist/lib/kubectl.js +47 -0
  70. package/dist/lib/kubectl.spec.d.ts +1 -0
  71. package/dist/lib/kubectl.spec.js +176 -0
  72. package/dist/lib/stdin.d.ts +1 -0
  73. package/dist/lib/stdin.js +16 -0
  74. package/dist/lib/stdin.spec.d.ts +1 -0
  75. package/dist/lib/stdin.spec.js +82 -0
  76. package/dist/lib/types.d.ts +39 -0
  77. package/dist/marketplaceServices.d.ts +15 -0
  78. package/dist/marketplaceServices.js +51 -0
  79. package/package.json +2 -1
@@ -0,0 +1,170 @@
1
+ import { jest } from '@jest/globals';
2
+ import { UNSUPPORTED_OUTPUT_FORMAT_MESSAGE } from './validation.js';
3
+ import output from '../../lib/output.js';
4
+ const mockExeca = jest.fn();
5
+ jest.unstable_mockModule('execa', () => ({
6
+ execa: mockExeca,
7
+ }));
8
+ const { createQueriesCommand } = await import('./index.js');
9
+ describe('queries list command', () => {
10
+ beforeEach(() => {
11
+ jest.clearAllMocks();
12
+ console.log = jest.fn();
13
+ jest.spyOn(output, 'warning').mockImplementation(() => { });
14
+ jest.spyOn(output, 'error').mockImplementation(() => { });
15
+ jest.spyOn(process, 'exit').mockImplementation(() => undefined);
16
+ });
17
+ it('should list all queries in text format by default', async () => {
18
+ const mockQueries = [
19
+ {
20
+ metadata: {
21
+ name: 'query-1',
22
+ creationTimestamp: '2024-01-01T00:00:00Z',
23
+ },
24
+ status: {
25
+ phase: 'done',
26
+ },
27
+ },
28
+ {
29
+ metadata: {
30
+ name: 'query-2',
31
+ creationTimestamp: '2024-01-02T00:00:00Z',
32
+ },
33
+ status: {
34
+ phase: 'running',
35
+ },
36
+ },
37
+ ];
38
+ mockExeca.mockResolvedValue({
39
+ stdout: JSON.stringify({ items: mockQueries }),
40
+ });
41
+ const command = createQueriesCommand({});
42
+ await command.parseAsync(['node', 'test', 'list']);
43
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/NAME.*STATUS/));
44
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-1/));
45
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-2/));
46
+ expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '-o', 'json'], { stdio: 'pipe' });
47
+ });
48
+ it('should list all queries in JSON format', async () => {
49
+ const mockQueries = [
50
+ {
51
+ metadata: {
52
+ name: 'query-1',
53
+ creationTimestamp: '2024-01-01T00:00:00Z',
54
+ },
55
+ },
56
+ {
57
+ metadata: {
58
+ name: 'query-2',
59
+ creationTimestamp: '2024-01-02T00:00:00Z',
60
+ },
61
+ },
62
+ ];
63
+ mockExeca.mockResolvedValue({
64
+ stdout: JSON.stringify({ items: mockQueries }),
65
+ });
66
+ const command = createQueriesCommand({});
67
+ await command.parseAsync(['node', 'test', '--output', 'json']);
68
+ expect(console.log).toHaveBeenCalledWith(JSON.stringify(mockQueries, null, 2));
69
+ expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '-o', 'json'], { stdio: 'pipe' });
70
+ });
71
+ it('should support sorting by creation timestamp', async () => {
72
+ const mockQueries = [
73
+ {
74
+ metadata: {
75
+ name: 'query-1',
76
+ creationTimestamp: '2024-01-01T00:00:00Z',
77
+ },
78
+ status: {
79
+ phase: 'done',
80
+ },
81
+ },
82
+ {
83
+ metadata: {
84
+ name: 'query-2',
85
+ creationTimestamp: '2024-01-02T00:00:00Z',
86
+ },
87
+ status: {
88
+ phase: 'running',
89
+ },
90
+ },
91
+ ];
92
+ mockExeca.mockResolvedValue({
93
+ stdout: JSON.stringify({ items: mockQueries }),
94
+ });
95
+ const command = createQueriesCommand({});
96
+ await command.parseAsync([
97
+ 'node',
98
+ 'test',
99
+ '--sort-by',
100
+ '.metadata.creationTimestamp',
101
+ ]);
102
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/NAME.*STATUS/));
103
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-1/));
104
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-2/));
105
+ expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '--sort-by=.metadata.creationTimestamp', '-o', 'json'], { stdio: 'pipe' });
106
+ });
107
+ it('should display warning when no queries exist', async () => {
108
+ mockExeca.mockResolvedValue({
109
+ stdout: JSON.stringify({ items: [] }),
110
+ });
111
+ const command = createQueriesCommand({});
112
+ await command.parseAsync(['node', 'test', 'list']);
113
+ expect(output.warning).toHaveBeenCalledWith('no queries available');
114
+ expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '-o', 'json'], { stdio: 'pipe' });
115
+ });
116
+ it('should handle errors when listing queries', async () => {
117
+ mockExeca.mockRejectedValue(new Error('kubectl connection failed'));
118
+ const command = createQueriesCommand({});
119
+ await command.parseAsync(['node', 'test', 'list']);
120
+ expect(output.error).toHaveBeenCalled();
121
+ expect(process.exit).toHaveBeenCalled();
122
+ });
123
+ it('should handle invalid output format gracefully', async () => {
124
+ const mockQueries = [
125
+ {
126
+ metadata: {
127
+ name: 'query-1',
128
+ creationTimestamp: '2024-01-01T00:00:00Z',
129
+ },
130
+ status: {
131
+ phase: 'done',
132
+ },
133
+ },
134
+ ];
135
+ mockExeca.mockResolvedValue({
136
+ stdout: JSON.stringify({ items: mockQueries }),
137
+ });
138
+ const command = createQueriesCommand({});
139
+ await command.parseAsync(['node', 'test', '--output', 'xml']);
140
+ expect(output.error).toHaveBeenCalledWith(expect.anything(), expect.stringMatching(UNSUPPORTED_OUTPUT_FORMAT_MESSAGE));
141
+ expect(mockExeca).not.toHaveBeenCalled();
142
+ expect(console.log).not.toHaveBeenCalledWith(expect.stringMatching(/query-1/));
143
+ expect(process.exit).toHaveBeenCalled();
144
+ });
145
+ it('should list many queries without truncation', async () => {
146
+ // Create 100 mock queries
147
+ const mockQueries = Array.from({ length: 100 }, (_, i) => ({
148
+ metadata: {
149
+ name: `query-${i + 1}`,
150
+ creationTimestamp: new Date(2024, 0, i + 1).toISOString(),
151
+ },
152
+ status: {
153
+ phase: i % 3 === 0 ? 'done' : i % 2 === 0 ? 'running' : 'initializing',
154
+ },
155
+ }));
156
+ mockExeca.mockResolvedValue({
157
+ stdout: JSON.stringify({ items: mockQueries }),
158
+ });
159
+ const command = createQueriesCommand({});
160
+ await command.parseAsync(['node', 'test', 'list']);
161
+ // Check for header and separator
162
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/NAME.*STATUS/));
163
+ // Verify all queries are logged
164
+ for (let i = 1; i <= 100; i++) {
165
+ expect(console.log).toHaveBeenCalledWith(expect.stringMatching(new RegExp(`query-${i}`)));
166
+ }
167
+ // Verify console.log was called: header + 100 queries
168
+ expect(console.log).toHaveBeenCalledTimes(101);
169
+ });
170
+ });
@@ -0,0 +1,2 @@
1
+ export declare const UNSUPPORTED_OUTPUT_FORMAT_MESSAGE = "unsupported \"output\" format";
2
+ export declare function assertSupportedOutputFormat(format: string | undefined): void;
@@ -0,0 +1,10 @@
1
+ import { InvalidArgumentError } from 'commander';
2
+ const SUPPORTED_OUTPUT_FORMATS = ['json', 'text'];
3
+ export const UNSUPPORTED_OUTPUT_FORMAT_MESSAGE = `unsupported "output" format`;
4
+ const VALID_OUTPUT_FORMATS_MESSAGE = `valid formats are: ${SUPPORTED_OUTPUT_FORMATS.join(', ')}`;
5
+ export function assertSupportedOutputFormat(format) {
6
+ if (format && !SUPPORTED_OUTPUT_FORMATS.includes(format)) {
7
+ const message = `${UNSUPPORTED_OUTPUT_FORMAT_MESSAGE}: "${format}". ${VALID_OUTPUT_FORMATS_MESSAGE}`;
8
+ throw new InvalidArgumentError(message);
9
+ }
10
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,27 @@
1
+ import { jest } from '@jest/globals';
2
+ import { InvalidArgumentError } from 'commander';
3
+ import { assertSupportedOutputFormat, UNSUPPORTED_OUTPUT_FORMAT_MESSAGE, } from './validation.js';
4
+ jest.spyOn(console, 'error').mockImplementation(() => { });
5
+ describe('queries validation', () => {
6
+ describe('assertSupportedOutputFormat', () => {
7
+ it('should not throw for supported formats', () => {
8
+ expect(() => assertSupportedOutputFormat('json')).not.toThrow();
9
+ expect(() => assertSupportedOutputFormat('text')).not.toThrow();
10
+ });
11
+ it('should not throw when format is undefined', () => {
12
+ expect(() => assertSupportedOutputFormat(undefined)).not.toThrow();
13
+ });
14
+ it('should throw InvalidArgumentError for unsupported format', () => {
15
+ expect(() => assertSupportedOutputFormat('xml')).toThrow(InvalidArgumentError);
16
+ });
17
+ it('should include format and supported formats in error message', () => {
18
+ expect(() => assertSupportedOutputFormat('xml')).toThrow(UNSUPPORTED_OUTPUT_FORMAT_MESSAGE);
19
+ });
20
+ it('should work with various invalid formats', () => {
21
+ const invalidFormats = ['yaml', 'csv', 'html', 'pdf'];
22
+ for (const format of invalidFormats) {
23
+ expect(() => assertSupportedOutputFormat(format)).toThrow(InvalidArgumentError);
24
+ }
25
+ });
26
+ });
27
+ });
@@ -8,7 +8,8 @@ export function createQueryCommand(_) {
8
8
  .description('Execute a single query against a model or agent')
9
9
  .argument('<target>', 'Query target (e.g., model/default, agent/my-agent)')
10
10
  .argument('<message>', 'Message to send')
11
- .action(async (target, message) => {
11
+ .option('-o, --output <format>', 'Output format: yaml, json, or name (prints only resource name)')
12
+ .action(async (target, message, options) => {
12
13
  const parsed = parseTarget(target);
13
14
  if (!parsed) {
14
15
  console.error(chalk.red('Invalid target format. Use: model/name or agent/name etc'));
@@ -18,6 +19,7 @@ export function createQueryCommand(_) {
18
19
  targetType: parsed.type,
19
20
  targetName: parsed.name,
20
21
  message,
22
+ outputFormat: options.output,
21
23
  });
22
24
  });
23
25
  return queryCommand;
@@ -42,6 +42,30 @@ describe('createQueryCommand', () => {
42
42
  targetType: 'model',
43
43
  targetName: 'default',
44
44
  message: 'Hello world',
45
+ outputFormat: undefined,
46
+ });
47
+ });
48
+ it('should pass output format option to executeQuery', async () => {
49
+ mockParseTarget.mockReturnValue({
50
+ type: 'model',
51
+ name: 'default',
52
+ });
53
+ mockExecuteQuery.mockResolvedValue(undefined);
54
+ const command = createQueryCommand({});
55
+ await command.parseAsync([
56
+ 'node',
57
+ 'test',
58
+ 'model/default',
59
+ 'Hello world',
60
+ '-o',
61
+ 'json',
62
+ ]);
63
+ expect(mockParseTarget).toHaveBeenCalledWith('model/default');
64
+ expect(mockExecuteQuery).toHaveBeenCalledWith({
65
+ targetType: 'model',
66
+ targetName: 'default',
67
+ message: 'Hello world',
68
+ outputFormat: 'json',
45
69
  });
46
70
  });
47
71
  it('should error on invalid target format', async () => {
@@ -5,6 +5,7 @@ import inquirer from 'inquirer';
5
5
  import { showNoClusterError } from '../../lib/startup.js';
6
6
  import output from '../../lib/output.js';
7
7
  import { getInstallableServices } from '../../arkServices.js';
8
+ import { isMarketplaceService, extractMarketplaceServiceName, getMarketplaceService, getAllMarketplaceServices, } from '../../marketplaceServices.js';
8
9
  async function uninstallService(service, verbose = false) {
9
10
  const helmArgs = ['uninstall', service.helmReleaseName, '--ignore-not-found'];
10
11
  // Only add namespace flag if service has explicit namespace
@@ -25,6 +26,32 @@ async function uninstallArk(config, serviceName, options = {}) {
25
26
  console.log(); // Add blank line after cluster info
26
27
  // If a specific service is requested, uninstall only that service
27
28
  if (serviceName) {
29
+ // Check if it's a marketplace service
30
+ if (isMarketplaceService(serviceName)) {
31
+ const marketplaceServiceName = extractMarketplaceServiceName(serviceName);
32
+ const service = getMarketplaceService(marketplaceServiceName);
33
+ if (!service) {
34
+ output.error(`marketplace service '${marketplaceServiceName}' not found`);
35
+ output.info('available marketplace services:');
36
+ const marketplaceServices = getAllMarketplaceServices();
37
+ for (const serviceName of Object.keys(marketplaceServices)) {
38
+ output.info(` marketplace/services/${serviceName}`);
39
+ }
40
+ process.exit(1);
41
+ }
42
+ output.info(`uninstalling marketplace service ${service.name}...`);
43
+ try {
44
+ await uninstallService(service, options.verbose);
45
+ output.success(`${service.name} uninstalled successfully`);
46
+ }
47
+ catch (error) {
48
+ output.error(`failed to uninstall ${service.name}`);
49
+ console.error(error);
50
+ process.exit(1);
51
+ }
52
+ return;
53
+ }
54
+ // Core ARK service
28
55
  const services = getInstallableServices();
29
56
  const service = Object.values(services).find((s) => s.name === serviceName);
30
57
  if (!service) {
@@ -18,10 +18,12 @@ const generateMessageId = () => {
18
18
  // Configure marked with terminal renderer for markdown output
19
19
  const configureMarkdown = () => {
20
20
  marked.setOptions({
21
+ // @ts-ignore - TerminalRenderer types are incomplete
21
22
  renderer: new TerminalRenderer({
22
23
  showSectionPrefix: false,
23
24
  width: 80,
24
25
  reflowText: true,
26
+ // @ts-ignore - preserveNewlines exists but not in types
25
27
  preserveNewlines: true,
26
28
  }),
27
29
  });
package/dist/index.js CHANGED
@@ -13,10 +13,14 @@ import { createClusterCommand } from './commands/cluster/index.js';
13
13
  import { createCompletionCommand } from './commands/completion/index.js';
14
14
  import { createDashboardCommand } from './commands/dashboard/index.js';
15
15
  import { createDocsCommand } from './commands/docs/index.js';
16
+ import { createEvaluationCommand } from './commands/evaluation/index.js';
16
17
  import { createGenerateCommand } from './commands/generate/index.js';
17
18
  import { createInstallCommand } from './commands/install/index.js';
19
+ import { createMarketplaceCommand } from './commands/marketplace/index.js';
20
+ import { createMemoryCommand } from './commands/memory/index.js';
18
21
  import { createModelsCommand } from './commands/models/index.js';
19
22
  import { createQueryCommand } from './commands/query/index.js';
23
+ import { createQueriesCommand } from './commands/queries/index.js';
20
24
  import { createUninstallCommand } from './commands/uninstall/index.js';
21
25
  import { createStatusCommand } from './commands/status/index.js';
22
26
  import { createConfigCommand } from './commands/config/index.js';
@@ -43,10 +47,14 @@ async function main() {
43
47
  program.addCommand(createCompletionCommand(config));
44
48
  program.addCommand(createDashboardCommand(config));
45
49
  program.addCommand(createDocsCommand(config));
50
+ program.addCommand(createEvaluationCommand(config));
46
51
  program.addCommand(createGenerateCommand(config));
47
52
  program.addCommand(createInstallCommand(config));
53
+ program.addCommand(createMarketplaceCommand(config));
54
+ program.addCommand(createMemoryCommand(config));
48
55
  program.addCommand(createModelsCommand(config));
49
56
  program.addCommand(createQueryCommand(config));
57
+ program.addCommand(createQueriesCommand(config));
50
58
  program.addCommand(createUninstallCommand(config));
51
59
  program.addCommand(createStatusCommand());
52
60
  program.addCommand(createConfigCommand(config));
@@ -47,6 +47,10 @@ export declare class ArkApiClient {
47
47
  getModels(): Promise<Model[]>;
48
48
  getTools(): Promise<Tool[]>;
49
49
  getTeams(): Promise<Team[]>;
50
+ getSessions(): Promise<any[]>;
51
+ deleteSession(sessionId: string): Promise<any>;
52
+ deleteQueryMessages(sessionId: string, queryId: string): Promise<any>;
53
+ deleteAllSessions(): Promise<any>;
50
54
  createChatCompletion(params: OpenAI.Chat.Completions.ChatCompletionCreateParams): Promise<OpenAI.Chat.Completions.ChatCompletion>;
51
55
  createChatCompletionStream(params: OpenAI.Chat.Completions.ChatCompletionCreateParams): AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>;
52
56
  }
@@ -84,6 +84,61 @@ export class ArkApiClient {
84
84
  throw new Error(`Failed to get teams: ${error instanceof Error ? error.message : 'Unknown error'}`);
85
85
  }
86
86
  }
87
+ async getSessions() {
88
+ try {
89
+ const response = await fetch(`${this.baseUrl}/v1/sessions`);
90
+ if (!response.ok) {
91
+ throw new Error(`HTTP error! status: ${response.status}`);
92
+ }
93
+ const data = (await response.json());
94
+ return data.items || [];
95
+ }
96
+ catch (error) {
97
+ throw new Error(`Failed to get sessions: ${error instanceof Error ? error.message : 'Unknown error'}`);
98
+ }
99
+ }
100
+ async deleteSession(sessionId) {
101
+ try {
102
+ const response = await fetch(`${this.baseUrl}/v1/sessions/${sessionId}`, {
103
+ method: 'DELETE',
104
+ });
105
+ if (!response.ok) {
106
+ throw new Error(`HTTP error! status: ${response.status}`);
107
+ }
108
+ return await response.json();
109
+ }
110
+ catch (error) {
111
+ throw new Error(`Failed to delete session: ${error instanceof Error ? error.message : 'Unknown error'}`);
112
+ }
113
+ }
114
+ async deleteQueryMessages(sessionId, queryId) {
115
+ try {
116
+ const response = await fetch(`${this.baseUrl}/v1/sessions/${sessionId}/queries/${queryId}/messages`, {
117
+ method: 'DELETE',
118
+ });
119
+ if (!response.ok) {
120
+ throw new Error(`HTTP error! status: ${response.status}`);
121
+ }
122
+ return await response.json();
123
+ }
124
+ catch (error) {
125
+ throw new Error(`Failed to delete query messages: ${error instanceof Error ? error.message : 'Unknown error'}`);
126
+ }
127
+ }
128
+ async deleteAllSessions() {
129
+ try {
130
+ const response = await fetch(`${this.baseUrl}/v1/sessions`, {
131
+ method: 'DELETE',
132
+ });
133
+ if (!response.ok) {
134
+ throw new Error(`HTTP error! status: ${response.status}`);
135
+ }
136
+ return await response.json();
137
+ }
138
+ catch (error) {
139
+ throw new Error(`Failed to delete all sessions: ${error instanceof Error ? error.message : 'Unknown error'}`);
140
+ }
141
+ }
87
142
  async createChatCompletion(params) {
88
143
  return (await this.openai.chat.completions.create({
89
144
  ...params,
@@ -91,6 +146,8 @@ export class ArkApiClient {
91
146
  }));
92
147
  }
93
148
  async *createChatCompletionStream(params) {
149
+ // Errors from OpenAI SDK will automatically propagate with proper error messages
150
+ // and kill the CLI, so no try/catch needed here
94
151
  const stream = await this.openai.chat.completions.create({
95
152
  ...params,
96
153
  stream: true,
@@ -6,6 +6,7 @@ export declare const ExitCodes: {
6
6
  readonly CliError: 1;
7
7
  readonly OperationError: 2;
8
8
  readonly Timeout: 3;
9
+ readonly EvaluationFailed: 4;
9
10
  };
10
11
  export declare enum ErrorCode {
11
12
  INVALID_INPUT = "INVALID_INPUT",
@@ -8,6 +8,7 @@ export const ExitCodes = {
8
8
  CliError: 1,
9
9
  OperationError: 2,
10
10
  Timeout: 3,
11
+ EvaluationFailed: 4,
11
12
  };
12
13
  export var ErrorCode;
13
14
  (function (ErrorCode) {
@@ -0,0 +1,16 @@
1
+ export interface DirectEvaluationOptions {
2
+ evaluatorName: string;
3
+ input: string;
4
+ output: string;
5
+ timeout?: string;
6
+ watchTimeout?: string;
7
+ }
8
+ export interface QueryEvaluationOptions {
9
+ evaluatorName: string;
10
+ queryName: string;
11
+ responseTarget?: string;
12
+ timeout?: string;
13
+ watchTimeout?: string;
14
+ }
15
+ export declare function executeDirectEvaluation(options: DirectEvaluationOptions): Promise<void>;
16
+ export declare function executeQueryEvaluation(options: QueryEvaluationOptions): Promise<void>;
@@ -0,0 +1,155 @@
1
+ import { execa } from 'execa';
2
+ import ora from 'ora';
3
+ import chalk from 'chalk';
4
+ import output from './output.js';
5
+ import { ExitCodes } from './errors.js';
6
+ import { parseDuration } from './duration.js';
7
+ async function waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay) {
8
+ const spinner = ora('Waiting for evaluation completion...').start();
9
+ try {
10
+ await execa('kubectl', [
11
+ 'wait',
12
+ '--for=condition=Completed',
13
+ `evaluation/${evaluationName}`,
14
+ `--timeout=${Math.floor(watchTimeoutMs / 1000)}s`,
15
+ ], { timeout: watchTimeoutMs });
16
+ }
17
+ catch (error) {
18
+ spinner.stop();
19
+ if (error instanceof Error && error.message.includes('timed out waiting')) {
20
+ console.error(chalk.red(`Evaluation did not complete within ${watchTimeoutDisplay}`));
21
+ process.exit(ExitCodes.Timeout);
22
+ }
23
+ throw error;
24
+ }
25
+ spinner.stop();
26
+ const { stdout } = await execa('kubectl', ['get', 'evaluation', evaluationName, '-o', 'json'], { stdio: 'pipe' });
27
+ const evaluation = JSON.parse(stdout);
28
+ const status = evaluation.status;
29
+ if (status?.phase === 'done') {
30
+ console.log(chalk.green('\nEvaluation completed successfully:'));
31
+ if (status.score !== undefined) {
32
+ console.log(`Score: ${status.score}`);
33
+ }
34
+ if (status.passed !== undefined) {
35
+ console.log(`Result: ${status.passed ? chalk.green('PASSED') : chalk.red('FAILED')}`);
36
+ }
37
+ if (status.message) {
38
+ console.log(`Message: ${status.message}`);
39
+ }
40
+ }
41
+ else if (status?.phase === 'error') {
42
+ console.error(chalk.red(status.message || 'Evaluation failed with unknown error'));
43
+ process.exit(ExitCodes.OperationError);
44
+ }
45
+ else {
46
+ output.warning(`Unexpected evaluation phase: ${status?.phase}`);
47
+ }
48
+ }
49
+ export async function executeDirectEvaluation(options) {
50
+ const spinner = ora('Creating evaluation...').start();
51
+ const queryTimeoutMs = options.timeout
52
+ ? parseDuration(options.timeout)
53
+ : parseDuration('5m');
54
+ const watchTimeoutMs = options.watchTimeout
55
+ ? parseDuration(options.watchTimeout)
56
+ : queryTimeoutMs + 60000;
57
+ const timestamp = Date.now();
58
+ const evaluationName = `cli-eval-${timestamp}`;
59
+ const evaluationManifest = {
60
+ apiVersion: 'ark.mckinsey.com/v1alpha1',
61
+ kind: 'Evaluation',
62
+ metadata: {
63
+ name: evaluationName,
64
+ },
65
+ spec: {
66
+ type: 'direct',
67
+ evaluator: {
68
+ name: options.evaluatorName,
69
+ },
70
+ config: {
71
+ input: options.input,
72
+ output: options.output,
73
+ },
74
+ ...(options.timeout && { timeout: options.timeout }),
75
+ ttl: '1h',
76
+ },
77
+ };
78
+ try {
79
+ spinner.text = 'Submitting evaluation...';
80
+ await execa('kubectl', ['apply', '-f', '-'], {
81
+ input: JSON.stringify(evaluationManifest),
82
+ stdio: ['pipe', 'pipe', 'pipe'],
83
+ });
84
+ spinner.stop();
85
+ const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
86
+ await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
87
+ }
88
+ catch (error) {
89
+ spinner.stop();
90
+ console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
91
+ process.exit(ExitCodes.CliError);
92
+ }
93
+ }
94
+ export async function executeQueryEvaluation(options) {
95
+ const spinner = ora('Creating evaluation...').start();
96
+ const queryTimeoutMs = options.timeout
97
+ ? parseDuration(options.timeout)
98
+ : parseDuration('5m');
99
+ const watchTimeoutMs = options.watchTimeout
100
+ ? parseDuration(options.watchTimeout)
101
+ : queryTimeoutMs + 60000;
102
+ const timestamp = Date.now();
103
+ const evaluationName = `cli-eval-${timestamp}`;
104
+ let responseTarget;
105
+ if (options.responseTarget) {
106
+ const parts = options.responseTarget.split(':');
107
+ if (parts.length === 2) {
108
+ responseTarget = {
109
+ type: parts[0],
110
+ name: parts[1],
111
+ };
112
+ }
113
+ else {
114
+ spinner.stop();
115
+ console.error(chalk.red('Invalid response-target format. Use: type:name (e.g., agent:my-agent)'));
116
+ process.exit(ExitCodes.CliError);
117
+ }
118
+ }
119
+ const evaluationManifest = {
120
+ apiVersion: 'ark.mckinsey.com/v1alpha1',
121
+ kind: 'Evaluation',
122
+ metadata: {
123
+ name: evaluationName,
124
+ },
125
+ spec: {
126
+ type: 'query',
127
+ evaluator: {
128
+ name: options.evaluatorName,
129
+ },
130
+ config: {
131
+ queryRef: {
132
+ name: options.queryName,
133
+ },
134
+ ...(responseTarget && { responseTarget }),
135
+ },
136
+ ...(options.timeout && { timeout: options.timeout }),
137
+ ttl: '1h',
138
+ },
139
+ };
140
+ try {
141
+ spinner.text = 'Submitting evaluation...';
142
+ await execa('kubectl', ['apply', '-f', '-'], {
143
+ input: JSON.stringify(evaluationManifest),
144
+ stdio: ['pipe', 'pipe', 'pipe'],
145
+ });
146
+ spinner.stop();
147
+ const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
148
+ await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
149
+ }
150
+ catch (error) {
151
+ spinner.stop();
152
+ console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
153
+ process.exit(ExitCodes.CliError);
154
+ }
155
+ }
@@ -9,11 +9,8 @@ export interface QueryOptions {
9
9
  timeout?: string;
10
10
  watchTimeout?: string;
11
11
  verbose?: boolean;
12
+ outputFormat?: string;
12
13
  }
13
- /**
14
- * Execute a query against any ARK target (model, agent, team)
15
- * This is the shared implementation used by all query commands
16
- */
17
14
  export declare function executeQuery(options: QueryOptions): Promise<void>;
18
15
  /**
19
16
  * Parse a target string like "model/default" or "agent/weather"