@agents-at-scale/ark 0.1.41 → 0.1.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arkServices.js +0 -9
- package/dist/commands/completion/index.js +46 -1
- package/dist/commands/evaluation/index.d.ts +3 -0
- package/dist/commands/evaluation/index.js +60 -0
- package/dist/commands/evaluation/index.spec.d.ts +1 -0
- package/dist/commands/evaluation/index.spec.js +161 -0
- package/dist/commands/generate/generators/team.js +4 -1
- package/dist/commands/install/index.js +27 -0
- package/dist/commands/marketplace/index.d.ts +4 -0
- package/dist/commands/marketplace/index.js +50 -0
- package/dist/commands/memory/index.d.ts +15 -0
- package/dist/commands/memory/index.js +130 -0
- package/dist/commands/memory/index.spec.d.ts +1 -0
- package/dist/commands/memory/index.spec.js +124 -0
- package/dist/commands/models/create.d.ts +5 -6
- package/dist/commands/models/create.js +14 -119
- package/dist/commands/models/create.spec.js +51 -0
- package/dist/commands/models/kubernetes/manifest-builder.d.ts +11 -0
- package/dist/commands/models/kubernetes/manifest-builder.js +109 -0
- package/dist/commands/models/kubernetes/secret-manager.d.ts +7 -0
- package/dist/commands/models/kubernetes/secret-manager.js +20 -0
- package/dist/commands/models/providers/azure.d.ts +31 -0
- package/dist/commands/models/providers/azure.js +82 -0
- package/dist/commands/models/providers/azure.spec.d.ts +1 -0
- package/dist/commands/models/providers/azure.spec.js +232 -0
- package/dist/commands/models/providers/bedrock.d.ts +37 -0
- package/dist/commands/models/providers/bedrock.js +105 -0
- package/dist/commands/models/providers/bedrock.spec.d.ts +1 -0
- package/dist/commands/models/providers/bedrock.spec.js +241 -0
- package/dist/commands/models/providers/factory.d.ts +18 -0
- package/dist/commands/models/providers/factory.js +31 -0
- package/dist/commands/models/providers/index.d.ts +17 -0
- package/dist/commands/models/providers/index.js +9 -0
- package/dist/commands/models/providers/openai.d.ts +28 -0
- package/dist/commands/models/providers/openai.js +68 -0
- package/dist/commands/models/providers/openai.spec.d.ts +1 -0
- package/dist/commands/models/providers/openai.spec.js +180 -0
- package/dist/commands/models/providers/types.d.ts +51 -0
- package/dist/commands/models/providers/types.js +1 -0
- package/dist/commands/queries/delete.d.ts +7 -0
- package/dist/commands/queries/delete.js +24 -0
- package/dist/commands/queries/delete.spec.d.ts +1 -0
- package/dist/commands/queries/delete.spec.js +74 -0
- package/dist/commands/queries/index.d.ts +3 -0
- package/dist/commands/queries/index.js +108 -0
- package/dist/commands/queries/list.d.ts +6 -0
- package/dist/commands/queries/list.js +66 -0
- package/dist/commands/queries/list.spec.d.ts +1 -0
- package/dist/commands/queries/list.spec.js +170 -0
- package/dist/commands/queries/validation.d.ts +2 -0
- package/dist/commands/queries/validation.js +10 -0
- package/dist/commands/queries/validation.spec.d.ts +1 -0
- package/dist/commands/queries/validation.spec.js +27 -0
- package/dist/commands/query/index.js +3 -1
- package/dist/commands/query/index.spec.js +24 -0
- package/dist/commands/uninstall/index.js +27 -0
- package/dist/components/ChatUI.js +2 -0
- package/dist/index.js +8 -0
- package/dist/lib/arkApiClient.d.ts +4 -0
- package/dist/lib/arkApiClient.js +57 -0
- package/dist/lib/errors.d.ts +1 -0
- package/dist/lib/errors.js +1 -0
- package/dist/lib/executeEvaluation.d.ts +16 -0
- package/dist/lib/executeEvaluation.js +155 -0
- package/dist/lib/executeQuery.d.ts +1 -4
- package/dist/lib/executeQuery.js +98 -68
- package/dist/lib/executeQuery.spec.js +176 -99
- package/dist/lib/kubectl.d.ts +15 -0
- package/dist/lib/kubectl.js +47 -0
- package/dist/lib/kubectl.spec.d.ts +1 -0
- package/dist/lib/kubectl.spec.js +176 -0
- package/dist/lib/stdin.d.ts +1 -0
- package/dist/lib/stdin.js +16 -0
- package/dist/lib/stdin.spec.d.ts +1 -0
- package/dist/lib/stdin.spec.js +82 -0
- package/dist/lib/types.d.ts +39 -0
- package/dist/marketplaceServices.d.ts +15 -0
- package/dist/marketplaceServices.js +51 -0
- package/package.json +2 -1
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { jest } from '@jest/globals';
|
|
2
|
+
import { UNSUPPORTED_OUTPUT_FORMAT_MESSAGE } from './validation.js';
|
|
3
|
+
import output from '../../lib/output.js';
|
|
4
|
+
const mockExeca = jest.fn();
|
|
5
|
+
jest.unstable_mockModule('execa', () => ({
|
|
6
|
+
execa: mockExeca,
|
|
7
|
+
}));
|
|
8
|
+
const { createQueriesCommand } = await import('./index.js');
|
|
9
|
+
describe('queries list command', () => {
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
jest.clearAllMocks();
|
|
12
|
+
console.log = jest.fn();
|
|
13
|
+
jest.spyOn(output, 'warning').mockImplementation(() => { });
|
|
14
|
+
jest.spyOn(output, 'error').mockImplementation(() => { });
|
|
15
|
+
jest.spyOn(process, 'exit').mockImplementation(() => undefined);
|
|
16
|
+
});
|
|
17
|
+
it('should list all queries in text format by default', async () => {
|
|
18
|
+
const mockQueries = [
|
|
19
|
+
{
|
|
20
|
+
metadata: {
|
|
21
|
+
name: 'query-1',
|
|
22
|
+
creationTimestamp: '2024-01-01T00:00:00Z',
|
|
23
|
+
},
|
|
24
|
+
status: {
|
|
25
|
+
phase: 'done',
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
metadata: {
|
|
30
|
+
name: 'query-2',
|
|
31
|
+
creationTimestamp: '2024-01-02T00:00:00Z',
|
|
32
|
+
},
|
|
33
|
+
status: {
|
|
34
|
+
phase: 'running',
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
];
|
|
38
|
+
mockExeca.mockResolvedValue({
|
|
39
|
+
stdout: JSON.stringify({ items: mockQueries }),
|
|
40
|
+
});
|
|
41
|
+
const command = createQueriesCommand({});
|
|
42
|
+
await command.parseAsync(['node', 'test', 'list']);
|
|
43
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/NAME.*STATUS/));
|
|
44
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-1/));
|
|
45
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-2/));
|
|
46
|
+
expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '-o', 'json'], { stdio: 'pipe' });
|
|
47
|
+
});
|
|
48
|
+
it('should list all queries in JSON format', async () => {
|
|
49
|
+
const mockQueries = [
|
|
50
|
+
{
|
|
51
|
+
metadata: {
|
|
52
|
+
name: 'query-1',
|
|
53
|
+
creationTimestamp: '2024-01-01T00:00:00Z',
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
metadata: {
|
|
58
|
+
name: 'query-2',
|
|
59
|
+
creationTimestamp: '2024-01-02T00:00:00Z',
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
];
|
|
63
|
+
mockExeca.mockResolvedValue({
|
|
64
|
+
stdout: JSON.stringify({ items: mockQueries }),
|
|
65
|
+
});
|
|
66
|
+
const command = createQueriesCommand({});
|
|
67
|
+
await command.parseAsync(['node', 'test', '--output', 'json']);
|
|
68
|
+
expect(console.log).toHaveBeenCalledWith(JSON.stringify(mockQueries, null, 2));
|
|
69
|
+
expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '-o', 'json'], { stdio: 'pipe' });
|
|
70
|
+
});
|
|
71
|
+
it('should support sorting by creation timestamp', async () => {
|
|
72
|
+
const mockQueries = [
|
|
73
|
+
{
|
|
74
|
+
metadata: {
|
|
75
|
+
name: 'query-1',
|
|
76
|
+
creationTimestamp: '2024-01-01T00:00:00Z',
|
|
77
|
+
},
|
|
78
|
+
status: {
|
|
79
|
+
phase: 'done',
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
metadata: {
|
|
84
|
+
name: 'query-2',
|
|
85
|
+
creationTimestamp: '2024-01-02T00:00:00Z',
|
|
86
|
+
},
|
|
87
|
+
status: {
|
|
88
|
+
phase: 'running',
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
];
|
|
92
|
+
mockExeca.mockResolvedValue({
|
|
93
|
+
stdout: JSON.stringify({ items: mockQueries }),
|
|
94
|
+
});
|
|
95
|
+
const command = createQueriesCommand({});
|
|
96
|
+
await command.parseAsync([
|
|
97
|
+
'node',
|
|
98
|
+
'test',
|
|
99
|
+
'--sort-by',
|
|
100
|
+
'.metadata.creationTimestamp',
|
|
101
|
+
]);
|
|
102
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/NAME.*STATUS/));
|
|
103
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-1/));
|
|
104
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/query-2/));
|
|
105
|
+
expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '--sort-by=.metadata.creationTimestamp', '-o', 'json'], { stdio: 'pipe' });
|
|
106
|
+
});
|
|
107
|
+
it('should display warning when no queries exist', async () => {
|
|
108
|
+
mockExeca.mockResolvedValue({
|
|
109
|
+
stdout: JSON.stringify({ items: [] }),
|
|
110
|
+
});
|
|
111
|
+
const command = createQueriesCommand({});
|
|
112
|
+
await command.parseAsync(['node', 'test', 'list']);
|
|
113
|
+
expect(output.warning).toHaveBeenCalledWith('no queries available');
|
|
114
|
+
expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', '-o', 'json'], { stdio: 'pipe' });
|
|
115
|
+
});
|
|
116
|
+
it('should handle errors when listing queries', async () => {
|
|
117
|
+
mockExeca.mockRejectedValue(new Error('kubectl connection failed'));
|
|
118
|
+
const command = createQueriesCommand({});
|
|
119
|
+
await command.parseAsync(['node', 'test', 'list']);
|
|
120
|
+
expect(output.error).toHaveBeenCalled();
|
|
121
|
+
expect(process.exit).toHaveBeenCalled();
|
|
122
|
+
});
|
|
123
|
+
it('should handle invalid output format gracefully', async () => {
|
|
124
|
+
const mockQueries = [
|
|
125
|
+
{
|
|
126
|
+
metadata: {
|
|
127
|
+
name: 'query-1',
|
|
128
|
+
creationTimestamp: '2024-01-01T00:00:00Z',
|
|
129
|
+
},
|
|
130
|
+
status: {
|
|
131
|
+
phase: 'done',
|
|
132
|
+
},
|
|
133
|
+
},
|
|
134
|
+
];
|
|
135
|
+
mockExeca.mockResolvedValue({
|
|
136
|
+
stdout: JSON.stringify({ items: mockQueries }),
|
|
137
|
+
});
|
|
138
|
+
const command = createQueriesCommand({});
|
|
139
|
+
await command.parseAsync(['node', 'test', '--output', 'xml']);
|
|
140
|
+
expect(output.error).toHaveBeenCalledWith(expect.anything(), expect.stringMatching(UNSUPPORTED_OUTPUT_FORMAT_MESSAGE));
|
|
141
|
+
expect(mockExeca).not.toHaveBeenCalled();
|
|
142
|
+
expect(console.log).not.toHaveBeenCalledWith(expect.stringMatching(/query-1/));
|
|
143
|
+
expect(process.exit).toHaveBeenCalled();
|
|
144
|
+
});
|
|
145
|
+
it('should list many queries without truncation', async () => {
|
|
146
|
+
// Create 100 mock queries
|
|
147
|
+
const mockQueries = Array.from({ length: 100 }, (_, i) => ({
|
|
148
|
+
metadata: {
|
|
149
|
+
name: `query-${i + 1}`,
|
|
150
|
+
creationTimestamp: new Date(2024, 0, i + 1).toISOString(),
|
|
151
|
+
},
|
|
152
|
+
status: {
|
|
153
|
+
phase: i % 3 === 0 ? 'done' : i % 2 === 0 ? 'running' : 'initializing',
|
|
154
|
+
},
|
|
155
|
+
}));
|
|
156
|
+
mockExeca.mockResolvedValue({
|
|
157
|
+
stdout: JSON.stringify({ items: mockQueries }),
|
|
158
|
+
});
|
|
159
|
+
const command = createQueriesCommand({});
|
|
160
|
+
await command.parseAsync(['node', 'test', 'list']);
|
|
161
|
+
// Check for header and separator
|
|
162
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(/NAME.*STATUS/));
|
|
163
|
+
// Verify all queries are logged
|
|
164
|
+
for (let i = 1; i <= 100; i++) {
|
|
165
|
+
expect(console.log).toHaveBeenCalledWith(expect.stringMatching(new RegExp(`query-${i}`)));
|
|
166
|
+
}
|
|
167
|
+
// Verify console.log was called: header + 100 queries
|
|
168
|
+
expect(console.log).toHaveBeenCalledTimes(101);
|
|
169
|
+
});
|
|
170
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { InvalidArgumentError } from 'commander';
|
|
2
|
+
const SUPPORTED_OUTPUT_FORMATS = ['json', 'text'];
|
|
3
|
+
export const UNSUPPORTED_OUTPUT_FORMAT_MESSAGE = `unsupported "output" format`;
|
|
4
|
+
const VALID_OUTPUT_FORMATS_MESSAGE = `valid formats are: ${SUPPORTED_OUTPUT_FORMATS.join(', ')}`;
|
|
5
|
+
export function assertSupportedOutputFormat(format) {
|
|
6
|
+
if (format && !SUPPORTED_OUTPUT_FORMATS.includes(format)) {
|
|
7
|
+
const message = `${UNSUPPORTED_OUTPUT_FORMAT_MESSAGE}: "${format}". ${VALID_OUTPUT_FORMATS_MESSAGE}`;
|
|
8
|
+
throw new InvalidArgumentError(message);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { jest } from '@jest/globals';
|
|
2
|
+
import { InvalidArgumentError } from 'commander';
|
|
3
|
+
import { assertSupportedOutputFormat, UNSUPPORTED_OUTPUT_FORMAT_MESSAGE, } from './validation.js';
|
|
4
|
+
jest.spyOn(console, 'error').mockImplementation(() => { });
|
|
5
|
+
describe('queries validation', () => {
|
|
6
|
+
describe('assertSupportedOutputFormat', () => {
|
|
7
|
+
it('should not throw for supported formats', () => {
|
|
8
|
+
expect(() => assertSupportedOutputFormat('json')).not.toThrow();
|
|
9
|
+
expect(() => assertSupportedOutputFormat('text')).not.toThrow();
|
|
10
|
+
});
|
|
11
|
+
it('should not throw when format is undefined', () => {
|
|
12
|
+
expect(() => assertSupportedOutputFormat(undefined)).not.toThrow();
|
|
13
|
+
});
|
|
14
|
+
it('should throw InvalidArgumentError for unsupported format', () => {
|
|
15
|
+
expect(() => assertSupportedOutputFormat('xml')).toThrow(InvalidArgumentError);
|
|
16
|
+
});
|
|
17
|
+
it('should include format and supported formats in error message', () => {
|
|
18
|
+
expect(() => assertSupportedOutputFormat('xml')).toThrow(UNSUPPORTED_OUTPUT_FORMAT_MESSAGE);
|
|
19
|
+
});
|
|
20
|
+
it('should work with various invalid formats', () => {
|
|
21
|
+
const invalidFormats = ['yaml', 'csv', 'html', 'pdf'];
|
|
22
|
+
for (const format of invalidFormats) {
|
|
23
|
+
expect(() => assertSupportedOutputFormat(format)).toThrow(InvalidArgumentError);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
});
|
|
@@ -8,7 +8,8 @@ export function createQueryCommand(_) {
|
|
|
8
8
|
.description('Execute a single query against a model or agent')
|
|
9
9
|
.argument('<target>', 'Query target (e.g., model/default, agent/my-agent)')
|
|
10
10
|
.argument('<message>', 'Message to send')
|
|
11
|
-
.
|
|
11
|
+
.option('-o, --output <format>', 'Output format: yaml, json, or name (prints only resource name)')
|
|
12
|
+
.action(async (target, message, options) => {
|
|
12
13
|
const parsed = parseTarget(target);
|
|
13
14
|
if (!parsed) {
|
|
14
15
|
console.error(chalk.red('Invalid target format. Use: model/name or agent/name etc'));
|
|
@@ -18,6 +19,7 @@ export function createQueryCommand(_) {
|
|
|
18
19
|
targetType: parsed.type,
|
|
19
20
|
targetName: parsed.name,
|
|
20
21
|
message,
|
|
22
|
+
outputFormat: options.output,
|
|
21
23
|
});
|
|
22
24
|
});
|
|
23
25
|
return queryCommand;
|
|
@@ -42,6 +42,30 @@ describe('createQueryCommand', () => {
|
|
|
42
42
|
targetType: 'model',
|
|
43
43
|
targetName: 'default',
|
|
44
44
|
message: 'Hello world',
|
|
45
|
+
outputFormat: undefined,
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
it('should pass output format option to executeQuery', async () => {
|
|
49
|
+
mockParseTarget.mockReturnValue({
|
|
50
|
+
type: 'model',
|
|
51
|
+
name: 'default',
|
|
52
|
+
});
|
|
53
|
+
mockExecuteQuery.mockResolvedValue(undefined);
|
|
54
|
+
const command = createQueryCommand({});
|
|
55
|
+
await command.parseAsync([
|
|
56
|
+
'node',
|
|
57
|
+
'test',
|
|
58
|
+
'model/default',
|
|
59
|
+
'Hello world',
|
|
60
|
+
'-o',
|
|
61
|
+
'json',
|
|
62
|
+
]);
|
|
63
|
+
expect(mockParseTarget).toHaveBeenCalledWith('model/default');
|
|
64
|
+
expect(mockExecuteQuery).toHaveBeenCalledWith({
|
|
65
|
+
targetType: 'model',
|
|
66
|
+
targetName: 'default',
|
|
67
|
+
message: 'Hello world',
|
|
68
|
+
outputFormat: 'json',
|
|
45
69
|
});
|
|
46
70
|
});
|
|
47
71
|
it('should error on invalid target format', async () => {
|
|
@@ -5,6 +5,7 @@ import inquirer from 'inquirer';
|
|
|
5
5
|
import { showNoClusterError } from '../../lib/startup.js';
|
|
6
6
|
import output from '../../lib/output.js';
|
|
7
7
|
import { getInstallableServices } from '../../arkServices.js';
|
|
8
|
+
import { isMarketplaceService, extractMarketplaceServiceName, getMarketplaceService, getAllMarketplaceServices, } from '../../marketplaceServices.js';
|
|
8
9
|
async function uninstallService(service, verbose = false) {
|
|
9
10
|
const helmArgs = ['uninstall', service.helmReleaseName, '--ignore-not-found'];
|
|
10
11
|
// Only add namespace flag if service has explicit namespace
|
|
@@ -25,6 +26,32 @@ async function uninstallArk(config, serviceName, options = {}) {
|
|
|
25
26
|
console.log(); // Add blank line after cluster info
|
|
26
27
|
// If a specific service is requested, uninstall only that service
|
|
27
28
|
if (serviceName) {
|
|
29
|
+
// Check if it's a marketplace service
|
|
30
|
+
if (isMarketplaceService(serviceName)) {
|
|
31
|
+
const marketplaceServiceName = extractMarketplaceServiceName(serviceName);
|
|
32
|
+
const service = getMarketplaceService(marketplaceServiceName);
|
|
33
|
+
if (!service) {
|
|
34
|
+
output.error(`marketplace service '${marketplaceServiceName}' not found`);
|
|
35
|
+
output.info('available marketplace services:');
|
|
36
|
+
const marketplaceServices = getAllMarketplaceServices();
|
|
37
|
+
for (const serviceName of Object.keys(marketplaceServices)) {
|
|
38
|
+
output.info(` marketplace/services/${serviceName}`);
|
|
39
|
+
}
|
|
40
|
+
process.exit(1);
|
|
41
|
+
}
|
|
42
|
+
output.info(`uninstalling marketplace service ${service.name}...`);
|
|
43
|
+
try {
|
|
44
|
+
await uninstallService(service, options.verbose);
|
|
45
|
+
output.success(`${service.name} uninstalled successfully`);
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
output.error(`failed to uninstall ${service.name}`);
|
|
49
|
+
console.error(error);
|
|
50
|
+
process.exit(1);
|
|
51
|
+
}
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
// Core ARK service
|
|
28
55
|
const services = getInstallableServices();
|
|
29
56
|
const service = Object.values(services).find((s) => s.name === serviceName);
|
|
30
57
|
if (!service) {
|
|
@@ -18,10 +18,12 @@ const generateMessageId = () => {
|
|
|
18
18
|
// Configure marked with terminal renderer for markdown output
|
|
19
19
|
const configureMarkdown = () => {
|
|
20
20
|
marked.setOptions({
|
|
21
|
+
// @ts-ignore - TerminalRenderer types are incomplete
|
|
21
22
|
renderer: new TerminalRenderer({
|
|
22
23
|
showSectionPrefix: false,
|
|
23
24
|
width: 80,
|
|
24
25
|
reflowText: true,
|
|
26
|
+
// @ts-ignore - preserveNewlines exists but not in types
|
|
25
27
|
preserveNewlines: true,
|
|
26
28
|
}),
|
|
27
29
|
});
|
package/dist/index.js
CHANGED
|
@@ -13,10 +13,14 @@ import { createClusterCommand } from './commands/cluster/index.js';
|
|
|
13
13
|
import { createCompletionCommand } from './commands/completion/index.js';
|
|
14
14
|
import { createDashboardCommand } from './commands/dashboard/index.js';
|
|
15
15
|
import { createDocsCommand } from './commands/docs/index.js';
|
|
16
|
+
import { createEvaluationCommand } from './commands/evaluation/index.js';
|
|
16
17
|
import { createGenerateCommand } from './commands/generate/index.js';
|
|
17
18
|
import { createInstallCommand } from './commands/install/index.js';
|
|
19
|
+
import { createMarketplaceCommand } from './commands/marketplace/index.js';
|
|
20
|
+
import { createMemoryCommand } from './commands/memory/index.js';
|
|
18
21
|
import { createModelsCommand } from './commands/models/index.js';
|
|
19
22
|
import { createQueryCommand } from './commands/query/index.js';
|
|
23
|
+
import { createQueriesCommand } from './commands/queries/index.js';
|
|
20
24
|
import { createUninstallCommand } from './commands/uninstall/index.js';
|
|
21
25
|
import { createStatusCommand } from './commands/status/index.js';
|
|
22
26
|
import { createConfigCommand } from './commands/config/index.js';
|
|
@@ -43,10 +47,14 @@ async function main() {
|
|
|
43
47
|
program.addCommand(createCompletionCommand(config));
|
|
44
48
|
program.addCommand(createDashboardCommand(config));
|
|
45
49
|
program.addCommand(createDocsCommand(config));
|
|
50
|
+
program.addCommand(createEvaluationCommand(config));
|
|
46
51
|
program.addCommand(createGenerateCommand(config));
|
|
47
52
|
program.addCommand(createInstallCommand(config));
|
|
53
|
+
program.addCommand(createMarketplaceCommand(config));
|
|
54
|
+
program.addCommand(createMemoryCommand(config));
|
|
48
55
|
program.addCommand(createModelsCommand(config));
|
|
49
56
|
program.addCommand(createQueryCommand(config));
|
|
57
|
+
program.addCommand(createQueriesCommand(config));
|
|
50
58
|
program.addCommand(createUninstallCommand(config));
|
|
51
59
|
program.addCommand(createStatusCommand());
|
|
52
60
|
program.addCommand(createConfigCommand(config));
|
|
@@ -47,6 +47,10 @@ export declare class ArkApiClient {
|
|
|
47
47
|
getModels(): Promise<Model[]>;
|
|
48
48
|
getTools(): Promise<Tool[]>;
|
|
49
49
|
getTeams(): Promise<Team[]>;
|
|
50
|
+
getSessions(): Promise<any[]>;
|
|
51
|
+
deleteSession(sessionId: string): Promise<any>;
|
|
52
|
+
deleteQueryMessages(sessionId: string, queryId: string): Promise<any>;
|
|
53
|
+
deleteAllSessions(): Promise<any>;
|
|
50
54
|
createChatCompletion(params: OpenAI.Chat.Completions.ChatCompletionCreateParams): Promise<OpenAI.Chat.Completions.ChatCompletion>;
|
|
51
55
|
createChatCompletionStream(params: OpenAI.Chat.Completions.ChatCompletionCreateParams): AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>;
|
|
52
56
|
}
|
package/dist/lib/arkApiClient.js
CHANGED
|
@@ -84,6 +84,61 @@ export class ArkApiClient {
|
|
|
84
84
|
throw new Error(`Failed to get teams: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
+
async getSessions() {
|
|
88
|
+
try {
|
|
89
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions`);
|
|
90
|
+
if (!response.ok) {
|
|
91
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
92
|
+
}
|
|
93
|
+
const data = (await response.json());
|
|
94
|
+
return data.items || [];
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
throw new Error(`Failed to get sessions: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async deleteSession(sessionId) {
|
|
101
|
+
try {
|
|
102
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions/${sessionId}`, {
|
|
103
|
+
method: 'DELETE',
|
|
104
|
+
});
|
|
105
|
+
if (!response.ok) {
|
|
106
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
107
|
+
}
|
|
108
|
+
return await response.json();
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
throw new Error(`Failed to delete session: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
async deleteQueryMessages(sessionId, queryId) {
|
|
115
|
+
try {
|
|
116
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions/${sessionId}/queries/${queryId}/messages`, {
|
|
117
|
+
method: 'DELETE',
|
|
118
|
+
});
|
|
119
|
+
if (!response.ok) {
|
|
120
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
121
|
+
}
|
|
122
|
+
return await response.json();
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
throw new Error(`Failed to delete query messages: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async deleteAllSessions() {
|
|
129
|
+
try {
|
|
130
|
+
const response = await fetch(`${this.baseUrl}/v1/sessions`, {
|
|
131
|
+
method: 'DELETE',
|
|
132
|
+
});
|
|
133
|
+
if (!response.ok) {
|
|
134
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
135
|
+
}
|
|
136
|
+
return await response.json();
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
throw new Error(`Failed to delete all sessions: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
87
142
|
async createChatCompletion(params) {
|
|
88
143
|
return (await this.openai.chat.completions.create({
|
|
89
144
|
...params,
|
|
@@ -91,6 +146,8 @@ export class ArkApiClient {
|
|
|
91
146
|
}));
|
|
92
147
|
}
|
|
93
148
|
async *createChatCompletionStream(params) {
|
|
149
|
+
// Errors from OpenAI SDK will automatically propagate with proper error messages
|
|
150
|
+
// and kill the CLI, so no try/catch needed here
|
|
94
151
|
const stream = await this.openai.chat.completions.create({
|
|
95
152
|
...params,
|
|
96
153
|
stream: true,
|
package/dist/lib/errors.d.ts
CHANGED
package/dist/lib/errors.js
CHANGED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface DirectEvaluationOptions {
|
|
2
|
+
evaluatorName: string;
|
|
3
|
+
input: string;
|
|
4
|
+
output: string;
|
|
5
|
+
timeout?: string;
|
|
6
|
+
watchTimeout?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface QueryEvaluationOptions {
|
|
9
|
+
evaluatorName: string;
|
|
10
|
+
queryName: string;
|
|
11
|
+
responseTarget?: string;
|
|
12
|
+
timeout?: string;
|
|
13
|
+
watchTimeout?: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function executeDirectEvaluation(options: DirectEvaluationOptions): Promise<void>;
|
|
16
|
+
export declare function executeQueryEvaluation(options: QueryEvaluationOptions): Promise<void>;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { execa } from 'execa';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
import output from './output.js';
|
|
5
|
+
import { ExitCodes } from './errors.js';
|
|
6
|
+
import { parseDuration } from './duration.js';
|
|
7
|
+
async function waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay) {
|
|
8
|
+
const spinner = ora('Waiting for evaluation completion...').start();
|
|
9
|
+
try {
|
|
10
|
+
await execa('kubectl', [
|
|
11
|
+
'wait',
|
|
12
|
+
'--for=condition=Completed',
|
|
13
|
+
`evaluation/${evaluationName}`,
|
|
14
|
+
`--timeout=${Math.floor(watchTimeoutMs / 1000)}s`,
|
|
15
|
+
], { timeout: watchTimeoutMs });
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
spinner.stop();
|
|
19
|
+
if (error instanceof Error && error.message.includes('timed out waiting')) {
|
|
20
|
+
console.error(chalk.red(`Evaluation did not complete within ${watchTimeoutDisplay}`));
|
|
21
|
+
process.exit(ExitCodes.Timeout);
|
|
22
|
+
}
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
spinner.stop();
|
|
26
|
+
const { stdout } = await execa('kubectl', ['get', 'evaluation', evaluationName, '-o', 'json'], { stdio: 'pipe' });
|
|
27
|
+
const evaluation = JSON.parse(stdout);
|
|
28
|
+
const status = evaluation.status;
|
|
29
|
+
if (status?.phase === 'done') {
|
|
30
|
+
console.log(chalk.green('\nEvaluation completed successfully:'));
|
|
31
|
+
if (status.score !== undefined) {
|
|
32
|
+
console.log(`Score: ${status.score}`);
|
|
33
|
+
}
|
|
34
|
+
if (status.passed !== undefined) {
|
|
35
|
+
console.log(`Result: ${status.passed ? chalk.green('PASSED') : chalk.red('FAILED')}`);
|
|
36
|
+
}
|
|
37
|
+
if (status.message) {
|
|
38
|
+
console.log(`Message: ${status.message}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else if (status?.phase === 'error') {
|
|
42
|
+
console.error(chalk.red(status.message || 'Evaluation failed with unknown error'));
|
|
43
|
+
process.exit(ExitCodes.OperationError);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
output.warning(`Unexpected evaluation phase: ${status?.phase}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export async function executeDirectEvaluation(options) {
|
|
50
|
+
const spinner = ora('Creating evaluation...').start();
|
|
51
|
+
const queryTimeoutMs = options.timeout
|
|
52
|
+
? parseDuration(options.timeout)
|
|
53
|
+
: parseDuration('5m');
|
|
54
|
+
const watchTimeoutMs = options.watchTimeout
|
|
55
|
+
? parseDuration(options.watchTimeout)
|
|
56
|
+
: queryTimeoutMs + 60000;
|
|
57
|
+
const timestamp = Date.now();
|
|
58
|
+
const evaluationName = `cli-eval-${timestamp}`;
|
|
59
|
+
const evaluationManifest = {
|
|
60
|
+
apiVersion: 'ark.mckinsey.com/v1alpha1',
|
|
61
|
+
kind: 'Evaluation',
|
|
62
|
+
metadata: {
|
|
63
|
+
name: evaluationName,
|
|
64
|
+
},
|
|
65
|
+
spec: {
|
|
66
|
+
type: 'direct',
|
|
67
|
+
evaluator: {
|
|
68
|
+
name: options.evaluatorName,
|
|
69
|
+
},
|
|
70
|
+
config: {
|
|
71
|
+
input: options.input,
|
|
72
|
+
output: options.output,
|
|
73
|
+
},
|
|
74
|
+
...(options.timeout && { timeout: options.timeout }),
|
|
75
|
+
ttl: '1h',
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
try {
|
|
79
|
+
spinner.text = 'Submitting evaluation...';
|
|
80
|
+
await execa('kubectl', ['apply', '-f', '-'], {
|
|
81
|
+
input: JSON.stringify(evaluationManifest),
|
|
82
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
83
|
+
});
|
|
84
|
+
spinner.stop();
|
|
85
|
+
const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
|
|
86
|
+
await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
spinner.stop();
|
|
90
|
+
console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
|
|
91
|
+
process.exit(ExitCodes.CliError);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
export async function executeQueryEvaluation(options) {
|
|
95
|
+
const spinner = ora('Creating evaluation...').start();
|
|
96
|
+
const queryTimeoutMs = options.timeout
|
|
97
|
+
? parseDuration(options.timeout)
|
|
98
|
+
: parseDuration('5m');
|
|
99
|
+
const watchTimeoutMs = options.watchTimeout
|
|
100
|
+
? parseDuration(options.watchTimeout)
|
|
101
|
+
: queryTimeoutMs + 60000;
|
|
102
|
+
const timestamp = Date.now();
|
|
103
|
+
const evaluationName = `cli-eval-${timestamp}`;
|
|
104
|
+
let responseTarget;
|
|
105
|
+
if (options.responseTarget) {
|
|
106
|
+
const parts = options.responseTarget.split(':');
|
|
107
|
+
if (parts.length === 2) {
|
|
108
|
+
responseTarget = {
|
|
109
|
+
type: parts[0],
|
|
110
|
+
name: parts[1],
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
spinner.stop();
|
|
115
|
+
console.error(chalk.red('Invalid response-target format. Use: type:name (e.g., agent:my-agent)'));
|
|
116
|
+
process.exit(ExitCodes.CliError);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const evaluationManifest = {
|
|
120
|
+
apiVersion: 'ark.mckinsey.com/v1alpha1',
|
|
121
|
+
kind: 'Evaluation',
|
|
122
|
+
metadata: {
|
|
123
|
+
name: evaluationName,
|
|
124
|
+
},
|
|
125
|
+
spec: {
|
|
126
|
+
type: 'query',
|
|
127
|
+
evaluator: {
|
|
128
|
+
name: options.evaluatorName,
|
|
129
|
+
},
|
|
130
|
+
config: {
|
|
131
|
+
queryRef: {
|
|
132
|
+
name: options.queryName,
|
|
133
|
+
},
|
|
134
|
+
...(responseTarget && { responseTarget }),
|
|
135
|
+
},
|
|
136
|
+
...(options.timeout && { timeout: options.timeout }),
|
|
137
|
+
ttl: '1h',
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
try {
|
|
141
|
+
spinner.text = 'Submitting evaluation...';
|
|
142
|
+
await execa('kubectl', ['apply', '-f', '-'], {
|
|
143
|
+
input: JSON.stringify(evaluationManifest),
|
|
144
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
145
|
+
});
|
|
146
|
+
spinner.stop();
|
|
147
|
+
const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
|
|
148
|
+
await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
spinner.stop();
|
|
152
|
+
console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
|
|
153
|
+
process.exit(ExitCodes.CliError);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
@@ -9,11 +9,8 @@ export interface QueryOptions {
|
|
|
9
9
|
timeout?: string;
|
|
10
10
|
watchTimeout?: string;
|
|
11
11
|
verbose?: boolean;
|
|
12
|
+
outputFormat?: string;
|
|
12
13
|
}
|
|
13
|
-
/**
|
|
14
|
-
* Execute a query against any ARK target (model, agent, team)
|
|
15
|
-
* This is the shared implementation used by all query commands
|
|
16
|
-
*/
|
|
17
14
|
export declare function executeQuery(options: QueryOptions): Promise<void>;
|
|
18
15
|
/**
|
|
19
16
|
* Parse a target string like "model/default" or "agent/weather"
|