maka-agent 0.2.0-dev.3.20260830 → 0.2.0-dev.5.20260830

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/cli-core.js +1 -0
  2. package/dist/eval-bundle-path.js +1 -1
  3. package/dist/pi-tui-runner.js +3 -1
  4. package/dist/runtime-host-cli.js +37 -3
  5. package/dist/runtime-host-onboarding.js +10 -1
  6. package/dist/runtime-host-peer-mesh-management-command.js +17 -0
  7. package/dist/runtime-host-setup-command.js +33 -15
  8. package/dist/runtime-host-systemd-service.js +9 -3
  9. package/dist/runtime-host-tui-context.js +2 -6
  10. package/native/runtime-host-peer/prebuilds/darwin-arm64/maka_runtime_host_peer.node +0 -0
  11. package/native/runtime-host-peer/prebuilds/linux-arm64/maka_runtime_host_peer.node +0 -0
  12. package/native/runtime-host-peer/prebuilds/linux-x64/maka_runtime_host_peer.node +0 -0
  13. package/native/runtime-host-peer/prebuilds/win32-x64/maka_runtime_host_peer.node +0 -0
  14. package/node_modules/@maka/eval/dist/external-subject.js +1 -1
  15. package/node_modules/@maka/eval/dist/harbor-external-subject.js +2 -2
  16. package/node_modules/@maka/runtime/dist/sandbox/macos-seatbelt.js +68 -9
  17. package/node_modules/@maka/runtime-host/dist/client/peer-client.js +22 -3
  18. package/node_modules/@maka/runtime-host/dist/operator/peer-mesh-management-frame.js +14 -0
  19. package/{packages/eval/dist/index.js → node_modules/@maka/runtime-host/dist/peer-mesh/display-name.js} +12 -10
  20. package/node_modules/@maka/runtime-host/dist/peer-mesh/model.js +32 -21
  21. package/node_modules/@maka/runtime-host/dist/peer-mesh/node.js +207 -47
  22. package/node_modules/@maka/runtime-host/dist/peer-mesh/owner.js +2 -0
  23. package/node_modules/@maka/runtime-host/dist/peer-mesh/store.js +18 -5
  24. package/node_modules/@maka/runtime-host/dist/protocol/index.js +5 -1
  25. package/node_modules/@maka/runtime-host/dist/protocol/peer-mesh.js +59 -2
  26. package/node_modules/@maka/runtime-host/dist/server/access-authority.js +11 -7
  27. package/node_modules/@maka/runtime-host/dist/server/connection-session.js +3 -0
  28. package/node_modules/@maka/runtime-host/dist/server/execution-service.js +1 -0
  29. package/node_modules/@maka/runtime-host/dist/server/host-kernel.js +1 -1
  30. package/node_modules/@maka/runtime-host/dist/server/peer-mesh-authority.js +19 -0
  31. package/package.json +1 -2
  32. package/packages/eval/dist/attempt-store.js +0 -138
  33. package/packages/eval/dist/cli.js +0 -119
  34. package/packages/eval/dist/experiment-directory.js +0 -59
  35. package/packages/eval/dist/experiment.js +0 -49
  36. package/packages/eval/dist/external-subject.js +0 -488
  37. package/packages/eval/dist/harbor-external-subject.js +0 -929
  38. package/packages/eval/dist/harbor-maka-subject.js +0 -83
  39. package/packages/eval/dist/harness-environment.js +0 -80
  40. package/packages/eval/dist/harness-executor.js +0 -1013
  41. package/packages/eval/dist/install-preflight.js +0 -201
  42. package/packages/eval/dist/maka-artifacts.js +0 -126
  43. package/packages/eval/dist/maka-runtime-policy.js +0 -42
  44. package/packages/eval/dist/maka-subject.js +0 -268
  45. package/packages/eval/dist/metering-checkpoint.js +0 -80
  46. package/packages/eval/dist/provider-admission.js +0 -39
  47. package/packages/eval/dist/provider-metering.js +0 -67
  48. package/packages/eval/dist/provider-web-tool-surface.js +0 -74
  49. package/packages/eval/dist/relay-result-frame.js +0 -38
  50. package/packages/eval/dist/result.js +0 -94
  51. package/packages/eval/dist/runner.js +0 -290
  52. package/packages/eval/dist/spec.js +0 -135
  53. package/packages/eval/dist/toolchain-verification.js +0 -130
  54. package/packages/eval/harbor/deepseek-codex-models.json +0 -69
  55. package/packages/eval/harbor/deepseek-harness-profile/cordis.patch.yml +0 -140
  56. package/packages/eval/harbor/deepseek-harness-profile/cordis.yml +0 -18
  57. package/packages/eval/harbor/deepseek-harness-profile/package.json +0 -10
  58. package/packages/eval/harbor/docker-compose-egress-proxy.yaml +0 -57
  59. package/packages/eval/harbor/egress-proxy/Dockerfile +0 -25
  60. package/packages/eval/harbor/egress-proxy/entrypoint.sh +0 -62
  61. package/packages/eval/harbor/egress-proxy/network-policy +0 -148
  62. package/packages/eval/harbor/egress_filter.py +0 -435
  63. package/packages/eval/harbor/eval_framework.py +0 -40
  64. package/packages/eval/harbor/relay_agent.py +0 -686
  65. package/packages/eval/harbor/run_trial.py +0 -223
  66. package/packages/eval/package.json +0 -20
@@ -18,6 +18,7 @@
18
18
  */
19
19
  import { assertExactKeys, requireCount, requireExactRecord, requireRecord, requireString, } from './codec.js';
20
20
  import { defineOperation } from './operation-spec.js';
21
+ import { canonicalPeerMeshDisplayName } from '../peer-mesh/display-name.js';
21
22
  import { PEER_MESH_MAX_MEMBERS, PEER_MESH_MAX_MESHES, PEER_MESH_MAX_ROUTE_HINTS, } from '../peer-mesh/limits.js';
22
23
  const PEER_ID_MAX_BYTES = 256;
23
24
  const MESH_ID_MAX_BYTES = 128;
@@ -93,6 +94,20 @@ export const PEER_MESH_OPERATION_SPECS = {
93
94
  decodeInput: decodePeerMeshTransitSetInput,
94
95
  decodeOutput: decodePeerMeshQueryResult,
95
96
  }),
97
+ 'peer.mesh.display-name.set': defineOperation({
98
+ mode: 'command',
99
+ availability: 'ready',
100
+ errors: MUTATION_ERRORS,
101
+ decodeInput: decodePeerMeshDisplayNameSetInput,
102
+ decodeOutput: decodePeerMeshQueryResult,
103
+ }),
104
+ 'peer.mesh.rename': defineOperation({
105
+ mode: 'command',
106
+ availability: 'ready',
107
+ errors: MUTATION_ERRORS,
108
+ decodeInput: decodePeerMeshRenameInput,
109
+ decodeOutput: decodePeerMeshQueryResult,
110
+ }),
96
111
  };
97
112
  function decodeEmptyInput(value) {
98
113
  requireExactRecord(value, 'Peer Mesh operation input', []);
@@ -104,6 +119,19 @@ function decodePeerMeshTargetInput(value) {
104
119
  meshId: requireString(record.meshId, 'Peer Mesh meshId', MESH_ID_MAX_BYTES),
105
120
  };
106
121
  }
122
+ function decodePeerMeshDisplayNameSetInput(value) {
123
+ const record = requireExactRecord(value, 'Peer Mesh display name input', ['displayName']);
124
+ return {
125
+ displayName: record.displayName === null ? null : canonicalPeerMeshDisplayName(record.displayName),
126
+ };
127
+ }
128
+ function decodePeerMeshRenameInput(value) {
129
+ const record = requireExactRecord(value, 'Peer Mesh rename input', ['meshId', 'displayName']);
130
+ return {
131
+ meshId: requireString(record.meshId, 'Peer Mesh meshId', MESH_ID_MAX_BYTES),
132
+ displayName: record.displayName === null ? null : canonicalPeerMeshDisplayName(record.displayName),
133
+ };
134
+ }
107
135
  function decodePeerMeshInviteInput(value) {
108
136
  return decodePeerMeshTargetInput(value);
109
137
  }
@@ -171,7 +199,13 @@ export function decodePeerMeshQueryResult(value) {
171
199
  const record = requireRecord(value, 'Peer Mesh query result');
172
200
  assertExactKeys(record, 'Peer Mesh query result', record.localPeerId === undefined
173
201
  ? ['available', 'meshes']
174
- : ['available', 'localPeerId', 'meshes', 'transit']);
202
+ : [
203
+ 'available',
204
+ 'localPeerId',
205
+ ...(record.localDisplayName === undefined ? [] : ['localDisplayName']),
206
+ 'meshes',
207
+ 'transit',
208
+ ]);
175
209
  if (typeof record.available !== 'boolean' ||
176
210
  !Array.isArray(record.meshes) ||
177
211
  record.meshes.length > PEER_MESH_MAX_MESHES) {
@@ -187,14 +221,19 @@ export function decodePeerMeshQueryResult(value) {
187
221
  ? {}
188
222
  : {
189
223
  localPeerId: requireString(localPeerId, 'Peer Mesh localPeerId', PEER_ID_MAX_BYTES),
224
+ ...(record.localDisplayName === undefined
225
+ ? {}
226
+ : { localDisplayName: canonicalPeerMeshDisplayName(record.localDisplayName) }),
190
227
  transit: decodePeerMeshTransitProjection(record.transit),
191
228
  }),
192
229
  meshes: Object.freeze(record.meshes.map(decodePeerMeshProjection)),
193
230
  };
194
231
  }
195
232
  export function decodePeerMeshProjection(value) {
233
+ const valueRecord = requireRecord(value, 'Peer Mesh projection');
196
234
  const record = requireExactRecord(value, 'Peer Mesh projection', [
197
235
  'meshId',
236
+ ...(valueRecord.displayName === undefined ? [] : ['displayName']),
198
237
  'role',
199
238
  'authorityPeerId',
200
239
  'revision',
@@ -214,6 +253,9 @@ export function decodePeerMeshProjection(value) {
214
253
  }
215
254
  return {
216
255
  meshId: requireString(record.meshId, 'Peer Mesh meshId', MESH_ID_MAX_BYTES),
256
+ ...(record.displayName === undefined
257
+ ? {}
258
+ : { displayName: canonicalPeerMeshDisplayName(record.displayName) }),
217
259
  role: record.role,
218
260
  authorityPeerId: requireString(record.authorityPeerId, 'Peer Mesh authorityPeerId', PEER_ID_MAX_BYTES),
219
261
  revision: requireCount(record.revision, 'Peer Mesh revision'),
@@ -250,7 +292,13 @@ function decodePeerMeshTransitProjection(value) {
250
292
  }
251
293
  function decodePeerMeshMemberProjection(value) {
252
294
  const record = requireRecord(value, 'Peer Mesh member route');
253
- assertExactKeys(record, 'Peer Mesh member route', record.expiresAt === undefined ? ['peerId', 'state'] : ['peerId', 'state', 'expiresAt']);
295
+ assertExactKeys(record, 'Peer Mesh member route', [
296
+ 'peerId',
297
+ 'state',
298
+ ...(record.endpointKind === undefined ? [] : ['endpointKind']),
299
+ ...(record.displayName === undefined ? [] : ['displayName']),
300
+ ...(record.expiresAt === undefined ? [] : ['expiresAt']),
301
+ ]);
254
302
  if (record.state !== 'local' &&
255
303
  record.state !== 'route_available' &&
256
304
  record.state !== 'coordination_only' &&
@@ -258,8 +306,17 @@ function decodePeerMeshMemberProjection(value) {
258
306
  record.state !== 'unknown') {
259
307
  throw new Error('Invalid Peer Mesh member route state');
260
308
  }
309
+ if (record.endpointKind !== undefined &&
310
+ record.endpointKind !== 'client' &&
311
+ record.endpointKind !== 'host') {
312
+ throw new Error('Invalid Peer Mesh member endpoint kind');
313
+ }
261
314
  return {
262
315
  peerId: requireString(record.peerId, 'Peer Mesh member route peerId', PEER_ID_MAX_BYTES),
316
+ ...(record.endpointKind === undefined ? {} : { endpointKind: record.endpointKind }),
317
+ ...(record.displayName === undefined
318
+ ? {}
319
+ : { displayName: canonicalPeerMeshDisplayName(record.displayName) }),
263
320
  state: record.state,
264
321
  ...(record.expiresAt === undefined
265
322
  ? {}
@@ -78,7 +78,9 @@ class FileRuntimeHostAccessAuthority {
78
78
  credentialId: match.credentialId,
79
79
  operationGrants: match.bindClientInstanceOnFinalize
80
80
  ? ['host.status', 'access.credential.finalize']
81
- : match.operationGrants,
81
+ : match.clientInstanceId
82
+ ? [...match.operationGrants, 'access.credential.finalize']
83
+ : match.operationGrants,
82
84
  canPublishClientCapabilities: !match.bindClientInstanceOnFinalize && match.canPublishClientCapabilities,
83
85
  canUseHostPaths: !match.bindClientInstanceOnFinalize && match.canUseHostPaths,
84
86
  ...(match.clientInstanceId ? { clientInstanceId: match.clientInstanceId } : {}),
@@ -476,21 +478,23 @@ class FileRuntimeHostAccessAuthority {
476
478
  await this.#commit(createNextAccessCredentialFile(this.#file, credentials, this.#file.sessionGrants), [current, ...pendingForPrincipal].map((credential) => credential.credentialId));
477
479
  return { credentialId, revoked: true };
478
480
  }
479
- finalize(credentialId, clientInstanceId) {
481
+ finalize(credentialId, clientInstanceId, connectionAlreadyFinalized) {
480
482
  return this.#mutate(async () => {
481
483
  const retained = this.#file.credentials.find((credential) => credential.credentialId === credentialId);
482
484
  if (!retained || retained.status === 'revoked') {
483
485
  throw new RuntimeHostAccessInputError('The current access credential is no longer active');
484
486
  }
485
- return this.#finalize(retained, clientInstanceId);
487
+ return this.#finalize(retained, clientInstanceId, connectionAlreadyFinalized);
486
488
  });
487
489
  }
488
- async #finalize(retained, clientInstanceId) {
490
+ async #finalize(retained, clientInstanceId, connectionAlreadyFinalized) {
489
491
  if (retained.status === 'active') {
490
492
  if (retained.clientInstanceId && retained.clientInstanceId !== clientInstanceId) {
491
493
  throw new RuntimeHostAccessInputError('The pairing candidate was claimed by another Client');
492
494
  }
493
- return { reconnectRequired: retained.clientInstanceId !== undefined };
495
+ return {
496
+ reconnectRequired: retained.clientInstanceId !== undefined && !connectionAlreadyFinalized,
497
+ };
494
498
  }
495
499
  if (Date.parse(retained.expiresAt) <= Date.now()) {
496
500
  await this.#expirePending();
@@ -776,7 +780,7 @@ export async function revokeAccessCredentialRotation(authority, input) {
776
780
  return accessPersistenceFailure(error, 'Access credential rotation revocation outcome is unknown', 'Access credential could not be revoked');
777
781
  }
778
782
  }
779
- export async function finalizeAccessCredential(authority, credentialId, clientInstanceId) {
783
+ export async function finalizeAccessCredential(authority, credentialId, clientInstanceId, credentialClientInstanceId) {
780
784
  if (!authority)
781
785
  return unavailable('finalize');
782
786
  if (!credentialId) {
@@ -800,7 +804,7 @@ export async function finalizeAccessCredential(authority, credentialId, clientIn
800
804
  try {
801
805
  return {
802
806
  ok: true,
803
- result: await authority.finalize(credentialId, clientInstanceId),
807
+ result: await authority.finalize(credentialId, clientInstanceId, credentialClientInstanceId === clientInstanceId),
804
808
  };
805
809
  }
806
810
  catch (error) {
@@ -164,6 +164,9 @@ export class RuntimeHostConnectionSession {
164
164
  ...(this.#options.connection.authority.credentialId
165
165
  ? { credentialId: this.#options.connection.authority.credentialId }
166
166
  : {}),
167
+ ...(this.#options.connection.authority.clientInstanceId
168
+ ? { credentialClientInstanceId: this.#options.connection.authority.clientInstanceId }
169
+ : {}),
167
170
  acquireResidency: () => admission.acquireResidency(),
168
171
  });
169
172
  admission.seal();
@@ -54,6 +54,7 @@ export async function startExecutionRuntimeHostService(options, dependencies = {
54
54
  peerOwner = await openRuntimeHostPeerMeshOwner({
55
55
  ...options.peer,
56
56
  dataRoot: options.peer.meshDataRoot,
57
+ endpointKind: 'host',
57
58
  });
58
59
  }
59
60
  catch (error) {
@@ -524,7 +524,7 @@ export class RuntimeHostKernel {
524
524
  'access.principal.revoke': async (input) => this.#settleAccessCredentialMutation(revokeAccessPrincipal(this.#options.accessAuthority, input)),
525
525
  'access.credential.rotation.prepare': async (input) => this.#settleAccessCredentialMutation(prepareAccessCredentialRotation(this.#options.accessAuthority, input)),
526
526
  'access.credential.rotation.revoke': async (input) => this.#settleAccessCredentialMutation(revokeAccessCredentialRotation(this.#options.accessAuthority, input)),
527
- 'access.credential.finalize': async (_input, context) => this.#settleAccessCredentialMutation(finalizeAccessCredential(this.#options.accessAuthority, context.credentialId, context.clientInstanceId)),
527
+ 'access.credential.finalize': async (_input, context) => this.#settleAccessCredentialMutation(finalizeAccessCredential(this.#options.accessAuthority, context.credentialId, context.clientInstanceId, context.credentialClientInstanceId)),
528
528
  'collaboration.invitation.prepare': async (input) => this.#settleAccessCredentialMutation(prepareCollaborationInvitation(this.#options.accessAuthority, this.rootId, input)),
529
529
  'collaboration.access.query': async (input) => this.#options.accessAuthority
530
530
  ? {
@@ -128,11 +128,28 @@ export function createPeerMeshOperationHandlers(mesh, options = {}) {
128
128
  return { ok: true, result: query() };
129
129
  });
130
130
  },
131
+ 'peer.mesh.display-name.set': async (input) => {
132
+ if (!mesh)
133
+ return unavailable();
134
+ return mutate(async () => {
135
+ await mesh.setDisplayName(input.displayName);
136
+ return { ok: true, result: query() };
137
+ });
138
+ },
139
+ 'peer.mesh.rename': async (input) => {
140
+ if (!mesh)
141
+ return unavailable();
142
+ return mutate(async () => {
143
+ await mesh.setMeshDisplayName(input.meshId, input.displayName);
144
+ return { ok: true, result: query() };
145
+ });
146
+ },
131
147
  };
132
148
  }
133
149
  export function projectPeerMeshStatus(status) {
134
150
  return Object.freeze({
135
151
  meshId: status.roster.roster.meshId,
152
+ ...(status.roster.roster.displayName ? { displayName: status.roster.roster.displayName } : {}),
136
153
  role: status.role,
137
154
  authorityPeerId: status.authority.peerId,
138
155
  revision: status.roster.roster.revision,
@@ -144,9 +161,11 @@ export function projectPeerMeshStatus(status) {
144
161
  export function projectPeerMeshQuery(mesh) {
145
162
  if (!mesh)
146
163
  return { available: false, meshes: [] };
164
+ const displayName = mesh.displayName();
147
165
  return Object.freeze({
148
166
  available: true,
149
167
  localPeerId: mesh.localPeerId(),
168
+ ...(displayName ? { localDisplayName: displayName } : {}),
150
169
  meshes: Object.freeze(mesh.status().map(projectPeerMeshStatus)),
151
170
  transit: projectTransitSnapshot(mesh.transitMeshId(), mesh.transitSnapshot()),
152
171
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "maka-agent",
3
- "version": "0.2.0-dev.3.20260830",
3
+ "version": "0.2.0-dev.5.20260830",
4
4
  "description": "Apache Maka (Incubating) developer snapshot; not an Apache release.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -39,7 +39,6 @@
39
39
  "files": [
40
40
  "dist",
41
41
  "native",
42
- "packages/eval",
43
42
  "README.md",
44
43
  "README.zh-CN.md",
45
44
  "LICENSE",
@@ -1,138 +0,0 @@
1
- /*
2
- * Licensed to the Apache Software Foundation (ASF) under one
3
- * or more contributor license agreements. See the NOTICE file
4
- * distributed with this work for additional information
5
- * regarding copyright ownership. The ASF licenses this file
6
- * to you under the Apache License, Version 2.0 (the
7
- * "License"); you may not use this file except in compliance
8
- * with the License. You may obtain a copy of the License at
9
- *
10
- * http://www.apache.org/licenses/LICENSE-2.0
11
- *
12
- * Unless required by applicable law or agreed to in writing,
13
- * software distributed under the License is distributed on an
14
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- * KIND, either express or implied. See the License for the
16
- * specific language governing permissions and limitations
17
- * under the License.
18
- */
19
- import { createHash, randomUUID } from 'node:crypto';
20
- import { link, mkdir, open, readFile, readdir, unlink } from 'node:fs/promises';
21
- import { join } from 'node:path';
22
- import { decodeEvalResult } from './result.js';
23
- export class FileAttemptStore {
24
- path;
25
- constructor(path) {
26
- this.path = path;
27
- }
28
- async runExclusive(operation) {
29
- await mkdir(this.path, { recursive: true });
30
- const path = join(this.path, '.writer.lock');
31
- let lock;
32
- try {
33
- lock = await open(path, 'wx', 0o600);
34
- }
35
- catch (error) {
36
- if (error.code === 'EEXIST') {
37
- throw new Error(`${path}: experiment already has an active writer`);
38
- }
39
- throw error;
40
- }
41
- try {
42
- return await operation();
43
- }
44
- finally {
45
- await lock.close();
46
- await unlink(path);
47
- }
48
- }
49
- async list(cellId) {
50
- const directory = this.#cellDirectory(cellId);
51
- let names;
52
- try {
53
- names = await readdir(directory);
54
- }
55
- catch (error) {
56
- if (error.code === 'ENOENT')
57
- return [];
58
- throw error;
59
- }
60
- const attempts = await Promise.all(names
61
- .filter((name) => /^\d{6}\.json$/u.test(name))
62
- .sort()
63
- .map(async (name) => {
64
- const attempt = decodeAttempt(JSON.parse(await readFile(join(directory, name), 'utf8')));
65
- const sequence = Number(name.slice(0, 6));
66
- if (attempt.cellId !== cellId || attempt.sequence !== sequence) {
67
- throw new Error('attempt file does not match its immutable identity');
68
- }
69
- return attempt;
70
- }));
71
- return attempts;
72
- }
73
- async append(attempt) {
74
- const record = decodeAttempt(attempt);
75
- const expected = ((await this.list(record.cellId)).at(-1)?.sequence ?? 0) + 1;
76
- if (record.sequence !== expected)
77
- throw new Error(`attempt sequence must be ${expected}`);
78
- const directory = this.#cellDirectory(record.cellId);
79
- await mkdir(directory, { recursive: true });
80
- const finalPath = join(directory, `${String(record.sequence).padStart(6, '0')}.json`);
81
- const temporaryPath = `${finalPath}.${randomUUID()}.tmp`;
82
- const temporary = await open(temporaryPath, 'wx', 0o600);
83
- try {
84
- await temporary.writeFile(`${JSON.stringify(record)}\n`);
85
- await temporary.sync();
86
- await temporary.close();
87
- await link(temporaryPath, finalPath);
88
- }
89
- finally {
90
- await temporary.close().catch(() => undefined);
91
- await unlink(temporaryPath).catch(() => undefined);
92
- }
93
- }
94
- #cellDirectory(cellId) {
95
- return join(this.path, createHash('sha256').update(cellId).digest('hex'));
96
- }
97
- }
98
- function decodeAttempt(value) {
99
- const attempt = exact(value, 'attempt', [
100
- 'cellId',
101
- 'sequence',
102
- 'startedAt',
103
- 'completedAt',
104
- 'result',
105
- ]);
106
- return {
107
- cellId: nonempty(attempt.cellId, 'attempt.cellId'),
108
- sequence: positiveInteger(attempt.sequence, 'attempt.sequence'),
109
- startedAt: nonnegative(attempt.startedAt, 'attempt.startedAt'),
110
- completedAt: nonnegative(attempt.completedAt, 'attempt.completedAt'),
111
- result: decodeEvalResult(attempt.result, 'attempt.result'),
112
- };
113
- }
114
- function exact(value, where, fields) {
115
- if (!value || typeof value !== 'object' || Array.isArray(value))
116
- throw new Error(`${where} must be an object`);
117
- const record = value;
118
- if (Object.keys(record).length !== fields.length ||
119
- fields.some((field) => !Object.hasOwn(record, field))) {
120
- throw new Error(`${where} fields are invalid`);
121
- }
122
- return record;
123
- }
124
- function nonempty(value, where) {
125
- if (typeof value !== 'string' || value.length === 0)
126
- throw new Error(`${where} is invalid`);
127
- return value;
128
- }
129
- function positiveInteger(value, where) {
130
- if (!Number.isSafeInteger(value) || value < 1)
131
- throw new Error(`${where} is invalid`);
132
- return value;
133
- }
134
- function nonnegative(value, where) {
135
- if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
136
- throw new Error(`${where} is invalid`);
137
- return value;
138
- }
@@ -1,119 +0,0 @@
1
- /*
2
- * Licensed to the Apache Software Foundation (ASF) under one
3
- * or more contributor license agreements. See the NOTICE file
4
- * distributed with this work for additional information
5
- * regarding copyright ownership. The ASF licenses this file
6
- * to you under the Apache License, Version 2.0 (the
7
- * "License"); you may not use this file except in compliance
8
- * with the License. You may obtain a copy of the License at
9
- *
10
- * http://www.apache.org/licenses/LICENSE-2.0
11
- *
12
- * Unless required by applicable law or agreed to in writing,
13
- * software distributed under the License is distributed on an
14
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- * KIND, either express or implied. See the License for the
16
- * specific language governing permissions and limitations
17
- * under the License.
18
- */
19
- import { readFile } from 'node:fs/promises';
20
- import { resolve } from 'node:path';
21
- import { createExternalSubjectAdapter } from './external-subject.js';
22
- import { createHarborExecutor, createPierExecutor, } from './harness-executor.js';
23
- import { openExperimentDirectory } from './experiment-directory.js';
24
- import { createMakaSubjectAdapter } from './maka-subject.js';
25
- import { runExperiment } from './runner.js';
26
- import { parseExperimentSpec } from './spec.js';
27
- const USAGE = 'usage: maka eval run <spec.json> --out <directory> [--cell <cell-id>]';
28
- export async function runMakaEvalCli(argv, overrides = {}) {
29
- const controller = overrides.signal ? undefined : new AbortController();
30
- const signal = overrides.signal ?? controller?.signal;
31
- let interrupted;
32
- const interrupt = () => {
33
- interrupted = 130;
34
- controller?.abort();
35
- };
36
- const terminate = () => {
37
- interrupted = 143;
38
- controller?.abort();
39
- };
40
- if (controller) {
41
- process.once('SIGINT', interrupt);
42
- process.once('SIGTERM', terminate);
43
- }
44
- try {
45
- const command = parseArgs(argv);
46
- if (command.kind === 'help') {
47
- (overrides.stdout ?? process.stdout.write.bind(process.stdout))(`${USAGE}\n`);
48
- return 0;
49
- }
50
- const specPath = resolve(command.specPath);
51
- const spec = parseExperimentSpec(JSON.parse(await readFile(specPath, 'utf8')));
52
- const directory = await openExperimentDirectory(resolve(command.outDir), spec);
53
- let executor;
54
- if (overrides.loadExecutor) {
55
- executor = overrides.loadExecutor(spec, specPath);
56
- }
57
- else {
58
- const builtin = builtinExecutor(spec, specPath);
59
- await builtin.preflight({
60
- subjectCredentialNames: [
61
- ...new Set(spec.subjects.flatMap((subject) => subject.credentials)),
62
- ],
63
- ...(signal ? { signal } : {}),
64
- });
65
- executor = builtin;
66
- }
67
- const results = await runExperiment({
68
- spec,
69
- store: directory.attempts,
70
- executor,
71
- subjects: overrides.subjects ?? [createMakaSubjectAdapter(), createExternalSubjectAdapter()],
72
- ...(command.cellIds.length > 0 ? { cellIds: command.cellIds } : {}),
73
- ...(signal ? { signal } : {}),
74
- });
75
- const incomplete = spec.tasks.length * spec.repetitions * spec.subjects.length - results.size;
76
- (overrides.stdout ?? process.stdout.write.bind(process.stdout))(`${JSON.stringify({ experimentId: spec.id, cells: results.size, incomplete })}\n`);
77
- return interrupted ?? (incomplete === 0 ? 0 : 1);
78
- }
79
- catch (error) {
80
- (overrides.stderr ?? process.stderr.write.bind(process.stderr))(`maka eval: ${error instanceof Error ? error.message : String(error)}\n${USAGE}\n`);
81
- return interrupted ?? 2;
82
- }
83
- finally {
84
- if (controller) {
85
- process.removeListener('SIGINT', interrupt);
86
- process.removeListener('SIGTERM', terminate);
87
- }
88
- }
89
- }
90
- function builtinExecutor(spec, specPath) {
91
- if (spec.executor.kind === 'harbor')
92
- return createHarborExecutor(spec.executor.config, specPath);
93
- if (spec.executor.kind === 'pier')
94
- return createPierExecutor(spec.executor.config, specPath);
95
- throw new Error(`unsupported executor: ${spec.executor.kind}`);
96
- }
97
- function parseArgs(argv) {
98
- if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h'))
99
- return { kind: 'help' };
100
- if (argv[0] !== 'run' || !argv[1] || argv[1].startsWith('-'))
101
- throw new Error(USAGE);
102
- let outDir;
103
- const cellIds = [];
104
- for (let index = 2; index < argv.length; index += 2) {
105
- const flag = argv[index];
106
- const value = argv[index + 1];
107
- if (!value || value.startsWith('-'))
108
- throw new Error(`${flag} requires a value`);
109
- if (flag === '--out')
110
- outDir = value;
111
- else if (flag === '--cell')
112
- cellIds.push(value);
113
- else
114
- throw new Error(`unexpected argument: ${flag}`);
115
- }
116
- if (!outDir)
117
- throw new Error('--out is required');
118
- return { kind: 'run', specPath: argv[1], outDir, cellIds };
119
- }
@@ -1,59 +0,0 @@
1
- /*
2
- * Licensed to the Apache Software Foundation (ASF) under one
3
- * or more contributor license agreements. See the NOTICE file
4
- * distributed with this work for additional information
5
- * regarding copyright ownership. The ASF licenses this file
6
- * to you under the Apache License, Version 2.0 (the
7
- * "License"); you may not use this file except in compliance
8
- * with the License. You may obtain a copy of the License at
9
- *
10
- * http://www.apache.org/licenses/LICENSE-2.0
11
- *
12
- * Unless required by applicable law or agreed to in writing,
13
- * software distributed under the License is distributed on an
14
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- * KIND, either express or implied. See the License for the
16
- * specific language governing permissions and limitations
17
- * under the License.
18
- */
19
- import { randomUUID } from 'node:crypto';
20
- import { link, mkdir, open, readFile, unlink } from 'node:fs/promises';
21
- import { join } from 'node:path';
22
- import { FileAttemptStore } from './attempt-store.js';
23
- import { parseExperimentSpec } from './spec.js';
24
- export async function openExperimentDirectory(root, spec) {
25
- await mkdir(root, { recursive: true });
26
- const specPath = join(root, 'experiment.json');
27
- const temporaryPath = join(root, `.experiment-${randomUUID()}.tmp`);
28
- const canonical = `${canonicalJson(spec)}\n`;
29
- const temporary = await open(temporaryPath, 'wx', 0o600);
30
- try {
31
- await temporary.writeFile(canonical);
32
- await temporary.sync();
33
- await temporary.close();
34
- await link(temporaryPath, specPath);
35
- }
36
- catch (error) {
37
- if (error.code !== 'EEXIST')
38
- throw error;
39
- const existing = parseExperimentSpec(JSON.parse(await readFile(specPath, 'utf8')));
40
- if (canonicalJson(existing) !== canonicalJson(spec))
41
- throw new Error('experiment spec differs');
42
- }
43
- finally {
44
- await temporary.close().catch(() => undefined);
45
- await unlink(temporaryPath).catch(() => undefined);
46
- }
47
- return { root, specPath, attempts: new FileAttemptStore(join(root, 'attempts')) };
48
- }
49
- function canonicalJson(value) {
50
- if (Array.isArray(value))
51
- return `[${value.map(canonicalJson).join(',')}]`;
52
- if (value && typeof value === 'object') {
53
- return `{${Object.entries(value)
54
- .sort(([left], [right]) => left.localeCompare(right))
55
- .map(([key, child]) => `${JSON.stringify(key)}:${canonicalJson(child)}`)
56
- .join(',')}}`;
57
- }
58
- return JSON.stringify(value);
59
- }
@@ -1,49 +0,0 @@
1
- /*
2
- * Licensed to the Apache Software Foundation (ASF) under one
3
- * or more contributor license agreements. See the NOTICE file
4
- * distributed with this work for additional information
5
- * regarding copyright ownership. The ASF licenses this file
6
- * to you under the Apache License, Version 2.0 (the
7
- * "License"); you may not use this file except in compliance
8
- * with the License. You may obtain a copy of the License at
9
- *
10
- * http://www.apache.org/licenses/LICENSE-2.0
11
- *
12
- * Unless required by applicable law or agreed to in writing,
13
- * software distributed under the License is distributed on an
14
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- * KIND, either express or implied. See the License for the
16
- * specific language governing permissions and limitations
17
- * under the License.
18
- */
19
- export function decodeJsonObject(value, where) {
20
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
21
- throw new Error(`${where} must be an object`);
22
- }
23
- return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, decodeJsonValue(child, `${where}.${key}`)]));
24
- }
25
- function decodeJsonValue(value, where) {
26
- if (value === null || typeof value === 'string' || typeof value === 'boolean')
27
- return value;
28
- if (typeof value === 'number' && Number.isFinite(value))
29
- return value;
30
- if (Array.isArray(value)) {
31
- return value.map((child, index) => decodeJsonValue(child, `${where}[${index}]`));
32
- }
33
- if (value && typeof value === 'object')
34
- return decodeJsonObject(value, where);
35
- throw new Error(`${where} must be JSON`);
36
- }
37
- export function expandExperiment(spec) {
38
- return spec.tasks.flatMap((task) => Array.from({ length: spec.repetitions }, (_, repetition) => repetition + 1).flatMap((repetition) => spec.subjects.map((subject) => ({
39
- id: `${task.id}::${repetition}::${subject.id}`,
40
- experimentId: spec.id,
41
- benchmark: spec.benchmark,
42
- executor: spec.executor,
43
- subject,
44
- task,
45
- repetition,
46
- budget: spec.budget,
47
- verifier: spec.verifier,
48
- }))));
49
- }