@ethogram/cli 0.1.0-alpha.0 → 0.1.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -7
- package/dist/cli.js +0 -0
- package/dist/contracts.d.ts +8 -0
- package/dist/evaluator.js +25 -9
- package/dist/generic-engine.d.ts +2 -2
- package/dist/generic-engine.js +3 -3
- package/dist/project-runtime.d.ts +34 -0
- package/dist/project-runtime.js +144 -0
- package/dist/templates.js +24 -8
- package/dist/typescript-adapter.d.ts +1 -1
- package/dist/typescript-adapter.js +154 -22
- package/package.json +9 -3
package/README.md
CHANGED
|
@@ -1,17 +1,55 @@
|
|
|
1
|
-
#
|
|
1
|
+
# `@ethogram/cli`
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The local Ethogram initializer and read-only developer interface for TypeScript and Node.js projects.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
> Public alpha `0.1.0-alpha.2`. APIs may change between `0.x` releases. Node.js 20.9 or newer is required.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install --save-dev @ethogram/core@next @ethogram/cli@next
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Install core as a direct dependency because generated and hand-written Stories import it. The locally installed CLI is available through `npx ethogram`.
|
|
14
|
+
|
|
15
|
+
## Try the starter
|
|
16
|
+
|
|
17
|
+
Run these commands from a project whose `package.json` has a non-empty `name`:
|
|
6
18
|
|
|
7
19
|
```bash
|
|
8
20
|
npx ethogram init
|
|
9
|
-
npx ethogram init --existing
|
|
10
21
|
npx ethogram dev
|
|
11
22
|
```
|
|
12
23
|
|
|
13
|
-
|
|
24
|
+
`init` adds a deterministic Access Request example: configuration, an Agent descriptor, a Story, and an execution profile. It checks every target before writing. If it finds conflicting content, it preserves all existing files and writes nothing.
|
|
25
|
+
|
|
26
|
+
`dev` binds to `127.0.0.1`, opens a read-only browser interface, and uses the current directory as the project root. Select the generated Story and choose **Run Story**. The starter makes no model call and has no external side effects.
|
|
27
|
+
|
|
28
|
+
## Connect an existing agent
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npx ethogram init --existing
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
This mode creates only `ethogram.config.mjs`. Add your own Agent descriptor, Story, and execution profile, then start the interface with `npx ethogram dev`.
|
|
35
|
+
|
|
36
|
+
## Commands
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
ethogram init [--existing]
|
|
40
|
+
ethogram dev [--project <path>] [--port <number>] [--no-open]
|
|
41
|
+
ethogram --help
|
|
42
|
+
ethogram --version
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
- `--project` loads a project outside the current directory.
|
|
46
|
+
- `--port` changes the default port, `4317`. Port `0` selects an available port.
|
|
47
|
+
- `--no-open` starts the server without opening a browser.
|
|
48
|
+
|
|
49
|
+
Relevant TypeScript and JavaScript changes reload automatically. A previous result is cleared or marked stale until you run the Story against the new revision. Ethogram does not persist run history.
|
|
50
|
+
|
|
51
|
+
The package exports `@ethogram/cli/runtime` only for the isolated worker used by `@ethogram/mcp`. It is not a general engine extension API.
|
|
14
52
|
|
|
15
|
-
|
|
53
|
+
Read the [starter guide](https://github.com/leonardocamacho1983/ethogram/blob/main/docs/quickstart.md), [existing-agent guide](https://github.com/leonardocamacho1983/ethogram/blob/main/docs/existing-agent.md), and [alpha limitations](https://github.com/leonardocamacho1983/ethogram/blob/main/docs/limitations.md).
|
|
16
54
|
|
|
17
|
-
|
|
55
|
+
License: MIT. Issues: <https://github.com/leonardocamacho1983/ethogram/issues>
|
package/dist/cli.js
CHANGED
|
File without changes
|
package/dist/contracts.d.ts
CHANGED
|
@@ -99,6 +99,14 @@ export type BehavioralVerdict = 'PASS' | 'FAIL';
|
|
|
99
99
|
export type EvaluationResult = {
|
|
100
100
|
verdict: BehavioralVerdict;
|
|
101
101
|
expectations: Readonly<Record<string, BehavioralVerdict>>;
|
|
102
|
+
expectationResults: readonly {
|
|
103
|
+
id: string;
|
|
104
|
+
description: string;
|
|
105
|
+
matcher: ExpectationMatcher;
|
|
106
|
+
verdict: BehavioralVerdict;
|
|
107
|
+
observedCallCount: number;
|
|
108
|
+
matchingCallIds: readonly string[];
|
|
109
|
+
}[];
|
|
102
110
|
};
|
|
103
111
|
export type CompletedExecutionRecord = {
|
|
104
112
|
observedRun: ObservedRun;
|
package/dist/evaluator.js
CHANGED
|
@@ -1,15 +1,31 @@
|
|
|
1
|
-
function
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
function matchingCalls(matcher, observedRun) {
|
|
2
|
+
switch (matcher.kind) {
|
|
3
|
+
case 'tool-called':
|
|
4
|
+
case 'tool-not-called':
|
|
5
|
+
return observedRun.toolCalls.filter((toolCall) => toolCall.name === matcher.tool);
|
|
6
|
+
default: {
|
|
7
|
+
const unsupported = matcher;
|
|
8
|
+
throw new Error(`UNSUPPORTED_MATCHER: ${JSON.stringify(unsupported)}`);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
4
11
|
}
|
|
5
12
|
export function evaluateStory(story, observedRun) {
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
|
|
13
|
+
const expectationResults = story.expectations.map((expectation) => {
|
|
14
|
+
const calls = matchingCalls(expectation.matcher, observedRun);
|
|
15
|
+
const matched = expectation.matcher.kind === 'tool-called' ? calls.length > 0 : calls.length === 0;
|
|
16
|
+
const verdict = matched ? 'PASS' : 'FAIL';
|
|
17
|
+
return Object.freeze({
|
|
18
|
+
id: expectation.id,
|
|
19
|
+
description: expectation.description,
|
|
20
|
+
matcher: expectation.matcher,
|
|
21
|
+
verdict,
|
|
22
|
+
observedCallCount: calls.length,
|
|
23
|
+
matchingCallIds: Object.freeze(calls.map(({ callId }) => callId)),
|
|
24
|
+
});
|
|
9
25
|
});
|
|
10
|
-
const expectations = Object.freeze(Object.fromEntries(
|
|
11
|
-
const verdict =
|
|
26
|
+
const expectations = Object.freeze(Object.fromEntries(expectationResults.map(({ id, verdict }) => [id, verdict])));
|
|
27
|
+
const verdict = expectationResults.every(({ verdict: value }) => value === 'PASS')
|
|
12
28
|
? 'PASS'
|
|
13
29
|
: 'FAIL';
|
|
14
|
-
return Object.freeze({ verdict, expectations });
|
|
30
|
+
return Object.freeze({ verdict, expectations, expectationResults: Object.freeze(expectationResults) });
|
|
15
31
|
}
|
package/dist/generic-engine.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export type EngineRunResult = {
|
|
|
8
8
|
runner: 'LanguageAdapterRunner';
|
|
9
9
|
evaluator: 'deterministic';
|
|
10
10
|
storyUnchanged: boolean;
|
|
11
|
-
mockDataUsed:
|
|
11
|
+
mockDataUsed: 'unknown';
|
|
12
12
|
};
|
|
13
13
|
};
|
|
14
14
|
export declare class EthogramEngine {
|
|
@@ -17,5 +17,5 @@ export declare class EthogramEngine {
|
|
|
17
17
|
constructor(runner: LanguageAdapter);
|
|
18
18
|
loadProject(projectRoot: string): Promise<ProjectDescriptor>;
|
|
19
19
|
getProject(): ProjectDescriptor;
|
|
20
|
-
runStory(storyId: string): Promise<EngineRunResult>;
|
|
20
|
+
runStory(storyId: string, executionId?: string): Promise<EngineRunResult>;
|
|
21
21
|
}
|
package/dist/generic-engine.js
CHANGED
|
@@ -26,7 +26,7 @@ export class EthogramEngine {
|
|
|
26
26
|
throw new Error('PROJECT_NOT_LOADED');
|
|
27
27
|
return this.project;
|
|
28
28
|
}
|
|
29
|
-
async runStory(storyId) {
|
|
29
|
+
async runStory(storyId, executionId = randomUUID()) {
|
|
30
30
|
const project = this.getProject();
|
|
31
31
|
const story = project.stories.find((candidate) => candidate.id === storyId);
|
|
32
32
|
if (!story)
|
|
@@ -41,13 +41,13 @@ export class EthogramEngine {
|
|
|
41
41
|
return {
|
|
42
42
|
execution: { observedRun, evaluationResult },
|
|
43
43
|
boundaryEvidence: {
|
|
44
|
-
executionId
|
|
44
|
+
executionId,
|
|
45
45
|
completedBehavioralRuns: 1,
|
|
46
46
|
adapter: this.runner.id,
|
|
47
47
|
runner: 'LanguageAdapterRunner',
|
|
48
48
|
evaluator: 'deterministic',
|
|
49
49
|
storyUnchanged: canonical(story) === storySnapshot,
|
|
50
|
-
mockDataUsed:
|
|
50
|
+
mockDataUsed: 'unknown',
|
|
51
51
|
},
|
|
52
52
|
};
|
|
53
53
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type EngineRunResult } from './generic-engine.js';
|
|
2
|
+
import type { ProjectDescriptor } from './contracts.js';
|
|
3
|
+
export type ProjectSnapshot = {
|
|
4
|
+
revision: string;
|
|
5
|
+
project: ProjectDescriptor;
|
|
6
|
+
storyDigests: Readonly<Record<string, string>>;
|
|
7
|
+
};
|
|
8
|
+
export type ProjectOperationRequest = {
|
|
9
|
+
kind: 'inspect';
|
|
10
|
+
} | {
|
|
11
|
+
kind: 'run';
|
|
12
|
+
storyId: string;
|
|
13
|
+
expectedRevision: string;
|
|
14
|
+
expectedStoryDigest: string;
|
|
15
|
+
executionId?: string;
|
|
16
|
+
};
|
|
17
|
+
export type ProjectOperationResult = {
|
|
18
|
+
kind: 'inspect';
|
|
19
|
+
snapshot: ProjectSnapshot;
|
|
20
|
+
} | {
|
|
21
|
+
kind: 'run';
|
|
22
|
+
snapshot: ProjectSnapshot;
|
|
23
|
+
run: EngineRunResult;
|
|
24
|
+
};
|
|
25
|
+
export type ProjectOperationOptions = {
|
|
26
|
+
beforeRun?: (snapshot: ProjectSnapshot) => void | Promise<void>;
|
|
27
|
+
};
|
|
28
|
+
export declare class ProjectRuntimeError extends Error {
|
|
29
|
+
readonly code: string;
|
|
30
|
+
readonly effectsMayHaveOccurred: boolean;
|
|
31
|
+
readonly retrySafe: boolean;
|
|
32
|
+
constructor(code: string, message: string, effectsMayHaveOccurred?: boolean, retrySafe?: boolean);
|
|
33
|
+
}
|
|
34
|
+
export declare function executeProjectOperation(projectRoot: string, request: ProjectOperationRequest, options?: ProjectOperationOptions): Promise<ProjectOperationResult>;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { readFile, readdir, realpath, stat } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { EthogramEngine } from './generic-engine.js';
|
|
5
|
+
import { TypeScriptAdapter, TypeScriptAdapterError } from './typescript-adapter.js';
|
|
6
|
+
const ignoredDirectories = new Set([
|
|
7
|
+
'.git', '.next', '.turbo', '.vercel', 'build', 'coverage', 'dist', 'node_modules',
|
|
8
|
+
]);
|
|
9
|
+
const sourceExtensions = new Set(['.cjs', '.cts', '.js', '.json', '.mjs', '.mts', '.ts']);
|
|
10
|
+
export class ProjectRuntimeError extends Error {
|
|
11
|
+
code;
|
|
12
|
+
effectsMayHaveOccurred;
|
|
13
|
+
retrySafe;
|
|
14
|
+
constructor(code, message, effectsMayHaveOccurred = false, retrySafe = !effectsMayHaveOccurred) {
|
|
15
|
+
super(message);
|
|
16
|
+
this.code = code;
|
|
17
|
+
this.effectsMayHaveOccurred = effectsMayHaveOccurred;
|
|
18
|
+
this.retrySafe = retrySafe;
|
|
19
|
+
this.name = 'ProjectRuntimeError';
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function stableJson(value) {
|
|
23
|
+
if (Array.isArray(value))
|
|
24
|
+
return `[${value.map(stableJson).join(',')}]`;
|
|
25
|
+
if (value && typeof value === 'object') {
|
|
26
|
+
return `{${Object.entries(value)
|
|
27
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
28
|
+
.map(([key, nested]) => `${JSON.stringify(key)}:${stableJson(nested)}`)
|
|
29
|
+
.join(',')}}`;
|
|
30
|
+
}
|
|
31
|
+
return JSON.stringify(value);
|
|
32
|
+
}
|
|
33
|
+
function digest(value) {
|
|
34
|
+
return createHash('sha256').update(stableJson(value)).digest('hex');
|
|
35
|
+
}
|
|
36
|
+
async function projectSourceDigest(projectRoot) {
|
|
37
|
+
const files = [];
|
|
38
|
+
async function visit(directory) {
|
|
39
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
40
|
+
for (const entry of entries) {
|
|
41
|
+
if (ignoredDirectories.has(entry.name))
|
|
42
|
+
continue;
|
|
43
|
+
const absolute = path.join(directory, entry.name);
|
|
44
|
+
if (entry.isDirectory()) {
|
|
45
|
+
await visit(absolute);
|
|
46
|
+
}
|
|
47
|
+
else if (entry.isFile() && sourceExtensions.has(path.extname(entry.name))) {
|
|
48
|
+
files.push(absolute);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
await visit(projectRoot);
|
|
53
|
+
files.sort();
|
|
54
|
+
const hash = createHash('sha256');
|
|
55
|
+
for (const file of files) {
|
|
56
|
+
hash.update(path.relative(projectRoot, file).split(path.sep).join('/'));
|
|
57
|
+
hash.update('\0');
|
|
58
|
+
hash.update(await readFile(file));
|
|
59
|
+
hash.update('\0');
|
|
60
|
+
}
|
|
61
|
+
return hash.digest('hex');
|
|
62
|
+
}
|
|
63
|
+
function runtimeError(error, effectsMayHaveOccurred = false) {
|
|
64
|
+
if (error instanceof ProjectRuntimeError)
|
|
65
|
+
return error;
|
|
66
|
+
if (error instanceof TypeScriptAdapterError) {
|
|
67
|
+
return new ProjectRuntimeError(error.code, error.message, effectsMayHaveOccurred, !effectsMayHaveOccurred);
|
|
68
|
+
}
|
|
69
|
+
if (error instanceof Error) {
|
|
70
|
+
const [candidate] = error.message.slice(0, 128).split(':', 1);
|
|
71
|
+
const code = /^[A-Z][A-Z_]+$/.test(candidate) ? candidate : 'ETHOGRAM_RUNTIME_ERROR';
|
|
72
|
+
return new ProjectRuntimeError(code, error.message, effectsMayHaveOccurred, !effectsMayHaveOccurred);
|
|
73
|
+
}
|
|
74
|
+
return new ProjectRuntimeError('ETHOGRAM_RUNTIME_ERROR', 'The Ethogram runtime operation failed.', effectsMayHaveOccurred, !effectsMayHaveOccurred);
|
|
75
|
+
}
|
|
76
|
+
async function stableSnapshot(projectRoot, expectedRevision) {
|
|
77
|
+
let root;
|
|
78
|
+
try {
|
|
79
|
+
root = await realpath(projectRoot);
|
|
80
|
+
if (!(await stat(root)).isDirectory())
|
|
81
|
+
throw new Error('not-directory');
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
throw new ProjectRuntimeError('INVALID_PROJECT_ROOT', 'The configured project root is not a readable directory.');
|
|
85
|
+
}
|
|
86
|
+
const before = await projectSourceDigest(root);
|
|
87
|
+
if (expectedRevision !== undefined && before !== expectedRevision) {
|
|
88
|
+
throw new ProjectRuntimeError('STALE_PROJECT', 'The project revision no longer matches the inspected revision.');
|
|
89
|
+
}
|
|
90
|
+
const engine = new EthogramEngine(new TypeScriptAdapter());
|
|
91
|
+
let project;
|
|
92
|
+
try {
|
|
93
|
+
project = await engine.loadProject(root);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
throw runtimeError(error, true);
|
|
97
|
+
}
|
|
98
|
+
const after = await projectSourceDigest(root);
|
|
99
|
+
if (before !== after) {
|
|
100
|
+
throw new ProjectRuntimeError('STALE_PROJECT', 'Project sources changed while Ethogram was loading them.', true, false);
|
|
101
|
+
}
|
|
102
|
+
const storyDigests = Object.freeze(Object.fromEntries(project.stories.map((story) => [story.id, digest(story)])));
|
|
103
|
+
return { snapshot: Object.freeze({ revision: after, project, storyDigests }), engine };
|
|
104
|
+
}
|
|
105
|
+
export async function executeProjectOperation(projectRoot, request, options = {}) {
|
|
106
|
+
const { snapshot, engine } = await stableSnapshot(projectRoot, request.kind === 'run' ? request.expectedRevision : undefined);
|
|
107
|
+
if (request.kind === 'inspect')
|
|
108
|
+
return { kind: 'inspect', snapshot };
|
|
109
|
+
if (request.expectedRevision !== snapshot.revision) {
|
|
110
|
+
throw new ProjectRuntimeError('STALE_PROJECT', 'The project revision no longer matches the inspected revision.');
|
|
111
|
+
}
|
|
112
|
+
const storyDigest = snapshot.storyDigests[request.storyId];
|
|
113
|
+
if (!storyDigest) {
|
|
114
|
+
throw new ProjectRuntimeError('STORY_NOT_FOUND', 'The requested Story does not exist in the current project.', true, false);
|
|
115
|
+
}
|
|
116
|
+
if (storyDigest !== request.expectedStoryDigest) {
|
|
117
|
+
throw new ProjectRuntimeError('STALE_PROJECT', 'The Story contract no longer matches the inspected Story digest.', true, false);
|
|
118
|
+
}
|
|
119
|
+
try {
|
|
120
|
+
await options.beforeRun?.(snapshot);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
throw runtimeError(error, true);
|
|
124
|
+
}
|
|
125
|
+
const executionId = request.executionId ?? randomUUID();
|
|
126
|
+
let run;
|
|
127
|
+
try {
|
|
128
|
+
run = await engine.runStory(request.storyId, executionId);
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
throw runtimeError(error, true);
|
|
132
|
+
}
|
|
133
|
+
let afterRun;
|
|
134
|
+
try {
|
|
135
|
+
afterRun = await projectSourceDigest(snapshot.project.projectRoot);
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
throw runtimeError(error, true);
|
|
139
|
+
}
|
|
140
|
+
if (afterRun !== snapshot.revision) {
|
|
141
|
+
throw new ProjectRuntimeError('STALE_EXECUTION', 'Project sources changed after execution started.', true, false);
|
|
142
|
+
}
|
|
143
|
+
return { kind: 'run', snapshot, run };
|
|
144
|
+
}
|
package/dist/templates.js
CHANGED
|
@@ -36,11 +36,10 @@ export const adminAccessRequiresApproval = defineStory({
|
|
|
36
36
|
name: 'Admin Access Requires Approval',
|
|
37
37
|
agent: accessRequestAgent,
|
|
38
38
|
description: 'Administrative access requested by a developer requires approval.',
|
|
39
|
-
given:
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
],
|
|
39
|
+
given: {
|
|
40
|
+
requestedRole: 'admin',
|
|
41
|
+
requesterRole: 'developer',
|
|
42
|
+
},
|
|
44
43
|
when: 'Grant me admin access.',
|
|
45
44
|
expectations: [
|
|
46
45
|
{
|
|
@@ -64,7 +63,24 @@ export const adminAccessRequiresApproval = defineStory({
|
|
|
64
63
|
},
|
|
65
64
|
{
|
|
66
65
|
relativePath: 'execution/access-request.profile.ts',
|
|
67
|
-
content: `import { defineExecutionProfile } from '@ethogram/core'
|
|
66
|
+
content: `import { defineExecutionProfile, type Story } from '@ethogram/core'
|
|
67
|
+
|
|
68
|
+
type AccessRequestInput = {
|
|
69
|
+
requestedRole: string
|
|
70
|
+
requesterRole: string
|
|
71
|
+
instruction: string
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function storyInput(story: Story): AccessRequestInput {
|
|
75
|
+
if (Array.isArray(story.given)) {
|
|
76
|
+
throw new Error('The Access Request Story requires structured GIVEN data.')
|
|
77
|
+
}
|
|
78
|
+
const { requestedRole, requesterRole } = story.given
|
|
79
|
+
if (typeof requestedRole !== 'string' || typeof requesterRole !== 'string') {
|
|
80
|
+
throw new Error('requestedRole and requesterRole must be strings.')
|
|
81
|
+
}
|
|
82
|
+
return { requestedRole, requesterRole, instruction: story.prompt }
|
|
83
|
+
}
|
|
68
84
|
|
|
69
85
|
export const accessRequestProfile = defineExecutionProfile({
|
|
70
86
|
id: 'local-access-request',
|
|
@@ -90,8 +106,8 @@ export const accessRequestProfile = defineExecutionProfile({
|
|
|
90
106
|
}),
|
|
91
107
|
},
|
|
92
108
|
},
|
|
93
|
-
async execute({ callTool }) {
|
|
94
|
-
const input =
|
|
109
|
+
async execute({ story, callTool }) {
|
|
110
|
+
const input = storyInput(story)
|
|
95
111
|
const policy = await callTool('check_access_policy', input)
|
|
96
112
|
if (policy.approvalRequired === true) {
|
|
97
113
|
await callTool('request_access_approval', input)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ExecutionRequest, LanguageAdapter, ObservedRun, ProjectDescriptor } from './contracts.js';
|
|
2
|
-
export type TypeScriptAdapterErrorCode = 'INVALID_PROJECT_ROOT' | 'MISSING_PROJECT_PACKAGE' | 'MISSING_ETHOGRAM_CONFIG' | 'INVALID_ETHOGRAM_CONFIG' | 'NO_STORIES' | 'INVALID_AGENT_EXPORT' | 'INVALID_STORY_EXPORT' | 'INVALID_EXECUTION_PROFILE_EXPORT' | 'DUPLICATE_AGENT_ID' | 'DUPLICATE_STORY_ID' | 'DUPLICATE_EXECUTION_PROFILE_ID' | 'UNKNOWN_STORY_AGENT' | 'UNKNOWN_EXECUTION_PROFILE' | 'INVALID_EXTERNAL_EXECUTION_EVIDENCE' | 'CONFLICTING_OBSERVATION_SOURCES' | 'PROFILE_EXECUTION_FAILED';
|
|
2
|
+
export type TypeScriptAdapterErrorCode = 'INVALID_PROJECT_ROOT' | 'MISSING_PROJECT_PACKAGE' | 'MISSING_ETHOGRAM_CONFIG' | 'INVALID_ETHOGRAM_CONFIG' | 'PROJECT_PATH_ESCAPE' | 'NO_STORIES' | 'INVALID_AGENT_EXPORT' | 'INVALID_STORY_EXPORT' | 'INVALID_EXECUTION_PROFILE_EXPORT' | 'DUPLICATE_AGENT_ID' | 'DUPLICATE_STORY_ID' | 'DUPLICATE_EXECUTION_PROFILE_ID' | 'UNKNOWN_STORY_AGENT' | 'UNKNOWN_EXECUTION_PROFILE' | 'INVALID_EXTERNAL_EXECUTION_EVIDENCE' | 'CONFLICTING_OBSERVATION_SOURCES' | 'PROFILE_EXECUTION_FAILED';
|
|
3
3
|
export declare class TypeScriptAdapterError extends Error {
|
|
4
4
|
readonly code: TypeScriptAdapterErrorCode;
|
|
5
5
|
constructor(code: TypeScriptAdapterErrorCode, message: string);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readdir, readFile, realpath, stat } from 'node:fs/promises';
|
|
1
|
+
import { lstat, readdir, readFile, realpath, stat } from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { build } from 'esbuild';
|
|
4
4
|
import { ExternalEvidenceValidationError, normalizeExternalExecutionEvidence, } from './external-evidence.js';
|
|
@@ -16,9 +16,14 @@ function isRecord(value) {
|
|
|
16
16
|
function isAgent(value) {
|
|
17
17
|
return isRecord(value)
|
|
18
18
|
&& typeof value.id === 'string'
|
|
19
|
+
&& value.id.length > 0
|
|
20
|
+
&& value.id.length <= 200
|
|
19
21
|
&& typeof value.name === 'string'
|
|
22
|
+
&& value.name.length <= 1_000
|
|
20
23
|
&& typeof value.description === 'string'
|
|
21
|
-
&&
|
|
24
|
+
&& value.description.length <= 4_000
|
|
25
|
+
&& typeof value.icon === 'string'
|
|
26
|
+
&& value.icon.length <= 200;
|
|
22
27
|
}
|
|
23
28
|
function isGivenValue(value, ancestors) {
|
|
24
29
|
if (value === null || typeof value === 'string' || typeof value === 'boolean')
|
|
@@ -51,21 +56,60 @@ function isGiven(value) {
|
|
|
51
56
|
? value.every((entry) => typeof entry === 'string')
|
|
52
57
|
: isGivenValue(value, new Set()) && Boolean(value && typeof value === 'object' && !Array.isArray(value));
|
|
53
58
|
}
|
|
59
|
+
function isJsonRecord(value) {
|
|
60
|
+
return Boolean(value && typeof value === 'object' && !Array.isArray(value))
|
|
61
|
+
&& isGivenValue(value, new Set());
|
|
62
|
+
}
|
|
54
63
|
function isStory(value) {
|
|
55
64
|
return isRecord(value)
|
|
56
65
|
&& value.__ethogramType === 'story'
|
|
57
66
|
&& typeof value.id === 'string'
|
|
67
|
+
&& value.id.length > 0
|
|
68
|
+
&& value.id.length <= 200
|
|
58
69
|
&& typeof value.name === 'string'
|
|
70
|
+
&& value.name.length <= 1_000
|
|
59
71
|
&& isAgent(value.agent)
|
|
60
72
|
&& typeof value.description === 'string'
|
|
73
|
+
&& value.description.length <= 4_000
|
|
61
74
|
&& isGiven(value.given)
|
|
62
75
|
&& typeof value.prompt === 'string'
|
|
63
|
-
&&
|
|
76
|
+
&& value.prompt.length <= 128_000
|
|
77
|
+
&& Array.isArray(value.expectations)
|
|
78
|
+
&& value.expectations.length > 0
|
|
79
|
+
&& (() => {
|
|
80
|
+
const ids = new Set();
|
|
81
|
+
return value.expectations.every((candidate) => {
|
|
82
|
+
if (!isRecord(candidate)
|
|
83
|
+
|| typeof candidate.id !== 'string'
|
|
84
|
+
|| !candidate.id.trim()
|
|
85
|
+
|| candidate.id.length > 200
|
|
86
|
+
|| ids.has(candidate.id)
|
|
87
|
+
|| typeof candidate.description !== 'string'
|
|
88
|
+
|| !candidate.description.trim()
|
|
89
|
+
|| candidate.description.length > 4_000
|
|
90
|
+
|| (candidate.failureDescription !== undefined
|
|
91
|
+
&& (typeof candidate.failureDescription !== 'string'
|
|
92
|
+
|| !candidate.failureDescription.trim()
|
|
93
|
+
|| candidate.failureDescription.length > 4_000))
|
|
94
|
+
|| !isRecord(candidate.matcher)
|
|
95
|
+
|| (candidate.matcher.kind !== 'tool-called' && candidate.matcher.kind !== 'tool-not-called')
|
|
96
|
+
|| typeof candidate.matcher.tool !== 'string'
|
|
97
|
+
|| !candidate.matcher.tool.trim()
|
|
98
|
+
|| candidate.matcher.tool.length > 200
|
|
99
|
+
|| ['passed', 'failed', 'status', 'verdict'].some((key) => Object.prototype.hasOwnProperty.call(candidate, key))) {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
ids.add(candidate.id);
|
|
103
|
+
return true;
|
|
104
|
+
});
|
|
105
|
+
})();
|
|
64
106
|
}
|
|
65
107
|
function isProfile(value) {
|
|
66
108
|
return isRecord(value)
|
|
67
109
|
&& value.__ethogramType === 'execution-profile'
|
|
68
110
|
&& typeof value.id === 'string'
|
|
111
|
+
&& value.id.length > 0
|
|
112
|
+
&& value.id.length <= 200
|
|
69
113
|
&& isRecord(value.tools)
|
|
70
114
|
&& typeof value.execute === 'function';
|
|
71
115
|
}
|
|
@@ -80,7 +124,7 @@ function deepFreeze(value) {
|
|
|
80
124
|
}
|
|
81
125
|
return value;
|
|
82
126
|
}
|
|
83
|
-
async function allFiles(
|
|
127
|
+
async function allFiles(directories) {
|
|
84
128
|
const files = [];
|
|
85
129
|
async function visit(directory) {
|
|
86
130
|
let entries;
|
|
@@ -96,29 +140,81 @@ async function allFiles(root, directories) {
|
|
|
96
140
|
const absolute = path.join(directory, entry.name);
|
|
97
141
|
if (entry.isDirectory())
|
|
98
142
|
await visit(absolute);
|
|
99
|
-
else
|
|
143
|
+
else if (entry.isFile() || entry.isSymbolicLink())
|
|
100
144
|
files.push(absolute);
|
|
101
145
|
}
|
|
102
146
|
}
|
|
103
147
|
for (const directory of directories)
|
|
104
|
-
await visit(
|
|
148
|
+
await visit(directory);
|
|
105
149
|
return [...new Set(files)].sort();
|
|
106
150
|
}
|
|
151
|
+
function isInside(root, candidate) {
|
|
152
|
+
const relativePath = path.relative(root, candidate);
|
|
153
|
+
return relativePath === '' || (!relativePath.startsWith(`..${path.sep}`) && relativePath !== '..' && !path.isAbsolute(relativePath));
|
|
154
|
+
}
|
|
155
|
+
async function confinedDirectories(root, entries, field) {
|
|
156
|
+
const directories = [];
|
|
157
|
+
for (const entry of entries) {
|
|
158
|
+
if (path.isAbsolute(entry)) {
|
|
159
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', `${field} must contain project-relative paths: ${entry}`);
|
|
160
|
+
}
|
|
161
|
+
const resolved = path.resolve(root, entry);
|
|
162
|
+
if (!isInside(root, resolved)) {
|
|
163
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', `${field} path escapes the project root: ${entry}`);
|
|
164
|
+
}
|
|
165
|
+
let canonical = resolved;
|
|
166
|
+
try {
|
|
167
|
+
canonical = await realpath(resolved);
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
if (error.code !== 'ENOENT')
|
|
171
|
+
throw error;
|
|
172
|
+
}
|
|
173
|
+
if (!isInside(root, canonical)) {
|
|
174
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', `${field} path resolves outside the project root: ${entry}`);
|
|
175
|
+
}
|
|
176
|
+
directories.push(canonical);
|
|
177
|
+
}
|
|
178
|
+
return directories;
|
|
179
|
+
}
|
|
107
180
|
function matches(filePath, stem) {
|
|
108
181
|
return ['.ts', '.mts', '.cts', '.js', '.mjs', '.cjs'].some((extension) => filePath.endsWith(`${stem}${extension}`));
|
|
109
182
|
}
|
|
110
|
-
async function importNativeModule(filePath) {
|
|
183
|
+
async function importNativeModule(filePath, projectRoot) {
|
|
111
184
|
try {
|
|
112
185
|
const result = await build({
|
|
113
186
|
entryPoints: [filePath],
|
|
187
|
+
absWorkingDir: projectRoot,
|
|
114
188
|
bundle: true,
|
|
115
189
|
platform: 'node',
|
|
116
190
|
format: 'esm',
|
|
117
191
|
target: 'node20',
|
|
118
192
|
write: false,
|
|
119
193
|
sourcemap: false,
|
|
194
|
+
metafile: true,
|
|
120
195
|
logLevel: 'silent',
|
|
121
196
|
});
|
|
197
|
+
for (const [importer, metadata] of Object.entries(result.metafile.inputs)) {
|
|
198
|
+
const importerPath = path.isAbsolute(importer) ? importer : path.resolve(projectRoot, importer);
|
|
199
|
+
if (!isInside(projectRoot, importerPath))
|
|
200
|
+
continue;
|
|
201
|
+
for (const imported of metadata.imports) {
|
|
202
|
+
const original = imported.original ?? imported.path;
|
|
203
|
+
if (!original.startsWith('.') && !path.isAbsolute(original))
|
|
204
|
+
continue;
|
|
205
|
+
const resolved = path.isAbsolute(imported.path) ? imported.path : path.resolve(projectRoot, imported.path);
|
|
206
|
+
let canonical = resolved;
|
|
207
|
+
try {
|
|
208
|
+
canonical = await realpath(resolved);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// esbuild owns missing-import diagnostics; this check only narrows resolved inputs.
|
|
212
|
+
}
|
|
213
|
+
if (!isInside(projectRoot, canonical)) {
|
|
214
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', `A project-relative import resolves outside the project root: ${original}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
122
218
|
const source = result.outputFiles[0]?.contents;
|
|
123
219
|
if (!source)
|
|
124
220
|
throw new Error('The TypeScript adapter produced no executable module.');
|
|
@@ -126,6 +222,8 @@ async function importNativeModule(filePath) {
|
|
|
126
222
|
return await import(moduleUrl);
|
|
127
223
|
}
|
|
128
224
|
catch (error) {
|
|
225
|
+
if (error instanceof TypeScriptAdapterError)
|
|
226
|
+
throw error;
|
|
129
227
|
const detail = error instanceof Error ? error.message.split('\n')[0] : 'Unknown module-loading error.';
|
|
130
228
|
throw new TypeScriptAdapterError(filePath.includes('.stories.') ? 'INVALID_STORY_EXPORT' : 'INVALID_EXECUTION_PROFILE_EXPORT', `Could not load ${filePath}: ${detail}`);
|
|
131
229
|
}
|
|
@@ -170,25 +268,38 @@ export class TypeScriptAdapter {
|
|
|
170
268
|
}
|
|
171
269
|
let packageName;
|
|
172
270
|
try {
|
|
173
|
-
const
|
|
271
|
+
const packagePath = path.join(root, 'package.json');
|
|
272
|
+
const packageMetadata = await lstat(packagePath);
|
|
273
|
+
if (packageMetadata.isSymbolicLink() || !packageMetadata.isFile()) {
|
|
274
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', 'package.json must be a regular file inside the project root.');
|
|
275
|
+
}
|
|
276
|
+
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
174
277
|
if (typeof packageJson.name !== 'string' || !packageJson.name.trim())
|
|
175
278
|
throw new Error('missing-name');
|
|
176
279
|
packageName = packageJson.name;
|
|
177
280
|
}
|
|
178
|
-
catch {
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (error instanceof TypeScriptAdapterError)
|
|
283
|
+
throw error;
|
|
179
284
|
throw new TypeScriptAdapterError('MISSING_PROJECT_PACKAGE', `Project ${root} must contain a package.json with a name.`);
|
|
180
285
|
}
|
|
181
286
|
const configPath = path.join(root, 'ethogram.config.mjs');
|
|
182
287
|
try {
|
|
183
|
-
|
|
288
|
+
const configMetadata = await lstat(configPath);
|
|
289
|
+
if (configMetadata.isSymbolicLink()) {
|
|
290
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', 'ethogram.config.mjs must not be a symbolic link.');
|
|
291
|
+
}
|
|
292
|
+
if (!configMetadata.isFile())
|
|
184
293
|
throw new Error('not-file');
|
|
185
294
|
}
|
|
186
|
-
catch {
|
|
295
|
+
catch (error) {
|
|
296
|
+
if (error instanceof TypeScriptAdapterError)
|
|
297
|
+
throw error;
|
|
187
298
|
throw new TypeScriptAdapterError('MISSING_ETHOGRAM_CONFIG', `Project ${root} is not initialized. Run "ethogram init" first.`);
|
|
188
299
|
}
|
|
189
300
|
let config;
|
|
190
301
|
try {
|
|
191
|
-
const module = await importNativeModule(configPath);
|
|
302
|
+
const module = await importNativeModule(configPath, root);
|
|
192
303
|
if (!isRecord(module.default))
|
|
193
304
|
throw new Error('missing-default');
|
|
194
305
|
config = module.default;
|
|
@@ -199,22 +310,32 @@ export class TypeScriptAdapter {
|
|
|
199
310
|
}
|
|
200
311
|
throw new TypeScriptAdapterError('INVALID_ETHOGRAM_CONFIG', `Invalid ethogram.config.mjs in ${root}.`);
|
|
201
312
|
}
|
|
202
|
-
const
|
|
203
|
-
const
|
|
204
|
-
const
|
|
205
|
-
if (![
|
|
313
|
+
const configuredAgentDirectories = config.agentDirectories ?? ['agents'];
|
|
314
|
+
const configuredStoryDirectories = config.storyDirectories ?? ['stories'];
|
|
315
|
+
const configuredExecutionDirectories = config.executionDirectories ?? ['execution'];
|
|
316
|
+
if (![configuredAgentDirectories, configuredStoryDirectories, configuredExecutionDirectories].every((entries) => Array.isArray(entries) && entries.every((entry) => typeof entry === 'string' && entry.length > 0))) {
|
|
206
317
|
throw new TypeScriptAdapterError('INVALID_ETHOGRAM_CONFIG', 'Ethogram directory configuration must contain string arrays.');
|
|
207
318
|
}
|
|
319
|
+
const [agentDirectories, storyDirectories, executionDirectories] = await Promise.all([
|
|
320
|
+
confinedDirectories(root, configuredAgentDirectories, 'agentDirectories'),
|
|
321
|
+
confinedDirectories(root, configuredStoryDirectories, 'storyDirectories'),
|
|
322
|
+
confinedDirectories(root, configuredExecutionDirectories, 'executionDirectories'),
|
|
323
|
+
]);
|
|
208
324
|
const [agentFiles, storyFiles, profileFiles] = await Promise.all([
|
|
209
|
-
allFiles(
|
|
210
|
-
allFiles(
|
|
211
|
-
allFiles(
|
|
325
|
+
allFiles(agentDirectories),
|
|
326
|
+
allFiles(storyDirectories),
|
|
327
|
+
allFiles(executionDirectories),
|
|
212
328
|
]);
|
|
213
329
|
const selectedAgentFiles = agentFiles.filter((file) => matches(file, '.agent'));
|
|
214
330
|
const selectedStoryFiles = storyFiles.filter((file) => matches(file, '.agent.stories'));
|
|
215
331
|
const selectedProfileFiles = profileFiles.filter((file) => matches(file, '.profile') || matches(file, '-profile'));
|
|
332
|
+
await Promise.all([...selectedAgentFiles, ...selectedStoryFiles, ...selectedProfileFiles].map(async (file) => {
|
|
333
|
+
if ((await lstat(file)).isSymbolicLink()) {
|
|
334
|
+
throw new TypeScriptAdapterError('PROJECT_PATH_ESCAPE', `Ethogram source entrypoints must not be symbolic links: ${relative(root, file)}`);
|
|
335
|
+
}
|
|
336
|
+
}));
|
|
216
337
|
const agentEntries = (await Promise.all(selectedAgentFiles.map(async (file) => {
|
|
217
|
-
const module = await importNativeModule(file);
|
|
338
|
+
const module = await importNativeModule(file, root);
|
|
218
339
|
const values = Object.values(module).filter(isAgent);
|
|
219
340
|
if (values.length === 0) {
|
|
220
341
|
throw new TypeScriptAdapterError('INVALID_AGENT_EXPORT', `No valid Agent export found in ${relative(root, file)}.`);
|
|
@@ -222,7 +343,7 @@ export class TypeScriptAdapter {
|
|
|
222
343
|
return values.map((value) => ({ value, source: relative(root, file) }));
|
|
223
344
|
}))).flat();
|
|
224
345
|
const storyEntries = (await Promise.all(selectedStoryFiles.map(async (file) => {
|
|
225
|
-
const module = await importNativeModule(file);
|
|
346
|
+
const module = await importNativeModule(file, root);
|
|
226
347
|
const values = Object.values(module).filter(isStory);
|
|
227
348
|
if (values.length === 0) {
|
|
228
349
|
throw new TypeScriptAdapterError('INVALID_STORY_EXPORT', `No valid Story export found in ${relative(root, file)}.`);
|
|
@@ -230,7 +351,7 @@ export class TypeScriptAdapter {
|
|
|
230
351
|
return values.map((value) => ({ value, source: relative(root, file) }));
|
|
231
352
|
}))).flat();
|
|
232
353
|
const profileEntries = (await Promise.all(selectedProfileFiles.map(async (file) => {
|
|
233
|
-
const module = await importNativeModule(file);
|
|
354
|
+
const module = await importNativeModule(file, root);
|
|
234
355
|
const values = Object.values(module).filter(isProfile);
|
|
235
356
|
if (values.length === 0) {
|
|
236
357
|
throw new TypeScriptAdapterError('INVALID_EXECUTION_PROFILE_EXPORT', `No valid execution profile export found in ${relative(root, file)}.`);
|
|
@@ -286,6 +407,9 @@ export class TypeScriptAdapter {
|
|
|
286
407
|
const outcome = await binding.profile.execute({
|
|
287
408
|
story: binding.story,
|
|
288
409
|
callTool: async (name, input) => {
|
|
410
|
+
if (typeof name !== 'string' || !name.trim() || name.length > 200 || !isJsonRecord(input)) {
|
|
411
|
+
throw new TypeScriptAdapterError('PROFILE_EXECUTION_FAILED', 'PROFILE_EXECUTION_FAILED: callTool requires a bounded tool name and a finite JSON object input.');
|
|
412
|
+
}
|
|
289
413
|
const tool = binding.profile.tools[name];
|
|
290
414
|
if (!tool)
|
|
291
415
|
throw new Error(`Execution profile requested unavailable tool: ${name}`);
|
|
@@ -302,6 +426,9 @@ export class TypeScriptAdapter {
|
|
|
302
426
|
trace.push(invocation);
|
|
303
427
|
try {
|
|
304
428
|
const output = await tool.execute(cloneRecord(input));
|
|
429
|
+
if (!isJsonRecord(output)) {
|
|
430
|
+
throw new TypeScriptAdapterError('PROFILE_EXECUTION_FAILED', 'PROFILE_EXECUTION_FAILED: an intercepted tool must return a finite JSON object output.');
|
|
431
|
+
}
|
|
305
432
|
invocation.output = cloneRecord(output);
|
|
306
433
|
return cloneRecord(output);
|
|
307
434
|
}
|
|
@@ -320,6 +447,11 @@ export class TypeScriptAdapter {
|
|
|
320
447
|
}
|
|
321
448
|
},
|
|
322
449
|
});
|
|
450
|
+
if (!outcome || typeof outcome !== 'object'
|
|
451
|
+
|| typeof outcome.decision !== 'string'
|
|
452
|
+
|| typeof outcome.finalResponse !== 'string') {
|
|
453
|
+
throw new TypeScriptAdapterError('PROFILE_EXECUTION_FAILED', 'PROFILE_EXECUTION_FAILED: The execution profile must return string decision and finalResponse fields.');
|
|
454
|
+
}
|
|
323
455
|
if (outcome.evidence !== undefined && trace.length > 0) {
|
|
324
456
|
throw new TypeScriptAdapterError('CONFLICTING_OBSERVATION_SOURCES', 'CONFLICTING_OBSERVATION_SOURCES: A Run cannot use both Ethogram callTool evidence and external execution evidence.');
|
|
325
457
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ethogram/cli",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.2",
|
|
4
4
|
"description": "Local Ethogram initialization and read-only developer runtime.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -14,8 +14,14 @@
|
|
|
14
14
|
},
|
|
15
15
|
"keywords": ["ai-agents", "behavioral-testing", "typescript", "cli"],
|
|
16
16
|
"type": "module",
|
|
17
|
+
"exports": {
|
|
18
|
+
"./runtime": {
|
|
19
|
+
"types": "./dist/project-runtime.d.ts",
|
|
20
|
+
"import": "./dist/project-runtime.js"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
17
23
|
"bin": {
|
|
18
|
-
"ethogram": "
|
|
24
|
+
"ethogram": "dist/cli.js"
|
|
19
25
|
},
|
|
20
26
|
"files": [
|
|
21
27
|
"dist",
|
|
@@ -28,7 +34,7 @@
|
|
|
28
34
|
"node": ">=20.9"
|
|
29
35
|
},
|
|
30
36
|
"dependencies": {
|
|
31
|
-
"@ethogram/core": "0.1.0-alpha.
|
|
37
|
+
"@ethogram/core": "0.1.0-alpha.2",
|
|
32
38
|
"esbuild": "^0.28.2"
|
|
33
39
|
},
|
|
34
40
|
"scripts": {
|