@ethogram/cli 0.1.0-alpha.0

package/README.md

@@ -0,0 +1,17 @@
+# Ethogram CLI
+
+Local Ethogram initialization and read-only developer runtime for TypeScript/Node projects.
+
+This package is prepared as `@ethogram/cli@0.1.0-alpha.0` but has not been published yet.
+
+```bash
+npx ethogram init
+npx ethogram init --existing
+npx ethogram dev
+```
+
+Use `ethogram init --existing` to create only `ethogram.config.mjs` in a project that already owns an agent implementation. Normal `ethogram init` adds the deterministic Access Request starter. Both modes preserve conflicting user-owned files and abort without partial writes.
+
+`ethogram dev` accepts `--project <path>`, `--port <number>`, and `--no-open`. It serves a localhost-only, code-first UI: project files are the source of truth and the UI does not save edits. Relevant source changes reload automatically, and previous execution evidence is cleared or marked stale until the Story is rerun.
+
+Node.js 20.9 or newer is required. Ethogram does not persist run history.

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+import { access, mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { constants } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { startDeveloperServer } from './server.js';
+import { existingProjectFiles, starterFiles } from './templates.js';
+import { TypeScriptAdapterError } from './typescript-adapter.js';
+const help = `Ethogram CLI (alpha)
+
+Usage:
+  ethogram init [--existing]
+  ethogram dev [--project <path>] [--port <number>] [--no-open]
+  ethogram --help
+  ethogram --version
+
+Commands:
+  init  Create a starter Agent, Story, and local execution profile in the current project.
+        Use --existing to create configuration only for an existing agent project.
+  dev   Start the local, read-only Ethogram developer UI. The current directory is the default project.
+        Agent, Story, and execution-profile files remain the source of truth and reload automatically.
+
+Examples:
+  npx ethogram init
+  npx ethogram init --existing
+  npx ethogram dev
+  npx ethogram dev --project ./my-agent-project --port 4317 --no-open
+`;
+async function version() {
+    const packagePath = fileURLToPath(new URL('../package.json', import.meta.url));
+    const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
+    return packageJson.version;
+}
+async function projectPackage(root) {
+    try {
+        const packagePath = path.join(root, 'package.json');
+        const value = JSON.parse(await readFile(packagePath, 'utf8'));
+        if (typeof value.name !== 'string' || !value.name.trim())
+            throw new Error('missing-name');
+        return { name: value.name };
+    }
+    catch {
+        throw new Error(`INIT_PROJECT_INVALID: ${root} must contain a package.json with a name.`);
+    }
+}
+async function initialize(options) {
+    const root = await realpath(process.cwd());
+    if (!(await stat(root)).isDirectory())
+        throw new Error(`INIT_PROJECT_INVALID: ${root} is not a directory.`);
+    const packageJson = await projectPackage(root);
+    const files = options.existing ? existingProjectFiles(packageJson.name) : starterFiles(packageJson.name);
+    const matching = [];
+    const missing = [];
+    const conflicts = [];
+    await access(root, constants.R_OK | constants.W_OK);
+    for (const file of files) {
+        const target = path.join(root, file.relativePath);
+        try {
+            const existing = await readFile(target, 'utf8');
+            if (existing === file.content)
+                matching.push(file.relativePath);
+            else
+                conflicts.push(file.relativePath);
+        }
+        catch (error) {
+            const code = error.code;
+            if (code === 'ENOENT')
+                missing.push(file.relativePath);
+            else
+                throw new Error(`INIT_PREFLIGHT_FAILED: Could not inspect ${file.relativePath}.`);
+        }
+    }
+    if (conflicts.length > 0) {
+        throw new Error(`INIT_CONFLICT: Existing files were preserved; nothing was written. Conflicts: ${conflicts.join(', ')}`);
+    }
+    for (const relativePath of missing) {
+        const file = files.find((candidate) => candidate.relativePath === relativePath);
+        if (!file)
+            continue;
+        const target = path.join(root, relativePath);
+        await mkdir(path.dirname(target), { recursive: true });
+        await writeFile(target, file.content, { encoding: 'utf8', flag: 'wx' });
+    }
+    if (missing.length === 0) {
+        process.stdout.write(`Ethogram is already initialized in ${root}. No files were changed.\n`);
+    }
+    else {
+        process.stdout.write(`Ethogram initialized in ${root}.\n`);
+        for (const relativePath of missing)
+            process.stdout.write(`Created ${relativePath}\n`);
+        for (const relativePath of matching)
+            process.stdout.write(`Preserved ${relativePath}\n`);
+    }
+    if (options.existing) {
+        process.stdout.write('Add an Agent descriptor, behavioral Story, and thin execution profile for your existing agent.\n');
+    }
+    process.stdout.write('Next: npx ethogram dev\n');
+}
+function parseDevArguments(args) {
+    let projectRoot = process.cwd();
+    let port = 4317;
+    let openBrowser = true;
+    for (let index = 0; index < args.length; index += 1) {
+        const argument = args[index];
+        if (argument === '--no-open') {
+            openBrowser = false;
+            continue;
+        }
+        if (argument === '--project') {
+            const value = args[index + 1];
+            if (!value)
+                throw new Error('CLI_USAGE: --project requires a path.');
+            projectRoot = path.resolve(value);
+            index += 1;
+            continue;
+        }
+        if (argument === '--port') {
+            const value = Number(args[index + 1]);
+            if (!Number.isInteger(value) || value < 0 || value > 65535) {
+                throw new Error('CLI_USAGE: --port requires an integer from 0 to 65535.');
+            }
+            port = value;
+            index += 1;
+            continue;
+        }
+        throw new Error(`CLI_USAGE: Unknown dev option ${argument}.`);
+    }
+    return { projectRoot, port, openBrowser };
+}
+async function main() {
+    const args = process.argv.slice(2);
+    const command = args[0];
+    if (!command || command === '--help' || command === '-h' || command === 'help') {
+        process.stdout.write(help);
+        return;
+    }
+    if (command === '--version' || command === '-v') {
+        process.stdout.write(`${await version()}\n`);
+        return;
+    }
+    if (command === 'init') {
+        const initArgs = args.slice(1);
+        if (initArgs.some((argument) => argument !== '--existing') || initArgs.filter((argument) => argument === '--existing').length > 1) {
+            throw new Error('CLI_USAGE: ethogram init accepts only the optional --existing flag.');
+        }
+        await initialize({ existing: initArgs.includes('--existing') });
+        return;
+    }
+    if (command === 'dev') {
+        await startDeveloperServer(parseDevArguments(args.slice(1)));
+        return;
+    }
+    throw new Error(`CLI_USAGE: Unknown command ${command}. Run ethogram --help.`);
+}
+main().catch((error) => {
+    if (error instanceof TypeScriptAdapterError) {
+        process.stderr.write(`Ethogram ${error.code}: ${error.message}\n`);
+    }
+    else if (error instanceof Error) {
+        process.stderr.write(`Ethogram error: ${error.message}\n`);
+    }
+    else {
+        process.stderr.write('Ethogram error: The command could not be completed.\n');
+    }
+    process.exitCode = 1;
+});

package/dist/contracts.d.ts

@@ -0,0 +1,114 @@
+export type AgentDescriptor = {
+    id: string;
+    name: string;
+    description: string;
+    icon: string;
+};
+export type ExpectationMatcher = {
+    kind: 'tool-called';
+    tool: string;
+} | {
+    kind: 'tool-not-called';
+    tool: string;
+};
+export type StoryExpectationDescriptor = {
+    id: string;
+    description: string;
+    failureDescription?: string;
+    matcher: ExpectationMatcher;
+};
+export type StoryGivenValue = string | number | boolean | null | readonly StoryGivenValue[] | {
+    readonly [key: string]: StoryGivenValue;
+};
+export type StoryGiven = string[] | Readonly<Record<string, StoryGivenValue>>;
+export type StoryDescriptor = {
+    id: string;
+    name: string;
+    agent: AgentDescriptor;
+    description: string;
+    given: StoryGiven;
+    prompt: string;
+    expectations: StoryExpectationDescriptor[];
+    source: string;
+    executable: boolean;
+};
+export type ProjectDescriptor = {
+    projectRoot: string;
+    name: string;
+    adapter: {
+        id: string;
+        label: string;
+    };
+    agents: AgentDescriptor[];
+    stories: StoryDescriptor[];
+};
+export type ExecutionRequest = {
+    story: StoryDescriptor;
+};
+type ObservedToolCallBase = {
+    callId: string;
+    name: string;
+    input: string;
+    duration?: string;
+    startedAt?: string;
+    endedAt?: string;
+};
+export type ObservedToolCall = (ObservedToolCallBase & {
+    status: 'success';
+    output?: string;
+    error?: never;
+}) | (ObservedToolCallBase & {
+    status: 'error';
+    output?: never;
+    error?: {
+        name?: string;
+        message: string;
+    };
+});
+export type ObservedTimelineStep = {
+    label: string;
+    detail: string;
+    duration?: string;
+};
+export type ObservedTokenUsage = {
+    availability: 'unavailable';
+} | {
+    availability: 'available';
+    inputTokens?: number;
+    outputTokens?: number;
+    totalTokens?: number;
+    reasoningTokens?: number;
+};
+export type ObservedRun = {
+    decision: string;
+    reason: string;
+    finalResponse: string;
+    toolCalls: ObservedToolCall[];
+    timeline: ObservedTimelineStep[];
+    evidence: {
+        provider?: string;
+        model?: string;
+        startedAt?: string;
+        endedAt?: string;
+        latencyMs?: number;
+        finishReason?: string;
+        tokenUsage: ObservedTokenUsage;
+    };
+};
+export type BehavioralVerdict = 'PASS' | 'FAIL';
+export type EvaluationResult = {
+    verdict: BehavioralVerdict;
+    expectations: Readonly<Record<string, BehavioralVerdict>>;
+};
+export type CompletedExecutionRecord = {
+    observedRun: ObservedRun;
+    evaluationResult: EvaluationResult;
+};
+export interface Runner {
+    run(request: ExecutionRequest): Promise<ObservedRun>;
+}
+export interface LanguageAdapter extends Runner {
+    readonly id: string;
+    loadProject(projectRoot: string): Promise<ProjectDescriptor>;
+}
+export {};

package/dist/contracts.js

@@ -0,0 +1 @@
+export {};

package/dist/evaluator.d.ts

@@ -0,0 +1,2 @@
+import type { EvaluationResult, ObservedRun, StoryDescriptor } from './contracts.js';
+export declare function evaluateStory(story: StoryDescriptor, observedRun: ObservedRun): EvaluationResult;

package/dist/evaluator.js

@@ -0,0 +1,15 @@
+function evaluateMatcher(matcher, observedRun) {
+    const called = observedRun.toolCalls.some((toolCall) => toolCall.name === matcher.tool);
+    return matcher.kind === 'tool-called' ? called : !called;
+}
+export function evaluateStory(story, observedRun) {
+    const expectationEntries = story.expectations.map((expectation) => {
+        const verdict = evaluateMatcher(expectation.matcher, observedRun) ? 'PASS' : 'FAIL';
+        return [expectation.id, verdict];
+    });
+    const expectations = Object.freeze(Object.fromEntries(expectationEntries));
+    const verdict = Object.values(expectations).every((value) => value === 'PASS')
+        ? 'PASS'
+        : 'FAIL';
+    return Object.freeze({ verdict, expectations });
+}

package/dist/external-evidence.d.ts

@@ -0,0 +1,7 @@
+import type { ObservedRun } from './contracts.js';
+export type NormalizedExternalEvidence = Pick<ObservedRun, 'toolCalls' | 'timeline' | 'evidence'>;
+export declare class ExternalEvidenceValidationError extends Error {
+    readonly code = "INVALID_EXTERNAL_EXECUTION_EVIDENCE";
+    constructor(message: string);
+}
+export declare function normalizeExternalExecutionEvidence(value: unknown): NormalizedExternalEvidence;

package/dist/external-evidence.js

@@ -0,0 +1,205 @@
+export class ExternalEvidenceValidationError extends Error {
+    code = 'INVALID_EXTERNAL_EXECUTION_EVIDENCE';
+    constructor(message) {
+        super(message);
+        this.name = 'ExternalEvidenceValidationError';
+    }
+}
+function isRecord(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+function fail(message) {
+    throw new ExternalEvidenceValidationError(message);
+}
+function requiredText(value, location) {
+    if (typeof value !== 'string' || !value.trim())
+        fail(`${location} must be a non-empty string.`);
+    return value;
+}
+function optionalText(record, key, location) {
+    if (!Object.prototype.hasOwnProperty.call(record, key))
+        return undefined;
+    return requiredText(record[key], `${location}.${key}`);
+}
+function optionalNonNegativeNumber(record, key, location) {
+    if (!Object.prototype.hasOwnProperty.call(record, key))
+        return undefined;
+    const value = record[key];
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
+        fail(`${location}.${key} must be a finite non-negative number.`);
+    }
+    return value;
+}
+function optionalNonNegativeInteger(record, key, location) {
+    const value = optionalNonNegativeNumber(record, key, location);
+    if (value !== undefined && !Number.isInteger(value))
+        fail(`${location}.${key} must be an integer.`);
+    return value;
+}
+function rejectEvaluationFields(record, location) {
+    for (const key of ['verdict', 'expectations', 'evaluationResult', 'passed', 'failed']) {
+        if (Object.prototype.hasOwnProperty.call(record, key)) {
+            fail(`${location} must not contain behavioral evaluation field "${key}".`);
+        }
+    }
+}
+function evidenceValue(value, location, ancestors) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean')
+        return value;
+    if (typeof value === 'number') {
+        if (!Number.isFinite(value))
+            fail(`${location} must contain only finite numbers.`);
+        return value;
+    }
+    if (!value || typeof value !== 'object')
+        fail(`${location} contains unsupported value type "${typeof value}".`);
+    if (ancestors.has(value))
+        fail(`${location} must not contain cyclic values.`);
+    ancestors.add(value);
+    try {
+        if (Array.isArray(value)) {
+            return value.map((entry, index) => {
+                if (!Object.prototype.hasOwnProperty.call(value, index))
+                    fail(`${location}[${index}] must not be sparse.`);
+                return evidenceValue(entry, `${location}[${index}]`, ancestors);
+            });
+        }
+        const prototype = Object.getPrototypeOf(value);
+        if (prototype !== Object.prototype && prototype !== null) {
+            fail(`${location} must contain only plain records and arrays.`);
+        }
+        const result = Object.create(null);
+        for (const key of Reflect.ownKeys(value)) {
+            if (typeof key !== 'string')
+                fail(`${location} must not contain symbol keys.`);
+            const descriptor = Object.getOwnPropertyDescriptor(value, key);
+            if (!descriptor || descriptor.get || descriptor.set)
+                fail(`${location}.${key} must be a plain data property.`);
+            result[key] = evidenceValue(descriptor.value, `${location}.${key}`, ancestors);
+        }
+        return result;
+    }
+    finally {
+        ancestors.delete(value);
+    }
+}
+function safeError(value, location) {
+    if (!isRecord(value))
+        fail(`${location} must be a record.`);
+    rejectEvaluationFields(value, location);
+    const message = requiredText(value.message, `${location}.message`);
+    const name = optionalText(value, 'name', location);
+    return { ...(name === undefined ? {} : { name }), message };
+}
+function tokenUsage(value) {
+    if (value === undefined)
+        return { availability: 'unavailable' };
+    if (!isRecord(value))
+        fail('External execution evidence tokenUsage must be a record.');
+    rejectEvaluationFields(value, 'External execution evidence tokenUsage');
+    const usage = {
+        inputTokens: optionalNonNegativeNumber(value, 'inputTokens', 'External execution evidence tokenUsage'),
+        outputTokens: optionalNonNegativeNumber(value, 'outputTokens', 'External execution evidence tokenUsage'),
+        totalTokens: optionalNonNegativeNumber(value, 'totalTokens', 'External execution evidence tokenUsage'),
+        reasoningTokens: optionalNonNegativeNumber(value, 'reasoningTokens', 'External execution evidence tokenUsage'),
+    };
+    if (Object.values(usage).every((entry) => entry === undefined))
+        return { availability: 'unavailable' };
+    return {
+        availability: 'available',
+        ...(usage.inputTokens === undefined ? {} : { inputTokens: usage.inputTokens }),
+        ...(usage.outputTokens === undefined ? {} : { outputTokens: usage.outputTokens }),
+        ...(usage.totalTokens === undefined ? {} : { totalTokens: usage.totalTokens }),
+        ...(usage.reasoningTokens === undefined ? {} : { reasoningTokens: usage.reasoningTokens }),
+    };
+}
+function normalizeToolCall(value, index) {
+    const location = `External execution evidence toolCalls[${index}]`;
+    if (!isRecord(value))
+        fail(`${location} must be a record.`);
+    rejectEvaluationFields(value, location);
+    const callId = requiredText(value.callId, `${location}.callId`);
+    const name = requiredText(value.name, `${location}.name`);
+    if (!Object.prototype.hasOwnProperty.call(value, 'input'))
+        fail(`${location}.input is required.`);
+    const input = JSON.stringify(evidenceValue(value.input, `${location}.input`, new Set()));
+    const sequence = optionalNonNegativeInteger(value, 'sequence', location);
+    if (sequence === undefined)
+        fail(`${location}.sequence is required.`);
+    optionalNonNegativeInteger(value, 'step', location);
+    const startedAt = optionalText(value, 'startedAt', location);
+    const endedAt = optionalText(value, 'endedAt', location);
+    const durationMs = optionalNonNegativeNumber(value, 'durationMs', location);
+    const base = {
+        callId,
+        name,
+        input,
+        ...(durationMs === undefined ? {} : { duration: `${durationMs}ms` }),
+        ...(startedAt === undefined ? {} : { startedAt }),
+        ...(endedAt === undefined ? {} : { endedAt }),
+    };
+    if (value.status === 'success') {
+        if (Object.prototype.hasOwnProperty.call(value, 'error'))
+            fail(`${location}.error is not allowed for a successful call.`);
+        if (!Object.prototype.hasOwnProperty.call(value, 'output'))
+            return { ...base, status: 'success' };
+        const output = JSON.stringify(evidenceValue(value.output, `${location}.output`, new Set()));
+        return { ...base, status: 'success', output };
+    }
+    if (value.status === 'error') {
+        if (Object.prototype.hasOwnProperty.call(value, 'output'))
+            fail(`${location}.output is not allowed for an error call.`);
+        if (!Object.prototype.hasOwnProperty.call(value, 'error'))
+            return { ...base, status: 'error' };
+        return { ...base, status: 'error', error: safeError(value.error, `${location}.error`) };
+    }
+    return fail(`${location}.status must be "success" or "error".`);
+}
+export function normalizeExternalExecutionEvidence(value) {
+    if (!isRecord(value))
+        fail('External execution evidence must be a record.');
+    rejectEvaluationFields(value, 'External execution evidence');
+    requiredText(value.source, 'External execution evidence source');
+    if (!Array.isArray(value.toolCalls))
+        fail('External execution evidence toolCalls must be an array.');
+    const toolCalls = value.toolCalls.map(normalizeToolCall);
+    const callIds = new Set();
+    const sequences = new Set();
+    value.toolCalls.forEach((rawCall, index) => {
+        const call = rawCall;
+        const callId = call.callId;
+        const sequence = call.sequence;
+        if (callIds.has(callId))
+            fail(`External execution evidence contains duplicate callId "${callId}".`);
+        if (sequences.has(sequence))
+            fail(`External execution evidence contains duplicate sequence "${sequence}".`);
+        callIds.add(callId);
+        sequences.add(sequence);
+        if (sequence !== index)
+            fail('External execution evidence sequences must be contiguous and zero-based.');
+    });
+    const timeline = toolCalls.map((call) => ({
+        label: `Tool completed: ${call.name}`,
+        detail: `Operational status: ${call.status}`,
+        ...(call.duration === undefined ? {} : { duration: call.duration }),
+    }));
+    const provider = optionalText(value, 'provider', 'External execution evidence');
+    const model = optionalText(value, 'model', 'External execution evidence');
+    const startedAt = optionalText(value, 'startedAt', 'External execution evidence');
+    const endedAt = optionalText(value, 'endedAt', 'External execution evidence');
+    const latencyMs = optionalNonNegativeNumber(value, 'latencyMs', 'External execution evidence');
+    const finishReason = optionalText(value, 'finishReason', 'External execution evidence');
+    return {
+        toolCalls,
+        timeline,
+        evidence: {
+            ...(provider === undefined ? {} : { provider }),
+            ...(model === undefined ? {} : { model }),
+            ...(startedAt === undefined ? {} : { startedAt }),
+            ...(endedAt === undefined ? {} : { endedAt }),
+            ...(latencyMs === undefined ? {} : { latencyMs }),
+            ...(finishReason === undefined ? {} : { finishReason }),
+            tokenUsage: tokenUsage(value.tokenUsage),
+        },
+    };
+}

package/dist/generic-engine.d.ts

@@ -0,0 +1,21 @@
+import type { CompletedExecutionRecord, LanguageAdapter, ProjectDescriptor } from './contracts.js';
+export type EngineRunResult = {
+    execution: CompletedExecutionRecord;
+    boundaryEvidence: {
+        executionId: string;
+        completedBehavioralRuns: 1;
+        adapter: string;
+        runner: 'LanguageAdapterRunner';
+        evaluator: 'deterministic';
+        storyUnchanged: boolean;
+        mockDataUsed: false;
+    };
+};
+export declare class EthogramEngine {
+    private readonly runner;
+    private project?;
+    constructor(runner: LanguageAdapter);
+    loadProject(projectRoot: string): Promise<ProjectDescriptor>;
+    getProject(): ProjectDescriptor;
+    runStory(storyId: string): Promise<EngineRunResult>;
+}

package/dist/generic-engine.js

@@ -0,0 +1,54 @@
+import { randomUUID } from 'node:crypto';
+import { evaluateStory } from './evaluator.js';
+function canonical(value) {
+    return JSON.stringify(value);
+}
+function deepFreeze(value) {
+    if (value && typeof value === 'object' && !Object.isFrozen(value)) {
+        Object.freeze(value);
+        for (const nested of Object.values(value))
+            deepFreeze(nested);
+    }
+    return value;
+}
+export class EthogramEngine {
+    runner;
+    project;
+    constructor(runner) {
+        this.runner = runner;
+    }
+    async loadProject(projectRoot) {
+        this.project = await this.runner.loadProject(projectRoot);
+        return this.project;
+    }
+    getProject() {
+        if (!this.project)
+            throw new Error('PROJECT_NOT_LOADED');
+        return this.project;
+    }
+    async runStory(storyId) {
+        const project = this.getProject();
+        const story = project.stories.find((candidate) => candidate.id === storyId);
+        if (!story)
+            throw new Error(`STORY_NOT_FOUND: ${storyId}`);
+        if (!story.executable)
+            throw new Error(`STORY_NOT_EXECUTABLE: ${storyId}`);
+        const storySnapshot = canonical(story);
+        deepFreeze(story);
+        const request = Object.freeze({ story });
+        const observedRun = await this.runner.run(request);
+        const evaluationResult = evaluateStory(story, observedRun);
+        return {
+            execution: { observedRun, evaluationResult },
+            boundaryEvidence: {
+                executionId: randomUUID(),
+                completedBehavioralRuns: 1,
+                adapter: this.runner.id,
+                runner: 'LanguageAdapterRunner',
+                evaluator: 'deterministic',
+                storyUnchanged: canonical(story) === storySnapshot,
+                mockDataUsed: false,
+            },
+        };
+    }
+}