@purista/harness 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +23 -0
- package/dist/agents/index.d.ts +34 -0
- package/dist/agents/index.js +301 -0
- package/dist/errors/catalog.d.ts +185 -0
- package/dist/errors/catalog.js +144 -0
- package/dist/errors/harness-error.d.ts +64 -0
- package/dist/errors/harness-error.js +58 -0
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +3 -0
- package/dist/errors/redaction.d.ts +5 -0
- package/dist/errors/redaction.js +64 -0
- package/dist/harness/defineHarness.d.ts +640 -0
- package/dist/harness/defineHarness.js +176 -0
- package/dist/harness/errors.d.ts +62 -0
- package/dist/harness/errors.js +67 -0
- package/dist/harness/types.d.ts +27 -0
- package/dist/harness/types.js +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +12 -0
- package/dist/logger/index.d.ts +2 -0
- package/dist/logger/index.js +2 -0
- package/dist/logger/json-logger.d.ts +31 -0
- package/dist/logger/json-logger.js +65 -0
- package/dist/logger/logger.d.ts +31 -0
- package/dist/logger/logger.js +1 -0
- package/dist/models/json.d.ts +6 -0
- package/dist/models/json.js +1 -0
- package/dist/models/registry.d.ts +112 -0
- package/dist/models/registry.js +286 -0
- package/dist/models/state.d.ts +64 -0
- package/dist/models/state.js +1 -0
- package/dist/ports/base-model-provider.d.ts +56 -0
- package/dist/ports/base-model-provider.js +343 -0
- package/dist/ports/capabilities.d.ts +70 -0
- package/dist/ports/capabilities.js +38 -0
- package/dist/ports/feedback.d.ts +29 -0
- package/dist/ports/feedback.js +1 -0
- package/dist/ports/harness-context.d.ts +20 -0
- package/dist/ports/harness-context.js +1 -0
- package/dist/ports/index.d.ts +6 -0
- package/dist/ports/index.js +6 -0
- package/dist/ports/model-provider.d.ts +280 -0
- package/dist/ports/model-provider.js +1 -0
- package/dist/ports/state.d.ts +72 -0
- package/dist/ports/state.js +24 -0
- package/dist/runtime/durable.d.ts +134 -0
- package/dist/runtime/durable.js +185 -0
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/steps.d.ts +22 -0
- package/dist/runtime/steps.js +51 -0
- package/dist/sandbox/index.d.ts +111 -0
- package/dist/sandbox/index.js +165 -0
- package/dist/sessions/index.d.ts +23 -0
- package/dist/sessions/index.js +718 -0
- package/dist/skills/index.d.ts +8 -0
- package/dist/skills/index.js +88 -0
- package/dist/state/in-memory.d.ts +35 -0
- package/dist/state/in-memory.js +140 -0
- package/dist/telemetry/index.d.ts +1 -0
- package/dist/telemetry/index.js +1 -0
- package/dist/telemetry/shim.d.ts +26 -0
- package/dist/telemetry/shim.js +120 -0
- package/dist/testing/capabilities.d.ts +11 -0
- package/dist/testing/capabilities.js +20 -0
- package/dist/testing/fakeModelProvider.d.ts +25 -0
- package/dist/testing/fakeModelProvider.js +79 -0
- package/dist/testing/feedback.d.ts +10 -0
- package/dist/testing/feedback.js +24 -0
- package/dist/testing/fixtures/mcp/fake-http-server.d.ts +8 -0
- package/dist/testing/fixtures/mcp/fake-http-server.js +95 -0
- package/dist/testing/index.d.ts +8 -0
- package/dist/testing/index.js +11 -0
- package/dist/testing/sandboxContract.d.ts +4 -0
- package/dist/testing/sandboxContract.js +74 -0
- package/dist/testing/sandboxSnapshot.d.ts +7 -0
- package/dist/testing/sandboxSnapshot.js +201 -0
- package/dist/testing/stateStoreContract.d.ts +2 -0
- package/dist/testing/stateStoreContract.js +109 -0
- package/dist/tools/index.d.ts +9 -0
- package/dist/tools/index.js +123 -0
- package/dist/tools/mcp/http.d.ts +2 -0
- package/dist/tools/mcp/http.js +109 -0
- package/dist/tools/mcp/index.d.ts +2 -0
- package/dist/tools/mcp/index.js +2 -0
- package/dist/tools/mcp/runner.d.ts +74 -0
- package/dist/tools/mcp/runner.js +238 -0
- package/dist/tools/mcp/schema.d.ts +41 -0
- package/dist/tools/mcp/schema.js +251 -0
- package/dist/tools/mcp/stdio.d.ts +2 -0
- package/dist/tools/mcp/stdio.js +122 -0
- package/dist/ulid/index.d.ts +6 -0
- package/dist/ulid/index.js +35 -0
- package/dist/workflows/index.d.ts +8 -0
- package/dist/workflows/index.js +26 -0
- package/package.json +75 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { JsonValue } from '../models/json.js';
|
|
2
|
+
import type { ResolvedSkill, SkillDefinition } from '../harness/defineHarness.js';
|
|
3
|
+
import type { SandboxSession } from '../sandbox/index.js';
|
|
4
|
+
export declare function loadSkillsSync(skills: Record<string, SkillDefinition>): Record<string, ResolvedSkill>;
|
|
5
|
+
export declare function loadSkills(skills: Record<string, SkillDefinition>): Promise<Record<string, ResolvedSkill>>;
|
|
6
|
+
export declare function mountSkillsOnce(session: SandboxSession, mounted: Set<string>, skills: Record<string, ResolvedSkill>, skillIds: readonly string[]): Promise<void>;
|
|
7
|
+
export declare function buildSkillIndex(skills: Record<string, ResolvedSkill>, ids: readonly string[]): string;
|
|
8
|
+
export declare function assertSerializable(value: unknown): asserts value is JsonValue;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import fsp from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { SkillManifestError, SkillNotFoundError } from '../errors/index.js';
|
|
5
|
+
function parseFrontmatter(content) {
|
|
6
|
+
if (!content.startsWith('---\n'))
|
|
7
|
+
throw new SkillManifestError('Invalid SKILL.md frontmatter', { reason: 'invalid_frontmatter', directory: '' });
|
|
8
|
+
const end = content.indexOf('\n---\n', 4);
|
|
9
|
+
if (end < 0)
|
|
10
|
+
throw new SkillManifestError('Invalid SKILL.md frontmatter', { reason: 'invalid_frontmatter', directory: '' });
|
|
11
|
+
const raw = content.slice(4, end);
|
|
12
|
+
const body = content.slice(end + 5);
|
|
13
|
+
const frontmatter = {};
|
|
14
|
+
for (const line of raw.split('\n')) {
|
|
15
|
+
const idx = line.indexOf(':');
|
|
16
|
+
if (idx < 0)
|
|
17
|
+
continue;
|
|
18
|
+
frontmatter[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
|
|
19
|
+
}
|
|
20
|
+
return { frontmatter, body };
|
|
21
|
+
}
|
|
22
|
+
function validateName(name) { return /^[a-z][a-z0-9-]*$/.test(name) && !/anthropic|claude|purista/i.test(name); }
|
|
23
|
+
export function loadSkillsSync(skills) {
|
|
24
|
+
const resolved = {};
|
|
25
|
+
for (const [key, def] of Object.entries(skills)) {
|
|
26
|
+
const stat = fs.existsSync(def.directory) ? fs.statSync(def.directory) : null;
|
|
27
|
+
if (!stat?.isDirectory())
|
|
28
|
+
throw new SkillManifestError('Skill directory missing', { reason: 'directory_missing', directory: def.directory, skill_id: key });
|
|
29
|
+
const skillPath = path.join(def.directory, 'SKILL.md');
|
|
30
|
+
const content = fs.existsSync(skillPath) ? fs.readFileSync(skillPath, 'utf8') : null;
|
|
31
|
+
if (!content)
|
|
32
|
+
throw new SkillManifestError('missing SKILL.md', { reason: 'missing_skill_md', directory: def.directory, skill_id: key });
|
|
33
|
+
const { frontmatter } = parseFrontmatter(content);
|
|
34
|
+
const name = frontmatter['name'] ?? '';
|
|
35
|
+
const description = frontmatter['description'] ?? '';
|
|
36
|
+
if (!validateName(name) || description.length < 1 || description.length > 1024) {
|
|
37
|
+
throw new SkillManifestError('invalid frontmatter', { reason: 'invalid_frontmatter', directory: def.directory, skill_id: key });
|
|
38
|
+
}
|
|
39
|
+
if (name !== key)
|
|
40
|
+
throw new SkillManifestError('name mismatch', { reason: 'name_mismatch', directory: def.directory, skill_id: key });
|
|
41
|
+
resolved[key] = { name, description, directory: def.directory, ...(frontmatter['version'] ? { version: frontmatter['version'] } : {}) };
|
|
42
|
+
}
|
|
43
|
+
return resolved;
|
|
44
|
+
}
|
|
45
|
+
export async function loadSkills(skills) {
|
|
46
|
+
return loadSkillsSync(skills);
|
|
47
|
+
}
|
|
48
|
+
async function readDirRecursive(root) {
|
|
49
|
+
const files = new Map();
|
|
50
|
+
const walk = async (dir) => {
|
|
51
|
+
const entries = await fsp.readdir(dir, { withFileTypes: true });
|
|
52
|
+
for (const entry of entries) {
|
|
53
|
+
const abs = path.join(dir, entry.name);
|
|
54
|
+
if (entry.isDirectory())
|
|
55
|
+
await walk(abs);
|
|
56
|
+
else if (entry.isFile())
|
|
57
|
+
files.set(path.posix.normalize(path.relative(root, abs).split(path.sep).join('/')), await fsp.readFile(abs));
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
await walk(root);
|
|
61
|
+
return files;
|
|
62
|
+
}
|
|
63
|
+
export async function mountSkillsOnce(session, mounted, skills, skillIds) {
|
|
64
|
+
for (const skillId of skillIds) {
|
|
65
|
+
if (mounted.has(skillId))
|
|
66
|
+
continue;
|
|
67
|
+
const skill = skills[skillId];
|
|
68
|
+
if (!skill)
|
|
69
|
+
throw new SkillNotFoundError('Skill not found.', { skill_id: skillId });
|
|
70
|
+
const files = await readDirRecursive(skill.directory);
|
|
71
|
+
await session.mount(files, `/skills/${skillId}`);
|
|
72
|
+
mounted.add(skillId);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
export function buildSkillIndex(skills, ids) {
|
|
76
|
+
if (ids.length === 0)
|
|
77
|
+
return '';
|
|
78
|
+
const lines = ids.map((id) => `- ${skills[id]?.name ?? id}: ${skills[id]?.description ?? ''}`);
|
|
79
|
+
return `\n\nAvailable skills (read /skills/<name>/SKILL.md for full instructions):\n${lines.join('\n')}`;
|
|
80
|
+
}
|
|
81
|
+
export function assertSerializable(value) {
|
|
82
|
+
try {
|
|
83
|
+
JSON.stringify(value);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
throw new SkillManifestError('Non-serializable value', { reason: 'invalid_frontmatter', directory: '' });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { Message, PersistedRunEvent, RunRecord, SessionRecord } from '../models/state.js';
|
|
2
|
+
import type { FinishRunPatch, StateStore } from '../ports/state.js';
|
|
3
|
+
/**
|
|
4
|
+
* In-process state store for local development and tests.
|
|
5
|
+
*/
|
|
6
|
+
export declare class InMemoryStateStore implements StateStore {
|
|
7
|
+
private readonly sessions;
|
|
8
|
+
private readonly messages;
|
|
9
|
+
private readonly runs;
|
|
10
|
+
private readonly events;
|
|
11
|
+
private readonly messageLocks;
|
|
12
|
+
getSession(id: string): Promise<SessionRecord | undefined>;
|
|
13
|
+
upsertSession(record: SessionRecord): Promise<void>;
|
|
14
|
+
closeSession(id: string): Promise<void>;
|
|
15
|
+
appendMessages(sessionId: string, messages: Message[]): Promise<void>;
|
|
16
|
+
listMessages(sessionId: string, opts?: {
|
|
17
|
+
limit?: number;
|
|
18
|
+
before?: string;
|
|
19
|
+
}): Promise<Message[]>;
|
|
20
|
+
clearMessages(sessionId: string): Promise<void>;
|
|
21
|
+
createRun(record: RunRecord): Promise<void>;
|
|
22
|
+
finishRun(runId: string, patch: FinishRunPatch): Promise<void>;
|
|
23
|
+
getRun(runId: string): Promise<RunRecord | undefined>;
|
|
24
|
+
listRuns(sessionId: string, opts?: {
|
|
25
|
+
limit?: number;
|
|
26
|
+
before?: string;
|
|
27
|
+
}): Promise<RunRecord[]>;
|
|
28
|
+
appendEvents(runId: string, events: PersistedRunEvent[]): Promise<void>;
|
|
29
|
+
listEvents(runId: string, opts?: {
|
|
30
|
+
limit?: number;
|
|
31
|
+
after?: string;
|
|
32
|
+
}): Promise<PersistedRunEvent[]>;
|
|
33
|
+
close(): Promise<void>;
|
|
34
|
+
private withMessageLock;
|
|
35
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { StateError } from '../errors/index.js';
|
|
2
|
+
class Mutex {
|
|
3
|
+
current = Promise.resolve();
|
|
4
|
+
async lock(fn) {
|
|
5
|
+
const prev = this.current;
|
|
6
|
+
let release;
|
|
7
|
+
this.current = new Promise((resolve) => { release = resolve; });
|
|
8
|
+
await prev;
|
|
9
|
+
try {
|
|
10
|
+
return await fn();
|
|
11
|
+
}
|
|
12
|
+
finally {
|
|
13
|
+
release?.();
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* In-process state store for local development and tests.
|
|
19
|
+
*/
|
|
20
|
+
export class InMemoryStateStore {
|
|
21
|
+
sessions = new Map();
|
|
22
|
+
messages = new Map();
|
|
23
|
+
runs = new Map();
|
|
24
|
+
events = new Map();
|
|
25
|
+
messageLocks = new Map();
|
|
26
|
+
async getSession(id) {
|
|
27
|
+
return this.sessions.get(id);
|
|
28
|
+
}
|
|
29
|
+
async upsertSession(record) {
|
|
30
|
+
this.sessions.set(record.id, record);
|
|
31
|
+
}
|
|
32
|
+
async closeSession(id) {
|
|
33
|
+
this.sessions.delete(id);
|
|
34
|
+
this.messages.delete(id);
|
|
35
|
+
for (const [runId, run] of this.runs) {
|
|
36
|
+
if (run.sessionId === id) {
|
|
37
|
+
this.runs.delete(runId);
|
|
38
|
+
this.events.delete(runId);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
async appendMessages(sessionId, messages) {
|
|
43
|
+
return this.withMessageLock(sessionId, async () => {
|
|
44
|
+
const current = this.messages.get(sessionId) ?? [];
|
|
45
|
+
const ids = new Set(current.map((msg) => msg.id));
|
|
46
|
+
for (const message of messages) {
|
|
47
|
+
if (ids.has(message.id)) {
|
|
48
|
+
throw new StateError('Duplicate message id.', { op: 'appendMessages', reason: 'duplicate_message_id' });
|
|
49
|
+
}
|
|
50
|
+
ids.add(message.id);
|
|
51
|
+
}
|
|
52
|
+
this.messages.set(sessionId, [...current, ...messages]);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
async listMessages(sessionId, opts = {}) {
|
|
56
|
+
let rows = [...(this.messages.get(sessionId) ?? [])]
|
|
57
|
+
.sort((a, b) => a.timestamp === b.timestamp ? a.id.localeCompare(b.id) : a.timestamp.localeCompare(b.timestamp));
|
|
58
|
+
if (opts.before) {
|
|
59
|
+
const beforeIndex = rows.findIndex((row) => row.id === opts.before);
|
|
60
|
+
if (beforeIndex >= 0) {
|
|
61
|
+
rows = rows.slice(0, beforeIndex);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (opts.limit !== undefined) {
|
|
65
|
+
rows = rows.slice(Math.max(0, rows.length - opts.limit));
|
|
66
|
+
}
|
|
67
|
+
return rows;
|
|
68
|
+
}
|
|
69
|
+
async clearMessages(sessionId) {
|
|
70
|
+
return this.withMessageLock(sessionId, async () => {
|
|
71
|
+
this.messages.delete(sessionId);
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
async createRun(record) {
|
|
75
|
+
this.runs.set(record.id, record);
|
|
76
|
+
}
|
|
77
|
+
async finishRun(runId, patch) {
|
|
78
|
+
const run = this.runs.get(runId);
|
|
79
|
+
if (!run)
|
|
80
|
+
return;
|
|
81
|
+
this.runs.set(runId, { ...run, ...patch });
|
|
82
|
+
}
|
|
83
|
+
async getRun(runId) {
|
|
84
|
+
return this.runs.get(runId);
|
|
85
|
+
}
|
|
86
|
+
async listRuns(sessionId, opts = {}) {
|
|
87
|
+
let rows = [...this.runs.values()]
|
|
88
|
+
.filter((run) => run.sessionId === sessionId)
|
|
89
|
+
.sort((a, b) => a.startedAt === b.startedAt ? b.id.localeCompare(a.id) : b.startedAt.localeCompare(a.startedAt));
|
|
90
|
+
if (opts.before) {
|
|
91
|
+
const beforeIndex = rows.findIndex((row) => row.id === opts.before);
|
|
92
|
+
if (beforeIndex >= 0) {
|
|
93
|
+
rows = rows.slice(beforeIndex + 1);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (opts.limit !== undefined) {
|
|
97
|
+
rows = rows.slice(0, opts.limit);
|
|
98
|
+
}
|
|
99
|
+
return rows;
|
|
100
|
+
}
|
|
101
|
+
async appendEvents(runId, events) {
|
|
102
|
+
const current = this.events.get(runId) ?? [];
|
|
103
|
+
this.events.set(runId, [...current, ...events]);
|
|
104
|
+
}
|
|
105
|
+
async listEvents(runId, opts = {}) {
|
|
106
|
+
let rows = [...(this.events.get(runId) ?? [])];
|
|
107
|
+
if (opts.after) {
|
|
108
|
+
const afterIndex = rows.findIndex((row) => row.id === opts.after);
|
|
109
|
+
if (afterIndex >= 0) {
|
|
110
|
+
rows = rows.slice(afterIndex + 1);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (opts.limit !== undefined) {
|
|
114
|
+
rows = rows.slice(0, opts.limit);
|
|
115
|
+
}
|
|
116
|
+
return rows;
|
|
117
|
+
}
|
|
118
|
+
async close() {
|
|
119
|
+
this.sessions.clear();
|
|
120
|
+
this.messages.clear();
|
|
121
|
+
this.runs.clear();
|
|
122
|
+
this.events.clear();
|
|
123
|
+
this.messageLocks.clear();
|
|
124
|
+
}
|
|
125
|
+
async withMessageLock(sessionId, fn) {
|
|
126
|
+
let lock = this.messageLocks.get(sessionId);
|
|
127
|
+
if (!lock) {
|
|
128
|
+
lock = new Mutex();
|
|
129
|
+
this.messageLocks.set(sessionId, lock);
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
return await lock.lock(fn);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
if (error instanceof StateError)
|
|
136
|
+
throw error;
|
|
137
|
+
throw new StateError('State store operation failed.', { op: 'appendMessages' }, error);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './shim.js';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './shim.js';
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** Attributes accepted by telemetry span/metric helpers. */
|
|
2
|
+
export type SpanAttrs = Record<string, string | number | boolean | string[] | undefined>;
|
|
3
|
+
/** Minimal telemetry abstraction used by harness internals and integrations. */
|
|
4
|
+
export interface TelemetryShim {
|
|
5
|
+
/** Creates a span, executes `fn`, and closes the span with success/error status. */
|
|
6
|
+
span<T>(name: string, attrs: SpanAttrs, fn: (span: import('@opentelemetry/api').Span) => Promise<T>): Promise<T>;
|
|
7
|
+
/** Records a histogram value with attributes. */
|
|
8
|
+
recordHistogram(name: string, value: number, attrs: SpanAttrs): void;
|
|
9
|
+
/** Records a counter increment/add value with attributes. */
|
|
10
|
+
recordCounter(name: string, value: number, attrs: SpanAttrs): void;
|
|
11
|
+
/** Injects the current active trace context into a W3C traceparent carrier. */
|
|
12
|
+
currentTraceparent(): string | undefined;
|
|
13
|
+
}
|
|
14
|
+
/** OpenTelemetry-backed implementation of {@link TelemetryShim}. */
|
|
15
|
+
export declare class OtelTelemetryShim implements TelemetryShim {
|
|
16
|
+
private readonly tracer;
|
|
17
|
+
private readonly meter;
|
|
18
|
+
private readonly histograms;
|
|
19
|
+
private readonly counters;
|
|
20
|
+
span<T>(name: string, attrs: SpanAttrs, fn: (span: import('@opentelemetry/api').Span) => Promise<T>): Promise<T>;
|
|
21
|
+
recordHistogram(name: string, value: number, attrs: SpanAttrs): void;
|
|
22
|
+
recordCounter(name: string, value: number, attrs: SpanAttrs): void;
|
|
23
|
+
currentTraceparent(): string | undefined;
|
|
24
|
+
}
|
|
25
|
+
/** Creates the default telemetry shim instance. */
|
|
26
|
+
export declare function createTelemetryShim(): TelemetryShim;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { SpanStatusCode, context, metrics, propagation, trace } from '@opentelemetry/api';
|
|
2
|
+
import { ATTR_ERROR_TYPE } from '@opentelemetry/semantic-conventions';
|
|
3
|
+
import { HarnessError } from '../errors/index.js';
|
|
4
|
+
import { sanitizeForLog } from '../errors/redaction.js';
|
|
5
|
+
function sanitizeAttrs(attrs) {
|
|
6
|
+
const out = {};
|
|
7
|
+
for (const [key, value] of Object.entries(attrs)) {
|
|
8
|
+
if (value === undefined)
|
|
9
|
+
continue;
|
|
10
|
+
if (Array.isArray(value)) {
|
|
11
|
+
out[key] = [...value];
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
out[key] = value;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
function errorAttributes(error) {
|
|
20
|
+
if (error instanceof HarnessError) {
|
|
21
|
+
const meta = asRecord(error.meta);
|
|
22
|
+
const providerBody = meta ? jsonAttr(sanitizeForLog(meta['providerBody'])) : undefined;
|
|
23
|
+
return {
|
|
24
|
+
[ATTR_ERROR_TYPE]: error.code,
|
|
25
|
+
'harness.error.code': error.code,
|
|
26
|
+
'harness.error.category': error.category,
|
|
27
|
+
'harness.error.retriable': error.retriable,
|
|
28
|
+
'harness.error.provider': stringAttr(meta?.['provider']),
|
|
29
|
+
'harness.error.model': stringAttr(meta?.['model']),
|
|
30
|
+
'harness.error.model_provider_status': numberAttr(meta?.['status']),
|
|
31
|
+
'harness.error.model_provider_code': stringAttr(meta?.['providerCode']),
|
|
32
|
+
'harness.error.model_provider_type': stringAttr(meta?.['providerType']),
|
|
33
|
+
'harness.error.model_provider_param': stringAttr(meta?.['providerParam']),
|
|
34
|
+
'harness.error.model_provider_request_id': stringAttr(meta?.['providerRequestId']),
|
|
35
|
+
'harness.error.model_provider_message': stringAttr(meta?.['providerMessage']),
|
|
36
|
+
'harness.error.model_provider_body': providerBody
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
const name = error instanceof Error ? error.name : 'Error';
|
|
40
|
+
return {
|
|
41
|
+
[ATTR_ERROR_TYPE]: name,
|
|
42
|
+
'harness.error.code': name,
|
|
43
|
+
'harness.error.category': 'internal',
|
|
44
|
+
'harness.error.retriable': false
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function asRecord(value) {
|
|
48
|
+
return value !== null && typeof value === 'object' ? value : undefined;
|
|
49
|
+
}
|
|
50
|
+
function stringAttr(value) {
|
|
51
|
+
return typeof value === 'string' && value.length > 0 ? value.slice(0, 4000) : undefined;
|
|
52
|
+
}
|
|
53
|
+
function numberAttr(value) {
|
|
54
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
55
|
+
}
|
|
56
|
+
function jsonAttr(value) {
|
|
57
|
+
if (value === undefined)
|
|
58
|
+
return undefined;
|
|
59
|
+
try {
|
|
60
|
+
return JSON.stringify(value).slice(0, 8000);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** OpenTelemetry-backed implementation of {@link TelemetryShim}. */
|
|
67
|
+
export class OtelTelemetryShim {
|
|
68
|
+
tracer = trace.getTracer('@purista/harness');
|
|
69
|
+
meter = metrics.getMeter('@purista/harness');
|
|
70
|
+
histograms = new Map();
|
|
71
|
+
counters = new Map();
|
|
72
|
+
async span(name, attrs, fn) {
|
|
73
|
+
return this.tracer.startActiveSpan(name, { attributes: sanitizeAttrs(attrs) }, async (span) => {
|
|
74
|
+
try {
|
|
75
|
+
const result = await fn(span);
|
|
76
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
span.setAttributes(sanitizeAttrs(errorAttributes(error)));
|
|
81
|
+
const recordedError = error instanceof HarnessError
|
|
82
|
+
? new Error(error.message)
|
|
83
|
+
: error instanceof Error
|
|
84
|
+
? new Error(error.message)
|
|
85
|
+
: new Error(String(error));
|
|
86
|
+
span.recordException(recordedError);
|
|
87
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: recordedError.message });
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
span.end();
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
recordHistogram(name, value, attrs) {
|
|
96
|
+
let histogram = this.histograms.get(name);
|
|
97
|
+
if (!histogram) {
|
|
98
|
+
histogram = this.meter.createHistogram(name);
|
|
99
|
+
this.histograms.set(name, histogram);
|
|
100
|
+
}
|
|
101
|
+
histogram.record(value, sanitizeAttrs(attrs));
|
|
102
|
+
}
|
|
103
|
+
recordCounter(name, value, attrs) {
|
|
104
|
+
let counter = this.counters.get(name);
|
|
105
|
+
if (!counter) {
|
|
106
|
+
counter = this.meter.createCounter(name);
|
|
107
|
+
this.counters.set(name, counter);
|
|
108
|
+
}
|
|
109
|
+
counter.add(value, sanitizeAttrs(attrs));
|
|
110
|
+
}
|
|
111
|
+
currentTraceparent() {
|
|
112
|
+
const carrier = {};
|
|
113
|
+
propagation.inject(context.active(), carrier);
|
|
114
|
+
return carrier['traceparent'];
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** Creates the default telemetry shim instance. */
|
|
118
|
+
export function createTelemetryShim() {
|
|
119
|
+
return new OtelTelemetryShim();
|
|
120
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type AdapterCapabilities, type AdapterCapability } from '../ports/capabilities.js';
|
|
2
|
+
/** Test adapter descriptor with a stable diagnostic id. */
|
|
3
|
+
export interface FakeCapabilityAdapter extends AdapterCapabilities {
|
|
4
|
+
readonly id: string;
|
|
5
|
+
}
|
|
6
|
+
/** Creates a fake adapter capability descriptor for tests. */
|
|
7
|
+
export declare function fakeCapabilityAdapter(capabilities: readonly AdapterCapability[], opts?: {
|
|
8
|
+
id?: string;
|
|
9
|
+
}): FakeCapabilityAdapter;
|
|
10
|
+
/** Shared contract for adapters that expose harness capabilities. */
|
|
11
|
+
export declare function adapterCapabilitiesContract(make: () => AdapterCapabilities | Promise<AdapterCapabilities>): void;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { collectAdapterCapabilities, validateAdapterCapabilities } from '../ports/capabilities.js';
|
|
3
|
+
/** Creates a fake adapter capability descriptor for tests. */
|
|
4
|
+
export function fakeCapabilityAdapter(capabilities, opts = {}) {
|
|
5
|
+
return {
|
|
6
|
+
id: opts.id ?? 'fake',
|
|
7
|
+
capabilities
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
/** Shared contract for adapters that expose harness capabilities. */
|
|
11
|
+
export function adapterCapabilitiesContract(make) {
|
|
12
|
+
describe('adapterCapabilitiesContract', () => {
|
|
13
|
+
it('declares a stable capability list', async () => {
|
|
14
|
+
const adapter = await make();
|
|
15
|
+
const capabilities = collectAdapterCapabilities([adapter]);
|
|
16
|
+
expect(capabilities).toEqual(adapter.capabilities);
|
|
17
|
+
expect(validateAdapterCapabilities(adapter.capabilities, capabilities).ok).toBe(true);
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { EmbeddingRequest, EmbeddingResponse, ModelProvider, ObjectRequest, ObjectResponse, ObjectStreamChunk, RerankRequest, RerankResponse, TextRequest, TextResponse, TextStreamChunk } from '../ports/model-provider.js';
|
|
2
|
+
import type { JsonValue } from '../models/json.js';
|
|
3
|
+
/** Deterministic model provider for harness tests and examples. */
|
|
4
|
+
export declare class FakeModelProvider implements ModelProvider {
|
|
5
|
+
private queue;
|
|
6
|
+
private textStreamQueue;
|
|
7
|
+
private objectStreamQueue;
|
|
8
|
+
readonly requests: Array<TextRequest | ObjectRequest | EmbeddingRequest | RerankRequest>;
|
|
9
|
+
readonly id = "fake";
|
|
10
|
+
readonly genAiSystem = "fake";
|
|
11
|
+
enqueueObject(response: ObjectResponse): void;
|
|
12
|
+
enqueueText(response: TextResponse): void;
|
|
13
|
+
enqueueEmbedding(response: EmbeddingResponse): void;
|
|
14
|
+
enqueueRerank(response: RerankResponse): void;
|
|
15
|
+
enqueueTextStream(chunks: TextStreamChunk[]): void;
|
|
16
|
+
enqueueObjectStream(chunks: ObjectStreamChunk[]): void;
|
|
17
|
+
/** Backward-compatible helper for older tests during the object migration. */
|
|
18
|
+
enqueue(response: ObjectResponse): void;
|
|
19
|
+
text(req: TextRequest): Promise<TextResponse>;
|
|
20
|
+
textStream(req: TextRequest): AsyncIterable<TextStreamChunk>;
|
|
21
|
+
object<T extends JsonValue = JsonValue>(req: ObjectRequest<T>): Promise<ObjectResponse<T>>;
|
|
22
|
+
objectStream<T extends JsonValue = JsonValue>(req: ObjectRequest<T>): AsyncIterable<ObjectStreamChunk<T>>;
|
|
23
|
+
embed(req: EmbeddingRequest): Promise<EmbeddingResponse>;
|
|
24
|
+
rerank(req: RerankRequest): Promise<RerankResponse>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/** Deterministic model provider for harness tests and examples. */
|
|
2
|
+
export class FakeModelProvider {
|
|
3
|
+
queue = [];
|
|
4
|
+
textStreamQueue = [];
|
|
5
|
+
objectStreamQueue = [];
|
|
6
|
+
requests = [];
|
|
7
|
+
id = 'fake';
|
|
8
|
+
genAiSystem = 'fake';
|
|
9
|
+
enqueueObject(response) { this.queue.push({ method: 'object', response }); }
|
|
10
|
+
enqueueText(response) { this.queue.push({ method: 'text', response }); }
|
|
11
|
+
enqueueEmbedding(response) { this.queue.push({ method: 'embed', response }); }
|
|
12
|
+
enqueueRerank(response) { this.queue.push({ method: 'rerank', response }); }
|
|
13
|
+
enqueueTextStream(chunks) { this.textStreamQueue.push(chunks); }
|
|
14
|
+
enqueueObjectStream(chunks) { this.objectStreamQueue.push(chunks); }
|
|
15
|
+
/** Backward-compatible helper for older tests during the object migration. */
|
|
16
|
+
enqueue(response) { this.enqueueObject(response); }
|
|
17
|
+
async text(req) {
|
|
18
|
+
this.requests.push(req);
|
|
19
|
+
const next = this.queue.shift();
|
|
20
|
+
if (next?.method === 'text')
|
|
21
|
+
return next.response;
|
|
22
|
+
if (next)
|
|
23
|
+
this.queue.unshift(next);
|
|
24
|
+
return { content: '', usage: emptyUsage(), toolCalls: [], finishReason: 'stop' };
|
|
25
|
+
}
|
|
26
|
+
async *textStream(req) {
|
|
27
|
+
this.requests.push(req);
|
|
28
|
+
for (const chunk of this.textStreamQueue.shift() ?? [{ kind: 'finish', usage: emptyUsage(), finishReason: 'stop' }]) {
|
|
29
|
+
yield chunk;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async object(req) {
|
|
33
|
+
this.requests.push(req);
|
|
34
|
+
const next = this.queue.shift();
|
|
35
|
+
if (next?.method === 'object')
|
|
36
|
+
return next.response;
|
|
37
|
+
if (next)
|
|
38
|
+
this.queue.unshift(next);
|
|
39
|
+
return { object: '', usage: emptyUsage(), toolCalls: [], finishReason: 'stop' };
|
|
40
|
+
}
|
|
41
|
+
async *objectStream(req) {
|
|
42
|
+
this.requests.push(req);
|
|
43
|
+
for (const chunk of this.objectStreamQueue.shift() ?? [{ kind: 'finish', object: '', usage: emptyUsage(), finishReason: 'stop' }]) {
|
|
44
|
+
yield chunk;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async embed(req) {
|
|
48
|
+
this.requests.push(req);
|
|
49
|
+
const next = this.queue.shift();
|
|
50
|
+
if (next?.method === 'embed')
|
|
51
|
+
return next.response;
|
|
52
|
+
if (next)
|
|
53
|
+
this.queue.unshift(next);
|
|
54
|
+
const inputCount = Array.isArray(req.input) ? req.input.length : 1;
|
|
55
|
+
return {
|
|
56
|
+
embeddings: Array.from({ length: inputCount }, (_, index) => ({ index, vector: [0] })),
|
|
57
|
+
usage: emptyUsage()
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
async rerank(req) {
|
|
61
|
+
this.requests.push(req);
|
|
62
|
+
const next = this.queue.shift();
|
|
63
|
+
if (next?.method === 'rerank')
|
|
64
|
+
return next.response;
|
|
65
|
+
if (next)
|
|
66
|
+
this.queue.unshift(next);
|
|
67
|
+
return {
|
|
68
|
+
results: req.documents.map((document, index) => ({
|
|
69
|
+
id: document.id,
|
|
70
|
+
index,
|
|
71
|
+
score: req.documents.length - index,
|
|
72
|
+
...(document.metadata ? { metadata: document.metadata } : {})
|
|
73
|
+
})).slice(0, req.topN ?? req.documents.length)
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function emptyUsage() {
|
|
78
|
+
return { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
|
|
79
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { FeedbackRecord, FeedbackTarget } from '../ports/feedback.js';
|
|
2
|
+
/** Minimal in-memory feedback recorder for tests and examples. */
|
|
3
|
+
export declare function createInMemoryFeedbackRecorder(): {
|
|
4
|
+
record(input: Omit<FeedbackRecord, "id" | "createdAt"> & {
|
|
5
|
+
id?: string;
|
|
6
|
+
createdAt?: string;
|
|
7
|
+
}): FeedbackRecord;
|
|
8
|
+
list(target?: FeedbackTarget): readonly FeedbackRecord[];
|
|
9
|
+
clear(): void;
|
|
10
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Minimal in-memory feedback recorder for tests and examples. */
|
|
2
|
+
export function createInMemoryFeedbackRecorder() {
|
|
3
|
+
const records = [];
|
|
4
|
+
return {
|
|
5
|
+
record(input) {
|
|
6
|
+
const record = {
|
|
7
|
+
...input,
|
|
8
|
+
id: input.id ?? `feedback_${records.length + 1}`,
|
|
9
|
+
createdAt: input.createdAt ?? new Date().toISOString()
|
|
10
|
+
};
|
|
11
|
+
records.push(record);
|
|
12
|
+
return record;
|
|
13
|
+
},
|
|
14
|
+
list(target) {
|
|
15
|
+
if (!target) {
|
|
16
|
+
return [...records];
|
|
17
|
+
}
|
|
18
|
+
return records.filter((record) => JSON.stringify(record.target) === JSON.stringify(target));
|
|
19
|
+
},
|
|
20
|
+
clear() {
|
|
21
|
+
records.splice(0, records.length);
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export interface FakeHttpMcpServer {
|
|
2
|
+
url: string;
|
|
3
|
+
close(): Promise<void>;
|
|
4
|
+
}
|
|
5
|
+
export interface FakeHttpMcpServerOptions {
|
|
6
|
+
requiredHeaders?: Record<string, string>;
|
|
7
|
+
}
|
|
8
|
+
export declare function startFakeHttpMcpServer(options?: FakeHttpMcpServerOptions): Promise<FakeHttpMcpServer>;
|