@purista/harness 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/agents/index.d.ts +5 -3
- package/dist/agents/index.js +84 -8
- package/dist/errors/catalog.d.ts +45 -5
- package/dist/errors/catalog.js +19 -0
- package/dist/errors/harness-error.d.ts +2 -0
- package/dist/eval/index.d.ts +57 -0
- package/dist/eval/index.js +181 -0
- package/dist/harness/defineHarness.d.ts +96 -20
- package/dist/harness/defineHarness.js +59 -2
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/memory/sandbox/index.d.ts +17 -0
- package/dist/memory/sandbox/index.js +122 -0
- package/dist/models/registry.js +32 -7
- package/dist/ports/capabilities.d.ts +46 -2
- package/dist/ports/harness-context.d.ts +4 -1
- package/dist/ports/index.d.ts +2 -0
- package/dist/ports/index.js +2 -0
- package/dist/ports/memory/facade.d.ts +5 -0
- package/dist/ports/memory/facade.js +123 -0
- package/dist/ports/memory/telemetry.d.ts +16 -0
- package/dist/ports/memory/telemetry.js +77 -0
- package/dist/ports/memory/types.d.ts +204 -0
- package/dist/ports/memory/types.js +1 -0
- package/dist/ports/memory/validation.d.ts +19 -0
- package/dist/ports/memory/validation.js +160 -0
- package/dist/ports/memory.d.ts +3 -0
- package/dist/ports/memory.js +3 -0
- package/dist/ports/workspace.d.ts +177 -0
- package/dist/ports/workspace.js +32 -0
- package/dist/runtime/durable.d.ts +3 -0
- package/dist/runtime/durable.js +2 -1
- package/dist/sessions/index.d.ts +2 -0
- package/dist/sessions/index.js +275 -68
- package/dist/skills/index.d.ts +2 -1
- package/dist/skills/index.js +263 -35
- package/dist/telemetry/shim.d.ts +20 -0
- package/dist/telemetry/shim.js +28 -0
- package/dist/testing/durableWorkspaceStoreContract.d.ts +3 -0
- package/dist/testing/durableWorkspaceStoreContract.js +41 -0
- package/dist/testing/fakeMemoryAdapter.d.ts +16 -0
- package/dist/testing/fakeMemoryAdapter.js +110 -0
- package/dist/testing/index.d.ts +5 -0
- package/dist/testing/index.js +4 -0
- package/dist/workspace/in-memory.d.ts +35 -0
- package/dist/workspace/in-memory.js +142 -0
- package/dist/workspace/index.d.ts +1 -0
- package/dist/workspace/index.js +1 -0
- package/package.json +12 -6
package/dist/skills/index.js
CHANGED
|
@@ -1,44 +1,177 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import fsp from 'node:fs/promises';
|
|
3
|
+
import os from 'node:os';
|
|
3
4
|
import path from 'node:path';
|
|
5
|
+
import { parseDocument } from 'yaml';
|
|
4
6
|
import { SkillManifestError, SkillNotFoundError } from '../errors/index.js';
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
7
|
+
const skillNamePattern = /^(?!-)(?!.*--)[a-z0-9-]{1,64}(?<!-)$/;
|
|
8
|
+
const skippedDirectories = new Set(['.git', 'node_modules', 'dist', 'build', '.next', '.astro']);
|
|
9
|
+
function diagnostic(code, message, opts = {}) {
|
|
10
|
+
return {
|
|
11
|
+
level: opts.level ?? 'error',
|
|
12
|
+
code,
|
|
13
|
+
message,
|
|
14
|
+
...(opts.skillName ? { skillName: opts.skillName } : {}),
|
|
15
|
+
...(opts.directory ? { directory: opts.directory } : {}),
|
|
16
|
+
...(opts.source ? { source: opts.source } : {})
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function throwManifest(diag, skillId, cause) {
|
|
20
|
+
throw new SkillManifestError(diag.message, {
|
|
21
|
+
reason: diag.code,
|
|
22
|
+
directory: diag.directory ?? '',
|
|
23
|
+
...(skillId ? { skill_id: skillId } : {}),
|
|
24
|
+
...(diag.source ? { source: diag.source } : {})
|
|
25
|
+
}, cause);
|
|
26
|
+
}
|
|
27
|
+
function extractFrontmatter(content, directory) {
|
|
28
|
+
if (!content.startsWith('---\n')) {
|
|
29
|
+
throwManifest(diagnostic('invalid_frontmatter', 'SKILL.md must start with YAML frontmatter.', { directory }));
|
|
30
|
+
}
|
|
31
|
+
const end = content.indexOf('\n---', 4);
|
|
32
|
+
if (end < 0) {
|
|
33
|
+
throwManifest(diagnostic('invalid_frontmatter', 'SKILL.md frontmatter is not terminated.', { directory }));
|
|
34
|
+
}
|
|
35
|
+
return content.slice(4, end);
|
|
36
|
+
}
|
|
37
|
+
function quoteColonScalars(raw) {
|
|
38
|
+
return raw.split('\n').map((line) => {
|
|
39
|
+
const match = /^([A-Za-z0-9_-]+):\s*(.+:.+)$/.exec(line);
|
|
40
|
+
if (!match)
|
|
41
|
+
return line;
|
|
42
|
+
const key = match[1] ?? '';
|
|
43
|
+
const value = match[2] ?? '';
|
|
44
|
+
const trimmed = value.trim();
|
|
45
|
+
if (trimmed.startsWith('"')
|
|
46
|
+
|| trimmed.startsWith("'")
|
|
47
|
+
|| trimmed.startsWith('|')
|
|
48
|
+
|| trimmed.startsWith('>')
|
|
49
|
+
|| trimmed.startsWith('{')
|
|
50
|
+
|| trimmed.startsWith('[')) {
|
|
51
|
+
return line;
|
|
52
|
+
}
|
|
53
|
+
return `${key}: ${JSON.stringify(trimmed)}`;
|
|
54
|
+
}).join('\n');
|
|
55
|
+
}
|
|
56
|
+
function parseYamlFrontmatter(raw, mode, directory) {
|
|
57
|
+
const first = parseDocument(raw, { strict: true });
|
|
58
|
+
if (!first.errors.length)
|
|
59
|
+
return { value: first.toJSON(), diagnostics: [] };
|
|
60
|
+
if (mode === 'lenient') {
|
|
61
|
+
const retried = parseDocument(quoteColonScalars(raw), { strict: true });
|
|
62
|
+
if (!retried.errors.length) {
|
|
63
|
+
return {
|
|
64
|
+
value: retried.toJSON(),
|
|
65
|
+
diagnostics: [diagnostic('invalid_frontmatter', 'Lenient skill parsing repaired YAML scalar quoting.', { level: 'warn', directory })]
|
|
66
|
+
};
|
|
67
|
+
}
|
|
19
68
|
}
|
|
20
|
-
|
|
69
|
+
throwManifest(diagnostic('invalid_frontmatter', 'Invalid SKILL.md YAML frontmatter.', { directory }), undefined, first.errors[0]);
|
|
70
|
+
}
|
|
71
|
+
function asRecord(value) {
|
|
72
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
73
|
+
}
|
|
74
|
+
function validateFrontmatter(value, mode, directory, expectedName, source) {
|
|
75
|
+
const data = asRecord(value);
|
|
76
|
+
const diagnostics = [];
|
|
77
|
+
const nameValue = data['name'];
|
|
78
|
+
const name = typeof nameValue === 'string' ? nameValue.trim() : '';
|
|
79
|
+
if (!skillNamePattern.test(name)) {
|
|
80
|
+
const diag = diagnostic('invalid_name', 'Skill name must be 1-64 lowercase ASCII letters, numbers, or hyphens with no leading, trailing, or consecutive hyphens.', { directory, source, skillName: name });
|
|
81
|
+
if (mode === 'strict')
|
|
82
|
+
throwManifest(diag, expectedName);
|
|
83
|
+
diagnostics.push(diag);
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
const descriptionValue = data['description'];
|
|
87
|
+
const description = typeof descriptionValue === 'string' ? descriptionValue.trim() : '';
|
|
88
|
+
if (description.length < 1 || description.length > 1024) {
|
|
89
|
+
const diag = diagnostic('missing_description', 'Skill description is required and must be 1-1024 characters.', { directory, source, skillName: name });
|
|
90
|
+
if (mode === 'strict')
|
|
91
|
+
throwManifest(diag, expectedName);
|
|
92
|
+
diagnostics.push(diag);
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
const parentName = path.basename(directory);
|
|
96
|
+
if (expectedName && expectedName !== name) {
|
|
97
|
+
const diag = diagnostic('name_mismatch', `Skill key "${expectedName}" must match frontmatter name "${name}".`, { directory, source, skillName: name, level: mode === 'strict' ? 'error' : 'warn' });
|
|
98
|
+
if (mode === 'strict')
|
|
99
|
+
throwManifest(diag, expectedName);
|
|
100
|
+
diagnostics.push(diag);
|
|
101
|
+
}
|
|
102
|
+
else if (!expectedName && parentName !== name) {
|
|
103
|
+
const diag = diagnostic('name_mismatch', `Skill directory "${parentName}" does not match frontmatter name "${name}".`, { directory, source, skillName: name, level: mode === 'strict' ? 'error' : 'warn' });
|
|
104
|
+
if (mode === 'strict')
|
|
105
|
+
throwManifest(diag, name);
|
|
106
|
+
diagnostics.push(diag);
|
|
107
|
+
}
|
|
108
|
+
const metadata = asRecord(data['metadata']);
|
|
109
|
+
const metadataOut = {};
|
|
110
|
+
for (const [key, val] of Object.entries(metadata)) {
|
|
111
|
+
if (typeof val === 'string')
|
|
112
|
+
metadataOut[key] = val;
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
frontmatter: {
|
|
116
|
+
name,
|
|
117
|
+
description,
|
|
118
|
+
...(typeof data['license'] === 'string' && data['license'].trim() ? { license: data['license'].trim() } : {}),
|
|
119
|
+
...(typeof data['compatibility'] === 'string' && data['compatibility'].trim() ? { compatibility: data['compatibility'].trim() } : {}),
|
|
120
|
+
...(Object.keys(metadataOut).length ? { metadata: metadataOut } : {}),
|
|
121
|
+
...(typeof data['allowed-tools'] === 'string' && data['allowed-tools'].trim() ? { 'allowed-tools': data['allowed-tools'].trim() } : {})
|
|
122
|
+
},
|
|
123
|
+
diagnostics
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function readSkill(directory, mode, expectedName, source) {
|
|
127
|
+
const stat = fs.existsSync(directory) ? fs.statSync(directory) : null;
|
|
128
|
+
if (!stat?.isDirectory()) {
|
|
129
|
+
const diag = diagnostic('directory_missing', 'Skill directory is missing.', { directory, source, skillName: expectedName });
|
|
130
|
+
if (mode === 'strict')
|
|
131
|
+
throwManifest(diag, expectedName);
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
const skillPath = path.resolve(directory, 'SKILL.md');
|
|
135
|
+
if (!fs.existsSync(skillPath)) {
|
|
136
|
+
const diag = diagnostic('missing_skill_md', 'Skill directory must contain SKILL.md.', { directory, source, skillName: expectedName });
|
|
137
|
+
if (mode === 'strict')
|
|
138
|
+
throwManifest(diag, expectedName);
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
const content = fs.readFileSync(skillPath, 'utf8');
|
|
142
|
+
const raw = extractFrontmatter(content, directory);
|
|
143
|
+
const parsed = parseYamlFrontmatter(raw, mode, directory);
|
|
144
|
+
const checked = validateFrontmatter(parsed.value, mode, directory, expectedName, source);
|
|
145
|
+
if (!checked)
|
|
146
|
+
return undefined;
|
|
147
|
+
const frontmatter = checked.frontmatter;
|
|
148
|
+
return {
|
|
149
|
+
name: frontmatter.name,
|
|
150
|
+
description: frontmatter.description,
|
|
151
|
+
directory: path.resolve(directory),
|
|
152
|
+
skillPath,
|
|
153
|
+
location: skillPath,
|
|
154
|
+
mountPath: `/skills/${frontmatter.name}`,
|
|
155
|
+
...(frontmatter.license ? { license: frontmatter.license } : {}),
|
|
156
|
+
...(frontmatter.compatibility ? { compatibility: frontmatter.compatibility } : {}),
|
|
157
|
+
...(frontmatter.metadata ? { metadata: frontmatter.metadata } : {}),
|
|
158
|
+
...(frontmatter['allowed-tools'] ? { allowedTools: frontmatter['allowed-tools'] } : {}),
|
|
159
|
+
trust: 'trusted',
|
|
160
|
+
...(source ? { source } : {}),
|
|
161
|
+
diagnostics: [...parsed.diagnostics, ...checked.diagnostics]
|
|
162
|
+
};
|
|
21
163
|
}
|
|
22
|
-
function validateName(name) { return /^[a-z][a-z0-9-]*$/.test(name) && !/anthropic|claude|purista/i.test(name); }
|
|
23
164
|
export function loadSkillsSync(skills) {
|
|
24
165
|
const resolved = {};
|
|
25
166
|
for (const [key, def] of Object.entries(skills)) {
|
|
26
|
-
const
|
|
27
|
-
if (!
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
const name = frontmatter['name'] ?? '';
|
|
35
|
-
const description = frontmatter['description'] ?? '';
|
|
36
|
-
if (!validateName(name) || description.length < 1 || description.length > 1024) {
|
|
37
|
-
throw new SkillManifestError('invalid frontmatter', { reason: 'invalid_frontmatter', directory: def.directory, skill_id: key });
|
|
38
|
-
}
|
|
39
|
-
if (name !== key)
|
|
40
|
-
throw new SkillManifestError('name mismatch', { reason: 'name_mismatch', directory: def.directory, skill_id: key });
|
|
41
|
-
resolved[key] = { name, description, directory: def.directory, ...(frontmatter['version'] ? { version: frontmatter['version'] } : {}) };
|
|
167
|
+
const skill = readSkill(path.resolve(def.directory), def.validationMode ?? 'strict', key, def.source);
|
|
168
|
+
if (!skill)
|
|
169
|
+
continue;
|
|
170
|
+
resolved[key] = {
|
|
171
|
+
...skill,
|
|
172
|
+
trust: def.trust ?? 'trusted',
|
|
173
|
+
...(def.source ? { source: def.source } : {})
|
|
174
|
+
};
|
|
42
175
|
}
|
|
43
176
|
return resolved;
|
|
44
177
|
}
|
|
@@ -61,6 +194,9 @@ async function readDirRecursive(root) {
|
|
|
61
194
|
return files;
|
|
62
195
|
}
|
|
63
196
|
export async function mountSkillsOnce(session, mounted, skills, skillIds) {
|
|
197
|
+
if (skillIds.length > 0 && typeof session.mount !== 'function') {
|
|
198
|
+
throw new SkillManifestError('Sandbox does not support skill mounting.', { reason: 'invalid_frontmatter', directory: '' });
|
|
199
|
+
}
|
|
64
200
|
for (const skillId of skillIds) {
|
|
65
201
|
if (mounted.has(skillId))
|
|
66
202
|
continue;
|
|
@@ -68,15 +204,107 @@ export async function mountSkillsOnce(session, mounted, skills, skillIds) {
|
|
|
68
204
|
if (!skill)
|
|
69
205
|
throw new SkillNotFoundError('Skill not found.', { skill_id: skillId });
|
|
70
206
|
const files = await readDirRecursive(skill.directory);
|
|
71
|
-
await session.mount(files,
|
|
207
|
+
await session.mount(files, skill.mountPath);
|
|
72
208
|
mounted.add(skillId);
|
|
73
209
|
}
|
|
74
210
|
}
|
|
75
211
|
export function buildSkillIndex(skills, ids) {
|
|
76
212
|
if (ids.length === 0)
|
|
77
213
|
return '';
|
|
78
|
-
const lines =
|
|
79
|
-
|
|
214
|
+
const lines = ['', '', 'Available skills:'];
|
|
215
|
+
for (const id of ids) {
|
|
216
|
+
const skill = skills[id];
|
|
217
|
+
if (!skill)
|
|
218
|
+
continue;
|
|
219
|
+
lines.push(`- ${skill.name}: ${skill.description}`);
|
|
220
|
+
lines.push(` Location: ${skill.mountPath}/SKILL.md`);
|
|
221
|
+
if (skill.compatibility)
|
|
222
|
+
lines.push(` Compatibility: ${skill.compatibility}`);
|
|
223
|
+
}
|
|
224
|
+
lines.push('', 'Use the read tool to load /skills/<name>/SKILL.md when a skill is relevant.');
|
|
225
|
+
lines.push('Relative paths in a skill are relative to /skills/<name>/.');
|
|
226
|
+
return lines.join('\n');
|
|
227
|
+
}
|
|
228
|
+
function shouldSkipDirectory(name, clientName) {
|
|
229
|
+
if (skippedDirectories.has(name))
|
|
230
|
+
return true;
|
|
231
|
+
if (!name.startsWith('.'))
|
|
232
|
+
return false;
|
|
233
|
+
return name !== '.agents' && name !== '.claude' && (clientName ? name !== `.${clientName}` : true);
|
|
234
|
+
}
|
|
235
|
+
async function findSkillDirectories(root, opts) {
|
|
236
|
+
const out = [];
|
|
237
|
+
let visited = 0;
|
|
238
|
+
const walk = async (dir, depth) => {
|
|
239
|
+
if (visited >= opts.maxDirectories) {
|
|
240
|
+
opts.diagnostics.push(diagnostic('scan_limit_reached', 'Skill discovery directory limit reached.', { directory: dir }));
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
visited += 1;
|
|
244
|
+
if (depth > opts.maxDepth)
|
|
245
|
+
return;
|
|
246
|
+
let entries;
|
|
247
|
+
try {
|
|
248
|
+
entries = await fsp.readdir(dir, { withFileTypes: true });
|
|
249
|
+
}
|
|
250
|
+
catch {
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
if (entries.some((entry) => entry.isFile() && entry.name === 'SKILL.md')) {
|
|
254
|
+
out.push(dir);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
for (const entry of entries) {
|
|
258
|
+
if (!entry.isDirectory() || shouldSkipDirectory(entry.name, opts.clientName))
|
|
259
|
+
continue;
|
|
260
|
+
await walk(path.join(dir, entry.name), depth + 1);
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
await walk(root, 0);
|
|
264
|
+
return out;
|
|
265
|
+
}
|
|
266
|
+
function addSkillConfig(target, name, def, diagnostics) {
|
|
267
|
+
if (target[name]) {
|
|
268
|
+
diagnostics.push(diagnostic('collision_shadowed', `Skill "${name}" was shadowed by a higher-precedence binding.`, { level: 'warn', skillName: name, directory: def.directory, source: def.source }));
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
target[name] = def;
|
|
272
|
+
}
|
|
273
|
+
export async function discoverSkills(options = {}) {
|
|
274
|
+
const diagnostics = [];
|
|
275
|
+
const skills = {};
|
|
276
|
+
const validationMode = options.validationMode ?? 'lenient';
|
|
277
|
+
const maxDepth = options.maxDepth ?? 6;
|
|
278
|
+
const maxDirectories = options.maxDirectories ?? 2000;
|
|
279
|
+
const trustedRoots = new Set((options.trustedProjectRoots ?? []).map((root) => path.resolve(root)));
|
|
280
|
+
const projectRoot = path.resolve(options.projectRoot ?? process.env['PWD'] ?? '.');
|
|
281
|
+
const roots = [];
|
|
282
|
+
if (options.includeUserAgentsDir)
|
|
283
|
+
roots.push({ root: path.join(os.homedir(), '.agents', 'skills'), trust: 'user', source: 'user_agents', trusted: true });
|
|
284
|
+
if (options.includeUserClientDir && options.clientName)
|
|
285
|
+
roots.push({ root: path.join(os.homedir(), `.${options.clientName}`, 'skills'), trust: 'user', source: 'user_client', trusted: true });
|
|
286
|
+
if (options.includeProjectAgentsDir ?? true)
|
|
287
|
+
roots.push({ root: path.join(projectRoot, '.agents', 'skills'), trust: 'project', source: 'project_agents', trusted: trustedRoots.has(projectRoot) });
|
|
288
|
+
if (options.includeProjectClientDir && options.clientName)
|
|
289
|
+
roots.push({ root: path.join(projectRoot, `.${options.clientName}`, 'skills'), trust: 'project', source: 'project_client', trusted: trustedRoots.has(projectRoot) });
|
|
290
|
+
if (options.includeClaudeCompatDir)
|
|
291
|
+
roots.push({ root: path.join(projectRoot, '.claude', 'skills'), trust: 'project', source: 'project_claude', trusted: trustedRoots.has(projectRoot) });
|
|
292
|
+
for (const rootInfo of roots) {
|
|
293
|
+
if (!fs.existsSync(rootInfo.root))
|
|
294
|
+
continue;
|
|
295
|
+
if (rootInfo.trust === 'project' && !rootInfo.trusted) {
|
|
296
|
+
diagnostics.push(diagnostic('untrusted_project_skill', 'Project skill discovery root is not trusted.', { level: 'warn', directory: rootInfo.root, source: rootInfo.source }));
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
for (const directory of await findSkillDirectories(rootInfo.root, { maxDepth, maxDirectories, clientName: options.clientName, diagnostics })) {
|
|
300
|
+
const skill = readSkill(directory, validationMode, undefined, rootInfo.source);
|
|
301
|
+
if (!skill)
|
|
302
|
+
continue;
|
|
303
|
+
addSkillConfig(skills, skill.name, { directory, validationMode, trust: rootInfo.trust, source: rootInfo.source }, diagnostics);
|
|
304
|
+
diagnostics.push(...skill.diagnostics);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return { skills, diagnostics };
|
|
80
308
|
}
|
|
81
309
|
export function assertSerializable(value) {
|
|
82
310
|
try {
|
package/dist/telemetry/shim.d.ts
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
/** Attributes accepted by telemetry span/metric helpers. */
|
|
2
2
|
export type SpanAttrs = Record<string, string | number | boolean | string[] | undefined>;
|
|
3
|
+
/** Developer-facing metric helper exposed in handler contexts. */
|
|
4
|
+
export interface Metrics {
|
|
5
|
+
/** Adds to a counter instrument. */
|
|
6
|
+
counter(name: string, value?: number, attrs?: SpanAttrs): void;
|
|
7
|
+
/** Records a histogram sample. */
|
|
8
|
+
histogram(name: string, value: number, attrs?: SpanAttrs): void;
|
|
9
|
+
/** Records the duration of an async operation in seconds. */
|
|
10
|
+
duration<T>(name: string, attrs: SpanAttrs | undefined, fn: () => Promise<T>): Promise<T>;
|
|
11
|
+
}
|
|
3
12
|
/** Minimal telemetry abstraction used by harness internals and integrations. */
|
|
4
13
|
export interface TelemetryShim {
|
|
5
14
|
/** Creates a span, executes `fn`, and closes the span with success/error status. */
|
|
@@ -10,6 +19,11 @@ export interface TelemetryShim {
|
|
|
10
19
|
recordCounter(name: string, value: number, attrs: SpanAttrs): void;
|
|
11
20
|
/** Injects the current active trace context into a W3C traceparent carrier. */
|
|
12
21
|
currentTraceparent(): string | undefined;
|
|
22
|
+
/** Runs `fn` with the supplied W3C Trace Context as the active parent context. */
|
|
23
|
+
withTraceContext?<T>(carrier: {
|
|
24
|
+
traceparent: string;
|
|
25
|
+
tracestate?: string;
|
|
26
|
+
}, fn: () => Promise<T>): Promise<T>;
|
|
13
27
|
}
|
|
14
28
|
/** OpenTelemetry-backed implementation of {@link TelemetryShim}. */
|
|
15
29
|
export declare class OtelTelemetryShim implements TelemetryShim {
|
|
@@ -21,6 +35,12 @@ export declare class OtelTelemetryShim implements TelemetryShim {
|
|
|
21
35
|
recordHistogram(name: string, value: number, attrs: SpanAttrs): void;
|
|
22
36
|
recordCounter(name: string, value: number, attrs: SpanAttrs): void;
|
|
23
37
|
currentTraceparent(): string | undefined;
|
|
38
|
+
withTraceContext<T>(carrier: {
|
|
39
|
+
traceparent: string;
|
|
40
|
+
tracestate?: string;
|
|
41
|
+
}, fn: () => Promise<T>): Promise<T>;
|
|
24
42
|
}
|
|
25
43
|
/** Creates the default telemetry shim instance. */
|
|
26
44
|
export declare function createTelemetryShim(): TelemetryShim;
|
|
45
|
+
/** Creates a scoped metrics helper with default attributes merged into every metric. */
|
|
46
|
+
export declare function createMetrics(telemetry: TelemetryShim, defaultAttrs?: SpanAttrs): Metrics;
|
package/dist/telemetry/shim.js
CHANGED
|
@@ -113,8 +113,36 @@ export class OtelTelemetryShim {
|
|
|
113
113
|
propagation.inject(context.active(), carrier);
|
|
114
114
|
return carrier['traceparent'];
|
|
115
115
|
}
|
|
116
|
+
async withTraceContext(carrier, fn) {
|
|
117
|
+
const extracted = propagation.extract(context.active(), {
|
|
118
|
+
traceparent: carrier.traceparent,
|
|
119
|
+
...(carrier.tracestate ? { tracestate: carrier.tracestate } : {})
|
|
120
|
+
});
|
|
121
|
+
return context.with(extracted, fn);
|
|
122
|
+
}
|
|
116
123
|
}
|
|
117
124
|
/** Creates the default telemetry shim instance. */
|
|
118
125
|
export function createTelemetryShim() {
|
|
119
126
|
return new OtelTelemetryShim();
|
|
120
127
|
}
|
|
128
|
+
/** Creates a scoped metrics helper with default attributes merged into every metric. */
|
|
129
|
+
export function createMetrics(telemetry, defaultAttrs = {}) {
|
|
130
|
+
const merge = (attrs) => ({ ...defaultAttrs, ...(attrs ?? {}) });
|
|
131
|
+
return {
|
|
132
|
+
counter(name, value = 1, attrs) {
|
|
133
|
+
telemetry.recordCounter(name, value, merge(attrs));
|
|
134
|
+
},
|
|
135
|
+
histogram(name, value, attrs) {
|
|
136
|
+
telemetry.recordHistogram(name, value, merge(attrs));
|
|
137
|
+
},
|
|
138
|
+
async duration(name, attrs, fn) {
|
|
139
|
+
const started = Date.now();
|
|
140
|
+
try {
|
|
141
|
+
return await fn();
|
|
142
|
+
}
|
|
143
|
+
finally {
|
|
144
|
+
telemetry.recordHistogram(name, (Date.now() - started) / 1000, merge(attrs));
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { validateDurableWorkspaceStore } from '../ports/workspace.js';
|
|
3
|
+
/** Shared Vitest contract for durable workspace store implementations. */
|
|
4
|
+
export function durableWorkspaceStoreContract(make) {
|
|
5
|
+
describe('durableWorkspaceStoreContract', () => {
|
|
6
|
+
it('validates metadata and round-trips checkpointed workspaces', async () => {
|
|
7
|
+
const adapter = await make();
|
|
8
|
+
validateDurableWorkspaceStore(adapter);
|
|
9
|
+
const signal = new AbortController().signal;
|
|
10
|
+
const handle = await adapter.startWorkspace({
|
|
11
|
+
sessionId: 'session-1',
|
|
12
|
+
runId: 'run-1',
|
|
13
|
+
agentId: 'agent-1',
|
|
14
|
+
attempt: 1,
|
|
15
|
+
idempotencyKey: 'start-1',
|
|
16
|
+
signal
|
|
17
|
+
});
|
|
18
|
+
const checkpoint = await adapter.pauseWorkspace({
|
|
19
|
+
handle,
|
|
20
|
+
stepId: 'step-1',
|
|
21
|
+
sequence: 1,
|
|
22
|
+
attempt: 1,
|
|
23
|
+
reason: 'step_completed',
|
|
24
|
+
idempotencyKey: 'pause-1',
|
|
25
|
+
signal
|
|
26
|
+
});
|
|
27
|
+
const resumed = await adapter.resumeWorkspace({
|
|
28
|
+
workspaceRef: handle.workspaceRef,
|
|
29
|
+
checkpointRef: checkpoint.checkpointRef,
|
|
30
|
+
sessionId: 'session-1',
|
|
31
|
+
runId: 'run-2',
|
|
32
|
+
attempt: 2,
|
|
33
|
+
idempotencyKey: 'resume-1',
|
|
34
|
+
signal
|
|
35
|
+
});
|
|
36
|
+
const inspection = await adapter.inspectWorkspace?.({ workspaceRef: resumed.workspaceRef, signal });
|
|
37
|
+
expect(resumed.workspaceRef).toBe(handle.workspaceRef);
|
|
38
|
+
expect(inspection?.checkpoints.map((item) => item.checkpointRef)).toEqual([checkpoint.checkpointRef]);
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { MemoryAdapter, MemoryOpenContext, MemoryScope, MemoryStore } from '../ports/memory.js';
|
|
2
|
+
/** Deterministic in-memory adapter for unit tests and adapter contract examples. */
|
|
3
|
+
export declare class FakeMemoryAdapter implements MemoryAdapter {
|
|
4
|
+
readonly info: {
|
|
5
|
+
id: string;
|
|
6
|
+
packageName: string;
|
|
7
|
+
capabilities: readonly ["memory.kv", "memory.list", "memory.delete", "memory.search", "memory.run", "memory.session", "memory.agent", "memory.user", "memory.tenant", "memory.persistent"];
|
|
8
|
+
};
|
|
9
|
+
readonly capabilities: readonly ["memory.kv", "memory.list", "memory.delete", "memory.search", "memory.run", "memory.session", "memory.agent", "memory.user", "memory.tenant", "memory.persistent"];
|
|
10
|
+
readonly openedScopes: MemoryScope[];
|
|
11
|
+
private readonly values;
|
|
12
|
+
configureHarnessContext(): void;
|
|
13
|
+
open(scope: MemoryScope, _ctx: MemoryOpenContext): Promise<MemoryStore>;
|
|
14
|
+
}
|
|
15
|
+
/** Shared contract for memory adapters. */
|
|
16
|
+
export declare function memoryAdapterContract(make: () => MemoryAdapter | Promise<MemoryAdapter>): void;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
/** Deterministic in-memory adapter for unit tests and adapter contract examples. */
|
|
3
|
+
export class FakeMemoryAdapter {
|
|
4
|
+
info = {
|
|
5
|
+
id: 'fake_memory',
|
|
6
|
+
packageName: '@purista/harness/testing',
|
|
7
|
+
capabilities: [
|
|
8
|
+
'memory.kv',
|
|
9
|
+
'memory.list',
|
|
10
|
+
'memory.delete',
|
|
11
|
+
'memory.search',
|
|
12
|
+
'memory.run',
|
|
13
|
+
'memory.session',
|
|
14
|
+
'memory.agent',
|
|
15
|
+
'memory.user',
|
|
16
|
+
'memory.tenant',
|
|
17
|
+
'memory.persistent'
|
|
18
|
+
]
|
|
19
|
+
};
|
|
20
|
+
capabilities = this.info.capabilities;
|
|
21
|
+
openedScopes = [];
|
|
22
|
+
values = new Map();
|
|
23
|
+
configureHarnessContext() {
|
|
24
|
+
// Fake adapter does not need inherited services, but implements the hook so contract users can assert it is called.
|
|
25
|
+
}
|
|
26
|
+
async open(scope, _ctx) {
|
|
27
|
+
this.openedScopes.push(scope);
|
|
28
|
+
const prefix = scopeKey(scope);
|
|
29
|
+
return {
|
|
30
|
+
get: async (key, ctx) => {
|
|
31
|
+
ctx.signal.throwIfAborted();
|
|
32
|
+
return this.values.get(`${prefix}:${key}`);
|
|
33
|
+
},
|
|
34
|
+
set: async (key, value, ctx) => {
|
|
35
|
+
ctx.signal.throwIfAborted();
|
|
36
|
+
this.values.set(`${prefix}:${key}`, value);
|
|
37
|
+
},
|
|
38
|
+
delete: async (key, ctx) => {
|
|
39
|
+
ctx.signal.throwIfAborted();
|
|
40
|
+
this.values.delete(`${prefix}:${key}`);
|
|
41
|
+
},
|
|
42
|
+
list: async (ctx) => {
|
|
43
|
+
ctx.signal.throwIfAborted();
|
|
44
|
+
const keys = [...this.values.keys()]
|
|
45
|
+
.filter((key) => key.startsWith(`${prefix}:`))
|
|
46
|
+
.map((key) => key.slice(prefix.length + 1))
|
|
47
|
+
.filter((key) => !ctx.opts?.prefix || key.startsWith(ctx.opts.prefix))
|
|
48
|
+
.filter((key) => !ctx.opts?.cursor || key > ctx.opts.cursor)
|
|
49
|
+
.sort()
|
|
50
|
+
.slice(0, ctx.opts?.limit);
|
|
51
|
+
return keys.map((key) => ({ key }));
|
|
52
|
+
},
|
|
53
|
+
search: async (query, ctx) => {
|
|
54
|
+
ctx.signal.throwIfAborted();
|
|
55
|
+
return [...this.values.entries()]
|
|
56
|
+
.filter(([key]) => key.startsWith(`${prefix}:`))
|
|
57
|
+
.map(([key, value]) => ({ key: key.slice(prefix.length + 1), value, score: JSON.stringify(value).includes(query.text) ? 1 : 0 }))
|
|
58
|
+
.filter((result) => result.score > 0)
|
|
59
|
+
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || a.key.localeCompare(b.key))
|
|
60
|
+
.slice(0, query.limit);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Shared contract for memory adapters. */
|
|
66
|
+
export function memoryAdapterContract(make) {
|
|
67
|
+
describe('memoryAdapterContract', () => {
|
|
68
|
+
it('round-trips scoped JSON values', async () => {
|
|
69
|
+
const adapter = await make();
|
|
70
|
+
const ctx = contractContext();
|
|
71
|
+
const store = await adapter.open({ kind: 'session', sessionId: 's1' }, ctx);
|
|
72
|
+
await store.set('foo', { a: 1 }, { ...ctx, scope: { kind: 'session', sessionId: 's1' }, operation: 'set' });
|
|
73
|
+
await expect(store.get('foo', { ...ctx, scope: { kind: 'session', sessionId: 's1' }, operation: 'get' })).resolves.toEqual({ a: 1 });
|
|
74
|
+
});
|
|
75
|
+
it('isolates scopes', async () => {
|
|
76
|
+
const adapter = await make();
|
|
77
|
+
const ctx = contractContext();
|
|
78
|
+
const one = await adapter.open({ kind: 'session', sessionId: 's1' }, ctx);
|
|
79
|
+
const two = await adapter.open({ kind: 'session', sessionId: 's2' }, ctx);
|
|
80
|
+
await one.set('foo', 'one', { ...ctx, scope: { kind: 'session', sessionId: 's1' }, operation: 'set' });
|
|
81
|
+
await expect(two.get('foo', { ...ctx, scope: { kind: 'session', sessionId: 's2' }, operation: 'get' })).resolves.toBeUndefined();
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function scopeKey(scope) {
|
|
86
|
+
return [
|
|
87
|
+
scope.kind,
|
|
88
|
+
scope.tenantId,
|
|
89
|
+
scope.userId,
|
|
90
|
+
scope.sessionId,
|
|
91
|
+
scope.runId,
|
|
92
|
+
scope.workflowId,
|
|
93
|
+
scope.agentId
|
|
94
|
+
].filter(Boolean).join(':');
|
|
95
|
+
}
|
|
96
|
+
function contractContext() {
|
|
97
|
+
const signal = new AbortController().signal;
|
|
98
|
+
return {
|
|
99
|
+
logger: { trace() { }, debug() { }, info() { }, warn() { }, error() { }, fatal() { }, child: () => contractContext().logger },
|
|
100
|
+
telemetry: {
|
|
101
|
+
span: async (_name, _attrs, fn) => fn({ setAttribute: () => undefined, setAttributes: () => undefined, addEvent: () => undefined, recordException: () => undefined, setStatus: () => undefined, end: () => undefined, spanContext: () => ({ traceId: '', spanId: '', traceFlags: 0 }), isRecording: () => false, updateName: () => undefined }),
|
|
102
|
+
recordHistogram() { },
|
|
103
|
+
recordCounter() { },
|
|
104
|
+
currentTraceparent: () => undefined
|
|
105
|
+
},
|
|
106
|
+
metrics: { counter() { }, histogram() { }, duration: async (_name, _attrs, fn) => fn() },
|
|
107
|
+
contentCaptureMode: 'NO_CONTENT',
|
|
108
|
+
signal
|
|
109
|
+
};
|
|
110
|
+
}
|
package/dist/testing/index.d.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
export { FakeModelProvider } from './fakeModelProvider.js';
|
|
2
|
+
export { FakeMemoryAdapter, memoryAdapterContract } from './fakeMemoryAdapter.js';
|
|
3
|
+
export { InMemoryDurableWorkspaceStore, inMemoryDurableWorkspaceStore } from '../workspace/index.js';
|
|
4
|
+
export { durableWorkspaceStoreContract } from './durableWorkspaceStoreContract.js';
|
|
2
5
|
export { adapterCapabilitiesContract, fakeCapabilityAdapter, type FakeCapabilityAdapter } from './capabilities.js';
|
|
3
6
|
export { createInMemoryFeedbackRecorder } from './feedback.js';
|
|
7
|
+
export { evaluateDeterministicScorer } from '../eval/index.js';
|
|
8
|
+
export type { DeterministicScorerDefinition, ScorerResult, ScorerTarget } from '../eval/index.js';
|
|
4
9
|
export { sandboxContract } from './sandboxContract.js';
|
|
5
10
|
export { fakeSnapshotSandbox, sandboxSnapshotContract } from './sandboxSnapshot.js';
|
|
6
11
|
export { stateStoreContract } from './stateStoreContract.js';
|
package/dist/testing/index.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { defineHarness } from '../harness/defineHarness.js';
|
|
2
2
|
export { FakeModelProvider } from './fakeModelProvider.js';
|
|
3
|
+
export { FakeMemoryAdapter, memoryAdapterContract } from './fakeMemoryAdapter.js';
|
|
4
|
+
export { InMemoryDurableWorkspaceStore, inMemoryDurableWorkspaceStore } from '../workspace/index.js';
|
|
5
|
+
export { durableWorkspaceStoreContract } from './durableWorkspaceStoreContract.js';
|
|
3
6
|
export { adapterCapabilitiesContract, fakeCapabilityAdapter } from './capabilities.js';
|
|
4
7
|
export { createInMemoryFeedbackRecorder } from './feedback.js';
|
|
8
|
+
export { evaluateDeterministicScorer } from '../eval/index.js';
|
|
5
9
|
export { sandboxContract } from './sandboxContract.js';
|
|
6
10
|
export { fakeSnapshotSandbox, sandboxSnapshotContract } from './sandboxSnapshot.js';
|
|
7
11
|
export { stateStoreContract } from './stateStoreContract.js';
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { DurableWorkspaceStore, WorkspaceAbortOptions, WorkspaceAbortResult, WorkspaceCheckpoint, WorkspaceCleanupOptions, WorkspaceCleanupResult, WorkspaceHandle, WorkspaceInspection, WorkspaceInspectionOptions, WorkspacePauseOptions, WorkspaceResumeOptions, WorkspaceStartOptions } from '../ports/workspace.js';
|
|
2
|
+
/** In-process durable workspace store for local development, examples, and tests. */
|
|
3
|
+
export declare class InMemoryDurableWorkspaceStore implements DurableWorkspaceStore {
|
|
4
|
+
readonly info: {
|
|
5
|
+
id: string;
|
|
6
|
+
packageName: string;
|
|
7
|
+
capabilities: readonly ["workspace_store.durable", "workspace_store.checkpoint", "workspace_store.resume", "workspace_store.abort", "workspace_store.cleanup", "workspace_store.inspect", "workspace_store.retention", "workspace_store.quota"];
|
|
8
|
+
policy: {
|
|
9
|
+
retention: {
|
|
10
|
+
pausedTtlMs: number;
|
|
11
|
+
terminalFailureTtlMs: number;
|
|
12
|
+
terminalSuccessTtlMs: number;
|
|
13
|
+
cleanupMode: "manual_only";
|
|
14
|
+
};
|
|
15
|
+
quota: {
|
|
16
|
+
maxActiveWorkspaces: number;
|
|
17
|
+
maxWorkspaceBytes: number;
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
readonly capabilities: readonly ["workspace_store.durable", "workspace_store.checkpoint", "workspace_store.resume", "workspace_store.abort", "workspace_store.cleanup", "workspace_store.inspect", "workspace_store.retention", "workspace_store.quota"];
|
|
22
|
+
private readonly workspaces;
|
|
23
|
+
private nextId;
|
|
24
|
+
configureHarnessContext(): void;
|
|
25
|
+
startWorkspace(opts: WorkspaceStartOptions): Promise<WorkspaceHandle>;
|
|
26
|
+
pauseWorkspace(opts: WorkspacePauseOptions): Promise<WorkspaceCheckpoint>;
|
|
27
|
+
resumeWorkspace(opts: WorkspaceResumeOptions): Promise<WorkspaceHandle>;
|
|
28
|
+
abortWorkspace(opts: WorkspaceAbortOptions): Promise<WorkspaceAbortResult>;
|
|
29
|
+
cleanupWorkspace(opts: WorkspaceCleanupOptions): Promise<WorkspaceCleanupResult>;
|
|
30
|
+
inspectWorkspace(opts: WorkspaceInspectionOptions): Promise<WorkspaceInspection>;
|
|
31
|
+
private findWorkspaceByCheckpoint;
|
|
32
|
+
private requireWorkspace;
|
|
33
|
+
}
|
|
34
|
+
/** Creates a fresh in-process durable workspace store. */
|
|
35
|
+
export declare function inMemoryDurableWorkspaceStore(): DurableWorkspaceStore;
|