@wix/evalforge-types 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +88 -15
- package/build/index.js.map +2 -2
- package/build/index.mjs +81 -15
- package/build/index.mjs.map +2 -2
- package/build/types/agent/adapter.d.ts +3 -3
- package/build/types/evaluation/eval-result.d.ts +2 -0
- package/build/types/evaluation/eval-run.d.ts +4 -0
- package/build/types/target/skill.d.ts +206 -23
- package/package.json +2 -2
package/build/index.js
CHANGED
|
@@ -45,6 +45,7 @@ __export(index_exports, {
|
|
|
45
45
|
CreateMcpInputSchema: () => CreateMcpInputSchema,
|
|
46
46
|
CreateProjectInputSchema: () => CreateProjectInputSchema,
|
|
47
47
|
CreateSkillInputSchema: () => CreateSkillInputSchema,
|
|
48
|
+
CreateSkillVersionInputSchema: () => CreateSkillVersionInputSchema,
|
|
48
49
|
CreateSkillsGroupInputSchema: () => CreateSkillsGroupInputSchema,
|
|
49
50
|
CreateSubAgentInputSchema: () => CreateSubAgentInputSchema,
|
|
50
51
|
CreateTemplateInputSchema: () => CreateTemplateInputSchema,
|
|
@@ -72,6 +73,8 @@ __export(index_exports, {
|
|
|
72
73
|
FileContentTestSchema: () => FileContentTestSchema,
|
|
73
74
|
FileModificationSchema: () => FileModificationSchema,
|
|
74
75
|
FilePresenceTestSchema: () => FilePresenceTestSchema,
|
|
76
|
+
GitHubSourceSchema: () => GitHubSourceSchema,
|
|
77
|
+
InitialVersionInputSchema: () => InitialVersionInputSchema,
|
|
75
78
|
LLMBreakdownStatsSchema: () => LLMBreakdownStatsSchema,
|
|
76
79
|
LLMStepType: () => LLMStepType,
|
|
77
80
|
LLMTestSchema: () => LLMTestSchema,
|
|
@@ -97,16 +100,20 @@ __export(index_exports, {
|
|
|
97
100
|
PlaywrightNLTestSchema: () => PlaywrightNLTestSchema,
|
|
98
101
|
ProjectSchema: () => ProjectSchema,
|
|
99
102
|
PromptResultSchema: () => PromptResultSchema,
|
|
103
|
+
SEMVER_REGEX: () => SEMVER_REGEX,
|
|
100
104
|
SKILL_FOLDER_NAME_REGEX: () => SKILL_FOLDER_NAME_REGEX,
|
|
101
105
|
SYSTEM_ASSERTIONS: () => SYSTEM_ASSERTIONS,
|
|
102
106
|
SYSTEM_ASSERTION_IDS: () => SYSTEM_ASSERTION_IDS,
|
|
103
107
|
ScenarioAssertionLinkSchema: () => ScenarioAssertionLinkSchema,
|
|
104
108
|
SiteConfigTestSchema: () => SiteConfigTestSchema,
|
|
109
|
+
SkillFileSchema: () => SkillFileSchema,
|
|
105
110
|
SkillMetadataSchema: () => SkillMetadataSchema,
|
|
106
111
|
SkillSchema: () => SkillSchema,
|
|
112
|
+
SkillVersionOriginSchema: () => SkillVersionOriginSchema,
|
|
107
113
|
SkillVersionSchema: () => SkillVersionSchema,
|
|
108
114
|
SkillWasCalledAssertionSchema: () => SkillWasCalledAssertionSchema,
|
|
109
115
|
SkillWasCalledConfigSchema: () => SkillWasCalledConfigSchema,
|
|
116
|
+
SkillWithLatestVersionSchema: () => SkillWithLatestVersionSchema,
|
|
110
117
|
SkillsGroupSchema: () => SkillsGroupSchema,
|
|
111
118
|
SubAgentSchema: () => SubAgentSchema,
|
|
112
119
|
TRACE_EVENT_PREFIX: () => TRACE_EVENT_PREFIX,
|
|
@@ -307,6 +314,8 @@ var UpdateAgentInputSchema = CreateAgentInputSchema.partial().extend({
|
|
|
307
314
|
// src/target/skill.ts
|
|
308
315
|
var import_zod5 = require("zod");
|
|
309
316
|
var SKILL_FOLDER_NAME_REGEX = /^[a-z0-9]+(-[a-z0-9]+)*$/;
|
|
317
|
+
var SEMVER_REGEX = /^\d+\.\d+\.\d+$/;
|
|
318
|
+
var SkillVersionOriginSchema = import_zod5.z.enum(["manual", "pr", "master"]);
|
|
310
319
|
function isValidSkillFolderName(name) {
|
|
311
320
|
return typeof name === "string" && name.length > 0 && SKILL_FOLDER_NAME_REGEX.test(name.trim());
|
|
312
321
|
}
|
|
@@ -316,36 +325,87 @@ var SkillMetadataSchema = import_zod5.z.object({
|
|
|
316
325
|
allowedTools: import_zod5.z.array(import_zod5.z.string()).optional(),
|
|
317
326
|
skills: import_zod5.z.array(import_zod5.z.string()).optional()
|
|
318
327
|
});
|
|
328
|
+
var GitHubSourceSchema = import_zod5.z.object({
|
|
329
|
+
/** GitHub org or user, e.g. "wix" */
|
|
330
|
+
owner: import_zod5.z.string().min(1),
|
|
331
|
+
/** Repository name, e.g. "skills" */
|
|
332
|
+
repo: import_zod5.z.string().min(1),
|
|
333
|
+
/** Folder path to the skill directory, e.g. "wix-cli/skills/wix-cli-dashboard-page" */
|
|
334
|
+
path: import_zod5.z.string().min(1),
|
|
335
|
+
/** Git ref (branch, tag, or SHA), e.g. "master" or "v1.2.0" */
|
|
336
|
+
ref: import_zod5.z.string().min(1)
|
|
337
|
+
});
|
|
338
|
+
var SkillFileSchema = import_zod5.z.object({
|
|
339
|
+
/** Relative path within the skill directory, e.g. "SKILL.md" or "references/API_SPEC.md" */
|
|
340
|
+
path: import_zod5.z.string().min(1),
|
|
341
|
+
/** File content (UTF-8 text) */
|
|
342
|
+
content: import_zod5.z.string()
|
|
343
|
+
});
|
|
319
344
|
var SkillVersionSchema = import_zod5.z.object({
|
|
320
345
|
id: import_zod5.z.string(),
|
|
346
|
+
projectId: import_zod5.z.string(),
|
|
321
347
|
skillId: import_zod5.z.string(),
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
348
|
+
/** Semver string (e.g. "1.2.0") or Falcon fingerprint */
|
|
349
|
+
version: import_zod5.z.string(),
|
|
350
|
+
/** How this version was created */
|
|
351
|
+
origin: SkillVersionOriginSchema,
|
|
352
|
+
/** Where this snapshot was taken from */
|
|
353
|
+
source: GitHubSourceSchema.optional(),
|
|
354
|
+
/** Frozen snapshot of all files in the skill directory */
|
|
355
|
+
files: import_zod5.z.array(SkillFileSchema).optional(),
|
|
356
|
+
/** Optional notes about this version (changelog, reason for change) */
|
|
357
|
+
notes: import_zod5.z.string().optional(),
|
|
358
|
+
createdAt: import_zod5.z.string()
|
|
359
|
+
});
|
|
360
|
+
var CreateSkillVersionInputSchema = import_zod5.z.object({
|
|
361
|
+
/** GitHub source to snapshot from. If not provided, uses the Skill's source. */
|
|
362
|
+
source: GitHubSourceSchema.optional(),
|
|
363
|
+
/** Version string for this snapshot (e.g. "1.0.0", "1.0.3"). */
|
|
364
|
+
version: import_zod5.z.string().min(1),
|
|
365
|
+
notes: import_zod5.z.string().optional(),
|
|
366
|
+
/** Origin of this version. Defaults to 'manual' in backend. */
|
|
367
|
+
origin: SkillVersionOriginSchema.optional(),
|
|
368
|
+
/** Pre-edited files to store directly (bypasses GitHub fetch when provided) */
|
|
369
|
+
files: import_zod5.z.array(SkillFileSchema).optional()
|
|
329
370
|
});
|
|
330
371
|
var SkillSchema = TargetSchema.extend({
|
|
331
|
-
/**
|
|
332
|
-
|
|
372
|
+
/** GitHub source reference for live content fetching */
|
|
373
|
+
source: GitHubSourceSchema.optional()
|
|
333
374
|
});
|
|
334
375
|
var KEBAB_CASE_MESSAGE = "Name must be in kebab-case (lowercase letters, numbers, hyphens only, e.g. my-skill)";
|
|
335
376
|
var SkillInputBaseSchema = SkillSchema.omit({
|
|
336
377
|
id: true,
|
|
337
378
|
createdAt: true,
|
|
338
379
|
updatedAt: true,
|
|
339
|
-
deleted: true
|
|
380
|
+
deleted: true,
|
|
381
|
+
description: true,
|
|
382
|
+
source: true
|
|
383
|
+
}).extend({
|
|
384
|
+
/** Optional - not stored on Skill; content description lives in SkillVersion */
|
|
385
|
+
description: import_zod5.z.string().optional(),
|
|
386
|
+
/** GitHub source reference for live content fetching */
|
|
387
|
+
source: GitHubSourceSchema.optional()
|
|
388
|
+
});
|
|
389
|
+
var InitialVersionInputSchema = import_zod5.z.object({
|
|
390
|
+
files: import_zod5.z.array(SkillFileSchema).optional(),
|
|
391
|
+
notes: import_zod5.z.string().optional(),
|
|
392
|
+
source: GitHubSourceSchema.optional(),
|
|
393
|
+
version: import_zod5.z.string().optional(),
|
|
394
|
+
origin: SkillVersionOriginSchema.optional()
|
|
395
|
+
});
|
|
396
|
+
var CreateSkillInputSchema = SkillInputBaseSchema.extend({
|
|
397
|
+
initialVersion: InitialVersionInputSchema.optional()
|
|
398
|
+
}).refine((data) => isValidSkillFolderName(data.name), {
|
|
399
|
+
message: KEBAB_CASE_MESSAGE,
|
|
400
|
+
path: ["name"]
|
|
340
401
|
});
|
|
341
|
-
var CreateSkillInputSchema = SkillInputBaseSchema.refine(
|
|
342
|
-
(data) => isValidSkillFolderName(data.name),
|
|
343
|
-
{ message: KEBAB_CASE_MESSAGE, path: ["name"] }
|
|
344
|
-
);
|
|
345
402
|
var UpdateSkillInputSchema = SkillInputBaseSchema.partial().refine(
|
|
346
403
|
(data) => data.name === void 0 || isValidSkillFolderName(data.name),
|
|
347
404
|
{ message: KEBAB_CASE_MESSAGE, path: ["name"] }
|
|
348
405
|
);
|
|
406
|
+
var SkillWithLatestVersionSchema = SkillSchema.extend({
|
|
407
|
+
latestVersion: SkillVersionSchema.optional()
|
|
408
|
+
});
|
|
349
409
|
|
|
350
410
|
// src/target/skills-group.ts
|
|
351
411
|
var import_zod6 = require("zod");
|
|
@@ -564,7 +624,7 @@ var import_zod19 = require("zod");
|
|
|
564
624
|
var SkillWasCalledAssertionSchema = import_zod19.z.object({
|
|
565
625
|
type: import_zod19.z.literal("skill_was_called"),
|
|
566
626
|
/** Names of the skills that must have been called (matched against trace Skill tool args) */
|
|
567
|
-
skillNames: import_zod19.z.array(import_zod19.z.string()).min(1)
|
|
627
|
+
skillNames: import_zod19.z.array(import_zod19.z.string().min(1)).min(1)
|
|
568
628
|
});
|
|
569
629
|
var BuildPassedAssertionSchema = import_zod19.z.object({
|
|
570
630
|
type: import_zod19.z.literal("build_passed"),
|
|
@@ -1034,6 +1094,8 @@ var EvalRunSchema = TenantEntitySchema.extend({
|
|
|
1034
1094
|
agentId: import_zod26.z.string().optional(),
|
|
1035
1095
|
/** Skills group ID for this run */
|
|
1036
1096
|
skillsGroupId: import_zod26.z.string().optional(),
|
|
1097
|
+
/** Map of skillId to skillVersionId for this run */
|
|
1098
|
+
skillVersions: import_zod26.z.record(import_zod26.z.string(), import_zod26.z.string()).optional(),
|
|
1037
1099
|
/** Scenario IDs to run */
|
|
1038
1100
|
scenarioIds: import_zod26.z.array(import_zod26.z.string()),
|
|
1039
1101
|
/** Current status */
|
|
@@ -1131,6 +1193,10 @@ var EvalRunResultSchema = import_zod27.z.object({
|
|
|
1131
1193
|
id: import_zod27.z.string(),
|
|
1132
1194
|
targetId: import_zod27.z.string(),
|
|
1133
1195
|
targetName: import_zod27.z.string().optional(),
|
|
1196
|
+
/** SkillVersion ID used for this evaluation (for version tracking) */
|
|
1197
|
+
skillVersionId: import_zod27.z.string().optional(),
|
|
1198
|
+
/** SkillVersion semver string (e.g., "1.0.0", "1.2.3") for display */
|
|
1199
|
+
skillVersion: import_zod27.z.string().optional(),
|
|
1134
1200
|
scenarioId: import_zod27.z.string(),
|
|
1135
1201
|
scenarioName: import_zod27.z.string(),
|
|
1136
1202
|
modelConfig: ModelConfigSchema.optional(),
|
|
@@ -1351,6 +1417,7 @@ function getSystemAssertion(id) {
|
|
|
1351
1417
|
CreateMcpInputSchema,
|
|
1352
1418
|
CreateProjectInputSchema,
|
|
1353
1419
|
CreateSkillInputSchema,
|
|
1420
|
+
CreateSkillVersionInputSchema,
|
|
1354
1421
|
CreateSkillsGroupInputSchema,
|
|
1355
1422
|
CreateSubAgentInputSchema,
|
|
1356
1423
|
CreateTemplateInputSchema,
|
|
@@ -1378,6 +1445,8 @@ function getSystemAssertion(id) {
|
|
|
1378
1445
|
FileContentTestSchema,
|
|
1379
1446
|
FileModificationSchema,
|
|
1380
1447
|
FilePresenceTestSchema,
|
|
1448
|
+
GitHubSourceSchema,
|
|
1449
|
+
InitialVersionInputSchema,
|
|
1381
1450
|
LLMBreakdownStatsSchema,
|
|
1382
1451
|
LLMStepType,
|
|
1383
1452
|
LLMTestSchema,
|
|
@@ -1403,16 +1472,20 @@ function getSystemAssertion(id) {
|
|
|
1403
1472
|
PlaywrightNLTestSchema,
|
|
1404
1473
|
ProjectSchema,
|
|
1405
1474
|
PromptResultSchema,
|
|
1475
|
+
SEMVER_REGEX,
|
|
1406
1476
|
SKILL_FOLDER_NAME_REGEX,
|
|
1407
1477
|
SYSTEM_ASSERTIONS,
|
|
1408
1478
|
SYSTEM_ASSERTION_IDS,
|
|
1409
1479
|
ScenarioAssertionLinkSchema,
|
|
1410
1480
|
SiteConfigTestSchema,
|
|
1481
|
+
SkillFileSchema,
|
|
1411
1482
|
SkillMetadataSchema,
|
|
1412
1483
|
SkillSchema,
|
|
1484
|
+
SkillVersionOriginSchema,
|
|
1413
1485
|
SkillVersionSchema,
|
|
1414
1486
|
SkillWasCalledAssertionSchema,
|
|
1415
1487
|
SkillWasCalledConfigSchema,
|
|
1488
|
+
SkillWithLatestVersionSchema,
|
|
1416
1489
|
SkillsGroupSchema,
|
|
1417
1490
|
SubAgentSchema,
|
|
1418
1491
|
TRACE_EVENT_PREFIX,
|