@seriousgaming/sdk-tools 1.11.0 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/guidance.md +4 -0
- package/dist/{cli-F2D4GGUA.js → cli-2BKEIIA3.js} +66 -0
- package/dist/{cli-ZGCG6C22.js → cli-2QGTDHRX.js} +21 -21
- package/dist/{cli-SRXDLMXY.js → cli-2SDG3SZ7.js} +21 -21
- package/dist/{cli-DH35TZ2Q.js → cli-2VXP34R5.js} +2 -2
- package/dist/{cli-P3NUTOPK.js → cli-2W4B6PD5.js} +1 -1
- package/dist/{cli-WJVPSVF2.js → cli-2X7DKSXN.js} +24 -24
- package/dist/{cli-ATAJSIXH.js → cli-2ZP4N7HK.js} +19 -19
- package/dist/{cli-5O7XENQY.js → cli-3GJYZH4C.js} +5 -5
- package/dist/{cli-H6PP2MZT.js → cli-3GO3M2MX.js} +17 -13
- package/dist/{cli-HMDYG6PS.js → cli-3LDLUNME.js} +17 -17
- package/dist/{cli-BZOKZ7LP.js → cli-3WY6L2WP.js} +27 -27
- package/dist/{cli-REAM7LSC.js → cli-44MBABRD.js} +25 -25
- package/dist/{cli-5TTJWM2R.js → cli-4IO7MKLR.js} +20 -20
- package/dist/{cli-HNDFANQG.js → cli-4U2PY7W3.js} +1 -1
- package/dist/{cli-WQ7JBZNW.js → cli-4WI4LTBI.js} +2 -2
- package/dist/{cli-OTBCA54D.js → cli-55EIGGZK.js} +1 -1
- package/dist/{cli-FYPW25VD.js → cli-5AQV6FTL.js} +18 -18
- package/dist/{cli-RKPQLB7J.js → cli-5CW7BEU7.js} +3 -3
- package/dist/{cli-27TQVLVF.js → cli-5JUVHNZF.js} +12 -12
- package/dist/{cli-F3YJKP6D.js → cli-5KERL6HV.js} +9 -4
- package/dist/{cli-MSNSXVGF.js → cli-5KSA24NQ.js} +24 -24
- package/dist/{cli-5VZBFMCK.js → cli-5PIFO3EN.js} +8 -0
- package/dist/{cli-FT3X3D4P.js → cli-5QN2Z43X.js} +20 -20
- package/dist/cli-5XSPAA5X.js +69 -0
- package/dist/{cli-B2QAJ5RQ.js → cli-62KAYGXO.js} +20 -20
- package/dist/{cli-QPV3TYOM.js → cli-6IMV5ZZU.js} +26 -26
- package/dist/{cli-UULZSQBV.js → cli-6KWCIAN2.js} +764 -66
- package/dist/{cli-DQE7ITIR.js → cli-6NWETIX5.js} +20 -20
- package/dist/{cli-UVIURDZC.js → cli-7224T6HG.js} +20 -20
- package/dist/{cli-L7F4XKCB.js → cli-774W3MN5.js} +26 -26
- package/dist/cli-7BUD3VAD.js +16 -0
- package/dist/{cli-XNOAO6DY.js → cli-7KEP57S7.js} +12 -12
- package/dist/{cli-5UKWX3IU.js → cli-7OYFJMA6.js} +18 -18
- package/dist/{cli-WNHMJM3Z.js → cli-7ZNEDQ3G.js} +20 -20
- package/dist/{cli-5RRPUL6V.js → cli-AFK6OUID.js} +12 -12
- package/dist/{cli-6P6V2EHY.js → cli-AIYQFWXN.js} +25 -25
- package/dist/{cli-ERJNDBDH.js → cli-ARAAEWIQ.js} +2 -2
- package/dist/cli-AVQYV3N4.js +36 -0
- package/dist/{cli-TI2FOFPA.js → cli-B54Z6CXH.js} +16 -16
- package/dist/{cli-EEO5ZPAZ.js → cli-B5TC4MEL.js} +5 -5
- package/dist/{cli-5Z4D3F6Y.js → cli-BCXGOLU3.js} +3 -3
- package/dist/{cli-G2GLW5Z4.js → cli-BHAODVTZ.js} +1 -1
- package/dist/{cli-4F24FUW6.js → cli-BPS2JG74.js} +20 -20
- package/dist/{cli-W3RG2MRI.js → cli-BVO55SR2.js} +2 -2
- package/dist/{cli-R5SNTPND.js → cli-CFH3JZP5.js} +21 -21
- package/dist/{cli-Q7QSWNY2.js → cli-CTK2OIK6.js} +20 -20
- package/dist/{cli-ATWJCB2G.js → cli-CW5MV4VX.js} +4 -4
- package/dist/{cli-TCQJL2GY.js → cli-D5VOSJZ7.js} +1 -1
- package/dist/{cli-KIZTASLL.js → cli-DAEPMK7A.js} +5 -5
- package/dist/{cli-44CM2AH2.js → cli-DAHR7PKB.js} +1 -1
- package/dist/{cli-XW3JLRSA.js → cli-DWDSBTYL.js} +1 -1
- package/dist/cli-DYSDPGXW.js +95 -0
- package/dist/{cli-QEFYDXMH.js → cli-DZCLRZXJ.js} +1 -1
- package/dist/{cli-TEBV2OIW.js → cli-EG2VP7IV.js} +116 -27
- package/dist/{cli-77T4LJQF.js → cli-EJ47IE45.js} +24 -24
- package/dist/{cli-QDBYFITU.js → cli-EK4OSJHA.js} +2 -2
- package/dist/{cli-GO64NABP.js → cli-FCJHNXDA.js} +1 -1
- package/dist/{cli-ZNQBF4Z3.js → cli-FNESSE3L.js} +1 -1
- package/dist/{cli-D3S7NJZ3.js → cli-FTLH4WCE.js} +17 -17
- package/dist/{cli-JHLWBRXB.js → cli-FYV77YS2.js} +18 -18
- package/dist/cli-GMJX6Q4Q.js +36 -0
- package/dist/{cli-7JUNA74L.js → cli-GRBVNGDW.js} +1 -1
- package/dist/{cli-XKAIDYIE.js → cli-HAFCFY77.js} +33 -19
- package/dist/{cli-OXLOGXT3.js → cli-HCJLYI24.js} +3 -3
- package/dist/{cli-XOYUTNER.js → cli-HCXS5EXU.js} +25 -25
- package/dist/cli-HEUNNHDH.js +36 -0
- package/dist/{cli-G3TU6CP6.js → cli-HNUTIWDZ.js} +19 -19
- package/dist/{cli-JXQLHJO2.js → cli-HXT25ETJ.js} +244 -243
- package/dist/{cli-5EH4WTH4.js → cli-IGEL7UJQ.js} +5 -5
- package/dist/{cli-TLSTSZAC.js → cli-IO32YA5N.js} +26 -26
- package/dist/cli-IPTGZDBB.js +36 -0
- package/dist/{cli-OBLEROSC.js → cli-J25TYFKB.js} +19 -19
- package/dist/{cli-5ATO27Z4.js → cli-J7XYMEMX.js} +20 -20
- package/dist/{cli-X3TSTTYI.js → cli-JC5OJRID.js} +2 -2
- package/dist/{cli-25SJZI7N.js → cli-JD563OF4.js} +126 -27
- package/dist/{cli-VXLHOZSO.js → cli-JRJF2WHU.js} +20 -20
- package/dist/{cli-UQC46YSV.js → cli-JTFLIMIG.js} +3 -3
- package/dist/{cli-K4HSQWXC.js → cli-K2BSJ2PK.js} +2 -2
- package/dist/{cli-OM32UURE.js → cli-K4QTYMN2.js} +25 -25
- package/dist/{cli-BJFYGVJV.js → cli-KFJDD4C4.js} +20 -20
- package/dist/{cli-WW54THXN.js → cli-KVQ2JPVD.js} +1 -1
- package/dist/{cli-LHMRUJVN.js → cli-L6AHMW2Z.js} +1 -1
- package/dist/{cli-IKSLPAZN.js → cli-LCYUBCVY.js} +4 -4
- package/dist/{cli-HGCOEL5T.js → cli-LEWSCDH5.js} +4 -4
- package/dist/{cli-XJ7JESC2.js → cli-LI6G7GND.js} +27 -27
- package/dist/{cli-OMTHTWYJ.js → cli-LU2GCWIU.js} +21 -21
- package/dist/{cli-UEQMK4WX.js → cli-M3Z2KOEB.js} +1 -1
- package/dist/{cli-226ZJ72R.js → cli-M6E3E3I2.js} +170 -3
- package/dist/{cli-NKKCDO6K.js → cli-MCRPA4Z6.js} +20 -20
- package/dist/{cli-BQ36DV5C.js → cli-MI2DEDMA.js} +3 -3
- package/dist/{cli-6O7YCK2M.js → cli-MITZ6SE7.js} +3 -3
- package/dist/{cli-LPFSJ625.js → cli-MTIYD3BX.js} +26 -26
- package/dist/{cli-5M6KDNVX.js → cli-MTZUJGBK.js} +2 -2
- package/dist/{cli-GSOV26O5.js → cli-NCLMUXLC.js} +14 -14
- package/dist/{cli-X52TXCNF.js → cli-NLS3DKWH.js} +26 -26
- package/dist/{cli-AH6AMAIB.js → cli-NMDGS7IE.js} +1 -1
- package/dist/cli-O3DJO2CS.js +30 -0
- package/dist/{cli-BC67QHD5.js → cli-O47D5BFN.js} +2 -2
- package/dist/{cli-MDVC5ZNM.js → cli-OBRZ36VP.js} +14 -14
- package/dist/{cli-C32Q5E5I.js → cli-P6V3NSU4.js} +1 -1
- package/dist/{cli-RMD5WSUQ.js → cli-PAMQMTHK.js} +21 -21
- package/dist/{cli-WNBP3LOD.js → cli-PKTHOCT3.js} +26 -26
- package/dist/cli-QQ432JEN.js +36 -0
- package/dist/{cli-YYXAGJQR.js → cli-QVNDPTBU.js} +20 -20
- package/dist/{cli-KJA6GLZI.js → cli-R2DFD6ZZ.js} +15 -15
- package/dist/{cli-6BLVI2JM.js → cli-RMVJMOWU.js} +111 -7
- package/dist/{cli-X64BNBQH.js → cli-RVZH462G.js} +585 -8
- package/dist/{cli-6AMK3DYS.js → cli-SEEOQVUL.js} +12 -12
- package/dist/{cli-QTCTWYCM.js → cli-SO354HPI.js} +3 -3
- package/dist/{cli-RHL4B4H7.js → cli-T76HDH7F.js} +88 -4
- package/dist/{cli-ZJW6HQ6N.js → cli-TIS7DWRR.js} +7 -7
- package/dist/{cli-6JWDFCUV.js → cli-TY3B6XS7.js} +21 -21
- package/dist/{cli-4FJL3KQZ.js → cli-UKEZJFIT.js} +2 -2
- package/dist/cli-VQAMW4RF.js +95 -0
- package/dist/{cli-ATHATKI5.js → cli-W4LMRVZT.js} +78 -47
- package/dist/cli-W7I5KGIB.js +84 -0
- package/dist/{cli-S3NHXAJM.js → cli-WCFZNEK2.js} +18 -18
- package/dist/{cli-B3EFE7ZF.js → cli-WXGJFA4F.js} +1 -1
- package/dist/{cli-PNS4KWWC.js → cli-XCG33I77.js} +24 -24
- package/dist/{cli-DBMEH4Y5.js → cli-YHLSG5HB.js} +26 -26
- package/dist/cli-YNH5DTHY.js +36 -0
- package/dist/cli-YPMJWPDA.js +69 -0
- package/dist/{cli-PXQYY6KM.js → cli-YSXIR7EM.js} +0 -0
- package/dist/cli-ZPFWO4VW.js +85 -0
- package/dist/cli.js +53 -53
- package/dist/client.js +1951 -289
- package/dist/core/assessment-moves.d.ts +32 -0
- package/dist/core/assessment-reports.d.ts +38 -0
- package/dist/core/assessment-settings.d.ts +15 -0
- package/dist/core/benchmark-samples.d.ts +28 -0
- package/dist/core/session-host.d.ts +4 -0
- package/dist/core/session-roster.d.ts +6 -0
- package/dist/core/session-timers.d.ts +40 -0
- package/dist/core/session.d.ts +14 -1
- package/dist/core/tick-timer.d.ts +18 -0
- package/dist/dev/console/assets/{index-Ch6HHb2T.js → index-zd--5H1f.js} +26 -26
- package/dist/dev/console/index.html +1 -1
- package/dist/dev/shell.d.ts +21 -0
- package/dist/dev/view/commands.d.ts +3 -1
- package/dist/docs/api/index.md +1 -1
- package/dist/docs/api/sdk-commands.md +29 -0
- package/dist/docs/api/sdk-e2e.md +30 -0
- package/dist/docs/api/sdk-projection.md +5 -0
- package/dist/docs/api/sdk-tools-client.md +30 -0
- package/dist/docs/api/sdk-ui.md +7 -0
- package/dist/docs/api/sdk.md +834 -19
- package/dist/docs/changelog.md +34 -0
- package/dist/docs/cli-error-codes.md +5 -1
- package/dist/docs/cli-reference.md +76 -2
- package/dist/docs/concepts/index.md +2 -0
- package/dist/docs/concepts/terminology.md +12 -0
- package/dist/docs/guides/assessments.md +248 -0
- package/dist/docs/guides/benchmarks.md +241 -0
- package/dist/docs/guides/engine.md +8 -0
- package/dist/docs/guides/index.md +2 -0
- package/dist/docs/guides/presentation.md +11 -9
- package/dist/docs/guides/session-hooks.md +24 -22
- package/dist/docs/guides/tick-controller.md +22 -1
- package/dist/docs/guides/ui.md +24 -0
- package/dist/docs/guides/upgrading.md +21 -0
- package/dist/docs/page-index.md +16 -12
- package/dist/docs/recipes/create-assessment.md +124 -0
- package/dist/docs/recipes/create-benchmark.md +115 -0
- package/dist/docs/recipes/index.md +2 -0
- package/dist/execution/context-composition.d.ts +39 -4
- package/dist/execution/engine-state.d.ts +13 -0
- package/dist/execution/executor.d.ts +16 -0
- package/dist/execution/logic-result.d.ts +13 -2
- package/dist/help-examples.d.ts +12 -0
- package/dist/host/adapter.d.ts +24 -1
- package/dist/host/protocol.d.ts +115 -1
- package/dist/host.js +29 -1
- package/dist/playwright.js +1945 -283
- package/dist/runtime/index.d.ts +3 -1
- package/dist/runtime/manifest-version.d.ts +1 -1
- package/dist/runtime.js +298 -16
- package/dist/schema.json +189 -0
- package/dist/templating/resource-metadata.d.ts +1 -1
- package/dist/utils/error-codes.d.ts +9 -1
- package/dist/validation/assessment-validation.d.ts +34 -0
- package/dist/validation/benchmark-validation.d.ts +20 -0
- package/dist/validation/schema-validation.d.ts +4 -0
- package/package.json +3 -3
- package/dist/cli-27Z5T52R.js +0 -30
- package/dist/cli-3VLBFWHN.js +0 -36
- package/dist/cli-BINVO5R7.js +0 -36
- package/dist/cli-HLI5UEKX.js +0 -36
- package/dist/cli-OF36EA62.js +0 -36
- package/dist/cli-OVIJRBFI.js +0 -36
- package/dist/cli-UIYKPSIY.js +0 -36
- package/dist/cli-WZNIW6ZO.js +0 -16
package/dist/runtime/index.d.ts
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* server's executor, session and build step are made of: the `uses` guard
|
|
10
10
|
* (§6.11), capability scope (§6.1), the §9.5 context merge, the §7.4 surface
|
|
11
11
|
* projection, the §11.8 to §11.9 tick-result check, the checks on what logic
|
|
12
|
-
* returns (§7.4, §10.2, §11.12),
|
|
12
|
+
* returns (§7.4, §10.2, §11.12), the benchmark and assessment write folds
|
|
13
|
+
* (§6.16, §6.17), and the manifest version and optional
|
|
13
14
|
* features this runtime serves. The shapes may change in a minor
|
|
14
15
|
* release.
|
|
15
16
|
*/
|
|
@@ -20,6 +21,7 @@ export { applyContextUpdate, deepMergeContext } from './context-merge.js';
|
|
|
20
21
|
export { declaredCapabilityIds, projectContext } from './context-projection.js';
|
|
21
22
|
export { validateTickControllerResult } from './tick-validation.js';
|
|
22
23
|
export { assertJsonSerialisable } from '../execution/engine-state.js';
|
|
24
|
+
export { foldAssessmentWrites, foldBenchmarkWrites } from '../execution/context-composition.js';
|
|
23
25
|
export { readAggregateResult, readEngineResult, readTickControllerResult, } from '../execution/logic-result.js';
|
|
24
26
|
export type { EngineResultRead } from '../execution/logic-result.js';
|
|
25
27
|
export type { ValidateTickResultOptions } from './tick-validation.js';
|
|
@@ -26,7 +26,7 @@ export declare const MANIFEST_VERSION = 3;
|
|
|
26
26
|
* section is stamped, it is the revision at the top of the appendix.
|
|
27
27
|
* `spec-revision.test.ts` holds the two together.
|
|
28
28
|
*/
|
|
29
|
-
export declare const SPEC_REVISION = "1.
|
|
29
|
+
export declare const SPEC_REVISION = "1.10";
|
|
30
30
|
/**
|
|
31
31
|
* Whether specification revision `revision` is later than `than`, both as
|
|
32
32
|
* `major.minor`. A runtime MAY warn when a package's `builtWith.specRevision`
|
package/dist/runtime.js
CHANGED
|
@@ -16,6 +16,169 @@ var MAX_EFFECTIVE_TICK_INTERVAL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
|
16
16
|
// ../sdk/lib/build/validators.js
|
|
17
17
|
var SCENARIO_GATE_READ_LIMIT_BYTES = 10 * 1024 * 1024;
|
|
18
18
|
|
|
19
|
+
// ../sdk/lib/build/benchmarks.js
|
|
20
|
+
var BENCHMARK_TAG_VALUE_PATTERN = /^[A-Za-z0-9:._-]{1,64}$/;
|
|
21
|
+
function resolveBenchmarkLevel(definition) {
|
|
22
|
+
return definition.level ?? "team";
|
|
23
|
+
}
|
|
24
|
+
function isRecord(value) {
|
|
25
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
26
|
+
}
|
|
27
|
+
function benchmarkWriteProblem(definition, write) {
|
|
28
|
+
if (write === null)
|
|
29
|
+
return null;
|
|
30
|
+
if (!isRecord(write))
|
|
31
|
+
return "must be an object, or null to remove the record";
|
|
32
|
+
for (const [name, value] of Object.entries(write)) {
|
|
33
|
+
if (value === void 0 || value === null)
|
|
34
|
+
continue;
|
|
35
|
+
if (name === "tags") {
|
|
36
|
+
if (!isRecord(value))
|
|
37
|
+
return `"tags" must be an object of tag values`;
|
|
38
|
+
const declared = definition.tags ?? [];
|
|
39
|
+
for (const [tag, tagValue] of Object.entries(value)) {
|
|
40
|
+
if (!declared.includes(tag)) {
|
|
41
|
+
return declared.length === 0 ? `tag "${tag}" is not declared: the benchmark declares no tags` : `tag "${tag}" is not declared; declared tags are ${declared.map((t) => `"${t}"`).join(", ")}`;
|
|
42
|
+
}
|
|
43
|
+
if (tagValue === void 0 || tagValue === null)
|
|
44
|
+
continue;
|
|
45
|
+
if (typeof tagValue !== "string" || !BENCHMARK_TAG_VALUE_PATTERN.test(tagValue)) {
|
|
46
|
+
return `tag "${tag}" must be 1 to 64 characters of letters, digits, ':', '.', '_' or '-', got ${JSON.stringify(tagValue)}`;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (name === "excluded") {
|
|
52
|
+
if (typeof value !== "boolean") {
|
|
53
|
+
return `"excluded" must be a boolean, got ${JSON.stringify(value)}`;
|
|
54
|
+
}
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
const field = definition.fields[name];
|
|
58
|
+
if (!field)
|
|
59
|
+
return `field "${name}" is not declared`;
|
|
60
|
+
if ("type" in field) {
|
|
61
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
62
|
+
return `field "${name}" must be a finite number, got ${JSON.stringify(value)}`;
|
|
63
|
+
}
|
|
64
|
+
} else if (typeof value !== "string" || !field.enum.includes(value)) {
|
|
65
|
+
return `field "${name}" must be one of ${field.enum.map((o) => JSON.stringify(o)).join(", ")}, got ${JSON.stringify(value)}`;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
function applyBenchmarkWrite(current, write) {
|
|
71
|
+
if (write === null)
|
|
72
|
+
return void 0;
|
|
73
|
+
const next = { ...current };
|
|
74
|
+
for (const [name, value] of Object.entries(write)) {
|
|
75
|
+
if (value === void 0)
|
|
76
|
+
continue;
|
|
77
|
+
if (value === null) {
|
|
78
|
+
delete next[name];
|
|
79
|
+
} else if (name === "tags" && isRecord(value)) {
|
|
80
|
+
const tags = Object.fromEntries(Object.entries(value).filter(([, tagValue]) => typeof tagValue === "string"));
|
|
81
|
+
if (Object.keys(tags).length === 0)
|
|
82
|
+
delete next.tags;
|
|
83
|
+
else
|
|
84
|
+
next.tags = tags;
|
|
85
|
+
} else {
|
|
86
|
+
next[name] = value;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return Object.keys(next).length === 0 ? void 0 : next;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ../sdk/lib/build/assessments.js
|
|
93
|
+
var ASSESSMENT_FEEDBACK_MAX_LENGTH = 500;
|
|
94
|
+
function isRecord2(value) {
|
|
95
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
96
|
+
}
|
|
97
|
+
function foldRecord(definition, current, write) {
|
|
98
|
+
if (write === null)
|
|
99
|
+
return { next: void 0 };
|
|
100
|
+
if (!isRecord2(write))
|
|
101
|
+
return { problem: "must be an object, or null to remove the result" };
|
|
102
|
+
const next = isRecord2(current) ? { ...current } : {};
|
|
103
|
+
for (const [name, value] of Object.entries(write)) {
|
|
104
|
+
if (value === void 0)
|
|
105
|
+
continue;
|
|
106
|
+
if (name === "score") {
|
|
107
|
+
if (value === null)
|
|
108
|
+
return { problem: '"score" cannot be removed; remove the result' };
|
|
109
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
110
|
+
return { problem: `"score" must be a finite number, got ${JSON.stringify(value)}` };
|
|
111
|
+
}
|
|
112
|
+
if (value < definition.min || value > definition.max) {
|
|
113
|
+
return {
|
|
114
|
+
problem: `"score" must be between ${definition.min} and ${definition.max}, got ${value}`
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
next.score = value;
|
|
118
|
+
} else if (name === "feedback") {
|
|
119
|
+
if (value === null) {
|
|
120
|
+
delete next.feedback;
|
|
121
|
+
} else if (typeof value !== "string") {
|
|
122
|
+
return { problem: `"feedback" must be a string, got ${JSON.stringify(value)}` };
|
|
123
|
+
} else if (value.length > ASSESSMENT_FEEDBACK_MAX_LENGTH) {
|
|
124
|
+
return {
|
|
125
|
+
problem: `"feedback" must be at most ${ASSESSMENT_FEEDBACK_MAX_LENGTH} characters, got ${value.length}`
|
|
126
|
+
};
|
|
127
|
+
} else {
|
|
128
|
+
next.feedback = value;
|
|
129
|
+
}
|
|
130
|
+
} else if (name === "final") {
|
|
131
|
+
if (value === null) {
|
|
132
|
+
delete next.final;
|
|
133
|
+
} else if (typeof value !== "boolean") {
|
|
134
|
+
return { problem: `"final" must be a boolean, got ${JSON.stringify(value)}` };
|
|
135
|
+
} else if (value) {
|
|
136
|
+
next.final = true;
|
|
137
|
+
} else {
|
|
138
|
+
delete next.final;
|
|
139
|
+
}
|
|
140
|
+
} else {
|
|
141
|
+
return {
|
|
142
|
+
problem: `"${name}" is not a field of a result; use score, feedback or final`
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (next.score === void 0) {
|
|
147
|
+
return { problem: 'a result needs a "score", and this write leaves it without one' };
|
|
148
|
+
}
|
|
149
|
+
return { next };
|
|
150
|
+
}
|
|
151
|
+
function foldAssessmentWrite(definition, current, write, participantIds) {
|
|
152
|
+
if (definition.level === "team")
|
|
153
|
+
return foldRecord(definition, current, write);
|
|
154
|
+
if (write === null)
|
|
155
|
+
return { next: void 0 };
|
|
156
|
+
if (!isRecord2(write)) {
|
|
157
|
+
return {
|
|
158
|
+
problem: "must be an object of results by participant id, or null to remove every result"
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
const records = isRecord2(current) ? { ...current } : {};
|
|
162
|
+
for (const [participantId, recordWrite] of Object.entries(write)) {
|
|
163
|
+
if (recordWrite === void 0)
|
|
164
|
+
continue;
|
|
165
|
+
if (recordWrite !== null && !participantIds.has(participantId)) {
|
|
166
|
+
return {
|
|
167
|
+
problem: `participant "${participantId}" is not on this team, so it cannot be scored here`
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
const folded = foldRecord(definition, records[participantId], recordWrite);
|
|
171
|
+
if ("problem" in folded) {
|
|
172
|
+
return { problem: `participant "${participantId}": ${folded.problem}` };
|
|
173
|
+
}
|
|
174
|
+
if (folded.next === void 0)
|
|
175
|
+
delete records[participantId];
|
|
176
|
+
else
|
|
177
|
+
records[participantId] = folded.next;
|
|
178
|
+
}
|
|
179
|
+
return { next: Object.keys(records).length === 0 ? void 0 : records };
|
|
180
|
+
}
|
|
181
|
+
|
|
19
182
|
// cli/runtime/guarded-callbacks.ts
|
|
20
183
|
var API_USE_TO_CALLBACK = {
|
|
21
184
|
generateText: "generateText",
|
|
@@ -157,7 +320,16 @@ function round(value) {
|
|
|
157
320
|
// cli/execution/engine-state.ts
|
|
158
321
|
var ENGINE_STATE_KEY = "$engineState";
|
|
159
322
|
var SHARED_ENGINE_STATE_KEY = "$sharedEngineState";
|
|
160
|
-
var
|
|
323
|
+
var BENCHMARKS_KEY = "$benchmarks";
|
|
324
|
+
var SHARED_BENCHMARKS_KEY = "$sharedBenchmarks";
|
|
325
|
+
var ASSESSMENTS_KEY = "$assessments";
|
|
326
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
327
|
+
ENGINE_STATE_KEY,
|
|
328
|
+
SHARED_ENGINE_STATE_KEY,
|
|
329
|
+
BENCHMARKS_KEY,
|
|
330
|
+
SHARED_BENCHMARKS_KEY,
|
|
331
|
+
ASSESSMENTS_KEY
|
|
332
|
+
]);
|
|
161
333
|
function isReservedKey(key) {
|
|
162
334
|
return RESERVED_KEYS.has(key);
|
|
163
335
|
}
|
|
@@ -203,11 +375,11 @@ function assertJsonSerialisable(label, value) {
|
|
|
203
375
|
}
|
|
204
376
|
|
|
205
377
|
// cli/execution/logic-result.ts
|
|
206
|
-
function
|
|
378
|
+
function isRecord3(value) {
|
|
207
379
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
208
380
|
}
|
|
209
381
|
function readCapabilities(label, result, allowed = ["capabilities", "deletes"]) {
|
|
210
|
-
if (!
|
|
382
|
+
if (!isRecord3(result)) {
|
|
211
383
|
throw new Error(`${label} must return an object of the form { capabilities: { ... } }`);
|
|
212
384
|
}
|
|
213
385
|
const unknownKeys = Object.keys(result).filter((key) => !allowed.includes(key));
|
|
@@ -221,7 +393,7 @@ function readCapabilities(label, result, allowed = ["capabilities", "deletes"])
|
|
|
221
393
|
if (capabilities === void 0 || capabilities === null) {
|
|
222
394
|
return {};
|
|
223
395
|
}
|
|
224
|
-
if (!
|
|
396
|
+
if (!isRecord3(capabilities)) {
|
|
225
397
|
throw new Error(`${label} returned "capabilities" that is not an object`);
|
|
226
398
|
}
|
|
227
399
|
const reserved = Object.keys(capabilities).find(isReservedKey);
|
|
@@ -233,7 +405,7 @@ function readCapabilities(label, result, allowed = ["capabilities", "deletes"])
|
|
|
233
405
|
return capabilities;
|
|
234
406
|
}
|
|
235
407
|
function readDeletes(label, result) {
|
|
236
|
-
if (!
|
|
408
|
+
if (!isRecord3(result)) return [];
|
|
237
409
|
const deletes = result.deletes;
|
|
238
410
|
if (deletes === void 0 || deletes === null) return [];
|
|
239
411
|
if (!Array.isArray(deletes)) {
|
|
@@ -249,18 +421,32 @@ function readDeletes(label, result) {
|
|
|
249
421
|
);
|
|
250
422
|
});
|
|
251
423
|
}
|
|
252
|
-
function readEngineResult(label, result, phase = "init") {
|
|
253
|
-
const allowed =
|
|
424
|
+
function readEngineResult(label, result, phase = "init", writesBenchmarks = false, writesAssessments = false) {
|
|
425
|
+
const allowed = [
|
|
426
|
+
"capabilities",
|
|
427
|
+
"deletes",
|
|
428
|
+
"state",
|
|
429
|
+
...phase === "tick" ? ["complete"] : [],
|
|
430
|
+
...writesBenchmarks ? ["benchmarks"] : [],
|
|
431
|
+
...writesAssessments ? ["assessments"] : []
|
|
432
|
+
];
|
|
254
433
|
const capabilities = readCapabilities(label, result, allowed);
|
|
255
434
|
const deletes = readDeletes(label, result);
|
|
256
|
-
const { state, complete } = result;
|
|
435
|
+
const { state, complete, benchmarks, assessments } = result;
|
|
257
436
|
if (complete !== void 0 && typeof complete !== "boolean") {
|
|
258
437
|
throw new Error(`${label} returned "complete" that is not a boolean`);
|
|
259
438
|
}
|
|
260
|
-
return {
|
|
439
|
+
return {
|
|
440
|
+
capabilities,
|
|
441
|
+
deletes,
|
|
442
|
+
state,
|
|
443
|
+
complete: complete === true,
|
|
444
|
+
...benchmarks === void 0 ? {} : { benchmarks },
|
|
445
|
+
...assessments === void 0 ? {} : { assessments }
|
|
446
|
+
};
|
|
261
447
|
}
|
|
262
|
-
function readAggregateResult(label, result, teamCount, phase = "init") {
|
|
263
|
-
if (!
|
|
448
|
+
function readAggregateResult(label, result, teamCount, phase = "init", writesBenchmarks = false, writesAssessments = false) {
|
|
449
|
+
if (!isRecord3(result)) {
|
|
264
450
|
throw new Error(
|
|
265
451
|
`${label} must return { shared: { capabilities }, teams: [{ capabilities }] }`
|
|
266
452
|
);
|
|
@@ -277,15 +463,27 @@ function readAggregateResult(label, result, teamCount, phase = "init") {
|
|
|
277
463
|
);
|
|
278
464
|
}
|
|
279
465
|
return {
|
|
280
|
-
shared: result.shared === void 0 ? { capabilities: {}, deletes: [], state: void 0, complete: false } : readEngineResult(
|
|
466
|
+
shared: result.shared === void 0 ? { capabilities: {}, deletes: [], state: void 0, complete: false } : readEngineResult(
|
|
467
|
+
`${label} "shared"`,
|
|
468
|
+
result.shared,
|
|
469
|
+
phase,
|
|
470
|
+
writesBenchmarks,
|
|
471
|
+
writesAssessments
|
|
472
|
+
),
|
|
281
473
|
teams: result.teams.map(
|
|
282
|
-
(team, i) => readEngineResult(
|
|
474
|
+
(team, i) => readEngineResult(
|
|
475
|
+
`${label} "teams[${i}]"`,
|
|
476
|
+
team,
|
|
477
|
+
phase,
|
|
478
|
+
writesBenchmarks,
|
|
479
|
+
writesAssessments
|
|
480
|
+
)
|
|
283
481
|
)
|
|
284
482
|
};
|
|
285
483
|
}
|
|
286
484
|
var TICK_CONTROLLER_KEYS = ["enabled", "intervalMs", "nextTickAt", "complete"];
|
|
287
485
|
function readTickControllerResult(label, result) {
|
|
288
|
-
if (!
|
|
486
|
+
if (!isRecord3(result)) {
|
|
289
487
|
throw new Error(`${label} must return an object of the form { enabled, intervalMs? }`);
|
|
290
488
|
}
|
|
291
489
|
const unknownKeys = Object.keys(result).filter((key) => !TICK_CONTROLLER_KEYS.includes(key));
|
|
@@ -303,9 +501,87 @@ function readTickControllerResult(label, result) {
|
|
|
303
501
|
return result;
|
|
304
502
|
}
|
|
305
503
|
|
|
504
|
+
// cli/execution/context-composition.ts
|
|
505
|
+
function foldAssessmentWrites(label, raw, current, participantIds, scope) {
|
|
506
|
+
if (raw === void 0) return void 0;
|
|
507
|
+
if (!isPlainObject2(raw)) {
|
|
508
|
+
scope.warn(
|
|
509
|
+
`${label} returned "assessments" that is not an object of writes by assessment. Every assessment write in it was dropped, and the round carried on.`
|
|
510
|
+
);
|
|
511
|
+
return void 0;
|
|
512
|
+
}
|
|
513
|
+
const writes = structuredClone(raw);
|
|
514
|
+
const results = structuredClone(isPlainObject2(current) ? current : {});
|
|
515
|
+
for (const [identifier, write] of Object.entries(writes)) {
|
|
516
|
+
if (write === void 0) continue;
|
|
517
|
+
const named = `${label} wrote assessment "${identifier}"`;
|
|
518
|
+
const definition = scope.assessments?.get(identifier);
|
|
519
|
+
if (!definition) {
|
|
520
|
+
scope.warn(
|
|
521
|
+
`${named}, which the engine does not declare in "assessments". The write was dropped.`
|
|
522
|
+
);
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
const folded = foldAssessmentWrite(definition, results[identifier], write, participantIds);
|
|
526
|
+
if ("problem" in folded) {
|
|
527
|
+
scope.warn(
|
|
528
|
+
`${named}: ${folded.problem}. The write was dropped, and the round carried on.`
|
|
529
|
+
);
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
if (folded.next === void 0) delete results[identifier];
|
|
533
|
+
else results[identifier] = folded.next;
|
|
534
|
+
}
|
|
535
|
+
return results;
|
|
536
|
+
}
|
|
537
|
+
function isPlainObject2(value) {
|
|
538
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
539
|
+
}
|
|
540
|
+
function foldBenchmarkWrites(label, raw, current, level, scope) {
|
|
541
|
+
if (raw === void 0) return void 0;
|
|
542
|
+
if (!isPlainObject2(raw)) {
|
|
543
|
+
scope.warn(
|
|
544
|
+
`${label} returned "benchmarks" that is not an object of writes by benchmark. Every benchmark write in it was dropped, and the round carried on.`
|
|
545
|
+
);
|
|
546
|
+
return void 0;
|
|
547
|
+
}
|
|
548
|
+
const writes = structuredClone(raw);
|
|
549
|
+
const records = structuredClone(isPlainObject2(current) ? current : {});
|
|
550
|
+
for (const [identifier, write] of Object.entries(writes)) {
|
|
551
|
+
if (write === void 0) continue;
|
|
552
|
+
const named = `${label} wrote benchmark "${identifier}"`;
|
|
553
|
+
const definition = scope.benchmarks.get(identifier);
|
|
554
|
+
if (!definition) {
|
|
555
|
+
scope.warn(
|
|
556
|
+
`${named}, which the engine does not declare in "benchmarks". The write was dropped.`
|
|
557
|
+
);
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
const declaredLevel = resolveBenchmarkLevel(definition);
|
|
561
|
+
if (declaredLevel !== level) {
|
|
562
|
+
scope.warn(
|
|
563
|
+
declaredLevel === "session" ? `${named}, which is session-level, so it is written under an aggregate result's "shared". The write was dropped.` : `${named}, which is per team, so it is written for each team rather than under "shared". The write was dropped.`
|
|
564
|
+
);
|
|
565
|
+
continue;
|
|
566
|
+
}
|
|
567
|
+
const problem = benchmarkWriteProblem(definition, write);
|
|
568
|
+
if (problem !== null) {
|
|
569
|
+
scope.warn(`${named}: ${problem}. The write was dropped, and the round carried on.`);
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
const next = applyBenchmarkWrite(
|
|
573
|
+
records[identifier],
|
|
574
|
+
write
|
|
575
|
+
);
|
|
576
|
+
if (next === void 0) delete records[identifier];
|
|
577
|
+
else records[identifier] = next;
|
|
578
|
+
}
|
|
579
|
+
return records;
|
|
580
|
+
}
|
|
581
|
+
|
|
306
582
|
// cli/runtime/manifest-version.ts
|
|
307
583
|
var MANIFEST_VERSION = 3;
|
|
308
|
-
var SPEC_REVISION = "1.
|
|
584
|
+
var SPEC_REVISION = "1.10";
|
|
309
585
|
function isLaterSpecRevision(revision, than) {
|
|
310
586
|
const [major, minor] = revision.split(".").map(Number);
|
|
311
587
|
const [thanMajor, thanMinor] = than.split(".").map(Number);
|
|
@@ -319,7 +595,11 @@ var SUPPORTED_FEATURES = [
|
|
|
319
595
|
// File reads in a validator's `Scenario` gate (§15.8).
|
|
320
596
|
"scenario-files",
|
|
321
597
|
// The `Scenario` gate judging a session as it starts, and refusing it (§15.8).
|
|
322
|
-
"session-validation"
|
|
598
|
+
"session-validation",
|
|
599
|
+
// Benchmarks: engine-written records pooled across sessions (§6.16).
|
|
600
|
+
"benchmarks",
|
|
601
|
+
// Assessments: engine-written scores per participant or team, in bands (§6.17).
|
|
602
|
+
"assessments"
|
|
323
603
|
];
|
|
324
604
|
function unsupportedFeatures(manifest) {
|
|
325
605
|
return (manifest.requires ?? []).filter((feature) => !SUPPORTED_FEATURES.includes(feature));
|
|
@@ -354,6 +634,8 @@ export {
|
|
|
354
634
|
createGuardedCallbacks,
|
|
355
635
|
declaredCapabilityIds,
|
|
356
636
|
deepMergeContext,
|
|
637
|
+
foldAssessmentWrites,
|
|
638
|
+
foldBenchmarkWrites,
|
|
357
639
|
isLaterSpecRevision,
|
|
358
640
|
projectContext,
|
|
359
641
|
readAggregateResult,
|
package/dist/schema.json
CHANGED
|
@@ -153,6 +153,24 @@
|
|
|
153
153
|
},
|
|
154
154
|
"uniqueItems": true
|
|
155
155
|
},
|
|
156
|
+
"benchmarks": {
|
|
157
|
+
"type": "array",
|
|
158
|
+
"description": "Identifiers of the benchmarks this engine writes. The engine alone writes benchmarks: it receives its current records as `benchmarks`, beside `context`, and returns writes under `benchmarks`, beside `capabilities`. A game that declares any lists the `benchmarks` feature in `requires`",
|
|
159
|
+
"items": {
|
|
160
|
+
"type": "string",
|
|
161
|
+
"pattern": "^[a-z][a-z0-9_]*$"
|
|
162
|
+
},
|
|
163
|
+
"uniqueItems": true
|
|
164
|
+
},
|
|
165
|
+
"assessments": {
|
|
166
|
+
"type": "array",
|
|
167
|
+
"description": "Identifiers of the assessments this engine scores. The engine alone writes assessments: it receives its team's current results as `assessments`, beside `context`, and returns writes under `assessments`, beside `capabilities`. A game that declares any lists the `assessments` feature in `requires`",
|
|
168
|
+
"items": {
|
|
169
|
+
"type": "string",
|
|
170
|
+
"pattern": "^[a-z][a-z0-9_]*$"
|
|
171
|
+
},
|
|
172
|
+
"uniqueItems": true
|
|
173
|
+
},
|
|
156
174
|
"aggregate": {
|
|
157
175
|
"type": "boolean",
|
|
158
176
|
"description": "Generated, not authored. True when the engine's logic exports InitAggregate and RunAggregate, in which case it runs once across all teams rather than per-team via Run, and may write to shared capabilities. The build derives this from the exports and writes it into the bundled manifest; authoring it in engine.sdk.json is an error.",
|
|
@@ -162,6 +180,163 @@
|
|
|
162
180
|
"required": ["identifier"],
|
|
163
181
|
"additionalProperties": false
|
|
164
182
|
},
|
|
183
|
+
"BenchmarkField": {
|
|
184
|
+
"description": "One field of a benchmark: a number, pooled as a distribution, or one of a short declared list of options, pooled as a count per option",
|
|
185
|
+
"oneOf": [
|
|
186
|
+
{
|
|
187
|
+
"type": "object",
|
|
188
|
+
"properties": {
|
|
189
|
+
"type": { "const": "number" },
|
|
190
|
+
"description": { "type": "string" }
|
|
191
|
+
},
|
|
192
|
+
"required": ["type"],
|
|
193
|
+
"additionalProperties": false
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
"type": "object",
|
|
197
|
+
"properties": {
|
|
198
|
+
"enum": {
|
|
199
|
+
"type": "array",
|
|
200
|
+
"description": "The options a record may hold. Closed and short, so nothing free-form leaves the session",
|
|
201
|
+
"items": {
|
|
202
|
+
"type": "string",
|
|
203
|
+
"pattern": "^[A-Za-z0-9_-]{1,64}$"
|
|
204
|
+
},
|
|
205
|
+
"minItems": 1,
|
|
206
|
+
"maxItems": 20,
|
|
207
|
+
"uniqueItems": true
|
|
208
|
+
},
|
|
209
|
+
"description": { "type": "string" }
|
|
210
|
+
},
|
|
211
|
+
"required": ["enum"],
|
|
212
|
+
"additionalProperties": false
|
|
213
|
+
}
|
|
214
|
+
]
|
|
215
|
+
},
|
|
216
|
+
"BenchmarkDefinition": {
|
|
217
|
+
"type": "object",
|
|
218
|
+
"description": "Definition of a benchmark: outcomes and decisions the engine records for each team (or the session), which a runtime pools anonymously across sessions so a user interface can compare a team with everyone who has played",
|
|
219
|
+
"properties": {
|
|
220
|
+
"identifier": {
|
|
221
|
+
"type": "string",
|
|
222
|
+
"description": "Identifier for the benchmark: its directory name",
|
|
223
|
+
"pattern": "^[a-z][a-z0-9_]*$"
|
|
224
|
+
},
|
|
225
|
+
"revision": {
|
|
226
|
+
"type": "integer",
|
|
227
|
+
"minimum": 1,
|
|
228
|
+
"description": "Results are pooled per revision. Bump it when a change makes earlier results no longer comparable; leave it alone across ordinary releases so the pool carries on"
|
|
229
|
+
},
|
|
230
|
+
"level": {
|
|
231
|
+
"type": "string",
|
|
232
|
+
"enum": ["team", "session"],
|
|
233
|
+
"default": "team",
|
|
234
|
+
"description": "'team' (default): one record per team, written by Run or under an aggregate result's teams. 'session': one record for the whole session, written only under an aggregate result's shared"
|
|
235
|
+
},
|
|
236
|
+
"tags": {
|
|
237
|
+
"type": "array",
|
|
238
|
+
"description": "The names a record can be grouped by. The engine sets each one's value, a short string, and a user interface filters on any combination of them",
|
|
239
|
+
"items": {
|
|
240
|
+
"type": "string",
|
|
241
|
+
"pattern": "^[A-Za-z][A-Za-z0-9_]*$"
|
|
242
|
+
},
|
|
243
|
+
"maxItems": 5,
|
|
244
|
+
"uniqueItems": true
|
|
245
|
+
},
|
|
246
|
+
"fields": {
|
|
247
|
+
"type": "object",
|
|
248
|
+
"description": "The benchmark's fields, flat. 'tags' and 'excluded' are reserved",
|
|
249
|
+
"additionalProperties": { "$ref": "#/$defs/BenchmarkField" },
|
|
250
|
+
"propertyNames": {
|
|
251
|
+
"pattern": "^[A-Za-z][A-Za-z0-9_]*$",
|
|
252
|
+
"not": { "enum": ["tags", "excluded"] }
|
|
253
|
+
},
|
|
254
|
+
"minProperties": 1,
|
|
255
|
+
"maxProperties": 20
|
|
256
|
+
}
|
|
257
|
+
},
|
|
258
|
+
"required": ["identifier", "revision", "fields"],
|
|
259
|
+
"additionalProperties": false
|
|
260
|
+
},
|
|
261
|
+
"SDKBenchmark": {
|
|
262
|
+
"type": "object",
|
|
263
|
+
"description": "Bundle containing a benchmark definition",
|
|
264
|
+
"properties": {
|
|
265
|
+
"definition": { "$ref": "#/$defs/BenchmarkDefinition" }
|
|
266
|
+
},
|
|
267
|
+
"required": ["definition"],
|
|
268
|
+
"additionalProperties": false
|
|
269
|
+
},
|
|
270
|
+
"AssessmentBand": {
|
|
271
|
+
"type": "object",
|
|
272
|
+
"description": "One band of an assessment's scores, such as 'developing' or 'expert': every score from `from` up to the next band's `from`",
|
|
273
|
+
"properties": {
|
|
274
|
+
"id": {
|
|
275
|
+
"type": "string",
|
|
276
|
+
"description": "The band's identifier, reported with each result in it. A user interface shows its own label for it",
|
|
277
|
+
"pattern": "^[a-z][a-z0-9_]*$"
|
|
278
|
+
},
|
|
279
|
+
"from": {
|
|
280
|
+
"type": "number",
|
|
281
|
+
"description": "The lowest score in the band. Bands are listed in ascending order of `from`, and the first starts at the assessment's `min`. A default: the built-in `assessments` settings let a scenario or a session move it"
|
|
282
|
+
},
|
|
283
|
+
"passing": {
|
|
284
|
+
"type": "boolean",
|
|
285
|
+
"description": "Whether a score in this band is a pass. A default, which the built-in `assessments` settings let a scenario or a session change",
|
|
286
|
+
"default": false
|
|
287
|
+
},
|
|
288
|
+
"description": {
|
|
289
|
+
"type": "string",
|
|
290
|
+
"description": "What the band means, for whoever reads the game's results"
|
|
291
|
+
}
|
|
292
|
+
},
|
|
293
|
+
"required": ["id", "from"],
|
|
294
|
+
"additionalProperties": false
|
|
295
|
+
},
|
|
296
|
+
"AssessmentDefinition": {
|
|
297
|
+
"type": "object",
|
|
298
|
+
"description": "Definition of an assessment: one measure the engine scores for each participant, or for each team, which a runtime may report to a learning management system. The score is the game's; the runtime places it in a band and decides pass or fail from the band",
|
|
299
|
+
"properties": {
|
|
300
|
+
"identifier": {
|
|
301
|
+
"type": "string",
|
|
302
|
+
"description": "Identifier for the assessment: its directory name",
|
|
303
|
+
"pattern": "^[a-z][a-z0-9_]*$"
|
|
304
|
+
},
|
|
305
|
+
"description": {
|
|
306
|
+
"type": "string",
|
|
307
|
+
"description": "What the assessment measures, for whoever reads the game's results"
|
|
308
|
+
},
|
|
309
|
+
"level": {
|
|
310
|
+
"type": "string",
|
|
311
|
+
"enum": ["participant", "team"],
|
|
312
|
+
"description": "'participant': one score per participant, written for each by participant id. 'team': one score per team, which every participant on the team receives"
|
|
313
|
+
},
|
|
314
|
+
"min": {
|
|
315
|
+
"type": "number",
|
|
316
|
+
"description": "The lowest score the engine may write"
|
|
317
|
+
},
|
|
318
|
+
"max": {
|
|
319
|
+
"type": "number",
|
|
320
|
+
"description": "The highest score the engine may write. Greater than `min`"
|
|
321
|
+
},
|
|
322
|
+
"bands": {
|
|
323
|
+
"type": "array",
|
|
324
|
+
"description": "The ranges a score falls into, ascending, the first starting at `min`: at most 10. Without bands, a result has no band and no pass or fail",
|
|
325
|
+
"items": { "$ref": "#/$defs/AssessmentBand" }
|
|
326
|
+
}
|
|
327
|
+
},
|
|
328
|
+
"required": ["identifier", "level", "min", "max"],
|
|
329
|
+
"additionalProperties": false
|
|
330
|
+
},
|
|
331
|
+
"SDKAssessment": {
|
|
332
|
+
"type": "object",
|
|
333
|
+
"description": "Bundle containing an assessment definition",
|
|
334
|
+
"properties": {
|
|
335
|
+
"definition": { "$ref": "#/$defs/AssessmentDefinition" }
|
|
336
|
+
},
|
|
337
|
+
"required": ["definition"],
|
|
338
|
+
"additionalProperties": false
|
|
339
|
+
},
|
|
165
340
|
"CapabilityDefinition": {
|
|
166
341
|
"type": "object",
|
|
167
342
|
"description": "Definition of a capability",
|
|
@@ -1190,6 +1365,14 @@
|
|
|
1190
1365
|
"type": "array",
|
|
1191
1366
|
"items": { "$ref": "#/$defs/SDKCapability" }
|
|
1192
1367
|
},
|
|
1368
|
+
"benchmarks": {
|
|
1369
|
+
"type": "array",
|
|
1370
|
+
"items": { "$ref": "#/$defs/SDKBenchmark" }
|
|
1371
|
+
},
|
|
1372
|
+
"assessments": {
|
|
1373
|
+
"type": "array",
|
|
1374
|
+
"items": { "$ref": "#/$defs/SDKAssessment" }
|
|
1375
|
+
},
|
|
1193
1376
|
"settings": {
|
|
1194
1377
|
"type": "array",
|
|
1195
1378
|
"items": { "$ref": "#/$defs/SDKSettings" }
|
|
@@ -1576,6 +1759,12 @@
|
|
|
1576
1759
|
"SDKPackage": { "$ref": "#/$defs/SDKPackage" },
|
|
1577
1760
|
"SDKEngine": { "$ref": "#/$defs/SDKEngine" },
|
|
1578
1761
|
"SDKCapability": { "$ref": "#/$defs/SDKCapability" },
|
|
1762
|
+
"BenchmarkField": { "$ref": "#/$defs/BenchmarkField" },
|
|
1763
|
+
"BenchmarkDefinition": { "$ref": "#/$defs/BenchmarkDefinition" },
|
|
1764
|
+
"SDKBenchmark": { "$ref": "#/$defs/SDKBenchmark" },
|
|
1765
|
+
"AssessmentBand": { "$ref": "#/$defs/AssessmentBand" },
|
|
1766
|
+
"AssessmentDefinition": { "$ref": "#/$defs/AssessmentDefinition" },
|
|
1767
|
+
"SDKAssessment": { "$ref": "#/$defs/SDKAssessment" },
|
|
1579
1768
|
"SDKScenario": { "$ref": "#/$defs/SDKScenario" },
|
|
1580
1769
|
"SDKUserInterface": { "$ref": "#/$defs/SDKUserInterface" },
|
|
1581
1770
|
"SDKSetup": {
|
|
@@ -45,7 +45,7 @@ export interface ReferenceSite {
|
|
|
45
45
|
}
|
|
46
46
|
export type ResourceKind = 'logic' | 'ui' | 'composition' | 'content';
|
|
47
47
|
/** Every registered resource type, so a stale type name is a compile error. */
|
|
48
|
-
export declare const RESOURCE_TYPES: readonly ["capability", "engine", "ui", "session", "waiting_room", "presentation", "setup", "scenario_editor", "scenario", "guide", "tick_controller", "hook", "validator", "endpoint", "settings"];
|
|
48
|
+
export declare const RESOURCE_TYPES: readonly ["capability", "benchmark", "assessment", "engine", "ui", "session", "waiting_room", "presentation", "setup", "scenario_editor", "scenario", "guide", "tick_controller", "hook", "validator", "endpoint", "settings"];
|
|
49
49
|
export type ResourceType = (typeof RESOURCE_TYPES)[number];
|
|
50
50
|
/**
|
|
51
51
|
* Where a resource's code sits inside its own directory, and what the
|