@bendyline/gezel 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checks/index.d.ts +1 -1
- package/dist/checks/index.js +2 -2
- package/dist/{index-4Q1N_3oe.d.ts → index-Cj_G8Y-S.d.ts} +169 -2
- package/dist/index.d.ts +6 -6
- package/dist/index.js +968 -13
- package/dist/markdown/index.d.ts +1 -1
- package/dist/markdown/index.js +7 -3
- package/dist/{report-action-C0KuyeNw.d.ts → report-action-UJ-pJpZO.d.ts} +5 -0
- package/dist/schemas/index.d.ts +2 -2
- package/dist/schemas/index.js +46 -12
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -673,7 +673,9 @@ var GateCheckSchema = z3.discriminatedUnion("kind", [
|
|
|
673
673
|
z3.object({
|
|
674
674
|
kind: z3.literal("markdownHeadingsMatch"),
|
|
675
675
|
file: z3.string().min(1),
|
|
676
|
-
outlineFile: z3.string().min(1)
|
|
676
|
+
outlineFile: z3.string().min(1),
|
|
677
|
+
/** Read outlineFile from artifacts while file uses the check's primary surface. */
|
|
678
|
+
outlineArtifact: z3.boolean().optional()
|
|
677
679
|
}),
|
|
678
680
|
/**
|
|
679
681
|
* Named facts in `file` must come from authorized sources: each fact
|
|
@@ -997,14 +999,16 @@ var CraftbookStepSchema = z5.object({
|
|
|
997
999
|
* Marks the parent step that triggers a declarative per-item fanout
|
|
998
1000
|
* (see {@link CraftbookSpawnSchema}). When this step activates on a
|
|
999
1001
|
* spawn-host task, the runtime reads the craftbook's `spawn.overFile`
|
|
1000
|
-
*
|
|
1002
|
+
* JSON array on its declared surface and spawns one child task per item — no model
|
|
1001
1003
|
* tool call. Inert unless the craftbook also declares `spawn`.
|
|
1002
1004
|
*/
|
|
1003
1005
|
spawnFanout: z5.boolean().optional()
|
|
1004
1006
|
});
|
|
1005
1007
|
var CraftbookSpawnSchema = z5.object({
|
|
1006
|
-
/**
|
|
1008
|
+
/** Surface-relative JSON file the parent produces; its array drives the fanout. */
|
|
1007
1009
|
overFile: z5.string().min(1),
|
|
1010
|
+
/** Read overFile from the artifacts drawer instead of the project workspace. */
|
|
1011
|
+
overArtifact: z5.boolean().optional(),
|
|
1008
1012
|
/** Dotted path to the array inside `overFile`. Absent → the file itself is the array. */
|
|
1009
1013
|
itemsPath: z5.string().optional(),
|
|
1010
1014
|
/** Entry step id of the child template. Defaults to the first `steps` entry. */
|
|
@@ -4392,7 +4396,7 @@ var ProjectSchema = z19.object({
|
|
|
4392
4396
|
/** Optional gezel that acts as the project's voorman (foreman). Surfaces in
|
|
4393
4397
|
* the project detail pane, flows into the system prompt when a session is
|
|
4394
4398
|
* scoped here. For solo projects (`mode === 'solo'`) this same field
|
|
4395
|
-
* holds the project's
|
|
4399
|
+
* holds the project's Builder — the data is unchanged, only the
|
|
4396
4400
|
* label flips. */
|
|
4397
4401
|
voormanGezelId: z19.string().optional(),
|
|
4398
4402
|
/**
|
|
@@ -4442,7 +4446,7 @@ var ProjectSchema = z19.object({
|
|
|
4442
4446
|
/**
|
|
4443
4447
|
* Project shape. `crew` (the default) is the original behavior — the
|
|
4444
4448
|
* voorman recruits and coordinates a team of specialists. `solo` is a
|
|
4445
|
-
* "job" — a single
|
|
4449
|
+
* "job" — a single Builder (stored in
|
|
4446
4450
|
* `voormanGezelId`) handles the whole project themselves; team-
|
|
4447
4451
|
* management MCP tools are stripped from their session, and the
|
|
4448
4452
|
* Meester is instructed not to nominate other gezels. Missing → `crew`
|
|
@@ -4452,7 +4456,7 @@ var ProjectSchema = z19.object({
|
|
|
4452
4456
|
mode: z19.enum(["crew", "solo"]).optional(),
|
|
4453
4457
|
/**
|
|
4454
4458
|
* Optional custom label for this project's lead gezel, overriding the
|
|
4455
|
-
* mode-based default ("Voorman" / "
|
|
4459
|
+
* mode-based default ("Voorman" / "Builder") everywhere the UI
|
|
4456
4460
|
* renders it. Set by a project type at adoption (e.g. checkers →
|
|
4457
4461
|
* "Opponent"); absent → the mode default. The data field stays
|
|
4458
4462
|
* `voormanGezelId` — only the label changes.
|
|
@@ -5389,6 +5393,15 @@ var TaskSchema = z23.object({
|
|
|
5389
5393
|
issueRef: z23.string().regex(/^BW-[1-9]\d*$/),
|
|
5390
5394
|
path: z23.string().min(1)
|
|
5391
5395
|
}),
|
|
5396
|
+
z23.object({
|
|
5397
|
+
/**
|
|
5398
|
+
* An invoke_craftbook call keyed to one persisted root chat turn.
|
|
5399
|
+
* The HTTP task boundary uses this opaque digest to return the first
|
|
5400
|
+
* still-active task when a provider continuation repeats the call.
|
|
5401
|
+
*/
|
|
5402
|
+
kind: z23.literal("craftbook-invocation"),
|
|
5403
|
+
key: z23.string().regex(/^craftbook-root-v1:[a-f0-9]{64}$/)
|
|
5404
|
+
}),
|
|
5392
5405
|
z23.object({
|
|
5393
5406
|
/**
|
|
5394
5407
|
* A host materialized from a gezel template's `suggestedCraftbooks`
|
|
@@ -5418,6 +5431,13 @@ var TaskSchema = z23.object({
|
|
|
5418
5431
|
params: z23.record(z23.string(), z23.unknown()).optional(),
|
|
5419
5432
|
at: z23.string()
|
|
5420
5433
|
}).optional(),
|
|
5434
|
+
/**
|
|
5435
|
+
* Naming presentation inherited from the session that launched this
|
|
5436
|
+
* workflow. Task handoffs create fresh sessions (and may be rehydrated
|
|
5437
|
+
* after restart), so they cannot rely on the launcher's session record
|
|
5438
|
+
* still being available when the next step starts.
|
|
5439
|
+
*/
|
|
5440
|
+
roleBasedNameOnlyMode: z23.boolean().optional(),
|
|
5421
5441
|
createdAt: z23.string(),
|
|
5422
5442
|
updatedAt: z23.string(),
|
|
5423
5443
|
createdBy: TaskAssigneeSchema
|
|
@@ -5472,6 +5492,8 @@ var CreateTaskRequestSchema = z23.object({
|
|
|
5472
5492
|
}).optional(),
|
|
5473
5493
|
fanout: NewTaskFanoutSchema.optional(),
|
|
5474
5494
|
createdBy: TaskAssigneeSchema.optional(),
|
|
5495
|
+
/** Preserve the launcher's naming presentation across task handoffs. */
|
|
5496
|
+
roleBasedNameOnlyMode: z23.boolean().optional(),
|
|
5475
5497
|
/**
|
|
5476
5498
|
* Enqueue the entry-step handoff immediately after create — the
|
|
5477
5499
|
* single-channel kickoff (there is no "tell a gezel about work"
|
|
@@ -5481,7 +5503,13 @@ var CreateTaskRequestSchema = z23.object({
|
|
|
5481
5503
|
* on cron/fanout hosts (their children dispatch via their own
|
|
5482
5504
|
* activation hooks — flag-dispatching the host would double-engage).
|
|
5483
5505
|
*/
|
|
5484
|
-
dispatchEntry: z23.boolean().optional()
|
|
5506
|
+
dispatchEntry: z23.boolean().optional(),
|
|
5507
|
+
/**
|
|
5508
|
+
* Internal invoke_craftbook idempotency digest. The task route converts
|
|
5509
|
+
* it into service-owned Task.origin provenance; ordinary create_task
|
|
5510
|
+
* callers omit it.
|
|
5511
|
+
*/
|
|
5512
|
+
craftbookInvocationKey: z23.string().regex(/^craftbook-root-v1:[a-f0-9]{64}$/).optional()
|
|
5485
5513
|
}).refine((v) => !!v.craftbookId !== !!(v.steps && v.steps.length > 0), {
|
|
5486
5514
|
message: "exactly one of craftbookId or steps must be provided for the main craftbook",
|
|
5487
5515
|
path: ["craftbookId"]
|
|
@@ -6397,7 +6425,7 @@ var ProjectTypeCompositionShape = {
|
|
|
6397
6425
|
tabVisibility: ProjectTabVisibilitySchema.optional(),
|
|
6398
6426
|
/**
|
|
6399
6427
|
* Project shape for instances of this type. `solo` marks a single-gezel
|
|
6400
|
-
* "
|
|
6428
|
+
* "Builder" experience (games, the chat room): the one roster gezel
|
|
6401
6429
|
* does everything, no separate overseer voorman is recruited, and the UI
|
|
6402
6430
|
* collapses the crew picker to that lead. Maps onto the project's `mode`
|
|
6403
6431
|
* at adoption. Omit for the default crew shape.
|
|
@@ -6405,9 +6433,9 @@ var ProjectTypeCompositionShape = {
|
|
|
6405
6433
|
mode: z26.enum(["crew", "solo"]).optional(),
|
|
6406
6434
|
/**
|
|
6407
6435
|
* Custom label for this type's project lead, shown wherever the generic
|
|
6408
|
-
* "Voorman" / "
|
|
6436
|
+
* "Voorman" / "Builder" would appear (the chat chip, project
|
|
6409
6437
|
* settings) — e.g. checkers uses "Opponent". Aimed at solo types that
|
|
6410
|
-
* want a domain word instead of "
|
|
6438
|
+
* want a domain word instead of "Builder". The underlying data field
|
|
6411
6439
|
* stays `voormanGezelId`; only the rendered label changes.
|
|
6412
6440
|
*/
|
|
6413
6441
|
leadLabel: z26.string().min(1).optional(),
|
|
@@ -11250,7 +11278,7 @@ var GezelConfigSchema = z37.object({
|
|
|
11250
11278
|
* crew with granular per-step craftbooks. Right for raw-completion
|
|
11251
11279
|
* local models, where gezel's loop IS the agent.
|
|
11252
11280
|
* - `flat`: route concrete asks to a single generalist (`start_job` →
|
|
11253
|
-
* solo
|
|
11281
|
+
* solo Builder, which already collapses the craftbook onto the
|
|
11254
11282
|
* one specialist). Right for self-orchestrating providers (codex-cli,
|
|
11255
11283
|
* anthropic-cli, copilot) that bring their own agent loop — the crew
|
|
11256
11284
|
* + granular steps are mostly redundant overhead for them (eval data:
|
|
@@ -12154,6 +12182,12 @@ var ListProjectsResponseSchema = z37.object({
|
|
|
12154
12182
|
var CreateProjectRequestSchema = z37.object({
|
|
12155
12183
|
name: z37.string().min(1),
|
|
12156
12184
|
description: z37.string().optional(),
|
|
12185
|
+
/**
|
|
12186
|
+
* Existing local folder to use as the project workspace. Folder-backed
|
|
12187
|
+
* projects are created with ambient Meester progress check-ins disabled;
|
|
12188
|
+
* the user can opt back in from Project Settings.
|
|
12189
|
+
*/
|
|
12190
|
+
workingDir: z37.string().min(1).optional(),
|
|
12157
12191
|
// about/missionObjectives are *encouraged, not required* at the wire level.
|
|
12158
12192
|
// The New Project dialog still enforces the 60/40 richness minimums for the
|
|
12159
12193
|
// blank/GitHub flows (where the user is authoring context from scratch), but
|
|
@@ -12166,7 +12200,7 @@ var CreateProjectRequestSchema = z37.object({
|
|
|
12166
12200
|
missionObjectives: z37.string().optional().describe('Concrete success criteria \u2014 usually a bullet list. What does "done" look like?'),
|
|
12167
12201
|
/**
|
|
12168
12202
|
* Project shape. `crew` (default) → traditional voorman-coordinates-
|
|
12169
|
-
* specialists. `solo` → a "job": one
|
|
12203
|
+
* specialists. `solo` → a "job": one Builder does
|
|
12170
12204
|
* everything themselves; team-management tools are filtered out for
|
|
12171
12205
|
* sessions on this project.
|
|
12172
12206
|
*/
|
|
@@ -23119,6 +23153,55 @@ function firstActionForKind(kind, path) {
|
|
|
23119
23153
|
var scorecard_default = {
|
|
23120
23154
|
schemaVersion: 1,
|
|
23121
23155
|
runs: [
|
|
23156
|
+
{
|
|
23157
|
+
id: "2026-08-13-m4max-llamacpp",
|
|
23158
|
+
provenance: {
|
|
23159
|
+
startedAt: "2026-08-13T07:05:36.620Z",
|
|
23160
|
+
device: {
|
|
23161
|
+
label: "Mac \xB7 Apple M4 Max",
|
|
23162
|
+
platform: "darwin",
|
|
23163
|
+
arch: "arm64",
|
|
23164
|
+
memoryGb: 64,
|
|
23165
|
+
osRelease: "darwin 25.5.0",
|
|
23166
|
+
cpuModel: "Apple M4 Max"
|
|
23167
|
+
},
|
|
23168
|
+
harnessCommit: "c5904085",
|
|
23169
|
+
gildeVersion: "0.1.23",
|
|
23170
|
+
count: 3,
|
|
23171
|
+
judgeModelId: null
|
|
23172
|
+
},
|
|
23173
|
+
suites: ["core", "productivity"],
|
|
23174
|
+
scenariosBySuite: {
|
|
23175
|
+
core: [
|
|
23176
|
+
"tictactoe",
|
|
23177
|
+
"petshop",
|
|
23178
|
+
"tankcombat",
|
|
23179
|
+
"schema-migration",
|
|
23180
|
+
"failing-tests-spec",
|
|
23181
|
+
"symptom-debug",
|
|
23182
|
+
"data-wrangle",
|
|
23183
|
+
"incident-postmortem",
|
|
23184
|
+
"ops-runbook-anomaly",
|
|
23185
|
+
"plan-and-estimate",
|
|
23186
|
+
"conflict-synthesis"
|
|
23187
|
+
],
|
|
23188
|
+
productivity: [
|
|
23189
|
+
"constrained-comms",
|
|
23190
|
+
"craftbook-week-plan",
|
|
23191
|
+
"craftbook-ab-test-readout",
|
|
23192
|
+
"craftbook-annotated-bibliography",
|
|
23193
|
+
"records-intake",
|
|
23194
|
+
"plan-and-estimate",
|
|
23195
|
+
"meeting-followup",
|
|
23196
|
+
"craftbook-spreadsheet-model",
|
|
23197
|
+
"conflict-synthesis",
|
|
23198
|
+
"docblocks-theme-roundtrip",
|
|
23199
|
+
"craftbook-research-to-document",
|
|
23200
|
+
"craftbook-powerpoint-deck",
|
|
23201
|
+
"wikipedia-research-brief"
|
|
23202
|
+
]
|
|
23203
|
+
}
|
|
23204
|
+
},
|
|
23122
23205
|
{
|
|
23123
23206
|
id: "2026-08-11-m4max-llamacpp",
|
|
23124
23207
|
provenance: {
|
|
@@ -23220,6 +23303,878 @@ var scorecard_default = {
|
|
|
23220
23303
|
}
|
|
23221
23304
|
],
|
|
23222
23305
|
results: [
|
|
23306
|
+
{
|
|
23307
|
+
modelId: "gemma4-26b-q4",
|
|
23308
|
+
label: "gemma4-26b-q4",
|
|
23309
|
+
engine: "llama-cpp",
|
|
23310
|
+
tier: "medium",
|
|
23311
|
+
parameterSize: "25.2B",
|
|
23312
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23313
|
+
suiteId: "core",
|
|
23314
|
+
performance: {
|
|
23315
|
+
prefillTokensPerSec: 1264,
|
|
23316
|
+
decodeTokensPerSec: 107.5,
|
|
23317
|
+
samples: 1
|
|
23318
|
+
},
|
|
23319
|
+
runtime: {
|
|
23320
|
+
contextTokens: 120832,
|
|
23321
|
+
peakMemoryMb: 42690
|
|
23322
|
+
},
|
|
23323
|
+
judge: {
|
|
23324
|
+
meanScore: 5.7,
|
|
23325
|
+
artifacts: 9,
|
|
23326
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23327
|
+
},
|
|
23328
|
+
cells: [
|
|
23329
|
+
{
|
|
23330
|
+
scenarioId: "tictactoe",
|
|
23331
|
+
trials: 3,
|
|
23332
|
+
successes: 3,
|
|
23333
|
+
nonModelFailures: 0,
|
|
23334
|
+
medianDurationMs: 127411
|
|
23335
|
+
},
|
|
23336
|
+
{
|
|
23337
|
+
scenarioId: "petshop",
|
|
23338
|
+
trials: 3,
|
|
23339
|
+
successes: 3,
|
|
23340
|
+
nonModelFailures: 0,
|
|
23341
|
+
medianDurationMs: 482159
|
|
23342
|
+
},
|
|
23343
|
+
{
|
|
23344
|
+
scenarioId: "tankcombat",
|
|
23345
|
+
trials: 3,
|
|
23346
|
+
successes: 3,
|
|
23347
|
+
nonModelFailures: 0,
|
|
23348
|
+
medianDurationMs: 290618
|
|
23349
|
+
},
|
|
23350
|
+
{
|
|
23351
|
+
scenarioId: "schema-migration",
|
|
23352
|
+
trials: 3,
|
|
23353
|
+
successes: 3,
|
|
23354
|
+
nonModelFailures: 0,
|
|
23355
|
+
medianDurationMs: 69039
|
|
23356
|
+
},
|
|
23357
|
+
{
|
|
23358
|
+
scenarioId: "failing-tests-spec",
|
|
23359
|
+
trials: 3,
|
|
23360
|
+
successes: 3,
|
|
23361
|
+
nonModelFailures: 0,
|
|
23362
|
+
medianDurationMs: 28770
|
|
23363
|
+
},
|
|
23364
|
+
{
|
|
23365
|
+
scenarioId: "symptom-debug",
|
|
23366
|
+
trials: 3,
|
|
23367
|
+
successes: 3,
|
|
23368
|
+
nonModelFailures: 0,
|
|
23369
|
+
medianDurationMs: 83940
|
|
23370
|
+
},
|
|
23371
|
+
{
|
|
23372
|
+
scenarioId: "data-wrangle",
|
|
23373
|
+
trials: 3,
|
|
23374
|
+
successes: 3,
|
|
23375
|
+
nonModelFailures: 0,
|
|
23376
|
+
medianDurationMs: 48055
|
|
23377
|
+
},
|
|
23378
|
+
{
|
|
23379
|
+
scenarioId: "incident-postmortem",
|
|
23380
|
+
trials: 3,
|
|
23381
|
+
successes: 3,
|
|
23382
|
+
nonModelFailures: 0,
|
|
23383
|
+
medianDurationMs: 189005
|
|
23384
|
+
},
|
|
23385
|
+
{
|
|
23386
|
+
scenarioId: "ops-runbook-anomaly",
|
|
23387
|
+
trials: 3,
|
|
23388
|
+
successes: 3,
|
|
23389
|
+
nonModelFailures: 0,
|
|
23390
|
+
medianDurationMs: 233624
|
|
23391
|
+
},
|
|
23392
|
+
{
|
|
23393
|
+
scenarioId: "plan-and-estimate",
|
|
23394
|
+
trials: 3,
|
|
23395
|
+
successes: 3,
|
|
23396
|
+
nonModelFailures: 0,
|
|
23397
|
+
medianDurationMs: 17679
|
|
23398
|
+
},
|
|
23399
|
+
{
|
|
23400
|
+
scenarioId: "conflict-synthesis",
|
|
23401
|
+
trials: 3,
|
|
23402
|
+
successes: 3,
|
|
23403
|
+
nonModelFailures: 0,
|
|
23404
|
+
medianDurationMs: 52710
|
|
23405
|
+
}
|
|
23406
|
+
]
|
|
23407
|
+
},
|
|
23408
|
+
{
|
|
23409
|
+
modelId: "muse-glimmer-30b-q4",
|
|
23410
|
+
label: "muse-glimmer-30b-q4",
|
|
23411
|
+
engine: "llama-cpp",
|
|
23412
|
+
tier: "medium",
|
|
23413
|
+
parameterSize: "30B",
|
|
23414
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23415
|
+
suiteId: "core",
|
|
23416
|
+
performance: {
|
|
23417
|
+
prefillTokensPerSec: 226,
|
|
23418
|
+
decodeTokensPerSec: 26,
|
|
23419
|
+
samples: 1
|
|
23420
|
+
},
|
|
23421
|
+
runtime: {
|
|
23422
|
+
contextTokens: 131072,
|
|
23423
|
+
peakMemoryMb: 19638
|
|
23424
|
+
},
|
|
23425
|
+
judge: {
|
|
23426
|
+
meanScore: 6.5,
|
|
23427
|
+
artifacts: 9,
|
|
23428
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23429
|
+
},
|
|
23430
|
+
cells: [
|
|
23431
|
+
{
|
|
23432
|
+
scenarioId: "tictactoe",
|
|
23433
|
+
trials: 3,
|
|
23434
|
+
successes: 3,
|
|
23435
|
+
nonModelFailures: 0,
|
|
23436
|
+
medianDurationMs: 210652
|
|
23437
|
+
},
|
|
23438
|
+
{
|
|
23439
|
+
scenarioId: "petshop",
|
|
23440
|
+
trials: 3,
|
|
23441
|
+
successes: 3,
|
|
23442
|
+
nonModelFailures: 0,
|
|
23443
|
+
medianDurationMs: 356601
|
|
23444
|
+
},
|
|
23445
|
+
{
|
|
23446
|
+
scenarioId: "tankcombat",
|
|
23447
|
+
trials: 3,
|
|
23448
|
+
successes: 3,
|
|
23449
|
+
nonModelFailures: 0,
|
|
23450
|
+
medianDurationMs: 272117
|
|
23451
|
+
},
|
|
23452
|
+
{
|
|
23453
|
+
scenarioId: "schema-migration",
|
|
23454
|
+
trials: 3,
|
|
23455
|
+
successes: 3,
|
|
23456
|
+
nonModelFailures: 0,
|
|
23457
|
+
medianDurationMs: 269225
|
|
23458
|
+
},
|
|
23459
|
+
{
|
|
23460
|
+
scenarioId: "failing-tests-spec",
|
|
23461
|
+
trials: 3,
|
|
23462
|
+
successes: 3,
|
|
23463
|
+
nonModelFailures: 0,
|
|
23464
|
+
medianDurationMs: 279796
|
|
23465
|
+
},
|
|
23466
|
+
{
|
|
23467
|
+
scenarioId: "symptom-debug",
|
|
23468
|
+
trials: 3,
|
|
23469
|
+
successes: 3,
|
|
23470
|
+
nonModelFailures: 0,
|
|
23471
|
+
medianDurationMs: 239896
|
|
23472
|
+
},
|
|
23473
|
+
{
|
|
23474
|
+
scenarioId: "data-wrangle",
|
|
23475
|
+
trials: 3,
|
|
23476
|
+
successes: 3,
|
|
23477
|
+
nonModelFailures: 0,
|
|
23478
|
+
medianDurationMs: 563414
|
|
23479
|
+
},
|
|
23480
|
+
{
|
|
23481
|
+
scenarioId: "incident-postmortem",
|
|
23482
|
+
trials: 3,
|
|
23483
|
+
successes: 2,
|
|
23484
|
+
nonModelFailures: 0,
|
|
23485
|
+
medianDurationMs: 4467148
|
|
23486
|
+
},
|
|
23487
|
+
{
|
|
23488
|
+
scenarioId: "ops-runbook-anomaly",
|
|
23489
|
+
trials: 3,
|
|
23490
|
+
successes: 2,
|
|
23491
|
+
nonModelFailures: 0,
|
|
23492
|
+
medianDurationMs: 474336
|
|
23493
|
+
},
|
|
23494
|
+
{
|
|
23495
|
+
scenarioId: "plan-and-estimate",
|
|
23496
|
+
trials: 3,
|
|
23497
|
+
successes: 3,
|
|
23498
|
+
nonModelFailures: 0,
|
|
23499
|
+
medianDurationMs: 150465
|
|
23500
|
+
},
|
|
23501
|
+
{
|
|
23502
|
+
scenarioId: "conflict-synthesis",
|
|
23503
|
+
trials: 3,
|
|
23504
|
+
successes: 2,
|
|
23505
|
+
nonModelFailures: 0,
|
|
23506
|
+
medianDurationMs: 404097
|
|
23507
|
+
}
|
|
23508
|
+
]
|
|
23509
|
+
},
|
|
23510
|
+
{
|
|
23511
|
+
modelId: "nemotron3.5-lightning-30b-q4",
|
|
23512
|
+
label: "nemotron3.5-lightning-30b-q4",
|
|
23513
|
+
engine: "llama-cpp",
|
|
23514
|
+
tier: "medium",
|
|
23515
|
+
parameterSize: "30B",
|
|
23516
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23517
|
+
suiteId: "core",
|
|
23518
|
+
performance: {
|
|
23519
|
+
prefillTokensPerSec: 1030,
|
|
23520
|
+
decodeTokensPerSec: 93.2,
|
|
23521
|
+
samples: 1
|
|
23522
|
+
},
|
|
23523
|
+
runtime: {
|
|
23524
|
+
contextTokens: 1048576,
|
|
23525
|
+
peakMemoryMb: 30085
|
|
23526
|
+
},
|
|
23527
|
+
judge: {
|
|
23528
|
+
meanScore: 5.6,
|
|
23529
|
+
artifacts: 9,
|
|
23530
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23531
|
+
},
|
|
23532
|
+
cells: [
|
|
23533
|
+
{
|
|
23534
|
+
scenarioId: "tictactoe",
|
|
23535
|
+
trials: 3,
|
|
23536
|
+
successes: 3,
|
|
23537
|
+
nonModelFailures: 0,
|
|
23538
|
+
medianDurationMs: 90179
|
|
23539
|
+
},
|
|
23540
|
+
{
|
|
23541
|
+
scenarioId: "petshop",
|
|
23542
|
+
trials: 3,
|
|
23543
|
+
successes: 3,
|
|
23544
|
+
nonModelFailures: 0,
|
|
23545
|
+
medianDurationMs: 899922
|
|
23546
|
+
},
|
|
23547
|
+
{
|
|
23548
|
+
scenarioId: "tankcombat",
|
|
23549
|
+
trials: 3,
|
|
23550
|
+
successes: 3,
|
|
23551
|
+
nonModelFailures: 0,
|
|
23552
|
+
medianDurationMs: 65485
|
|
23553
|
+
},
|
|
23554
|
+
{
|
|
23555
|
+
scenarioId: "schema-migration",
|
|
23556
|
+
trials: 3,
|
|
23557
|
+
successes: 3,
|
|
23558
|
+
nonModelFailures: 0,
|
|
23559
|
+
medianDurationMs: 133904
|
|
23560
|
+
},
|
|
23561
|
+
{
|
|
23562
|
+
scenarioId: "failing-tests-spec",
|
|
23563
|
+
trials: 3,
|
|
23564
|
+
successes: 2,
|
|
23565
|
+
nonModelFailures: 0,
|
|
23566
|
+
medianDurationMs: 230269
|
|
23567
|
+
},
|
|
23568
|
+
{
|
|
23569
|
+
scenarioId: "symptom-debug",
|
|
23570
|
+
trials: 3,
|
|
23571
|
+
successes: 3,
|
|
23572
|
+
nonModelFailures: 0,
|
|
23573
|
+
medianDurationMs: 94023
|
|
23574
|
+
},
|
|
23575
|
+
{
|
|
23576
|
+
scenarioId: "data-wrangle",
|
|
23577
|
+
trials: 3,
|
|
23578
|
+
successes: 2,
|
|
23579
|
+
nonModelFailures: 0,
|
|
23580
|
+
medianDurationMs: 319197
|
|
23581
|
+
},
|
|
23582
|
+
{
|
|
23583
|
+
scenarioId: "incident-postmortem",
|
|
23584
|
+
trials: 3,
|
|
23585
|
+
successes: 1,
|
|
23586
|
+
nonModelFailures: 0,
|
|
23587
|
+
medianDurationMs: 223455
|
|
23588
|
+
},
|
|
23589
|
+
{
|
|
23590
|
+
scenarioId: "ops-runbook-anomaly",
|
|
23591
|
+
trials: 3,
|
|
23592
|
+
successes: 3,
|
|
23593
|
+
nonModelFailures: 0,
|
|
23594
|
+
medianDurationMs: 93099
|
|
23595
|
+
},
|
|
23596
|
+
{
|
|
23597
|
+
scenarioId: "plan-and-estimate",
|
|
23598
|
+
trials: 3,
|
|
23599
|
+
successes: 2,
|
|
23600
|
+
nonModelFailures: 0,
|
|
23601
|
+
medianDurationMs: 68902
|
|
23602
|
+
},
|
|
23603
|
+
{
|
|
23604
|
+
scenarioId: "conflict-synthesis",
|
|
23605
|
+
trials: 3,
|
|
23606
|
+
successes: 1,
|
|
23607
|
+
nonModelFailures: 0,
|
|
23608
|
+
medianDurationMs: 254006
|
|
23609
|
+
}
|
|
23610
|
+
]
|
|
23611
|
+
},
|
|
23612
|
+
{
|
|
23613
|
+
modelId: "qwen3.6-35b-a3b-q4",
|
|
23614
|
+
label: "qwen3.6-35b-a3b-q4",
|
|
23615
|
+
engine: "llama-cpp",
|
|
23616
|
+
tier: "medium",
|
|
23617
|
+
parameterSize: "35B",
|
|
23618
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23619
|
+
suiteId: "core",
|
|
23620
|
+
performance: {
|
|
23621
|
+
prefillTokensPerSec: 1104,
|
|
23622
|
+
decodeTokensPerSec: 71.3,
|
|
23623
|
+
samples: 1
|
|
23624
|
+
},
|
|
23625
|
+
runtime: {
|
|
23626
|
+
contextTokens: 262144,
|
|
23627
|
+
peakMemoryMb: 26502
|
|
23628
|
+
},
|
|
23629
|
+
judge: {
|
|
23630
|
+
meanScore: 8.1,
|
|
23631
|
+
artifacts: 9,
|
|
23632
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23633
|
+
},
|
|
23634
|
+
cells: [
|
|
23635
|
+
{
|
|
23636
|
+
scenarioId: "tictactoe",
|
|
23637
|
+
trials: 3,
|
|
23638
|
+
successes: 3,
|
|
23639
|
+
nonModelFailures: 0,
|
|
23640
|
+
medianDurationMs: 92213
|
|
23641
|
+
},
|
|
23642
|
+
{
|
|
23643
|
+
scenarioId: "petshop",
|
|
23644
|
+
trials: 3,
|
|
23645
|
+
successes: 3,
|
|
23646
|
+
nonModelFailures: 0,
|
|
23647
|
+
medianDurationMs: 162363
|
|
23648
|
+
},
|
|
23649
|
+
{
|
|
23650
|
+
scenarioId: "tankcombat",
|
|
23651
|
+
trials: 3,
|
|
23652
|
+
successes: 3,
|
|
23653
|
+
nonModelFailures: 0,
|
|
23654
|
+
medianDurationMs: 247709
|
|
23655
|
+
},
|
|
23656
|
+
{
|
|
23657
|
+
scenarioId: "schema-migration",
|
|
23658
|
+
trials: 3,
|
|
23659
|
+
successes: 2,
|
|
23660
|
+
nonModelFailures: 0,
|
|
23661
|
+
medianDurationMs: 186639
|
|
23662
|
+
},
|
|
23663
|
+
{
|
|
23664
|
+
scenarioId: "failing-tests-spec",
|
|
23665
|
+
trials: 3,
|
|
23666
|
+
successes: 3,
|
|
23667
|
+
nonModelFailures: 0,
|
|
23668
|
+
medianDurationMs: 28425
|
|
23669
|
+
},
|
|
23670
|
+
{
|
|
23671
|
+
scenarioId: "symptom-debug",
|
|
23672
|
+
trials: 3,
|
|
23673
|
+
successes: 3,
|
|
23674
|
+
nonModelFailures: 0,
|
|
23675
|
+
medianDurationMs: 37627
|
|
23676
|
+
},
|
|
23677
|
+
{
|
|
23678
|
+
scenarioId: "data-wrangle",
|
|
23679
|
+
trials: 3,
|
|
23680
|
+
successes: 2,
|
|
23681
|
+
nonModelFailures: 0,
|
|
23682
|
+
medianDurationMs: 47569
|
|
23683
|
+
},
|
|
23684
|
+
{
|
|
23685
|
+
scenarioId: "incident-postmortem",
|
|
23686
|
+
trials: 3,
|
|
23687
|
+
successes: 3,
|
|
23688
|
+
nonModelFailures: 0,
|
|
23689
|
+
medianDurationMs: 73048
|
|
23690
|
+
},
|
|
23691
|
+
{
|
|
23692
|
+
scenarioId: "ops-runbook-anomaly",
|
|
23693
|
+
trials: 3,
|
|
23694
|
+
successes: 3,
|
|
23695
|
+
nonModelFailures: 0,
|
|
23696
|
+
medianDurationMs: 98132
|
|
23697
|
+
},
|
|
23698
|
+
{
|
|
23699
|
+
scenarioId: "plan-and-estimate",
|
|
23700
|
+
trials: 3,
|
|
23701
|
+
successes: 3,
|
|
23702
|
+
nonModelFailures: 0,
|
|
23703
|
+
medianDurationMs: 27443
|
|
23704
|
+
},
|
|
23705
|
+
{
|
|
23706
|
+
scenarioId: "conflict-synthesis",
|
|
23707
|
+
trials: 3,
|
|
23708
|
+
successes: 2,
|
|
23709
|
+
nonModelFailures: 0,
|
|
23710
|
+
medianDurationMs: 82923
|
|
23711
|
+
}
|
|
23712
|
+
]
|
|
23713
|
+
},
|
|
23714
|
+
{
|
|
23715
|
+
modelId: "gemma4-26b-q4",
|
|
23716
|
+
label: "gemma4-26b-q4",
|
|
23717
|
+
engine: "llama-cpp",
|
|
23718
|
+
tier: "medium",
|
|
23719
|
+
parameterSize: "25.2B",
|
|
23720
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23721
|
+
suiteId: "productivity",
|
|
23722
|
+
performance: {
|
|
23723
|
+
prefillTokensPerSec: 1264,
|
|
23724
|
+
decodeTokensPerSec: 107.5,
|
|
23725
|
+
samples: 1
|
|
23726
|
+
},
|
|
23727
|
+
runtime: {
|
|
23728
|
+
contextTokens: 120832,
|
|
23729
|
+
peakMemoryMb: 41762
|
|
23730
|
+
},
|
|
23731
|
+
judge: {
|
|
23732
|
+
meanScore: 6,
|
|
23733
|
+
artifacts: 19,
|
|
23734
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23735
|
+
},
|
|
23736
|
+
cells: [
|
|
23737
|
+
{
|
|
23738
|
+
scenarioId: "constrained-comms",
|
|
23739
|
+
trials: 3,
|
|
23740
|
+
successes: 3,
|
|
23741
|
+
nonModelFailures: 0,
|
|
23742
|
+
medianDurationMs: 22693
|
|
23743
|
+
},
|
|
23744
|
+
{
|
|
23745
|
+
scenarioId: "craftbook-week-plan",
|
|
23746
|
+
trials: 3,
|
|
23747
|
+
successes: 3,
|
|
23748
|
+
nonModelFailures: 0,
|
|
23749
|
+
medianDurationMs: 58001
|
|
23750
|
+
},
|
|
23751
|
+
{
|
|
23752
|
+
scenarioId: "craftbook-ab-test-readout",
|
|
23753
|
+
trials: 3,
|
|
23754
|
+
successes: 0,
|
|
23755
|
+
nonModelFailures: 0,
|
|
23756
|
+
medianDurationMs: 196546
|
|
23757
|
+
},
|
|
23758
|
+
{
|
|
23759
|
+
scenarioId: "craftbook-annotated-bibliography",
|
|
23760
|
+
trials: 3,
|
|
23761
|
+
successes: 3,
|
|
23762
|
+
nonModelFailures: 0,
|
|
23763
|
+
medianDurationMs: 33179
|
|
23764
|
+
},
|
|
23765
|
+
{
|
|
23766
|
+
scenarioId: "records-intake",
|
|
23767
|
+
trials: 3,
|
|
23768
|
+
successes: 3,
|
|
23769
|
+
nonModelFailures: 0,
|
|
23770
|
+
medianDurationMs: 58377
|
|
23771
|
+
},
|
|
23772
|
+
{
|
|
23773
|
+
scenarioId: "plan-and-estimate",
|
|
23774
|
+
trials: 3,
|
|
23775
|
+
successes: 3,
|
|
23776
|
+
nonModelFailures: 0,
|
|
23777
|
+
medianDurationMs: 17469
|
|
23778
|
+
},
|
|
23779
|
+
{
|
|
23780
|
+
scenarioId: "meeting-followup",
|
|
23781
|
+
trials: 3,
|
|
23782
|
+
successes: 3,
|
|
23783
|
+
nonModelFailures: 0,
|
|
23784
|
+
medianDurationMs: 48022
|
|
23785
|
+
},
|
|
23786
|
+
{
|
|
23787
|
+
scenarioId: "craftbook-spreadsheet-model",
|
|
23788
|
+
trials: 3,
|
|
23789
|
+
successes: 0,
|
|
23790
|
+
nonModelFailures: 0,
|
|
23791
|
+
medianDurationMs: 191950
|
|
23792
|
+
},
|
|
23793
|
+
{
|
|
23794
|
+
scenarioId: "conflict-synthesis",
|
|
23795
|
+
trials: 3,
|
|
23796
|
+
successes: 3,
|
|
23797
|
+
nonModelFailures: 0,
|
|
23798
|
+
medianDurationMs: 48088
|
|
23799
|
+
},
|
|
23800
|
+
{
|
|
23801
|
+
scenarioId: "docblocks-theme-roundtrip",
|
|
23802
|
+
trials: 3,
|
|
23803
|
+
successes: 3,
|
|
23804
|
+
nonModelFailures: 0,
|
|
23805
|
+
medianDurationMs: 27505
|
|
23806
|
+
},
|
|
23807
|
+
{
|
|
23808
|
+
scenarioId: "craftbook-research-to-document",
|
|
23809
|
+
trials: 3,
|
|
23810
|
+
successes: 3,
|
|
23811
|
+
nonModelFailures: 0,
|
|
23812
|
+
medianDurationMs: 32944
|
|
23813
|
+
},
|
|
23814
|
+
{
|
|
23815
|
+
scenarioId: "craftbook-powerpoint-deck",
|
|
23816
|
+
trials: 3,
|
|
23817
|
+
successes: 0,
|
|
23818
|
+
nonModelFailures: 0,
|
|
23819
|
+
medianDurationMs: 937028
|
|
23820
|
+
},
|
|
23821
|
+
{
|
|
23822
|
+
scenarioId: "wikipedia-research-brief",
|
|
23823
|
+
trials: 3,
|
|
23824
|
+
successes: 1,
|
|
23825
|
+
nonModelFailures: 0,
|
|
23826
|
+
medianDurationMs: 81272
|
|
23827
|
+
}
|
|
23828
|
+
]
|
|
23829
|
+
},
|
|
23830
|
+
{
|
|
23831
|
+
modelId: "muse-glimmer-30b-q4",
|
|
23832
|
+
label: "muse-glimmer-30b-q4",
|
|
23833
|
+
engine: "llama-cpp",
|
|
23834
|
+
tier: "medium",
|
|
23835
|
+
parameterSize: "30B",
|
|
23836
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23837
|
+
suiteId: "productivity",
|
|
23838
|
+
performance: {
|
|
23839
|
+
prefillTokensPerSec: 228,
|
|
23840
|
+
decodeTokensPerSec: 26,
|
|
23841
|
+
samples: 2
|
|
23842
|
+
},
|
|
23843
|
+
runtime: {
|
|
23844
|
+
contextTokens: 131072,
|
|
23845
|
+
peakMemoryMb: 19648
|
|
23846
|
+
},
|
|
23847
|
+
judge: {
|
|
23848
|
+
meanScore: 5.5,
|
|
23849
|
+
artifacts: 25,
|
|
23850
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23851
|
+
},
|
|
23852
|
+
cells: [
|
|
23853
|
+
{
|
|
23854
|
+
scenarioId: "constrained-comms",
|
|
23855
|
+
trials: 3,
|
|
23856
|
+
successes: 3,
|
|
23857
|
+
nonModelFailures: 0,
|
|
23858
|
+
medianDurationMs: 168777
|
|
23859
|
+
},
|
|
23860
|
+
{
|
|
23861
|
+
scenarioId: "craftbook-week-plan",
|
|
23862
|
+
trials: 3,
|
|
23863
|
+
successes: 0,
|
|
23864
|
+
nonModelFailures: 0,
|
|
23865
|
+
medianDurationMs: 635096
|
|
23866
|
+
},
|
|
23867
|
+
{
|
|
23868
|
+
scenarioId: "craftbook-ab-test-readout",
|
|
23869
|
+
trials: 3,
|
|
23870
|
+
successes: 1,
|
|
23871
|
+
nonModelFailures: 0,
|
|
23872
|
+
medianDurationMs: 505316
|
|
23873
|
+
},
|
|
23874
|
+
{
|
|
23875
|
+
scenarioId: "craftbook-annotated-bibliography",
|
|
23876
|
+
trials: 3,
|
|
23877
|
+
successes: 3,
|
|
23878
|
+
nonModelFailures: 0,
|
|
23879
|
+
medianDurationMs: 153322
|
|
23880
|
+
},
|
|
23881
|
+
{
|
|
23882
|
+
scenarioId: "records-intake",
|
|
23883
|
+
trials: 3,
|
|
23884
|
+
successes: 2,
|
|
23885
|
+
nonModelFailures: 0,
|
|
23886
|
+
medianDurationMs: 2593201
|
|
23887
|
+
},
|
|
23888
|
+
{
|
|
23889
|
+
scenarioId: "plan-and-estimate",
|
|
23890
|
+
trials: 3,
|
|
23891
|
+
successes: 3,
|
|
23892
|
+
nonModelFailures: 0,
|
|
23893
|
+
medianDurationMs: 185158
|
|
23894
|
+
},
|
|
23895
|
+
{
|
|
23896
|
+
scenarioId: "meeting-followup",
|
|
23897
|
+
trials: 3,
|
|
23898
|
+
successes: 2,
|
|
23899
|
+
nonModelFailures: 0,
|
|
23900
|
+
medianDurationMs: 680419
|
|
23901
|
+
},
|
|
23902
|
+
{
|
|
23903
|
+
scenarioId: "craftbook-spreadsheet-model",
|
|
23904
|
+
trials: 3,
|
|
23905
|
+
successes: 0,
|
|
23906
|
+
nonModelFailures: 0,
|
|
23907
|
+
medianDurationMs: 541672
|
|
23908
|
+
},
|
|
23909
|
+
{
|
|
23910
|
+
scenarioId: "conflict-synthesis",
|
|
23911
|
+
trials: 3,
|
|
23912
|
+
successes: 2,
|
|
23913
|
+
nonModelFailures: 0,
|
|
23914
|
+
medianDurationMs: 344063
|
|
23915
|
+
},
|
|
23916
|
+
{
|
|
23917
|
+
scenarioId: "docblocks-theme-roundtrip",
|
|
23918
|
+
trials: 3,
|
|
23919
|
+
successes: 3,
|
|
23920
|
+
nonModelFailures: 0,
|
|
23921
|
+
medianDurationMs: 248997
|
|
23922
|
+
},
|
|
23923
|
+
{
|
|
23924
|
+
scenarioId: "craftbook-research-to-document",
|
|
23925
|
+
trials: 3,
|
|
23926
|
+
successes: 3,
|
|
23927
|
+
nonModelFailures: 0,
|
|
23928
|
+
medianDurationMs: 274010
|
|
23929
|
+
},
|
|
23930
|
+
{
|
|
23931
|
+
scenarioId: "craftbook-powerpoint-deck",
|
|
23932
|
+
trials: 3,
|
|
23933
|
+
successes: 0,
|
|
23934
|
+
nonModelFailures: 0,
|
|
23935
|
+
medianDurationMs: 3009529
|
|
23936
|
+
},
|
|
23937
|
+
{
|
|
23938
|
+
scenarioId: "wikipedia-research-brief",
|
|
23939
|
+
trials: 3,
|
|
23940
|
+
successes: 3,
|
|
23941
|
+
nonModelFailures: 0,
|
|
23942
|
+
medianDurationMs: 353289
|
|
23943
|
+
}
|
|
23944
|
+
]
|
|
23945
|
+
},
|
|
23946
|
+
{
|
|
23947
|
+
modelId: "nemotron3.5-lightning-30b-q4",
|
|
23948
|
+
label: "nemotron3.5-lightning-30b-q4",
|
|
23949
|
+
engine: "llama-cpp",
|
|
23950
|
+
tier: "medium",
|
|
23951
|
+
parameterSize: "30B",
|
|
23952
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
23953
|
+
suiteId: "productivity",
|
|
23954
|
+
performance: {
|
|
23955
|
+
prefillTokensPerSec: 1030,
|
|
23956
|
+
decodeTokensPerSec: 93.2,
|
|
23957
|
+
samples: 1
|
|
23958
|
+
},
|
|
23959
|
+
runtime: {
|
|
23960
|
+
contextTokens: 1048576,
|
|
23961
|
+
peakMemoryMb: 29452
|
|
23962
|
+
},
|
|
23963
|
+
judge: {
|
|
23964
|
+
meanScore: 5.6,
|
|
23965
|
+
artifacts: 27,
|
|
23966
|
+
judgeModel: "claude-sonnet-4-6"
|
|
23967
|
+
},
|
|
23968
|
+
cells: [
|
|
23969
|
+
{
|
|
23970
|
+
scenarioId: "constrained-comms",
|
|
23971
|
+
trials: 3,
|
|
23972
|
+
successes: 0,
|
|
23973
|
+
nonModelFailures: 0,
|
|
23974
|
+
medianDurationMs: 91133
|
|
23975
|
+
},
|
|
23976
|
+
{
|
|
23977
|
+
scenarioId: "craftbook-week-plan",
|
|
23978
|
+
trials: 3,
|
|
23979
|
+
successes: 3,
|
|
23980
|
+
nonModelFailures: 0,
|
|
23981
|
+
medianDurationMs: 113088
|
|
23982
|
+
},
|
|
23983
|
+
{
|
|
23984
|
+
scenarioId: "craftbook-ab-test-readout",
|
|
23985
|
+
trials: 3,
|
|
23986
|
+
successes: 3,
|
|
23987
|
+
nonModelFailures: 0,
|
|
23988
|
+
medianDurationMs: 283896
|
|
23989
|
+
},
|
|
23990
|
+
{
|
|
23991
|
+
scenarioId: "craftbook-annotated-bibliography",
|
|
23992
|
+
trials: 3,
|
|
23993
|
+
successes: 3,
|
|
23994
|
+
nonModelFailures: 0,
|
|
23995
|
+
medianDurationMs: 73049
|
|
23996
|
+
},
|
|
23997
|
+
{
|
|
23998
|
+
scenarioId: "records-intake",
|
|
23999
|
+
trials: 3,
|
|
24000
|
+
successes: 1,
|
|
24001
|
+
nonModelFailures: 0,
|
|
24002
|
+
medianDurationMs: 395444
|
|
24003
|
+
},
|
|
24004
|
+
{
|
|
24005
|
+
scenarioId: "plan-and-estimate",
|
|
24006
|
+
trials: 3,
|
|
24007
|
+
successes: 1,
|
|
24008
|
+
nonModelFailures: 0,
|
|
24009
|
+
medianDurationMs: 105986
|
|
24010
|
+
},
|
|
24011
|
+
{
|
|
24012
|
+
scenarioId: "meeting-followup",
|
|
24013
|
+
trials: 3,
|
|
24014
|
+
successes: 3,
|
|
24015
|
+
nonModelFailures: 0,
|
|
24016
|
+
medianDurationMs: 113144
|
|
24017
|
+
},
|
|
24018
|
+
{
|
|
24019
|
+
scenarioId: "craftbook-spreadsheet-model",
|
|
24020
|
+
trials: 3,
|
|
24021
|
+
successes: 3,
|
|
24022
|
+
nonModelFailures: 0,
|
|
24023
|
+
medianDurationMs: 233654
|
|
24024
|
+
},
|
|
24025
|
+
{
|
|
24026
|
+
scenarioId: "conflict-synthesis",
|
|
24027
|
+
trials: 3,
|
|
24028
|
+
successes: 1,
|
|
24029
|
+
nonModelFailures: 0,
|
|
24030
|
+
medianDurationMs: 469600
|
|
24031
|
+
},
|
|
24032
|
+
{
|
|
24033
|
+
scenarioId: "docblocks-theme-roundtrip",
|
|
24034
|
+
trials: 3,
|
|
24035
|
+
successes: 1,
|
|
24036
|
+
nonModelFailures: 0,
|
|
24037
|
+
medianDurationMs: 109277
|
|
24038
|
+
},
|
|
24039
|
+
{
|
|
24040
|
+
scenarioId: "craftbook-research-to-document",
|
|
24041
|
+
trials: 3,
|
|
24042
|
+
successes: 3,
|
|
24043
|
+
nonModelFailures: 0,
|
|
24044
|
+
medianDurationMs: 130992
|
|
24045
|
+
},
|
|
24046
|
+
{
|
|
24047
|
+
scenarioId: "craftbook-powerpoint-deck",
|
|
24048
|
+
trials: 3,
|
|
24049
|
+
successes: 0,
|
|
24050
|
+
nonModelFailures: 0,
|
|
24051
|
+
medianDurationMs: 791082
|
|
24052
|
+
},
|
|
24053
|
+
{
|
|
24054
|
+
scenarioId: "wikipedia-research-brief",
|
|
24055
|
+
trials: 3,
|
|
24056
|
+
successes: 1,
|
|
24057
|
+
nonModelFailures: 0,
|
|
24058
|
+
medianDurationMs: 136065
|
|
24059
|
+
}
|
|
24060
|
+
]
|
|
24061
|
+
},
|
|
24062
|
+
{
|
|
24063
|
+
modelId: "qwen3.6-35b-a3b-q4",
|
|
24064
|
+
label: "qwen3.6-35b-a3b-q4",
|
|
24065
|
+
engine: "llama-cpp",
|
|
24066
|
+
tier: "medium",
|
|
24067
|
+
parameterSize: "35B",
|
|
24068
|
+
runId: "2026-08-13-m4max-llamacpp",
|
|
24069
|
+
suiteId: "productivity",
|
|
24070
|
+
performance: {
|
|
24071
|
+
prefillTokensPerSec: 1104,
|
|
24072
|
+
decodeTokensPerSec: 71.3,
|
|
24073
|
+
samples: 1
|
|
24074
|
+
},
|
|
24075
|
+
runtime: {
|
|
24076
|
+
contextTokens: 262144,
|
|
24077
|
+
peakMemoryMb: 25657
|
|
24078
|
+
},
|
|
24079
|
+
judge: {
|
|
24080
|
+
meanScore: 6.6,
|
|
24081
|
+
artifacts: 27,
|
|
24082
|
+
judgeModel: "claude-sonnet-4-6"
|
|
24083
|
+
},
|
|
24084
|
+
cells: [
|
|
24085
|
+
{
|
|
24086
|
+
scenarioId: "constrained-comms",
|
|
24087
|
+
trials: 3,
|
|
24088
|
+
successes: 3,
|
|
24089
|
+
nonModelFailures: 0,
|
|
24090
|
+
medianDurationMs: 22314
|
|
24091
|
+
},
|
|
24092
|
+
{
|
|
24093
|
+
scenarioId: "craftbook-week-plan",
|
|
24094
|
+
trials: 3,
|
|
24095
|
+
successes: 3,
|
|
24096
|
+
nonModelFailures: 0,
|
|
24097
|
+
medianDurationMs: 88382
|
|
24098
|
+
},
|
|
24099
|
+
{
|
|
24100
|
+
scenarioId: "craftbook-ab-test-readout",
|
|
24101
|
+
trials: 3,
|
|
24102
|
+
successes: 3,
|
|
24103
|
+
nonModelFailures: 0,
|
|
24104
|
+
medianDurationMs: 303967
|
|
24105
|
+
},
|
|
24106
|
+
{
|
|
24107
|
+
scenarioId: "craftbook-annotated-bibliography",
|
|
24108
|
+
trials: 3,
|
|
24109
|
+
successes: 3,
|
|
24110
|
+
nonModelFailures: 0,
|
|
24111
|
+
medianDurationMs: 47530
|
|
24112
|
+
},
|
|
24113
|
+
{
|
|
24114
|
+
scenarioId: "records-intake",
|
|
24115
|
+
trials: 3,
|
|
24116
|
+
successes: 3,
|
|
24117
|
+
nonModelFailures: 0,
|
|
24118
|
+
medianDurationMs: 22365
|
|
24119
|
+
},
|
|
24120
|
+
{
|
|
24121
|
+
scenarioId: "plan-and-estimate",
|
|
24122
|
+
trials: 3,
|
|
24123
|
+
successes: 3,
|
|
24124
|
+
nonModelFailures: 0,
|
|
24125
|
+
medianDurationMs: 32491
|
|
24126
|
+
},
|
|
24127
|
+
{
|
|
24128
|
+
scenarioId: "meeting-followup",
|
|
24129
|
+
trials: 3,
|
|
24130
|
+
successes: 3,
|
|
24131
|
+
nonModelFailures: 0,
|
|
24132
|
+
medianDurationMs: 89751
|
|
24133
|
+
},
|
|
24134
|
+
{
|
|
24135
|
+
scenarioId: "craftbook-spreadsheet-model",
|
|
24136
|
+
trials: 3,
|
|
24137
|
+
successes: 3,
|
|
24138
|
+
nonModelFailures: 0,
|
|
24139
|
+
medianDurationMs: 228470
|
|
24140
|
+
},
|
|
24141
|
+
{
|
|
24142
|
+
scenarioId: "conflict-synthesis",
|
|
24143
|
+
trials: 3,
|
|
24144
|
+
successes: 1,
|
|
24145
|
+
nonModelFailures: 0,
|
|
24146
|
+
medianDurationMs: 211329
|
|
24147
|
+
},
|
|
24148
|
+
{
|
|
24149
|
+
scenarioId: "docblocks-theme-roundtrip",
|
|
24150
|
+
trials: 3,
|
|
24151
|
+
successes: 3,
|
|
24152
|
+
nonModelFailures: 0,
|
|
24153
|
+
medianDurationMs: 17446
|
|
24154
|
+
},
|
|
24155
|
+
{
|
|
24156
|
+
scenarioId: "craftbook-research-to-document",
|
|
24157
|
+
trials: 3,
|
|
24158
|
+
successes: 3,
|
|
24159
|
+
nonModelFailures: 0,
|
|
24160
|
+
medianDurationMs: 22407
|
|
24161
|
+
},
|
|
24162
|
+
{
|
|
24163
|
+
scenarioId: "craftbook-powerpoint-deck",
|
|
24164
|
+
trials: 3,
|
|
24165
|
+
successes: 1,
|
|
24166
|
+
nonModelFailures: 0,
|
|
24167
|
+
medianDurationMs: 1493404
|
|
24168
|
+
},
|
|
24169
|
+
{
|
|
24170
|
+
scenarioId: "wikipedia-research-brief",
|
|
24171
|
+
trials: 3,
|
|
24172
|
+
successes: 1,
|
|
24173
|
+
nonModelFailures: 0,
|
|
24174
|
+
medianDurationMs: 125222
|
|
24175
|
+
}
|
|
24176
|
+
]
|
|
24177
|
+
},
|
|
23223
24178
|
{
|
|
23224
24179
|
modelId: "gemma4-26b-q4",
|
|
23225
24180
|
label: "gemma4-26b-q4",
|
|
@@ -27094,7 +28049,7 @@ function visibleCatalogItems(items, showWorkInProgressFeatures, connectorIds = c
|
|
|
27094
28049
|
}
|
|
27095
28050
|
|
|
27096
28051
|
// src/index.ts
|
|
27097
|
-
var GEZEL_VERSION = "1.0.
|
|
28052
|
+
var GEZEL_VERSION = "1.0.4";
|
|
27098
28053
|
var GEZEL_CONTENT_COMPAT = "1.26226";
|
|
27099
28054
|
function nowIso() {
|
|
27100
28055
|
return (/* @__PURE__ */ new Date()).toISOString();
|