chofex-cli 0.1.173 → 0.1.177
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/index.js +421 -31
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -24,6 +24,11 @@ chofex challenge notebook
|
|
|
24
24
|
chofex challenge test --source ./shipping.js
|
|
25
25
|
chofex challenge evaluate --source ./shipping.js
|
|
26
26
|
chofex challenge ranking
|
|
27
|
+
chofex challenge init --challenge broken-agent
|
|
28
|
+
cd broken-agent && npm test
|
|
29
|
+
chofex challenge test --challenge broken-agent --source ./scheduler.js
|
|
30
|
+
chofex challenge evaluate --challenge broken-agent --source ./scheduler.js
|
|
31
|
+
chofex challenge ranking --challenge broken-agent
|
|
27
32
|
chofex confirm
|
|
28
33
|
```
|
|
29
34
|
|
|
@@ -34,7 +39,7 @@ to the latest published version.
|
|
|
34
39
|
optional phone number, bio, portfolio URL, shipped project, LinkedIn and GitHub
|
|
35
40
|
URLs, and Terms and Conditions. Use `--input` to submit a completed JSON
|
|
36
41
|
application.
|
|
37
|
-
The
|
|
42
|
+
The technical challenges do not block submission, but every challenge winner must submit
|
|
38
43
|
an application before being accepted. Top results receive a direct pass; other
|
|
39
44
|
scores give organizers another review metric.
|
|
40
45
|
|
package/dist/index.js
CHANGED
|
@@ -58552,6 +58552,7 @@ var runWith2 = (command, config) => {
|
|
|
58552
58552
|
};
|
|
58553
58553
|
// ../../packages/challenges-contract/src/index.ts
|
|
58554
58554
|
var blackBoxChallengeSlug = "black-box";
|
|
58555
|
+
var brokenAgentChallengeSlug = "broken-agent";
|
|
58555
58556
|
var challengeCatalog = [
|
|
58556
58557
|
{
|
|
58557
58558
|
slug: blackBoxChallengeSlug,
|
|
@@ -58573,21 +58574,21 @@ var challengeCatalog = [
|
|
|
58573
58574
|
solutionKind: "javascript_source"
|
|
58574
58575
|
},
|
|
58575
58576
|
{
|
|
58576
|
-
slug:
|
|
58577
|
+
slug: brokenAgentChallengeSlug,
|
|
58577
58578
|
number: 2,
|
|
58578
58579
|
code: "02",
|
|
58579
|
-
theme: "
|
|
58580
|
-
title: "The
|
|
58581
|
-
summary: "
|
|
58582
|
-
coreSkill: "
|
|
58583
|
-
format: "
|
|
58584
|
-
formatLabel: "
|
|
58580
|
+
theme: "Broken Agent",
|
|
58581
|
+
title: "The Scheduler",
|
|
58582
|
+
summary: "Take a plausible AI-generated job scheduler whose public tests already pass and harden it against production reality.",
|
|
58583
|
+
coreSkill: "Production correctness & reliability",
|
|
58584
|
+
format: "accuracy",
|
|
58585
|
+
formatLabel: "Production readiness score",
|
|
58585
58586
|
opensAt: "2026-09-25T05:00:00.000Z",
|
|
58586
58587
|
queryLimit: 0,
|
|
58587
|
-
evaluationLimit:
|
|
58588
|
-
hiddenSampleSize:
|
|
58589
|
-
playable:
|
|
58590
|
-
solutionKind: "
|
|
58588
|
+
evaluationLimit: 5,
|
|
58589
|
+
hiddenSampleSize: 100,
|
|
58590
|
+
playable: true,
|
|
58591
|
+
solutionKind: "javascript_source"
|
|
58591
58592
|
},
|
|
58592
58593
|
{
|
|
58593
58594
|
slug: "make-it-fast",
|
|
@@ -58690,7 +58691,7 @@ var formatChallengeOpeningInPeru = (opensAt) => {
|
|
|
58690
58691
|
var challengeOpeningNotice = (title, opensAt) => `${title} opens ${formatChallengeOpeningInPeru(opensAt)}. Queries and evaluations are disabled until then; no attempts will be consumed.`;
|
|
58691
58692
|
var ChallengeSlugSchema = exports_Schema.Literals([
|
|
58692
58693
|
"black-box",
|
|
58693
|
-
"
|
|
58694
|
+
"broken-agent",
|
|
58694
58695
|
"make-it-fast",
|
|
58695
58696
|
"power-grid",
|
|
58696
58697
|
"agent-arena"
|
|
@@ -58718,7 +58719,38 @@ var ChallengeScoreSchema = exports_Schema.Struct({
|
|
|
58718
58719
|
sampleSize: exports_Schema.Number,
|
|
58719
58720
|
meanError: exports_Schema.Number,
|
|
58720
58721
|
queriesUsed: exports_Schema.Number,
|
|
58721
|
-
runtimeMs: exports_Schema.Number
|
|
58722
|
+
runtimeMs: exports_Schema.Number,
|
|
58723
|
+
evaluationsUsed: exports_Schema.optional(exports_Schema.Number),
|
|
58724
|
+
breakdown: exports_Schema.optional(exports_Schema.Struct({
|
|
58725
|
+
coreBehavior: exports_Schema.Struct({
|
|
58726
|
+
earned: exports_Schema.Number,
|
|
58727
|
+
available: exports_Schema.Number
|
|
58728
|
+
}),
|
|
58729
|
+
persistence: exports_Schema.Struct({
|
|
58730
|
+
earned: exports_Schema.Number,
|
|
58731
|
+
available: exports_Schema.Number
|
|
58732
|
+
}),
|
|
58733
|
+
concurrency: exports_Schema.Struct({
|
|
58734
|
+
earned: exports_Schema.Number,
|
|
58735
|
+
available: exports_Schema.Number
|
|
58736
|
+
}),
|
|
58737
|
+
failureRecovery: exports_Schema.Struct({
|
|
58738
|
+
earned: exports_Schema.Number,
|
|
58739
|
+
available: exports_Schema.Number
|
|
58740
|
+
}),
|
|
58741
|
+
idempotency: exports_Schema.Struct({
|
|
58742
|
+
earned: exports_Schema.Number,
|
|
58743
|
+
available: exports_Schema.Number
|
|
58744
|
+
}),
|
|
58745
|
+
regressionSafety: exports_Schema.Struct({
|
|
58746
|
+
earned: exports_Schema.Number,
|
|
58747
|
+
available: exports_Schema.Number
|
|
58748
|
+
}),
|
|
58749
|
+
performance: exports_Schema.Struct({
|
|
58750
|
+
earned: exports_Schema.Number,
|
|
58751
|
+
available: exports_Schema.Number
|
|
58752
|
+
})
|
|
58753
|
+
}))
|
|
58722
58754
|
});
|
|
58723
58755
|
var ChallengeProgressStatus = exports_Schema.Literals([
|
|
58724
58756
|
"not_started",
|
|
@@ -58785,7 +58817,8 @@ var ChallengeEvaluationSummarySchema = exports_Schema.Struct({
|
|
|
58785
58817
|
shareCode: exports_Schema.String,
|
|
58786
58818
|
rank: exports_Schema.optional(exports_Schema.Number),
|
|
58787
58819
|
percentile: exports_Schema.optional(exports_Schema.Number),
|
|
58788
|
-
createdAt: exports_Schema.String
|
|
58820
|
+
createdAt: exports_Schema.String,
|
|
58821
|
+
breakdown: exports_Schema.optional(ChallengeScoreSchema.fields.breakdown)
|
|
58789
58822
|
});
|
|
58790
58823
|
var ChallengeAttemptViewSchema = exports_Schema.Struct({
|
|
58791
58824
|
challenge: ChallengeCatalogItemSchema,
|
|
@@ -58802,6 +58835,7 @@ var ChallengeQueryResultSchema = exports_Schema.Struct({
|
|
|
58802
58835
|
queriesLimit: exports_Schema.Number
|
|
58803
58836
|
});
|
|
58804
58837
|
var ChallengeLocalTestResultSchema = exports_Schema.Struct({
|
|
58838
|
+
kind: exports_Schema.optional(exports_Schema.Literals(["black_box", "broken_agent"])),
|
|
58805
58839
|
matchedObservations: exports_Schema.Number,
|
|
58806
58840
|
observationCount: exports_Schema.Number,
|
|
58807
58841
|
accuracy: exports_Schema.Number,
|
|
@@ -58827,7 +58861,8 @@ var ChallengeEvaluationResultSchema = exports_Schema.Struct({
|
|
|
58827
58861
|
evaluationsRemaining: exports_Schema.Number,
|
|
58828
58862
|
evaluationsLimit: exports_Schema.Number,
|
|
58829
58863
|
rankingPath: exports_Schema.String,
|
|
58830
|
-
shareText: exports_Schema.String
|
|
58864
|
+
shareText: exports_Schema.String,
|
|
58865
|
+
breakdown: exports_Schema.optional(ChallengeScoreSchema.fields.breakdown)
|
|
58831
58866
|
});
|
|
58832
58867
|
var ChallengeRankingEntrySchema = exports_Schema.Struct({
|
|
58833
58868
|
rank: exports_Schema.Number,
|
|
@@ -59709,6 +59744,7 @@ var getChallengeRanking = (options, slug) => publicRequest(options, `/api/v1/cha
|
|
|
59709
59744
|
|
|
59710
59745
|
// src/challenge-input.ts
|
|
59711
59746
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
59747
|
+
var defaultChallengeSlug = "black-box";
|
|
59712
59748
|
var readStdin = async () => {
|
|
59713
59749
|
const chunks = [];
|
|
59714
59750
|
for await (const chunk of process.stdin) {
|
|
@@ -59724,9 +59760,12 @@ var readTextFile = (path4) => exports_Effect.tryPromise({
|
|
|
59724
59760
|
},
|
|
59725
59761
|
catch: (error2) => cliError("INVALID_INPUT_FILE", `Could not read ${path4}: ${String(error2)}`)
|
|
59726
59762
|
});
|
|
59727
|
-
var javascriptSourceFromPath = (path4) => {
|
|
59763
|
+
var javascriptSourceFromPath = (path4, challenge = defaultChallengeSlug) => {
|
|
59728
59764
|
if (!path4) {
|
|
59729
|
-
|
|
59765
|
+
let expectedFunction = "calculateShipping(input)";
|
|
59766
|
+
if (challenge === "broken-agent")
|
|
59767
|
+
expectedFunction = "createScheduler(dependencies)";
|
|
59768
|
+
return exports_Effect.fail(cliError("SOURCE_REQUIRED", `Pass --source <file.js> with function ${expectedFunction}`));
|
|
59730
59769
|
}
|
|
59731
59770
|
return readTextFile(path4).pipe(exports_Effect.map((source) => ({ kind: "javascript_source", source })));
|
|
59732
59771
|
};
|
|
@@ -59778,7 +59817,6 @@ var shipmentInput = (path4, flags) => {
|
|
|
59778
59817
|
}
|
|
59779
59818
|
return interactiveShipment();
|
|
59780
59819
|
};
|
|
59781
|
-
var defaultChallengeSlug = "black-box";
|
|
59782
59820
|
|
|
59783
59821
|
// src/brand.ts
|
|
59784
59822
|
var eventName = "Hack the Andes";
|
|
@@ -59834,6 +59872,32 @@ var challengeShowText = (attempt) => {
|
|
|
59834
59872
|
const { challenge, progress } = attempt;
|
|
59835
59873
|
const queriesRemaining = progress.queriesLimit - progress.queriesUsed;
|
|
59836
59874
|
const evaluationsRemaining = progress.evaluationsLimit - progress.evaluationsUsed;
|
|
59875
|
+
if (challenge.slug === "broken-agent") {
|
|
59876
|
+
let caseStatus2 = "READY TO AUDIT";
|
|
59877
|
+
if (attempt.latestEvaluation)
|
|
59878
|
+
caseStatus2 = "EVALUATED";
|
|
59879
|
+
const lines3 = [
|
|
59880
|
+
`${challenge.title.toUpperCase()} — CASE #${challenge.code}`,
|
|
59881
|
+
challenge.summary,
|
|
59882
|
+
"",
|
|
59883
|
+
`CASE STATUS: ${caseStatus2}`,
|
|
59884
|
+
"",
|
|
59885
|
+
"The public tests are green. Your job is to make the scheduler trustworthy under production conditions.",
|
|
59886
|
+
"AI tools are allowed.",
|
|
59887
|
+
"",
|
|
59888
|
+
`Evaluations ${evaluationsRemaining} / ${progress.evaluationsLimit} remaining`
|
|
59889
|
+
];
|
|
59890
|
+
if (progress.bestAccuracy !== undefined) {
|
|
59891
|
+
lines3.push("", `Best score ${percent(progress.bestAccuracy)}`);
|
|
59892
|
+
if (progress.rank !== undefined)
|
|
59893
|
+
lines3.push(`Rank #${progress.rank}`);
|
|
59894
|
+
if (progress.shareCode)
|
|
59895
|
+
lines3.push(`Share code #${progress.shareCode}`);
|
|
59896
|
+
}
|
|
59897
|
+
lines3.push("", "YOUR NEXT MOVE", "Read the contract, audit scheduler.js, and keep the visible suite green.", " cd broken-agent && npm test", " chofex challenge test --challenge broken-agent --source ./broken-agent/scheduler.js");
|
|
59898
|
+
return lines3.join(`
|
|
59899
|
+
`);
|
|
59900
|
+
}
|
|
59837
59901
|
let evidence = `${attempt.observations.length} observations`;
|
|
59838
59902
|
if (attempt.observations.length === 1)
|
|
59839
59903
|
evidence = "1 observation";
|
|
@@ -59996,6 +60060,26 @@ var notebookCsvText = (observations) => {
|
|
|
59996
60060
|
`);
|
|
59997
60061
|
};
|
|
59998
60062
|
var challengeTestText = (result3) => {
|
|
60063
|
+
if (result3.kind === "broken_agent") {
|
|
60064
|
+
const lines3 = [
|
|
60065
|
+
"PUBLIC SUITE — BROKEN AGENT",
|
|
60066
|
+
`${result3.matchedObservations} / ${result3.observationCount} visible behaviors passing`
|
|
60067
|
+
];
|
|
60068
|
+
if (result3.accuracy === 1) {
|
|
60069
|
+
lines3.push("", "Everything passes.", "Unfortunately, that does not mean the scheduler is production-correct.", "Reason about concurrency, restarts, failures, and idempotency before evaluating.", "", "Next: spend one official evaluation when you would ship it", " chofex challenge evaluate --challenge broken-agent --source ./broken-agent/scheduler.js");
|
|
60070
|
+
return lines3.join(`
|
|
60071
|
+
`);
|
|
60072
|
+
}
|
|
60073
|
+
if (result3.mismatches.length > 0) {
|
|
60074
|
+
lines3.push("", "Visible failures:");
|
|
60075
|
+
for (const mismatch of result3.mismatches) {
|
|
60076
|
+
lines3.push(` ${String(mismatch.actual)}`);
|
|
60077
|
+
}
|
|
60078
|
+
}
|
|
60079
|
+
lines3.push("", "Fix the public regressions before using an official evaluation.");
|
|
60080
|
+
return lines3.join(`
|
|
60081
|
+
`);
|
|
60082
|
+
}
|
|
59999
60083
|
let verdict = "KEEP WORKING";
|
|
60000
60084
|
if (result3.accuracy === 1)
|
|
60001
60085
|
verdict = "NOTEBOOK MATCHED";
|
|
@@ -60022,6 +60106,34 @@ var challengeTestText = (result3) => {
|
|
|
60022
60106
|
`);
|
|
60023
60107
|
};
|
|
60024
60108
|
var challengeEvaluateText = (result3) => {
|
|
60109
|
+
if (result3.breakdown) {
|
|
60110
|
+
const capabilityLines = [
|
|
60111
|
+
["Core behavior", result3.breakdown.coreBehavior],
|
|
60112
|
+
["Persistence", result3.breakdown.persistence],
|
|
60113
|
+
["Concurrency", result3.breakdown.concurrency],
|
|
60114
|
+
["Failure recovery", result3.breakdown.failureRecovery],
|
|
60115
|
+
["Idempotency", result3.breakdown.idempotency],
|
|
60116
|
+
["Regression safety", result3.breakdown.regressionSafety],
|
|
60117
|
+
["Performance", result3.breakdown.performance]
|
|
60118
|
+
];
|
|
60119
|
+
const lines3 = [
|
|
60120
|
+
"OFFICIAL VERDICT — BROKEN AGENT — PRODUCTION READINESS",
|
|
60121
|
+
`Score ${(result3.accuracy * 100).toFixed(2)} / ${result3.sampleSize}`,
|
|
60122
|
+
`Runtime ${result3.runtimeMs} ms`,
|
|
60123
|
+
"",
|
|
60124
|
+
"Capability scores",
|
|
60125
|
+
...capabilityLines.map(([label, score]) => `${label.padEnd(20)}${score.earned.toFixed(2)} / ${score.available}`)
|
|
60126
|
+
];
|
|
60127
|
+
if (result3.rank !== undefined) {
|
|
60128
|
+
lines3.push(`Rank #${result3.rank}`);
|
|
60129
|
+
}
|
|
60130
|
+
lines3.push(`Official evaluations remaining ${result3.evaluationsRemaining} / ${result3.evaluationsLimit}`, "", result3.shareText, "", "Next: inspect the leaderboard", " chofex challenge ranking --challenge broken-agent");
|
|
60131
|
+
if (result3.evaluationsRemaining > 0) {
|
|
60132
|
+
lines3.push("", "The evaluator reports capabilities, not individual hidden cases. Reason before submitting again.", " chofex challenge test --challenge broken-agent --source ./broken-agent/scheduler.js");
|
|
60133
|
+
}
|
|
60134
|
+
return lines3.join(`
|
|
60135
|
+
`);
|
|
60136
|
+
}
|
|
60025
60137
|
const lines2 = [
|
|
60026
60138
|
"OFFICIAL VERDICT — BLACK BOX REPLICATION",
|
|
60027
60139
|
`Accuracy ${percent(result3.accuracy)}`,
|
|
@@ -60050,6 +60162,18 @@ var challengeRankingText = (ranking, now3 = new Date) => {
|
|
|
60050
60162
|
if (ranking.entries.length === 0) {
|
|
60051
60163
|
lines2.push("No official evaluations yet.");
|
|
60052
60164
|
return lines2.join(`
|
|
60165
|
+
`);
|
|
60166
|
+
}
|
|
60167
|
+
if (ranking.challenge.slug === "broken-agent") {
|
|
60168
|
+
lines2.push("Rank Score Points Runtime Name");
|
|
60169
|
+
for (const entry of ranking.entries.slice(0, 20)) {
|
|
60170
|
+
const rank = String(entry.rank).padStart(4, " ");
|
|
60171
|
+
const accuracy = percent(entry.accuracy).padStart(8, " ");
|
|
60172
|
+
const points = `${entry.exactCount}/${entry.sampleSize}`.padStart(11, " ");
|
|
60173
|
+
const runtime = `${entry.runtimeMs}ms`.padStart(8, " ");
|
|
60174
|
+
lines2.push(`${rank} ${accuracy} ${points} ${runtime} ${entry.displayName} #${entry.shareCode}`);
|
|
60175
|
+
}
|
|
60176
|
+
return lines2.join(`
|
|
60053
60177
|
`);
|
|
60054
60178
|
}
|
|
60055
60179
|
lines2.push("Rank Accuracy Exact Queries Name");
|
|
@@ -60086,7 +60210,216 @@ var registrationPartsText = (result3) => {
|
|
|
60086
60210
|
};
|
|
60087
60211
|
|
|
60088
60212
|
// src/challenge-scaffold.ts
|
|
60089
|
-
import { writeFile as writeFile3 } from "node:fs/promises";
|
|
60213
|
+
import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
|
|
60214
|
+
|
|
60215
|
+
// ../../packages/challenges-contract/src/broken-agent.ts
|
|
60216
|
+
var brokenAgentScaffoldDirectory = "broken-agent";
|
|
60217
|
+
var brokenAgentSolutionPath = `${brokenAgentScaffoldDirectory}/scheduler.js`;
|
|
60218
|
+
var brokenAgentPackageJson = `${JSON.stringify({
|
|
60219
|
+
name: "broken-agent-challenge",
|
|
60220
|
+
private: true,
|
|
60221
|
+
type: "commonjs",
|
|
60222
|
+
scripts: { test: "node --test scheduler.test.js" }
|
|
60223
|
+
}, null, 2)}
|
|
60224
|
+
`;
|
|
60225
|
+
var brokenAgentStarterSource = `/**
|
|
60226
|
+
* Broken Agent — The Scheduler
|
|
60227
|
+
*
|
|
60228
|
+
* A coding agent implemented this scheduler and declared the feature complete.
|
|
60229
|
+
* Every public test passes. The implementation is not production-correct.
|
|
60230
|
+
* Keep the createScheduler interface intact and harden the implementation.
|
|
60231
|
+
*/
|
|
60232
|
+
function createScheduler({ store, clock, execute }) {
|
|
60233
|
+
return {
|
|
60234
|
+
async schedule(input) {
|
|
60235
|
+
const job = { ...input, status: "pending" };
|
|
60236
|
+
await store.transaction((tx) => tx.put(job));
|
|
60237
|
+
return job;
|
|
60238
|
+
},
|
|
60239
|
+
|
|
60240
|
+
async cancel(id) {
|
|
60241
|
+
return store.transaction((tx) => {
|
|
60242
|
+
const job = tx.get(id);
|
|
60243
|
+
if (!job || job.status !== "pending") return false;
|
|
60244
|
+
tx.put({ ...job, status: "cancelled" });
|
|
60245
|
+
return true;
|
|
60246
|
+
});
|
|
60247
|
+
},
|
|
60248
|
+
|
|
60249
|
+
async list() {
|
|
60250
|
+
return store.transaction((tx) => tx.list());
|
|
60251
|
+
},
|
|
60252
|
+
|
|
60253
|
+
async runDue() {
|
|
60254
|
+
const now = clock.now().getTime();
|
|
60255
|
+
const jobs = await store.transaction((tx) => tx.list());
|
|
60256
|
+
for (const job of jobs) {
|
|
60257
|
+
if (job.status !== "pending" || Date.parse(job.runAt) > now) continue;
|
|
60258
|
+
await execute(job);
|
|
60259
|
+
await store.transaction((tx) =>
|
|
60260
|
+
tx.put({ ...job, status: "completed" }),
|
|
60261
|
+
);
|
|
60262
|
+
}
|
|
60263
|
+
},
|
|
60264
|
+
};
|
|
60265
|
+
}
|
|
60266
|
+
|
|
60267
|
+
module.exports = { createScheduler };
|
|
60268
|
+
`;
|
|
60269
|
+
var brokenAgentReadme = `# Broken Agent — The Scheduler
|
|
60270
|
+
|
|
60271
|
+
> A coding agent was asked to implement a feature. It says it is done. It is
|
|
60272
|
+
> not. Your job is to ship it.
|
|
60273
|
+
|
|
60274
|
+
The repository starts with all public tests passing. Repair \`scheduler.js\`
|
|
60275
|
+
without changing its exported \`createScheduler(dependencies)\` interface.
|
|
60276
|
+
|
|
60277
|
+
## Commands
|
|
60278
|
+
|
|
60279
|
+
\`\`\`sh
|
|
60280
|
+
npm test
|
|
60281
|
+
chofex challenge test --challenge broken-agent --source ./broken-agent/scheduler.js
|
|
60282
|
+
chofex challenge evaluate --challenge broken-agent --source ./broken-agent/scheduler.js
|
|
60283
|
+
\`\`\`
|
|
60284
|
+
|
|
60285
|
+
Local and public tests are unlimited. You have **5 official evaluations**
|
|
60286
|
+
against hidden production scenarios. AI tools are explicitly allowed.
|
|
60287
|
+
|
|
60288
|
+
## Public contract
|
|
60289
|
+
|
|
60290
|
+
\`createScheduler({ store, clock, execute, workerId })\` returns:
|
|
60291
|
+
|
|
60292
|
+
- \`schedule({ id, runAt, payload })\`
|
|
60293
|
+
- \`cancel(id)\`
|
|
60294
|
+
- \`list()\`
|
|
60295
|
+
- \`runDue()\`
|
|
60296
|
+
|
|
60297
|
+
### Jobs
|
|
60298
|
+
|
|
60299
|
+
- \`id\` matches \`^[A-Za-z0-9_-]{1,64}$\`.
|
|
60300
|
+
- \`runAt\` is an ISO-8601 timestamp with an explicit timezone.
|
|
60301
|
+
- \`payload\` is JSON-serializable.
|
|
60302
|
+
- Scheduling an identical job ID is idempotent. Reusing an ID for different
|
|
60303
|
+
work must reject without changing the original job.
|
|
60304
|
+
- Past timestamps are valid and become due immediately.
|
|
60305
|
+
- \`list()\` returns jobs ordered by \`runAt\`, then \`id\`.
|
|
60306
|
+
|
|
60307
|
+
### Durability and concurrency
|
|
60308
|
+
|
|
60309
|
+
- The supplied store survives scheduler instances and process restarts.
|
|
60310
|
+
- Several workers may call \`runDue()\` concurrently against the same store.
|
|
60311
|
+
- A worker claim lasts 30 seconds. Another worker may reclaim the job when
|
|
60312
|
+
\`clock.now()\` reaches that lease deadline.
|
|
60313
|
+
- A pending job may be cancelled. A running or terminal job may not.
|
|
60314
|
+
- Failed executions retry, without preventing other due jobs from running.
|
|
60315
|
+
- A job becomes \`failed\` after 3 unsuccessful attempts.
|
|
60316
|
+
- The execution adapter applies effects idempotently by job ID. A worker may
|
|
60317
|
+
stop after the effect is applied but before completion is recorded; retrying
|
|
60318
|
+
must eventually complete the job without duplicating the effect.
|
|
60319
|
+
|
|
60320
|
+
The store exposes only \`transaction(action)\`. Inside an action, \`tx\`
|
|
60321
|
+
provides synchronous \`get(id)\`, \`put(job)\`, \`delete(id)\`, and \`list()\`
|
|
60322
|
+
operations. Transactions are serialized. Do not keep a transaction open while
|
|
60323
|
+
executing a job.
|
|
60324
|
+
|
|
60325
|
+
\`clock.now()\` returns a \`Date\`. \`execute(job)\` returns a promise.
|
|
60326
|
+
|
|
60327
|
+
## Evaluation
|
|
60328
|
+
|
|
60329
|
+
The hidden evaluator checks the published contract under deterministic
|
|
60330
|
+
concurrency, restart, failure, idempotency, compatibility, and load scenarios.
|
|
60331
|
+
It reports capability-level scores, not individual hidden-test failures.
|
|
60332
|
+
|
|
60333
|
+
- Core behavior: 10 points
|
|
60334
|
+
- Persistence: 15 points
|
|
60335
|
+
- Concurrency: 20 points
|
|
60336
|
+
- Failure recovery: 20 points
|
|
60337
|
+
- Idempotency: 15 points
|
|
60338
|
+
- Regression safety: 15 points
|
|
60339
|
+
- Performance: 5 points, available only when the load jobs complete correctly
|
|
60340
|
+
|
|
60341
|
+
Ranking order is total score, fewer official evaluations used, runtime, then
|
|
60342
|
+
official submission time. Submission time is only the final tie-breaker.
|
|
60343
|
+
`;
|
|
60344
|
+
var brokenAgentPublicTestSource = `const test = require("node:test");
|
|
60345
|
+
const assert = require("node:assert/strict");
|
|
60346
|
+
const { createScheduler } = require("./scheduler.js");
|
|
60347
|
+
|
|
60348
|
+
const clone = (value) => {
|
|
60349
|
+
if (value === undefined) return undefined;
|
|
60350
|
+
return JSON.parse(JSON.stringify(value));
|
|
60351
|
+
};
|
|
60352
|
+
|
|
60353
|
+
class MemoryStore {
|
|
60354
|
+
constructor() {
|
|
60355
|
+
this.jobs = new Map();
|
|
60356
|
+
this.tail = Promise.resolve();
|
|
60357
|
+
}
|
|
60358
|
+
|
|
60359
|
+
transaction(action) {
|
|
60360
|
+
const run = this.tail.then(() => action({
|
|
60361
|
+
get: (id) => clone(this.jobs.get(id)),
|
|
60362
|
+
put: (job) => {
|
|
60363
|
+
this.jobs.set(job.id, clone(job));
|
|
60364
|
+
return clone(job);
|
|
60365
|
+
},
|
|
60366
|
+
delete: (id) => this.jobs.delete(id),
|
|
60367
|
+
list: () => [...this.jobs.values()].map(clone),
|
|
60368
|
+
}));
|
|
60369
|
+
this.tail = run.then(() => undefined, () => undefined);
|
|
60370
|
+
return run;
|
|
60371
|
+
}
|
|
60372
|
+
}
|
|
60373
|
+
|
|
60374
|
+
const setup = () => {
|
|
60375
|
+
const store = new MemoryStore();
|
|
60376
|
+
const now = new Date("2026-10-17T15:00:00.000Z");
|
|
60377
|
+
const executed = [];
|
|
60378
|
+
const scheduler = createScheduler({
|
|
60379
|
+
store,
|
|
60380
|
+
clock: { now: () => new Date(now) },
|
|
60381
|
+
execute: async (job) => executed.push(job.id),
|
|
60382
|
+
workerId: "public-worker",
|
|
60383
|
+
});
|
|
60384
|
+
return { store, scheduler, executed };
|
|
60385
|
+
};
|
|
60386
|
+
|
|
60387
|
+
const job = (id, runAt = "2026-10-17T15:00:00.000Z") => ({
|
|
60388
|
+
id,
|
|
60389
|
+
runAt,
|
|
60390
|
+
payload: { message: id },
|
|
60391
|
+
});
|
|
60392
|
+
|
|
60393
|
+
test("schedules and lists a job", async () => {
|
|
60394
|
+
const { scheduler } = setup();
|
|
60395
|
+
await scheduler.schedule(job("job-1"));
|
|
60396
|
+
assert.equal((await scheduler.list()).length, 1);
|
|
60397
|
+
});
|
|
60398
|
+
|
|
60399
|
+
test("executes a due job", async () => {
|
|
60400
|
+
const { scheduler, executed } = setup();
|
|
60401
|
+
await scheduler.schedule(job("job-1"));
|
|
60402
|
+
await scheduler.runDue();
|
|
60403
|
+
assert.deepEqual(executed, ["job-1"]);
|
|
60404
|
+
});
|
|
60405
|
+
|
|
60406
|
+
test("does not execute a future job", async () => {
|
|
60407
|
+
const { scheduler, executed } = setup();
|
|
60408
|
+
await scheduler.schedule(job("future", "2026-10-17T16:00:00.000Z"));
|
|
60409
|
+
await scheduler.runDue();
|
|
60410
|
+
assert.deepEqual(executed, []);
|
|
60411
|
+
});
|
|
60412
|
+
|
|
60413
|
+
test("cancels a pending job", async () => {
|
|
60414
|
+
const { scheduler, executed } = setup();
|
|
60415
|
+
await scheduler.schedule(job("cancel-me"));
|
|
60416
|
+
assert.equal(await scheduler.cancel("cancel-me"), true);
|
|
60417
|
+
await scheduler.runDue();
|
|
60418
|
+
assert.deepEqual(executed, []);
|
|
60419
|
+
});
|
|
60420
|
+
`;
|
|
60421
|
+
|
|
60422
|
+
// src/challenge-scaffold.ts
|
|
60090
60423
|
var solutionPath = "shipping.js";
|
|
60091
60424
|
var starterSource = `/**
|
|
60092
60425
|
* The Shipping Machine
|
|
@@ -60107,7 +60440,36 @@ function calculateShipping(input) {
|
|
|
60107
60440
|
return 0;
|
|
60108
60441
|
}
|
|
60109
60442
|
`;
|
|
60110
|
-
var createChallengeScaffold = exports_Effect.fn("createChallengeScaffold")(function* () {
|
|
60443
|
+
var createChallengeScaffold = exports_Effect.fn("createChallengeScaffold")(function* (challengeSlug = "black-box") {
|
|
60444
|
+
const challenge = challengeBySlug(challengeSlug);
|
|
60445
|
+
if (!challenge?.playable) {
|
|
60446
|
+
return yield* exports_Effect.fail(cliError("CHALLENGE_NOT_AVAILABLE", `Challenge ${challengeSlug} is not available`));
|
|
60447
|
+
}
|
|
60448
|
+
if (challengeSlug === "broken-agent") {
|
|
60449
|
+
const status2 = yield* exports_Effect.tryPromise({
|
|
60450
|
+
try: async () => {
|
|
60451
|
+
await mkdir2(brokenAgentScaffoldDirectory);
|
|
60452
|
+
await Promise.all([
|
|
60453
|
+
writeFile3(brokenAgentSolutionPath, brokenAgentStarterSource, "utf8"),
|
|
60454
|
+
writeFile3(`${brokenAgentScaffoldDirectory}/scheduler.test.js`, brokenAgentPublicTestSource, "utf8"),
|
|
60455
|
+
writeFile3(`${brokenAgentScaffoldDirectory}/README.md`, brokenAgentReadme, "utf8"),
|
|
60456
|
+
writeFile3(`${brokenAgentScaffoldDirectory}/package.json`, brokenAgentPackageJson, "utf8")
|
|
60457
|
+
]);
|
|
60458
|
+
return "created";
|
|
60459
|
+
},
|
|
60460
|
+
catch: (error2) => error2
|
|
60461
|
+
}).pipe(exports_Effect.catch((error2) => {
|
|
60462
|
+
if (exports_Predicate.hasProperty(error2, "code") && error2.code === "EEXIST") {
|
|
60463
|
+
return exports_Effect.succeed("exists");
|
|
60464
|
+
}
|
|
60465
|
+
return exports_Effect.fail(cliError("SCAFFOLD_FAILED", `Could not create ${brokenAgentScaffoldDirectory}: ${String(error2)}`));
|
|
60466
|
+
}));
|
|
60467
|
+
return {
|
|
60468
|
+
challenge: challengeSlug,
|
|
60469
|
+
path: brokenAgentScaffoldDirectory,
|
|
60470
|
+
status: status2
|
|
60471
|
+
};
|
|
60472
|
+
}
|
|
60111
60473
|
const status = yield* exports_Effect.tryPromise({
|
|
60112
60474
|
try: async () => {
|
|
60113
60475
|
await writeFile3(solutionPath, starterSource, {
|
|
@@ -60123,7 +60485,7 @@ var createChallengeScaffold = exports_Effect.fn("createChallengeScaffold")(funct
|
|
|
60123
60485
|
}
|
|
60124
60486
|
return exports_Effect.fail(cliError("SCAFFOLD_FAILED", `Could not create ${solutionPath}: ${String(error2)}`));
|
|
60125
60487
|
}));
|
|
60126
|
-
return { path: solutionPath, status };
|
|
60488
|
+
return { challenge: challengeSlug, path: solutionPath, status };
|
|
60127
60489
|
});
|
|
60128
60490
|
|
|
60129
60491
|
// src/cli-root.ts
|
|
@@ -60364,7 +60726,7 @@ var challengeQuickstartText = (participationState, launchNotice) => {
|
|
|
60364
60726
|
};
|
|
60365
60727
|
var optionalString = (name, description) => exports_Flag.string(name).pipe(exports_Flag.optional, exports_Flag.withDescription(description));
|
|
60366
60728
|
var challengeFlag = exports_Flag.string("challenge").pipe(exports_Flag.withDefault(defaultChallengeSlug), exports_Flag.withDescription("Challenge slug (default: black-box)"));
|
|
60367
|
-
var sourceFlag = optionalString("source", "JavaScript
|
|
60729
|
+
var sourceFlag = optionalString("source", "JavaScript challenge solution file");
|
|
60368
60730
|
var inputFlag = optionalString("input", "JSON file, or - for stdin");
|
|
60369
60731
|
var formatFlag = exports_Flag.choice("format", ["table", "json", "csv"]).pipe(exports_Flag.withDefault("table"), exports_Flag.withDescription("Notebook format"));
|
|
60370
60732
|
var numberFromOption = (value3) => {
|
|
@@ -60381,6 +60743,30 @@ var booleanFromOption = (value3) => {
|
|
|
60381
60743
|
return false;
|
|
60382
60744
|
};
|
|
60383
60745
|
var challengeInitText = (result3) => {
|
|
60746
|
+
if (result3.challenge === "broken-agent") {
|
|
60747
|
+
if (result3.status === "exists") {
|
|
60748
|
+
return [
|
|
60749
|
+
`${result3.path} already exists. Left it unchanged.`,
|
|
60750
|
+
"",
|
|
60751
|
+
"Next: keep hardening your scheduler, then run the public tests",
|
|
60752
|
+
` cd ${result3.path} && npm test`,
|
|
60753
|
+
` chofex challenge test --challenge broken-agent --source ./${result3.path}/scheduler.js`
|
|
60754
|
+
].join(`
|
|
60755
|
+
`);
|
|
60756
|
+
}
|
|
60757
|
+
return [
|
|
60758
|
+
`Created ${result3.path}`,
|
|
60759
|
+
"The previous agent says the scheduler is done. Everything passes.",
|
|
60760
|
+
"Unfortunately, the agent was wrong.",
|
|
60761
|
+
"",
|
|
60762
|
+
"Start with the public contract and tests:",
|
|
60763
|
+
` cd ${result3.path} && npm test`,
|
|
60764
|
+
"",
|
|
60765
|
+
"Then repair scheduler.js and test it safely:",
|
|
60766
|
+
` chofex challenge test --challenge broken-agent --source ./${result3.path}/scheduler.js`
|
|
60767
|
+
].join(`
|
|
60768
|
+
`);
|
|
60769
|
+
}
|
|
60384
60770
|
if (result3.status === "exists") {
|
|
60385
60771
|
return [
|
|
60386
60772
|
`${result3.path} already exists. Left it unchanged.`,
|
|
@@ -60412,28 +60798,32 @@ var listCommand = exports_Command.make("list", {}, exports_Effect.fn("challengeL
|
|
|
60412
60798
|
description: "See which challenge is currently playable"
|
|
60413
60799
|
}
|
|
60414
60800
|
]));
|
|
60415
|
-
var initCommand = exports_Command.make("init", {}, exports_Effect.fn("challengeInitCommand")(function* () {
|
|
60801
|
+
var initCommand = exports_Command.make("init", { challenge: challengeFlag }, exports_Effect.fn("challengeInitCommand")(function* ({ challenge }) {
|
|
60416
60802
|
const options = yield* root;
|
|
60417
|
-
const participationState = participationStateFor(
|
|
60803
|
+
const participationState = participationStateFor(challenge);
|
|
60418
60804
|
if (participationState !== "open") {
|
|
60419
|
-
const message = launchNoticeFor(
|
|
60805
|
+
const message = launchNoticeFor(challenge) ?? "El challenge todavía no está disponible.";
|
|
60420
60806
|
let code = "CHALLENGE_NOT_OPEN";
|
|
60421
60807
|
if (participationState === "closed")
|
|
60422
60808
|
code = "CHALLENGE_CLOSED";
|
|
60423
60809
|
yield* execute(options.output, exports_Effect.fail(cliError(code, message)), challengeInitText);
|
|
60424
60810
|
return;
|
|
60425
60811
|
}
|
|
60426
|
-
const operation = createChallengeScaffold().pipe(exports_Effect.map((data) => ({
|
|
60812
|
+
const operation = createChallengeScaffold(challenge).pipe(exports_Effect.map((data) => ({
|
|
60427
60813
|
version: 1,
|
|
60428
60814
|
ok: true,
|
|
60429
60815
|
requestId: crypto.randomUUID(),
|
|
60430
60816
|
data
|
|
60431
60817
|
})));
|
|
60432
60818
|
yield* execute(options.output, operation, challengeInitText);
|
|
60433
|
-
})).pipe(exports_Command.withDescription("Create a
|
|
60819
|
+
})).pipe(exports_Command.withDescription("Create a challenge starter without overwriting existing work."), exports_Command.withExamples([
|
|
60434
60820
|
{
|
|
60435
60821
|
command: "chofex challenge init",
|
|
60436
60822
|
description: "Prepare a safe JavaScript solution file"
|
|
60823
|
+
},
|
|
60824
|
+
{
|
|
60825
|
+
command: "chofex challenge init --challenge broken-agent",
|
|
60826
|
+
description: "Prepare the Broken Agent starter repository"
|
|
60437
60827
|
}
|
|
60438
60828
|
]));
|
|
60439
60829
|
var showCommand = exports_Command.make("show", { challenge: challengeFlag }, exports_Effect.fn("challengeShowCommand")(function* ({ challenge }) {
|
|
@@ -60512,11 +60902,11 @@ var testCommand = exports_Command.make("test", { challenge: challengeFlag, sourc
|
|
|
60512
60902
|
const options = yield* root;
|
|
60513
60903
|
const token = exports_Option.getOrUndefined(options.token);
|
|
60514
60904
|
const operation = exports_Effect.gen(function* () {
|
|
60515
|
-
const solution = yield* javascriptSourceFromPath(exports_Option.getOrUndefined(source));
|
|
60905
|
+
const solution = yield* javascriptSourceFromPath(exports_Option.getOrUndefined(source), challenge);
|
|
60516
60906
|
return yield* testChallenge({ apiUrl: options.apiUrl, token }, challenge, solution);
|
|
60517
60907
|
});
|
|
60518
60908
|
yield* execute(options.output, operation, challengeTestText);
|
|
60519
|
-
})).pipe(exports_Command.withDescription("
|
|
60909
|
+
})).pipe(exports_Command.withDescription("Run the challenge's public tests. Safe to repeat; official evaluations are not consumed. Requires sign-in."), exports_Command.withExamples([
|
|
60520
60910
|
{
|
|
60521
60911
|
command: "chofex challenge test --source ./shipping.js",
|
|
60522
60912
|
description: "Test a JavaScript replacement against your notebook"
|
|
@@ -60526,11 +60916,11 @@ var evaluateCommand = exports_Command.make("evaluate", { challenge: challengeFla
|
|
|
60526
60916
|
const options = yield* root;
|
|
60527
60917
|
const token = exports_Option.getOrUndefined(options.token);
|
|
60528
60918
|
const operation = exports_Effect.gen(function* () {
|
|
60529
|
-
const solution = yield* javascriptSourceFromPath(exports_Option.getOrUndefined(source));
|
|
60919
|
+
const solution = yield* javascriptSourceFromPath(exports_Option.getOrUndefined(source), challenge);
|
|
60530
60920
|
return yield* evaluateChallenge({ apiUrl: options.apiUrl, token }, challenge, solution);
|
|
60531
60921
|
});
|
|
60532
60922
|
yield* execute(options.output, operation, challengeEvaluateText);
|
|
60533
|
-
})).pipe(exports_Command.withDescription("Score
|
|
60923
|
+
})).pipe(exports_Command.withDescription("Score a solution on hidden cases. This consumes one limited official evaluation. Requires sign-in."), exports_Command.withExamples([
|
|
60534
60924
|
{
|
|
60535
60925
|
command: "chofex challenge evaluate --source ./shipping.js",
|
|
60536
60926
|
description: "Spend one official evaluation when your solution is ready"
|