icoa-cli 2.19.712 → 2.19.713

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- import{createHash as e}from"node:crypto";import{appendFileSync as t,existsSync as s,mkdirSync as n,readdirSync as r,readFileSync as a,statSync as o,writeFileSync as i}from"node:fs";import{basename as c,join as l,resolve as u}from"node:path";import{gzipSync as d}from"node:zlib";import chalk from"chalk";import{renderPanel as h}from"../lib/guide-panel.js";import{logCommand as f}from"../lib/logger.js";import{parseIpynb as m,serializeIpynb as b}from"../lib/notebook-doc.js";import{BASELINE_FILE as p,INLINE_LIMIT_BYTES as g,LOWER_IS_BETTER as w,METRICS as v,TASK_FILES as k,baselineCells as y,capsulePlan as $,checkTask as S,parseTaskMeta as B,scoreSubmission as j,studentBundle as _,tarArchive as z,validName as A,verdictLines as x}from"../lib/setter.js";import{getCliVersion as G}from"../lib/version.js";const N=(e="")=>console.log(e),O=e=>console.log(chalk.red(` ✗ ${e}`)),C=e=>console.log(chalk.green(` ✓ ${e}`)),J=e=>console.log(chalk.gray(` ${e}`)),P=(e,t)=>console.log(chalk.gray(" ▸ next, type: ")+chalk.bold.green(e)+chalk.gray(` ${t}`));function F(e){if(e){const t=u(e);return s(l(t,"task.json"))?t:null}const t=process.cwd();if(s(l(t,"task.json")))return t;try{const e=r(t).filter(e=>!e.startsWith(".")&&s(l(t,e,"task.json")));if(1===e.length)return l(t,e[0])}catch{}return null}function M(e,t){const n=l(e,t);if(s(n)){if(o(n).size>419430400)throw new Error(`${t} is over 400 MB — too large to check here; tell the organiser, this one goes through the dataset bucket`);return a(n,"utf-8")}}function q(e){const{meta:t,problems:s}=B(M(e,"task.json")??"");for(const e of s)O(e);return t}function L(e,t){return S(t,{train:M(e,"train.csv"),testA:M(e,"test_a.csv"),labelsA:M(e,"labels_a.csv"),testB:M(e,"test_b.csv"),labelsB:M(e,"labels_b.csv")})}function T(e,s,r){try{n(l(e,".setter"),{recursive:!0}),t(l(e,".setter","ledger.jsonl"),`${JSON.stringify({at:(new Date).toISOString(),cli:G(),action:s,...r})}\n`)}catch{}}function U(e){try{return a(l(e,".setter","ledger.jsonl"),"utf-8").split("\n").filter(e=>e.trim()).map(e=>JSON.parse(e))}catch{return[]}}const D=e=>Number.isFinite(e)?e.toFixed(4):String(e),I=e=>e>=1048576?`${(e/1024/1024).toFixed(1)} MB`:`${Math.max(1,Math.round(e/1024))} KB`;function V(e){try{return JSON.parse(a(e,"utf-8"))}catch{return null}}export async function runSetter(t=[]){const[S,...B]=t,M=(S||"").toLowerCase();f(`setter ${M}`.trim());try{""===M||"help"===M||"?"===M||"guide"===M||"status"===M?function(e){N();for(const t of h(function(e){const t=t=>!!e&&s(l(e,t));return{id:"setter",title:{en:"Problem setter",zh:"命题工具"},what:{en:"Build one task folder and test it yourself, start to finish, on this machine: the data checks, the files a contestant gets, the A and B scores of your own solutions. Nothing leaves your computer until you hand the archive over.",zh:"在这台机器上把一道题从头到尾自己测完:数据检查、学生拿到的文件、你自己的解在 A 榜和 B 榜上的分数。在你亲手把压缩包交出去之前,没有任何东西离开你的电脑。"},steps:[{verb:"setter new my-task",what:{en:"make the folder: task.json and a baseline notebook that already runs",zh:"建文件夹:task.json 和一份能直接跑的 baseline notebook"},done:()=>t("task.json")},{verb:"setter check",what:{en:"after you put train.csv, test_a.csv + labels_a.csv, test_b.csv + labels_b.csv in it: the builder's checks",zh:"把 train.csv、test_a.csv + labels_a.csv、test_b.csv + labels_b.csv 放进去之后:跑建赛脚本的那套检查"},done:()=>t(".setter/check.json")},{verb:"setter student",what:{en:"write student/ — exactly what a contestant downloads",zh:"生成 student/ —— 学生下载到的就是这些"},done:()=>t("student/test.csv")},{verb:"ipynb",what:{en:"open student/baseline.ipynb and run all, as a contestant would — it writes submission.csv",zh:"打开 student/baseline.ipynb 敲 run all,像学生一样跑 —— 它会写出 submission.csv"},done:()=>t("student/submission.csv")},{verb:"setter score",what:{en:"A score, B score and the difference (setter score ref.csv for another solution)",zh:"A 榜分、B 榜分和差值(setter score ref.csv 给另一份解算分)"},done:()=>t(".setter/scores.json")},{verb:"setter plan 3 1.8",what:{en:"GPU tasks: minutes and peak GiB of your reference run → capsule size, seats per card",zh:"GPU 题:参考解的分钟数和峰值显存 → 胶囊档位、每张卡坐几人"},optional:!0,done:()=>t(".setter/plan.json")},{verb:"setter pack",what:{en:"one archive + report to send to the organiser",zh:"打成一个压缩包 + 报告,发给组织者"},done:()=>!!e&&r(e).some(e=>e.endsWith("-handin.tar.gz"))}],notes:[{en:`Metrics: ${v.join(" · ")}. Files are plain CSV: commas, no quotes. Publishing stays with the organiser.`,zh:`指标:${v.join(" · ")}。文件是普通 CSV:逗号分隔,不带引号。发布由组织者来做。`}],leave:{verb:"menu",what:{en:"main menu",zh:"主菜单"}}}}(e),"full"))N(t);J(e?`task folder: ${e}`:"no task folder here yet — setter new <name> makes one in the current directory"),N()}(F(B[0])):"new"===M||"init"===M||"create"===M?function(e,t){if(!e||!A(e))return O("give the task a short name: 3–40 characters of a–z, 0–9 and -"),void J("example: setter new sydney-rain (add a metric: setter new sydney-rain rmse)");const r=(t||"accuracy").toLowerCase();if(!v.includes(r))return void O(`metric "${t}" is not one the judge has — choose ${v.join(" / ")}`);const a=u(e);if(s(a))return O(`${e}/ already exists — nothing was changed`),void P("setter check",`(inside ${e}/, or: setter check ${e})`);n(a,{recursive:!0});const o={name:e,title:e,metric:r,label:"label",id:"id",per_day:5};i(l(a,"task.json"),`${JSON.stringify({...o,description:""},null,2)}\n`),i(l(a,p),b(y(o).map(e=>({...e,result:null})))),C(`made ${e}/ with task.json and ${p}`),J('now put your data in it (plain CSV, an "id" column in every file):'),J(" train.csv id + features + label"),J(" test_a.csv labels_a.csv the public-board rows, and id,label for them"),J(" test_b.csv labels_b.csv the private-board rows, and id,label for them"),J('edit task.json if your label column is not called "label" or the title should read better.'),P("setter check",`(inside ${e}/, or: setter check ${e})`)}(B[0],B[1]):"check"===M||"verify"===M||"validate"===M?function(e){const t=F(e);if(!t)return O("no task folder found (a folder with task.json)"),P("setter new my-task","makes one here"),null;const r=q(t);if(!r)return null;let a;try{a=L(t,r)}catch(e){return O(e.message),null}if(N(),N(chalk.bold.white(` ${r.title}`)+chalk.gray(` · ${r.name} · metric ${r.metric} · predict "${r.label}"`)),!a.ok){for(const e of a.problems)O(e);return J("fix the first line above and run setter check again — nothing was written."),null}if(C(`train ${a.nTrain} rows · A ${a.nA} rows · B ${a.nB} rows · ${a.features.length} feature columns`),a.classes.length){const e=a.classes.reduce((e,[,t])=>e+t,0);J(`classes in train: ${a.classes.map(([t,s])=>`${t} ${(100*s/e).toFixed(1)}%`).join(" · ")}`)}for(const e of a.warnings)N(chalk.yellow(` ⚠ ${e}`));const c=["train.csv","test_a.csv","test_b.csv"].reduce((e,s)=>e+o(l(t,s)).size,0);c>g?N(chalk.yellow(` ⚠ data is ${I(c)} — over the ${I(g)} a competition carries inline; the organiser will host it in the dataset bucket`)):J(`data size ${I(c)} — fits the inline bundle`),s(l(t,p))||N(chalk.yellow(` ⚠ no ${p} yet — every task ships a baseline the contestants start from`)),n(l(t,".setter"),{recursive:!0}),i(l(t,".setter","check.json"),`${JSON.stringify({at:(new Date).toISOString(),cli:G(),n_train:a.nTrain,n_a:a.nA,n_b:a.nB,features:a.features.length,warnings:a.warnings,data_bytes:c},null,2)}\n`),T(t,"check",{n_train:a.nTrain,n_a:a.nA,n_b:a.nB,features:a.features.length,data_bytes:c,warnings:a.warnings.length}),P("setter student","writes student/ — what a contestant downloads")}(B[0]):"student"===M||"bundle"===M||"build"===M?function(e){const t=F(e);if(!t)return void O("no task folder found — setter new my-task makes one");const r=q(t);if(!r)return;let o;try{o=L(t,r)}catch(e){return void O(e.message)}if(!o.ok){for(const e of o.problems)O(e);return void P("setter check","shows the same problems with the fix")}const u=l(t,"student");n(u,{recursive:!0});const d=_(r,o);i(l(u,"train.csv"),d.train),i(l(u,"test.csv"),d.test),i(l(u,"sample_submission.csv"),d.sample);const h=l(t,p);s(h)&&i(l(u,p),a(h)),T(t,"student",{rows:o.nA+o.nB,baseline:s(h)}),C(`student/ written: train.csv · test.csv (${o.nA+o.nB} rows, A and B mixed) · sample_submission.csv${s(h)?` · ${p}`:""}`),J("no labels and no A/B list are in student/ — it is safe to show to a tester."),s(h)?P(`ipynb ${l(c(t),"student",p)}`,"then run all — it writes student/submission.csv"):N(chalk.yellow(` ⚠ put your ${p} in the task folder and run setter student again`))}(B[0]):"score"===M||"grade"===M?function(t){let r,o;for(const e of t)s(l(u(e),"task.json"))?r=e:o=e;const d=F(r);if(!d)return void O("no task folder found — setter new my-task makes one");const h=q(d);if(!h)return;const f=o?s(u(o))?u(o):l(d,o):l(d,"student","submission.csv");if(!s(f))return O(`no submission at ${f}`),void P(`ipynb ${l(c(d),"student",p)}`,"run all writes student/submission.csv");let m;try{m=L(d,h)}catch(e){return void O(e.message)}if(!m.ok){for(const e of m.problems)O(e);return}const b=a(f,"utf-8"),g=j(h,m,b);if(!g.ok){for(const e of g.problems)O(e);return}const v=w.has(h.metric)?"lower is better":"higher is better";N(),N(chalk.bold.white(` ${c(f)}`)+chalk.gray(` · ${h.metric} (${v})`)),N(chalk.white(` A public board ${String(m.nA).padStart(7)} rows `)+chalk.bold.white(D(g.a))),N(chalk.white(` B private board ${String(m.nB).padStart(7)} rows `)+chalk.bold.white(D(g.b))),N(chalk.gray(` all ${String(m.nA+m.nB).padStart(7)} rows ${D(g.all)} A looks ${g.gap>=0?"better":"worse"} than B by ${D(Math.abs(g.gap))}`));const k=l(d,".setter","scores.json");let y=[];try{y=JSON.parse(a(k,"utf-8"))}catch{y=[]}const $=f.startsWith(d)?f.slice(d.length+1):f,S={file:$,sha:e("sha256").update(b).digest("hex").slice(0,16),a:g.a,b:g.b,all:g.all,gap:g.gap,at:(new Date).toISOString()};y=[...y.filter(e=>e.file!==$),S],T(d,"score",{file:$,sha:S.sha,metric:h.metric,a:g.a,b:g.b,all:g.all}),n(l(d,".setter"),{recursive:!0}),i(k,`${JSON.stringify(y,null,2)}\n`);const B=y.find(e=>e.file===l("student","submission.csv")),_=y.filter(e=>e!==B),z=w.has(h.metric),A=_.sort((e,t)=>z?e.b-t.b:t.b-e.b)[0];if(B){const e=e=>({ok:!0,problems:[],a:e.a,b:e.b,all:e.all,gap:e.gap,nMissing:0});N();for(const t of x(h,e(B),A?e(A):null))J(`· ${t}`);return A?void P("setter pack","one archive + report for the organiser"):void P("setter score ref.csv","score your own better solution too — that shows how much room contestants have")}J("· this is not the baseline run (student/submission.csv) — score that one too to see the room between them"),P(`ipynb ${l(c(d),"student",p)}`,"then run all, then setter score")}(B):"plan"===M||"capsule"===M||"capacity"===M?function(e){const t=F(),s="cpu"===e[0]?.toLowerCase(),r=e.filter(e=>"cpu"!==e.toLowerCase()).map(Number),a=r[0],o=s?0:r[1],c=s?r[1]:r[2];if(!Number.isFinite(a)||a<=0||!s&&(!Number.isFinite(o)||o<=0))return O("give the two numbers your reference run measured: minutes and peak GPU memory in GiB"),J("example: setter plan 3 1.8 200 (3 minutes, 1.8 GiB peak, 200 contestants at once)"),J(" setter plan cpu 2 200 (a task that needs no GPU)"),void J("where the numbers come from: run the reference on a seat (gpufortask → ipynb → gpu on) and type selfcheck.");const u=$(o,a,!s,Number.isFinite(c)?c:void 0);if(N(),"cpu"===u.tier)C(`CPU task · ${a} min per run — contestants run it on their own machine, no seat needed`);else{if(!u.fits)return void O(`peak ${o} GiB does not fit the largest capsule (cap ${u.capGib} GiB) — shrink the batch or the model; a task every card can run beats one that only fits the biggest`);C(`${u.squares}-square capsule · peak ${o} GiB under the ${u.capGib} GiB cap · ${a} min per run`),N(chalk.white(" card seats at once runs per card-hour")+(u.cards?chalk.white(` cards for ${c} at once`):""));for(const[e,t]of[["l4","L4 24 GB"],["ada","Ada 20 GB"],["t4","T4 16 GB"]])u.perCard[e]?N(chalk.white(` ${t} ${String(u.perCard[e]).padStart(6)} ${String(u.runsPerCardHour?.[e]??"-").padStart(6)}`)+(u.cards?chalk.white(` ${String(u.cards[e]).padStart(6)}`):"")):N(chalk.gray(` ${t} — (a 3-square task runs on L4 only, so every contestant gets the same card)`));J("every contestant of one task gets the same capsule size on the same card type — that is what makes the times comparable."),J("a planning estimate from fixed capsule sizes; the organiser calibrates the real number on the fleet before the event.")}t&&(n(l(t,".setter"),{recursive:!0}),i(l(t,".setter","plan.json"),`${JSON.stringify({minutes:a,peak_gib:o,contestants:c??null,...u},null,2)}\n`),T(t,"plan",{minutes:a,peak_gib:o,squares:u.squares,tier:u.tier}),J("saved with the task — setter pack carries it to the organiser."))}(B):"pack"===M||"handin"===M||"hand-in"===M||"package"===M?function(t){const n=F(t);if(!n)return void O("no task folder found — setter new my-task makes one");const r=q(n);if(!r)return;let o;try{o=L(n,r)}catch(e){return void O(e.message)}if(!o.ok){for(const e of o.problems)O(e);return void P("setter check","the archive is only made from a folder that passes")}if(!s(l(n,p)))return void O(`${p} is missing — every task ships a baseline`);try{m(a(l(n,p),"utf-8"))}catch(e){return void O(`${p} does not open as a notebook: ${e.message}`)}let u=[];try{u=JSON.parse(a(l(n,".setter","scores.json"),"utf-8"))}catch{u=[]}u.some(e=>e.file===l("student","submission.csv"))||N(chalk.yellow(" ⚠ the baseline has not been scored — the organiser will ask for it; run it and setter score first if you can"));const h=[...k];for(const e of["description.md","ref.csv","reference.ipynb"])s(l(n,e))&&h.push(e);const f=h.map(e=>({name:`${r.name}/${e}`,data:new Uint8Array(a(l(n,e)))})),b={task:r,packed_at:(new Date).toISOString(),cli:G(),rows:{train:o.nTrain,a:o.nA,b:o.nB},features:o.features.length,warnings:o.warnings,scores:u,capsule:V(l(n,".setter","plan.json")),ledger:U(n),files:f.map(t=>({name:t.name,bytes:t.data.length,sha256:e("sha256").update(t.data).digest("hex")}))};f.push({name:`${r.name}/setter-report.json`,data:(new TextEncoder).encode(`${JSON.stringify(b,null,2)}\n`)});const g=l(n,`${r.name}-handin.tar.gz`),w=d(z(f));i(g,w),C(`${c(g)} · ${I(w.length)} · ${f.length} files`),J(g),N(chalk.yellow(" ⚠ this archive contains the B labels — send it only to the organiser, never to a shared channel.")),J("the organiser runs the same checks again, builds the competition, and you then walk it as a contestant: kaggle <name>.")}(B[0]):(O(`setter does not know "${S}"`),J("it knows: new · check · student · score · plan · pack (bare setter shows the steps)"))}catch(e){O(e.message)}}export function registerSetterCommand(e){e.command("setter [args...]").description("Problem setter: build a task folder and test it end to end locally (new · check · student · score · pack)").action(async(e=[])=>{await runSetter(e)})}
1
+ import{createHash as e}from"node:crypto";import{appendFileSync as t,existsSync as s,mkdirSync as n,readdirSync as r,readFileSync as a,statSync as o,writeFileSync as i}from"node:fs";import{basename as c,join as l,resolve as u}from"node:path";import{gzipSync as d}from"node:zlib";import chalk from"chalk";import{renderPanel as h}from"../lib/guide-panel.js";import{logCommand as f}from"../lib/logger.js";import{parseIpynb as m,serializeIpynb as b}from"../lib/notebook-doc.js";import{BASELINE_FILE as p,INLINE_LIMIT_BYTES as g,LOWER_IS_BETTER as w,METRICS as v,metricGroups as y,TASK_FILES as k,baselineCells as $,capsulePlan as S,checkTask as _,parseTaskMeta as B,scoreSubmission as j,studentBundle as z,tarArchive as A,validName as x,verdictLines as G}from"../lib/setter.js";import{getCliVersion as N}from"../lib/version.js";const O=(e="")=>console.log(e),C=e=>console.log(chalk.red(` ✗ ${e}`)),F=e=>console.log(chalk.green(` ✓ ${e}`)),J=e=>console.log(chalk.gray(` ${e}`)),q=(e,t)=>console.log(chalk.gray(" ▸ next, type: ")+chalk.bold.green(e)+chalk.gray(` ${t}`));function P(e){if(e){const t=u(e);return s(l(t,"task.json"))?t:null}const t=process.cwd();if(s(l(t,"task.json")))return t;try{const e=r(t).filter(e=>!e.startsWith(".")&&s(l(t,e,"task.json")));if(1===e.length)return l(t,e[0])}catch{}return null}function M(e,t){const n=l(e,t);if(s(n)){if(o(n).size>419430400)throw new Error(`${t} is over 400 MB — too large to check here; tell the organiser, this one goes through the dataset bucket`);return a(n,"utf-8")}}function L(e){const{meta:t,problems:s}=B(M(e,"task.json")??"");for(const e of s)C(e);return t}function T(e,t){return _(t,{train:M(e,"train.csv"),testA:M(e,"test_a.csv"),labelsA:M(e,"labels_a.csv"),testB:M(e,"test_b.csv"),labelsB:M(e,"labels_b.csv")})}function U(e,s,r){try{n(l(e,".setter"),{recursive:!0}),t(l(e,".setter","ledger.jsonl"),`${JSON.stringify({at:(new Date).toISOString(),cli:N(),action:s,...r})}\n`)}catch{}}function D(e){try{return a(l(e,".setter","ledger.jsonl"),"utf-8").split("\n").filter(e=>e.trim()).map(e=>JSON.parse(e))}catch{return[]}}const I=e=>Number.isFinite(e)?e.toFixed(4):String(e),E=e=>e>=1048576?`${(e/1024/1024).toFixed(1)} MB`:`${Math.max(1,Math.round(e/1024))} KB`;function V(){const e={classes:"label = a class · submit the predicted class","probability of 1":"label = 0 or 1 · submit the probability of 1",numbers:"label = a number · submit a number"};O();for(const t of y()){O(chalk.bold.white(` ${t.kind}`)+chalk.gray(` · ${e[t.kind]}`));for(const e of t.names){const t=w.has(e)?"lower is better ":"higher is better";O(chalk.cyan(` ${e.padEnd(18)}`)+chalk.gray(`${t} ${W[e]??""}`))}}O(),J('use one: setter new <name> <metric> · or edit "metric" in task.json'),J("a metric that is not here (several values per row, images, text): tell the organiser"),O()}const W={balanced_accuracy:"mean recall per class",macro_f1:"F1 per class, plain mean",weighted_f1:"F1 per class, weighted by class size",mcc:"Matthews correlation",f1:"labels 0 / 1, F1 of class 1",qwk:"whole-number grades, quadratic weighted kappa",auc:"area under the ROC curve",ap:"average precision (area under precision–recall)",logloss:"binary cross-entropy",brier:"mean squared error of the probability",rmsle:"labels 0 or more",mape:"avoid when labels can be 0",smape:"symmetric percentage error, 0 to 2",r2:"coefficient of determination",pearson:"linear correlation",spearman:"rank correlation"};function H(e){try{return JSON.parse(a(e,"utf-8"))}catch{return null}}export async function runSetter(t=[]){const[y,..._]=t,B=(y||"").toLowerCase();f(`setter ${B}`.trim());try{""===B||"help"===B||"?"===B||"guide"===B||"status"===B?function(e){O();for(const t of h(function(e){const t=t=>!!e&&s(l(e,t));return{id:"setter",title:{en:"Problem setter",zh:"命题工具"},what:{en:"Build one task folder and test it yourself, start to finish, on this machine: the data checks, the files a contestant gets, the A and B scores of your own solutions. Nothing leaves your computer until you hand the archive over.",zh:"在这台机器上把一道题从头到尾自己测完:数据检查、学生拿到的文件、你自己的解在 A 榜和 B 榜上的分数。在你亲手把压缩包交出去之前,没有任何东西离开你的电脑。"},steps:[{verb:"setter new my-task",what:{en:"make the folder: task.json and a baseline notebook that already runs",zh:"建文件夹:task.json 和一份能直接跑的 baseline notebook"},done:()=>t("task.json")},{verb:"setter check",what:{en:"after you put train.csv, test_a.csv + labels_a.csv, test_b.csv + labels_b.csv in it: the builder's checks",zh:"把 train.csv、test_a.csv + labels_a.csv、test_b.csv + labels_b.csv 放进去之后:跑建赛脚本的那套检查"},done:()=>t(".setter/check.json")},{verb:"setter student",what:{en:"write student/ — exactly what a contestant downloads",zh:"生成 student/ —— 学生下载到的就是这些"},done:()=>t("student/test.csv")},{verb:"ipynb",what:{en:"open student/baseline.ipynb and run all, as a contestant would — it writes submission.csv",zh:"打开 student/baseline.ipynb 敲 run all,像学生一样跑 —— 它会写出 submission.csv"},done:()=>t("student/submission.csv")},{verb:"setter score",what:{en:"A score, B score and the difference (setter score ref.csv for another solution)",zh:"A 榜分、B 榜分和差值(setter score ref.csv 给另一份解算分)"},done:()=>t(".setter/scores.json")},{verb:"setter plan 3 1.8",what:{en:"GPU tasks: minutes and peak GiB of your reference run → capsule size, seats per card",zh:"GPU 题:参考解的分钟数和峰值显存 → 胶囊档位、每张卡坐几人"},optional:!0,done:()=>t(".setter/plan.json")},{verb:"setter pack",what:{en:"one archive + report to send to the organiser",zh:"打成一个压缩包 + 报告,发给组织者"},done:()=>!!e&&r(e).some(e=>e.endsWith("-handin.tar.gz"))}],notes:[{en:`${v.length} metrics — setter metrics lists them. Files are plain CSV: commas, no quotes. Publishing stays with the organiser.`,zh:`${v.length} 个指标,敲 setter metrics 看全表。文件是普通 CSV:逗号分隔,不带引号。发布由组织者来做。`}],leave:{verb:"menu",what:{en:"main menu",zh:"主菜单"}}}}(e),"full"))O(t);J(e?`task folder: ${e}`:"no task folder here yet — setter new <name> makes one in the current directory"),O()}(P(_[0])):"new"===B||"init"===B||"create"===B?function(e,t){if(!e||!x(e))return C("give the task a short name: 3–40 characters of a–z, 0–9 and -"),void J("example: setter new sydney-rain (add a metric: setter new sydney-rain rmse)");const r=(t||"accuracy").toLowerCase();if(!v.includes(r))return C(`metric "${t}" is not one the judge has`),void V();const a=u(e);if(s(a))return C(`${e}/ already exists — nothing was changed`),void q("setter check",`(inside ${e}/, or: setter check ${e})`);n(a,{recursive:!0});const o={name:e,title:e,metric:r,label:"label",id:"id",per_day:5};i(l(a,"task.json"),`${JSON.stringify({...o,description:""},null,2)}\n`),i(l(a,p),b($(o).map(e=>({...e,result:null})))),F(`made ${e}/ with task.json and ${p}`),J('now put your data in it (plain CSV, an "id" column in every file):'),J(" train.csv id + features + label"),J(" test_a.csv labels_a.csv the public-board rows, and id,label for them"),J(" test_b.csv labels_b.csv the private-board rows, and id,label for them"),J('edit task.json if your label column is not called "label" or the title should read better.'),q("setter check",`(inside ${e}/, or: setter check ${e})`)}(_[0],_[1]):"metrics"===B||"metric"===B?V():"check"===B||"verify"===B||"validate"===B?function(e){const t=P(e);if(!t)return C("no task folder found (a folder with task.json)"),q("setter new my-task","makes one here"),null;const r=L(t);if(!r)return null;let a;try{a=T(t,r)}catch(e){return C(e.message),null}if(O(),O(chalk.bold.white(` ${r.title}`)+chalk.gray(` · ${r.name} · metric ${r.metric} · predict "${r.label}"`)),!a.ok){for(const e of a.problems)C(e);return J("fix the first line above and run setter check again — nothing was written."),null}if(F(`train ${a.nTrain} rows · A ${a.nA} rows · B ${a.nB} rows · ${a.features.length} feature columns`),a.classes.length){const e=a.classes.reduce((e,[,t])=>e+t,0);J(`classes in train: ${a.classes.map(([t,s])=>`${t} ${(100*s/e).toFixed(1)}%`).join(" · ")}`)}for(const e of a.warnings)O(chalk.yellow(` ⚠ ${e}`));const c=["train.csv","test_a.csv","test_b.csv"].reduce((e,s)=>e+o(l(t,s)).size,0);c>g?O(chalk.yellow(` ⚠ data is ${E(c)} — over the ${E(g)} a competition carries inline; the organiser will host it in the dataset bucket`)):J(`data size ${E(c)} — fits the inline bundle`),s(l(t,p))||O(chalk.yellow(` ⚠ no ${p} yet — every task ships a baseline the contestants start from`)),n(l(t,".setter"),{recursive:!0}),i(l(t,".setter","check.json"),`${JSON.stringify({at:(new Date).toISOString(),cli:N(),n_train:a.nTrain,n_a:a.nA,n_b:a.nB,features:a.features.length,warnings:a.warnings,data_bytes:c},null,2)}\n`),U(t,"check",{n_train:a.nTrain,n_a:a.nA,n_b:a.nB,features:a.features.length,data_bytes:c,warnings:a.warnings.length}),q("setter student","writes student/ — what a contestant downloads")}(_[0]):"student"===B||"bundle"===B||"build"===B?function(e){const t=P(e);if(!t)return void C("no task folder found — setter new my-task makes one");const r=L(t);if(!r)return;let o;try{o=T(t,r)}catch(e){return void C(e.message)}if(!o.ok){for(const e of o.problems)C(e);return void q("setter check","shows the same problems with the fix")}const u=l(t,"student");n(u,{recursive:!0});const d=z(r,o);i(l(u,"train.csv"),d.train),i(l(u,"test.csv"),d.test),i(l(u,"sample_submission.csv"),d.sample);const h=l(t,p);s(h)&&i(l(u,p),a(h)),U(t,"student",{rows:o.nA+o.nB,baseline:s(h)}),F(`student/ written: train.csv · test.csv (${o.nA+o.nB} rows, A and B mixed) · sample_submission.csv${s(h)?` · ${p}`:""}`),J("no labels and no A/B list are in student/ — it is safe to show to a tester."),s(h)?q(`ipynb ${l(c(t),"student",p)}`,"then run all — it writes student/submission.csv"):O(chalk.yellow(` ⚠ put your ${p} in the task folder and run setter student again`))}(_[0]):"score"===B||"grade"===B?function(t){let r,o;for(const e of t)s(l(u(e),"task.json"))?r=e:o=e;const d=P(r);if(!d)return void C("no task folder found — setter new my-task makes one");const h=L(d);if(!h)return;const f=o?s(u(o))?u(o):l(d,o):l(d,"student","submission.csv");if(!s(f))return C(`no submission at ${f}`),void q(`ipynb ${l(c(d),"student",p)}`,"run all writes student/submission.csv");let m;try{m=T(d,h)}catch(e){return void C(e.message)}if(!m.ok){for(const e of m.problems)C(e);return}const b=a(f,"utf-8"),g=j(h,m,b);if(!g.ok){for(const e of g.problems)C(e);return}const v=w.has(h.metric)?"lower is better":"higher is better";O(),O(chalk.bold.white(` ${c(f)}`)+chalk.gray(` · ${h.metric} (${v})`)),O(chalk.white(` A public board ${String(m.nA).padStart(7)} rows `)+chalk.bold.white(I(g.a))),O(chalk.white(` B private board ${String(m.nB).padStart(7)} rows `)+chalk.bold.white(I(g.b))),O(chalk.gray(` all ${String(m.nA+m.nB).padStart(7)} rows ${I(g.all)} A looks ${g.gap>=0?"better":"worse"} than B by ${I(Math.abs(g.gap))}`));const y=l(d,".setter","scores.json");let k=[];try{k=JSON.parse(a(y,"utf-8"))}catch{k=[]}const $=f.startsWith(d)?f.slice(d.length+1):f,S={file:$,sha:e("sha256").update(b).digest("hex").slice(0,16),a:g.a,b:g.b,all:g.all,gap:g.gap,at:(new Date).toISOString()};k=[...k.filter(e=>e.file!==$),S],U(d,"score",{file:$,sha:S.sha,metric:h.metric,a:g.a,b:g.b,all:g.all}),n(l(d,".setter"),{recursive:!0}),i(y,`${JSON.stringify(k,null,2)}\n`);const _=k.find(e=>e.file===l("student","submission.csv")),B=k.filter(e=>e!==_),z=w.has(h.metric),A=B.sort((e,t)=>z?e.b-t.b:t.b-e.b)[0];if(_){const e=e=>({ok:!0,problems:[],a:e.a,b:e.b,all:e.all,gap:e.gap,nMissing:0});O();for(const t of G(h,e(_),A?e(A):null))J(`· ${t}`);return A?void q("setter pack","one archive + report for the organiser"):void q("setter score ref.csv","score your own better solution too — that shows how much room contestants have")}J("· this is not the baseline run (student/submission.csv) — score that one too to see the room between them"),q(`ipynb ${l(c(d),"student",p)}`,"then run all, then setter score")}(_):"plan"===B||"capsule"===B||"capacity"===B?function(e){const t=P(),s="cpu"===e[0]?.toLowerCase(),r=e.filter(e=>"cpu"!==e.toLowerCase()).map(Number),a=r[0],o=s?0:r[1],c=s?r[1]:r[2];if(!Number.isFinite(a)||a<=0||!s&&(!Number.isFinite(o)||o<=0))return C("give the two numbers your reference run measured: minutes and peak GPU memory in GiB"),J("example: setter plan 3 1.8 200 (3 minutes, 1.8 GiB peak, 200 contestants at once)"),J(" setter plan cpu 2 200 (a task that needs no GPU)"),void J("where the numbers come from: run the reference on a seat (gpufortask → ipynb → gpu on) and type selfcheck.");const u=S(o,a,!s,Number.isFinite(c)?c:void 0);if(O(),"cpu"===u.tier)F(`CPU task · ${a} min per run — contestants run it on their own machine, no seat needed`);else{if(!u.fits)return void C(`peak ${o} GiB does not fit the largest capsule (cap ${u.capGib} GiB) — shrink the batch or the model; a task every card can run beats one that only fits the biggest`);F(`${u.squares}-square capsule · peak ${o} GiB under the ${u.capGib} GiB cap · ${a} min per run`),O(chalk.white(" card seats at once runs per card-hour")+(u.cards?chalk.white(` cards for ${c} at once`):""));for(const[e,t]of[["l4","L4 24 GB"],["ada","Ada 20 GB"],["t4","T4 16 GB"]])u.perCard[e]?O(chalk.white(` ${t} ${String(u.perCard[e]).padStart(6)} ${String(u.runsPerCardHour?.[e]??"-").padStart(6)}`)+(u.cards?chalk.white(` ${String(u.cards[e]).padStart(6)}`):"")):O(chalk.gray(` ${t} — (a 3-square task runs on L4 only, so every contestant gets the same card)`));J("every contestant of one task gets the same capsule size on the same card type — that is what makes the times comparable."),J("a planning estimate from fixed capsule sizes; the organiser calibrates the real number on the fleet before the event.")}t&&(n(l(t,".setter"),{recursive:!0}),i(l(t,".setter","plan.json"),`${JSON.stringify({minutes:a,peak_gib:o,contestants:c??null,...u},null,2)}\n`),U(t,"plan",{minutes:a,peak_gib:o,squares:u.squares,tier:u.tier}),J("saved with the task — setter pack carries it to the organiser."))}(_):"pack"===B||"handin"===B||"hand-in"===B||"package"===B?function(t){const n=P(t);if(!n)return void C("no task folder found — setter new my-task makes one");const r=L(n);if(!r)return;let o;try{o=T(n,r)}catch(e){return void C(e.message)}if(!o.ok){for(const e of o.problems)C(e);return void q("setter check","the archive is only made from a folder that passes")}if(!s(l(n,p)))return void C(`${p} is missing — every task ships a baseline`);try{m(a(l(n,p),"utf-8"))}catch(e){return void C(`${p} does not open as a notebook: ${e.message}`)}let u=[];try{u=JSON.parse(a(l(n,".setter","scores.json"),"utf-8"))}catch{u=[]}u.some(e=>e.file===l("student","submission.csv"))||O(chalk.yellow(" ⚠ the baseline has not been scored — the organiser will ask for it; run it and setter score first if you can"));const h=[...k];for(const e of["description.md","ref.csv","reference.ipynb"])s(l(n,e))&&h.push(e);const f=h.map(e=>({name:`${r.name}/${e}`,data:new Uint8Array(a(l(n,e)))})),b={task:r,packed_at:(new Date).toISOString(),cli:N(),rows:{train:o.nTrain,a:o.nA,b:o.nB},features:o.features.length,warnings:o.warnings,scores:u,capsule:H(l(n,".setter","plan.json")),ledger:D(n),files:f.map(t=>({name:t.name,bytes:t.data.length,sha256:e("sha256").update(t.data).digest("hex")}))};f.push({name:`${r.name}/setter-report.json`,data:(new TextEncoder).encode(`${JSON.stringify(b,null,2)}\n`)});const g=l(n,`${r.name}-handin.tar.gz`),w=d(A(f));i(g,w),F(`${c(g)} · ${E(w.length)} · ${f.length} files`),J(g),O(chalk.yellow(" ⚠ this archive contains the B labels — send it only to the organiser, never to a shared channel.")),J("the organiser runs the same checks again, builds the competition, and you then walk it as a contestant: kaggle <name>.")}(_[0]):(C(`setter does not know "${y}"`),J("it knows: new · check · student · score · plan · pack (bare setter shows the steps)"))}catch(e){C(e.message)}}export function registerSetterCommand(e){e.command("setter [args...]").description("Problem setter: build a task folder and test it end to end locally (new · check · student · score · pack)").action(async(e=[])=>{await runSetter(e)})}
@@ -1 +1 @@
1
- import{createHash as e}from"node:crypto";export const METRICS=["accuracy","macro_f1","rmse","mse"];export const LOWER_IS_BETTER=new Set(["rmse","mse"]);export function parseCsv(e){const t=[],s=e.replace(/^/,"").split(/\r?\n/).filter(e=>""!==e.trim());if(0===s.length)return{table:{header:[],rows:[]},problems:["file is empty"]};e.includes('"')&&t.push('quoted fields (") are not supported — write plain comma-separated values');const n=s[0].split(",").map(e=>e.trim()),r=[];for(let e=1;e<s.length;e++){const a=s[e].split(",").map(e=>e.trim());if(a.length!==n.length){t.push(`row ${e+1} has ${a.length} fields, the header has ${n.length}`);break}r.push(a)}return{table:{header:n,rows:r},problems:t}}export function validName(e){return/^[a-z0-9][a-z0-9-]{1,38}[a-z0-9]$/.test(e)}export function parseTaskMeta(e){let t;try{t=JSON.parse(e)}catch{return{meta:null,problems:["task.json is not valid JSON"]}}const s=[],n=String(t.name??""),r=String(t.metric??""),a=String(t.label??"");if(validName(n)||s.push("task.json name: 3–40 characters of a–z, 0–9 and -"),METRICS.includes(r)||s.push(`task.json metric: one of ${METRICS.join(" / ")}`),a||s.push("task.json label: the name of the column to predict"),s.length)return{meta:null,problems:s};const i=Number(t.per_day);return{meta:{name:n,title:String(t.title||n),metric:r,label:a,id:String(t.id||"id"),per_day:Number.isFinite(i)&&i>=1?Math.floor(i):5,description:"string"==typeof t.description?t.description:void 0},problems:[]}}const t={train:"train.csv",testA:"test_a.csv",labelsA:"labels_a.csv",testB:"test_b.csv",labelsB:"labels_b.csv"};export function checkTask(e,s){const n={ok:!1,problems:[],warnings:[],nTrain:0,nA:0,nB:0,features:[],classes:[]},r={};for(const e of Object.keys(t)){const a=s[e];if(void 0===a){n.problems.push(`${t[e]} is missing`);continue}const{table:i,problems:o}=parseCsv(a);for(const s of o)n.problems.push(`${t[e]}: ${s}`);r[e]=i}if(n.problems.length)return n;const a=r.train,i=e.id;if(!a.header.includes(i)||!a.header.includes(e.label))return n.problems.push(`train.csv needs both "${i}" and "${e.label}" columns — its header is ${a.header.join(", ")}`),n;const o=a.header.filter(t=>t!==e.label);n.features=o.filter(e=>e!==i);const l=a.header.indexOf(i),c=a.header.indexOf(e.label),d=new Set;for(const e of a.rows){if(d.has(e[l]))return n.problems.push(`train.csv: duplicate id ${e[l]}`),n;d.add(e[l])}const h=new Map,u=new Map,p={A:[],B:[]};for(const[s,a,l]of[["A","testA","labelsA"],["B","testB","labelsB"]]){const c=r[a],d=r[l],m=t[a],b=t[l];if(c.header.includes(e.label))return n.problems.push(`${m} carries the label column "${e.label}" — remove it, labels go in ${b}`),n;if(c.header.join(",")!==o.join(","))return n.problems.push(`${m} columns differ from train.csv minus the label — test: ${c.header.join(", ")} · train: ${o.join(", ")}`),n;if(2!==d.header.length||d.header[0]!==i||d.header[1]!==e.label)return n.problems.push(`${b} must have exactly two columns (${i}, ${e.label}) — got ${d.header.join(", ")}`),n;const f=new Set,g=c.header.indexOf(i);for(const e of c.rows){if(f.has(e[g]))return n.problems.push(`${m} has duplicate ids (${e[g]})`),n;f.add(e[g])}const $=new Map;for(const e of d.rows){if(""===e[0]||""===e[1])return n.problems.push(`${b}: empty id or label near id ${e[0]||"(blank)"}`),n;if($.has(e[0]))return n.problems.push(`${b}: duplicate id ${e[0]}`),n;$.set(e[0],e[1])}const w=[...f].filter(e=>$.has(e)).length;if(w!==f.size||w!==$.size)return n.problems.push(`${m}: ${f.size} ids in the test file, ${$.size} in ${b}, ${w} in both — they must be the same set`),n;for(const e of f){const t=u.get(e);if(t)return n.problems.push(`id ${e} is in both ${t} and ${m}`),n;u.set(e,m),h.set(e,$.get(e)),p[s].push(e)}}const m=[...u.keys()].filter(e=>d.has(e));if(m.length)return n.problems.push(`${m.length} train ids are reused in test (e.g. ${m.slice(0,3).join(", ")}) — ids must be unique across files`),n;if(n.nTrain=a.rows.length,n.nA=p.A.length,n.nB=p.B.length,0===n.nA||0===n.nB)return n.problems.push("one board is empty — both test_a.csv and test_b.csv need rows"),n;if(LOWER_IS_BETTER.has(e.metric)){const t=[...h.values()].find(e=>!Number.isFinite(Number(e)));if(void 0!==t)return n.problems.push(`metric ${e.metric} needs numeric labels — found "${t}"`),n}else{const t=new Map;for(const e of a.rows)t.set(e[c],(t.get(e[c])||0)+1);n.classes=[...t.entries()].sort((e,t)=>t[1]-e[1]).slice(0,12);const s=new Set(t.keys()),r=[...new Set(h.values())].filter(e=>!s.has(e));r.length&&n.warnings.push(`test labels contain ${r.length} value(s) never seen in train (e.g. ${r.slice(0,3).join(", ")})`),t.size>Math.max(50,n.nTrain/2)&&n.warnings.push(`${t.size} distinct label values with a classification metric — is "${e.label}" really a class?`)}return(n.nA<100||n.nB<100)&&n.warnings.push(`a board with under 100 rows (A ${n.nA}, B ${n.nB}) moves a lot on a single row`),n.nB<n.nA&&n.warnings.push(`B (${n.nB}) is smaller than A (${n.nA}) — the final ranking usually rests on the larger set`),n.ok=!0,n.parsed={train:a,testA:r.testA,testB:r.testB,labels:h,aIds:p.A,bIds:p.B},n}const s=(e,t)=>`${[e,...t].map(e=>e.join(",")).join("\n")}\n`;export function studentBundle(t,n){if(!n.ok||!n.parsed)throw new Error("check the task first");const{train:r,testA:a,testB:i}=n.parsed,o=a.header.indexOf(t.id),l=s=>e("sha256").update(`${t.name}:order:${s}`).digest("hex"),c=[...a.rows,...i.rows].sort((e,t)=>l(e[o])<l(t[o])?-1:1),d=r.header.indexOf(t.label),h=LOWER_IS_BETTER.has(t.metric)?"0":n.classes[0]?.[0]??r.rows[0]?.[d]??"0";return{train:s(r.header,r.rows),test:s(a.header,c),sample:s([t.id,t.label],c.map(e=>[e[o],h]))}}function n(e,t){if(0===t.size)return Number.POSITIVE_INFINITY;let s=0;for(const[n,r]of t){const t=e.get(n);s+=((void 0!==t&&""!==t.trim()&&Number.isFinite(Number(t))?Number(t):0)-Number(r))**2}return s/t.size}export function metricValue(e,t,s){return"accuracy"===e?function(e,t){if(0===t.size)return 0;let s=0;for(const[n,r]of t)(e.get(n)??"").trim()===r.trim()&&s++;return s/t.size}(t,s):"macro_f1"===e?function(e,t){const s=[...new Set([...t.values()].map(e=>e.trim()))].sort();if(0===s.length)return 0;let n=0;for(const r of s){let s=0,a=0,i=0;for(const[n,o]of t){const t=(e.get(n)??"").trim(),l=o.trim();t===r&&l===r?s++:t===r&&l!==r?a++:t!==r&&l===r&&i++}const o=2*s+a+i;n+=o?2*s/o:0}return n/s.length}(t,s):"mse"===e?n(t,s):Math.sqrt(n(t,s))}export function scoreSubmission(e,t,s){const n={ok:!1,problems:[],a:0,b:0,all:0,gap:0,nMissing:0};if(!t.ok||!t.parsed)return n.problems.push("check the task first"),n;const{table:r,problems:a}=parseCsv(s);if(a.length)return n.problems.push(...a),n;if(2!==r.header.length||r.header[0]!==e.id||r.header[1]!==e.label)return n.problems.push(`the submission needs exactly two columns: ${e.id},${e.label} — got ${r.header.join(",")}`),n;const i=new Map;for(const e of r.rows){if(i.has(e[0]))return n.problems.push(`duplicate id ${e[0]} in the submission`),n;i.set(e[0],e[1])}const{labels:o,aIds:l,bIds:c}=t.parsed,d=[...i.keys()].filter(e=>!o.has(e));if(d.length)return n.problems.push(`${d.length} ids are not in the test set (e.g. ${d.slice(0,3).join(", ")})`),n;if(n.nMissing=o.size-i.size,n.nMissing>0)return n.problems.push(`${n.nMissing} test rows have no prediction — the judge refuses an incomplete submission`),n;const h=e=>new Map(e.map(e=>[e,o.get(e)]));return n.a=metricValue(e.metric,i,h(l)),n.b=metricValue(e.metric,i,h(c)),n.all=metricValue(e.metric,i,o),n.gap=LOWER_IS_BETTER.has(e.metric)?n.b-n.a:n.a-n.b,n.ok=!0,n}export function verdictLines(e,t,s){const n=[],r=LOWER_IS_BETTER.has(e.metric),a=(e,t)=>r?e<t:e>t,i=Math.abs(t.gap);if(i/Math.max(Math.abs(t.all),1e-9)>.05?n.push(`A and B disagree by ${i.toFixed(4)} on the baseline — check that the two test sets come from the same distribution`):n.push(`A and B agree on the baseline (difference ${i.toFixed(4)})`),s){if(a(s.b,t.b)){const e=Math.abs(s.b-t.b);n.push(`room to climb on B: ${e.toFixed(4)} (baseline ${t.b.toFixed(4)} → reference ${s.b.toFixed(4)})`),e<2*i&&n.push("that room is smaller than twice the A/B difference — the ranking may come down to noise")}else n.push("the reference does NOT beat the baseline on B — contestants have nothing to climb");a(s.a,t.a)!==a(s.b,t.b)&&n.push("baseline and reference rank differently on A and on B — the public board would mislead")}return n}export function tarArchive(e){const t=[],s=new TextEncoder;for(const n of e){const e=new Uint8Array(512),r=(t,n,r)=>{e.set(s.encode(t).subarray(0,r),n)};if(s.encode(n.name).length>99)throw new Error(`file name too long for the archive: ${n.name}`);r(n.name,0,100),r("0000644\0",100,8),r("0000000\0",108,8),r("0000000\0",116,8),r(`${n.data.length.toString(8).padStart(11,"0")}\0`,124,12),r("00000000000\0",136,12),r(" ",148,8),r("0",156,1),r("ustar\0",257,6),r("00",263,2);let a=0;for(const t of e)a+=t;r(`${a.toString(8).padStart(6,"0")}\0 `,148,8),t.push(e,n.data);const i=(512-n.data.length%512)%512;i&&t.push(new Uint8Array(i))}t.push(new Uint8Array(1024));const n=t.reduce((e,t)=>e+t.length,0),r=new Uint8Array(n);let a=0;for(const e of t)r.set(e,a),a+=e.length;return r}export const BASELINE_FILE="baseline.ipynb";export const TASK_FILES=["task.json","train.csv","test_a.csv","labels_a.csv","test_b.csv","labels_b.csv",BASELINE_FILE];export const INLINE_LIMIT_BYTES=5242880;export function baselineCells(e){const t=LOWER_IS_BETTER.has(e.metric);return[{kind:"markdown",source:`# ${e.title}\n\nDescribe the task here: what one row is, what "${e.label}" means, and how a submission is scored (**${e.metric}**).\n\nThis notebook is the baseline every contestant starts from. Keep it short and make sure it runs top to bottom.`},{kind:"code",source:'import pandas as pd\n\ntrain = pd.read_csv("train.csv")\ntest = pd.read_csv("test.csv")\nprint(train.shape, test.shape)\ntrain.head()'},{kind:"code",source:t?`# Baseline: predict the training mean for every row.\npred = [train["${e.label}"].mean()] * len(test)`:`# Baseline: predict the most common class for every row.\npred = [train["${e.label}"].mode()[0]] * len(test)`},{kind:"code",source:`pd.DataFrame({"${e.id}": test["${e.id}"], "${e.label}": pred}).to_csv("submission.csv", index=False)\nprint("wrote submission.csv —", len(pred), "rows")`}]}export const SQUARE_GIB=3.5;export const CARD_UNITS={l4:6,ada:5,t4:4};export function capsulePlan(e,t,s,n){if(!s)return{tier:"cpu",squares:0,fits:!0,capGib:0,perCard:{l4:0,ada:0,t4:0}};const r=[1,2,3].find(t=>e<3.5*t-.5);if(!r)return{tier:"gpu",squares:3,fits:!1,capGib:10,perCard:{l4:0,ada:0,t4:0}};const a=(e,t)=>3===r&&"l4"!==t?0:Math.floor(e/r),i={l4:a(CARD_UNITS.l4,"l4"),ada:a(CARD_UNITS.ada,"ada"),t4:a(CARD_UNITS.t4,"t4")},o={tier:"gpu",squares:r,fits:!0,capGib:3.5*r-.5,perCard:i},l=e=>({l4:e(i.l4),ada:e(i.ada),t4:e(i.t4)});return n&&n>0&&(o.cards=l(e=>e?Math.ceil(n/e):0)),t>0&&(o.runsPerCardHour=l(e=>Math.floor(60*e/t))),o}
1
+ import{createHash as e}from"node:crypto";export const METRIC_KIND={accuracy:"class",balanced_accuracy:"class",macro_f1:"class",weighted_f1:"class",mcc:"class",f1:"binary",qwk:"grade",auc:"prob",ap:"prob",logloss:"prob",brier:"prob",mse:"reg",rmse:"reg",mae:"reg",rmsle:"reg",mape:"reg",smape:"reg",r2:"reg",pearson:"reg",spearman:"reg"};export const METRICS=Object.keys(METRIC_KIND);export const LOWER_IS_BETTER=new Set(["mse","rmse","mae","rmsle","mape","smape","logloss","brier"]);export function metricGroups(){const e=(...e)=>METRICS.filter(t=>e.includes(METRIC_KIND[t]));return[{kind:"classes",names:e("class","binary","grade")},{kind:"probability of 1",names:e("prob")},{kind:"numbers",names:e("reg")}]}export function parseCsv(e){const t=[],r=e.replace(/^/,"").split(/\r?\n/).filter(e=>""!==e.trim());if(0===r.length)return{table:{header:[],rows:[]},problems:["file is empty"]};e.includes('"')&&t.push('quoted fields (") are not supported — write plain comma-separated values');const s=r[0].split(",").map(e=>e.trim()),n=[];for(let e=1;e<r.length;e++){const a=r[e].split(",").map(e=>e.trim());if(a.length!==s.length){t.push(`row ${e+1} has ${a.length} fields, the header has ${s.length}`);break}n.push(a)}return{table:{header:s,rows:n},problems:t}}export function validName(e){return/^[a-z0-9][a-z0-9-]{1,38}[a-z0-9]$/.test(e)}export function parseTaskMeta(e){let t;try{t=JSON.parse(e)}catch{return{meta:null,problems:["task.json is not valid JSON"]}}const r=[],s=String(t.name??""),n=String(t.metric??""),a=String(t.label??"");if(validName(s)||r.push("task.json name: 3–40 characters of a–z, 0–9 and -"),METRICS.includes(n)||r.push(`task.json metric: one of ${METRICS.join(" / ")}`),a||r.push("task.json label: the name of the column to predict"),r.length)return{meta:null,problems:r};const o=Number(t.per_day);return{meta:{name:s,title:String(t.title||s),metric:n,label:a,id:String(t.id||"id"),per_day:Number.isFinite(o)&&o>=1?Math.floor(o):5,description:"string"==typeof t.description?t.description:void 0},problems:[]}}const t={train:"train.csv",testA:"test_a.csv",labelsA:"labels_a.csv",testB:"test_b.csv",labelsB:"labels_b.csv"};export function checkTask(e,r){const s={ok:!1,problems:[],warnings:[],nTrain:0,nA:0,nB:0,features:[],classes:[]},n={};for(const e of Object.keys(t)){const a=r[e];if(void 0===a){s.problems.push(`${t[e]} is missing`);continue}const{table:o,problems:i}=parseCsv(a);for(const r of i)s.problems.push(`${t[e]}: ${r}`);n[e]=o}if(s.problems.length)return s;const a=n.train,o=e.id;if(!a.header.includes(o)||!a.header.includes(e.label))return s.problems.push(`train.csv needs both "${o}" and "${e.label}" columns — its header is ${a.header.join(", ")}`),s;const i=a.header.filter(t=>t!==e.label);s.features=i.filter(e=>e!==o);const l=a.header.indexOf(o),c=a.header.indexOf(e.label),u=new Set;for(const e of a.rows){if(u.has(e[l]))return s.problems.push(`train.csv: duplicate id ${e[l]}`),s;u.add(e[l])}const m=new Map,p=new Map,h={A:[],B:[]};for(const[r,a,l]of[["A","testA","labelsA"],["B","testB","labelsB"]]){const c=n[a],u=n[l],d=t[a],f=t[l];if(c.header.includes(e.label))return s.problems.push(`${d} carries the label column "${e.label}" — remove it, labels go in ${f}`),s;if(c.header.join(",")!==i.join(","))return s.problems.push(`${d} columns differ from train.csv minus the label — test: ${c.header.join(", ")} · train: ${i.join(", ")}`),s;if(2!==u.header.length||u.header[0]!==o||u.header[1]!==e.label)return s.problems.push(`${f} must have exactly two columns (${o}, ${e.label}) — got ${u.header.join(", ")}`),s;const b=new Set,g=c.header.indexOf(o);for(const e of c.rows){if(b.has(e[g]))return s.problems.push(`${d} has duplicate ids (${e[g]})`),s;b.add(e[g])}const $=new Map;for(const e of u.rows){if(""===e[0]||""===e[1])return s.problems.push(`${f}: empty id or label near id ${e[0]||"(blank)"}`),s;if($.has(e[0]))return s.problems.push(`${f}: duplicate id ${e[0]}`),s;$.set(e[0],e[1])}const w=[...b].filter(e=>$.has(e)).length;if(w!==b.size||w!==$.size)return s.problems.push(`${d}: ${b.size} ids in the test file, ${$.size} in ${f}, ${w} in both — they must be the same set`),s;for(const e of b){const t=p.get(e);if(t)return s.problems.push(`id ${e} is in both ${t} and ${d}`),s;p.set(e,d),m.set(e,$.get(e)),h[r].push(e)}}const d=[...p.keys()].filter(e=>u.has(e));if(d.length)return s.problems.push(`${d.length} train ids are reused in test (e.g. ${d.slice(0,3).join(", ")}) — ids must be unique across files`),s;if(s.nTrain=a.rows.length,s.nA=h.A.length,s.nB=h.B.length,0===s.nA||0===s.nB)return s.problems.push("one board is empty — both test_a.csv and test_b.csv need rows"),s;const f=METRIC_KIND[e.metric];if("reg"!==f){const t=new Map;for(const e of a.rows)t.set(e[c],(t.get(e[c])||0)+1);s.classes=[...t.entries()].sort((e,t)=>t[1]-e[1]).slice(0,12);const r=new Set(t.keys()),n=[...new Set(m.values())].filter(e=>!r.has(e));n.length&&s.warnings.push(`test labels contain ${n.length} value(s) never seen in train (e.g. ${n.slice(0,3).join(", ")})`),t.size>Math.max(50,s.nTrain/2)&&s.warnings.push(`${t.size} distinct label values with a classification metric — is "${e.label}" really a class?`);const o=[...t.keys(),...m.values()];if("binary"===f||"prob"===f){const t=o.find(e=>""===e.trim()||0!==Number(e)&&1!==Number(e));if(void 0!==t)return s.problems.push(`metric ${e.metric} needs labels that are 0 or 1 — found "${t}"`),s}if("grade"===f){const t=o.find(e=>""===e.trim()||!Number.isInteger(Number(e)));if(void 0!==t)return s.problems.push(`metric ${e.metric} needs whole-number grades as labels — found "${t}"`),s}}else{const t=[...m.values()].find(e=>!Number.isFinite(Number(e)));if(void 0!==t)return s.problems.push(`metric ${e.metric} needs numeric labels — found "${t}"`),s;const r=[...m.values()].map(Number);if("rmsle"===e.metric&&r.some(e=>e<0))return s.problems.push("metric rmsle needs labels that are 0 or more — found a negative one"),s;"mape"===e.metric&&r.some(e=>0===e)&&s.warnings.push("some labels are exactly 0 — mape divides by the label, one such row dominates the score (smape or mae are safer)")}return(s.nA<100||s.nB<100)&&s.warnings.push(`a board with under 100 rows (A ${s.nA}, B ${s.nB}) moves a lot on a single row`),s.nB<s.nA&&s.warnings.push(`B (${s.nB}) is smaller than A (${s.nA}) — the final ranking usually rests on the larger set`),s.ok=!0,s.parsed={train:a,testA:n.testA,testB:n.testB,labels:m,aIds:h.A,bIds:h.B},s}const r=(e,t)=>`${[e,...t].map(e=>e.join(",")).join("\n")}\n`;export function studentBundle(t,s){if(!s.ok||!s.parsed)throw new Error("check the task first");const{train:n,testA:a,testB:o}=s.parsed,i=a.header.indexOf(t.id),l=r=>e("sha256").update(`${t.name}:order:${r}`).digest("hex"),c=[...a.rows,...o.rows].sort((e,t)=>l(e[i])<l(t[i])?-1:1),u=n.header.indexOf(t.label),m=METRIC_KIND[t.metric],p="prob"===m?"0.5":"reg"===m||"binary"===m?"0":s.classes[0]?.[0]??n.rows[0]?.[u]??"0";return{train:r(n.header,n.rows),test:r(a.header,c),sample:r([t.id,t.label],c.map(e=>[e[i],p]))}}function s(e,t){if(0===t.size)return Number.POSITIVE_INFINITY;let r=0;for(const[s,n]of t){const t=e.get(s);r+=((void 0!==t&&""!==t.trim()&&Number.isFinite(Number(t))?Number(t):0)-Number(n))**2}return r/t.size}function n(e){if(void 0===e||""===e.trim())return 0;const t=Number(e);return Number.isFinite(t)?t:0}function a(e){return void 0!==e&&""!==e.trim()&&1===Number(e)}function o(...[e,t]){return[...t].map(([t,r])=>[n(e.get(t)),Number(r)])}function i(...[e,t]){return[...t].map(([t,r])=>[n(e.get(t)),a(r)?1:0])}function l(...[e,t]){return[...t].map(([t,r])=>[(e.get(t)??"").trim(),r.trim()])}const c=e=>e.reduce((e,t)=>e+t,0);function u(e,t){return 0===e.length?Number.POSITIVE_INFINITY:c(e.map(([e,r])=>t(e,r)))/e.length}function m(e,t){const r=e.length;if(0===r)return 0;const s=c(e)/r,n=c(t)/r,a=c(e.map((e,r)=>(e-s)*(t[r]-n))),o=c(e.map(e=>(e-s)**2)),i=c(t.map(e=>(e-n)**2));return 0===o||0===i?0:a/Math.sqrt(o*i)}function p(e){const t=e.map((e,t)=>t).sort((t,r)=>e[t]-e[r]),r=new Array(e.length).fill(0);let s=0;for(;s<t.length;){let n=s;for(;n+1<t.length&&e[t[n+1]]===e[t[s]];)n++;for(let e=s;e<=n;e++)r[t[e]]=(s+n)/2+1;s=n+1}return r}function h(e){return[...new Set(e.map(([,e])=>e))].sort().map(t=>({lab:t,tp:e.filter(([e,r])=>e===t&&r===t).length,fp:e.filter(([e,r])=>e===t&&r!==t).length,fn:e.filter(([e,r])=>e!==t&&r===t).length}))}export function metricValue(e,t,r){switch(e){case"accuracy":return function(e,t){if(0===t.size)return 0;let r=0;for(const[s,n]of t)(e.get(s)??"").trim()===n.trim()&&r++;return r/t.size}(t,r);case"macro_f1":return function(e,t){const r=[...new Set([...t.values()].map(e=>e.trim()))].sort();if(0===r.length)return 0;let s=0;for(const n of r){let r=0,a=0,o=0;for(const[s,i]of t){const t=(e.get(s)??"").trim(),l=i.trim();t===n&&l===n?r++:t===n&&l!==n?a++:t!==n&&l===n&&o++}const i=2*r+a+o;s+=i?2*r/i:0}return s/r.length}(t,r);case"mse":return s(t,r);case"rmse":return Math.sqrt(s(t,r));case"balanced_accuracy":return function(e){const t=h(e);return 0===t.length?0:c(t.map(e=>e.tp/(e.tp+e.fn)))/t.length}(l(t,r));case"weighted_f1":return function(e){const t=h(e);return 0===t.length?0:c(t.map(e=>{const t=2*e.tp+e.fp+e.fn;return(e.tp+e.fn)*(t?2*e.tp/t:0)}))/e.length}(l(t,r));case"mcc":return function(e){const t=e.length;if(0===t)return 0;const r=new Map,s=new Map;let n=0;for(const[t,a]of e)r.set(a,(r.get(a)||0)+1),s.set(t,(s.get(t)||0)+1),t===a&&n++;const a=n*t-c([...r].map(([e,t])=>t*(s.get(e)||0))),o=t*t-c([...r.values()].map(e=>e*e)),i=t*t-c([...s.values()].map(e=>e*e));return 0===o||0===i?0:a/Math.sqrt(o*i)}(l(t,r));case"f1":return function(...[e,t]){let r=0,s=0,n=0;for(const[o,i]of t){const t=a(e.get(o)),l=a(i);t&&l?r++:t&&!l?s++:l&&!t&&n++}const o=2*r+s+n;return o?2*r/o:0}(t,r);case"qwk":return function(...[e,t]){const r=[...t.values()].map(e=>Math.floor(Number(e)+.5));if(0===r.length)return 0;const s=Math.min(...r),a=Math.max(...r),o=a-s+1;if(o<2)return 0;const i=[...t.keys()].map(t=>Math.min(Math.max(Math.floor(n(e.get(t))+.5),s),a)),l=new Array(o).fill(0),c=new Array(o).fill(0);let u=0;r.forEach((e,t)=>{l[e-s]++,c[i[t]-s]++,u+=(i[t]-e)**2});let m=0;for(let e=0;e<o;e++)for(let t=0;t<o;t++)m+=(e-t)**2*l[e]*c[t];return m/=r.length,0===m?0:1-u/m}(t,r);case"auc":return function(e){const t=c(e.map(([,e])=>e)),r=e.length-t;if(0===t||0===r)return.5;const s=p(e.map(([e])=>e));return(c(s.filter((t,r)=>1===e[r][1]))-t*(t+1)/2)/(t*r)}(i(t,r));case"ap":return function(e){const t=[...e].sort((e,t)=>t[0]-e[0]),r=c(t.map(([,e])=>e));if(0===r)return 0;let s=0,n=0,a=0,o=0,i=0;for(;i<t.length;){let e=i;for(;e<t.length&&t[e][0]===t[i][0];)n+=t[e][1],a++,e++;const l=n/r;s+=n/a*(l-o),o=l,i=e}return s}(i(t,r));case"logloss":return u(i(t,r),(e,t)=>{const r=Math.min(Math.max(e,1e-15),1-1e-15);return 1===t?-Math.log(r):-Math.log(1-r)});case"brier":return u(i(t,r),(e,t)=>(e-t)**2);case"mae":return u(o(t,r),(e,t)=>Math.abs(e-t));case"rmsle":return Math.sqrt(u(o(t,r),(e,t)=>(Math.log1p(Math.max(e,0))-Math.log1p(Math.max(t,0)))**2));case"mape":return u(o(t,r),(e,t)=>Math.abs(e-t)/Math.max(Math.abs(t),2220446049250313e-31));case"smape":return u(o(t,r),(e,t)=>{const r=Math.abs(e)+Math.abs(t);return r?2*Math.abs(e-t)/r:0});case"r2":return function(e){if(0===e.length)return 0;const t=c(e.map(([,e])=>e))/e.length,r=c(e.map(([,e])=>(e-t)**2));return 0===r?0:1-c(e.map(([e,t])=>(e-t)**2))/r}(o(t,r));case"pearson":{const e=o(t,r);return m(e.map(([e])=>e),e.map(([,e])=>e))}case"spearman":{const e=o(t,r);return m(p(e.map(([e])=>e)),p(e.map(([,e])=>e)))}}}export function scoreSubmission(e,t,r){const s={ok:!1,problems:[],a:0,b:0,all:0,gap:0,nMissing:0};if(!t.ok||!t.parsed)return s.problems.push("check the task first"),s;const{table:n,problems:a}=parseCsv(r);if(a.length)return s.problems.push(...a),s;if(2!==n.header.length||n.header[0]!==e.id||n.header[1]!==e.label)return s.problems.push(`the submission needs exactly two columns: ${e.id},${e.label} — got ${n.header.join(",")}`),s;const o=new Map;for(const e of n.rows){if(o.has(e[0]))return s.problems.push(`duplicate id ${e[0]} in the submission`),s;o.set(e[0],e[1])}const{labels:i,aIds:l,bIds:c}=t.parsed,u=[...o.keys()].filter(e=>!i.has(e));if(u.length)return s.problems.push(`${u.length} ids are not in the test set (e.g. ${u.slice(0,3).join(", ")})`),s;if(s.nMissing=i.size-o.size,s.nMissing>0)return s.problems.push(`${s.nMissing} test rows have no prediction — the judge refuses an incomplete submission`),s;const m=e=>new Map(e.map(e=>[e,i.get(e)]));return s.a=metricValue(e.metric,o,m(l)),s.b=metricValue(e.metric,o,m(c)),s.all=metricValue(e.metric,o,i),s.gap=LOWER_IS_BETTER.has(e.metric)?s.b-s.a:s.a-s.b,s.ok=!0,s}export function verdictLines(e,t,r){const s=[],n=LOWER_IS_BETTER.has(e.metric),a=(e,t)=>n?e<t:e>t,o=Math.abs(t.gap);if(o/Math.max(Math.abs(t.all),1e-9)>.05?s.push(`A and B disagree by ${o.toFixed(4)} on the baseline — check that the two test sets come from the same distribution`):s.push(`A and B agree on the baseline (difference ${o.toFixed(4)})`),r){if(a(r.b,t.b)){const e=Math.abs(r.b-t.b);s.push(`room to climb on B: ${e.toFixed(4)} (baseline ${t.b.toFixed(4)} → reference ${r.b.toFixed(4)})`),e<2*o&&s.push("that room is smaller than twice the A/B difference — the ranking may come down to noise")}else s.push("the reference does NOT beat the baseline on B — contestants have nothing to climb");a(r.a,t.a)!==a(r.b,t.b)&&s.push("baseline and reference rank differently on A and on B — the public board would mislead")}return s}export function tarArchive(e){const t=[],r=new TextEncoder;for(const s of e){const e=new Uint8Array(512),n=(t,s,n)=>{e.set(r.encode(t).subarray(0,n),s)};if(r.encode(s.name).length>99)throw new Error(`file name too long for the archive: ${s.name}`);n(s.name,0,100),n("0000644\0",100,8),n("0000000\0",108,8),n("0000000\0",116,8),n(`${s.data.length.toString(8).padStart(11,"0")}\0`,124,12),n("00000000000\0",136,12),n(" ",148,8),n("0",156,1),n("ustar\0",257,6),n("00",263,2);let a=0;for(const t of e)a+=t;n(`${a.toString(8).padStart(6,"0")}\0 `,148,8),t.push(e,s.data);const o=(512-s.data.length%512)%512;o&&t.push(new Uint8Array(o))}t.push(new Uint8Array(1024));const s=t.reduce((e,t)=>e+t.length,0),n=new Uint8Array(s);let a=0;for(const e of t)n.set(e,a),a+=e.length;return n}export const BASELINE_FILE="baseline.ipynb";export const TASK_FILES=["task.json","train.csv","test_a.csv","labels_a.csv","test_b.csv","labels_b.csv",BASELINE_FILE];export const INLINE_LIMIT_BYTES=5242880;export function baselineCells(e){const t=METRIC_KIND[e.metric],r="reg"===t||"prob"===t;return[{kind:"markdown",source:`# ${e.title}\n\nDescribe the task here: what one row is, what "${e.label}" means, and how a submission is scored (**${e.metric}**).\n\nThis notebook is the baseline every contestant starts from. Keep it short and make sure it runs top to bottom.`},{kind:"code",source:'import pandas as pd\n\ntrain = pd.read_csv("train.csv")\ntest = pd.read_csv("test.csv")\nprint(train.shape, test.shape)\ntrain.head()'},{kind:"code",source:r?`# Baseline: predict the training mean for every row${"prob"===t?" (the share of 1s — a probability)":""}.\npred = [train["${e.label}"].mean()] * len(test)`:`# Baseline: predict the most common class for every row.\npred = [train["${e.label}"].mode()[0]] * len(test)`},{kind:"code",source:`pd.DataFrame({"${e.id}": test["${e.id}"], "${e.label}": pred}).to_csv("submission.csv", index=False)\nprint("wrote submission.csv —", len(pred), "rows")`}]}export const SQUARE_GIB=3.5;export const CARD_UNITS={l4:6,ada:5,t4:4};export function capsulePlan(e,t,r,s){if(!r)return{tier:"cpu",squares:0,fits:!0,capGib:0,perCard:{l4:0,ada:0,t4:0}};const n=[1,2,3].find(t=>e<3.5*t-.5);if(!n)return{tier:"gpu",squares:3,fits:!1,capGib:10,perCard:{l4:0,ada:0,t4:0}};const a=(e,t)=>3===n&&"l4"!==t?0:Math.floor(e/n),o={l4:a(CARD_UNITS.l4,"l4"),ada:a(CARD_UNITS.ada,"ada"),t4:a(CARD_UNITS.t4,"t4")},i={tier:"gpu",squares:n,fits:!0,capGib:3.5*n-.5,perCard:o},l=e=>({l4:e(o.l4),ada:e(o.ada),t4:e(o.t4)});return s&&s>0&&(i.cards=l(e=>e?Math.ceil(s/e):0)),t>0&&(i.runsPerCardHour=l(e=>Math.floor(60*e/t))),i}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "icoa-cli",
3
- "version": "2.19.712",
3
+ "version": "2.19.713",
4
4
  "description": "ICOA CLI — AI-native competition runtime of the International Cyber Olympiad in AI: learn, practise, and compete in AI & cybersecurity from the terminal (AI4CTF · CTF4AI · CTF4EAI)",
5
5
  "author": "Charlie Zhu",
6
6
  "type": "module",