icoa-cli 2.19.713 → 2.19.714
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/setter/icoa_metrics.py +426 -0
- package/dist/commands/ai4ctf.js +1 -1
- package/dist/commands/ctf4ai-demo.js +1 -1
- package/dist/commands/ctf4vla.js +1 -1
- package/dist/commands/exam.js +1 -1
- package/dist/commands/kaggle.js +1 -1
- package/dist/commands/setter.js +1 -1
- package/dist/lib/kaggle.js +1 -1
- package/dist/lib/setter-scorer.js +101 -0
- package/dist/lib/setter.js +1 -1
- package/package.json +1 -1
package/dist/commands/kaggle.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{existsSync as o,mkdirSync as e,readFileSync as n,readdirSync as t,writeFileSync as l}from"node:fs";import{join as s}from"node:path";import chalk from"chalk";import{makeReplAsker as r}from"../lib/repl-asker.js";import{getConfig as a}from"../lib/config.js";import{logCommand as i}from"../lib/logger.js";import{openFile as c}from"../lib/open-file.js";import{serializeIpynb as g}from"../lib/notebook-doc.js";import{loadLearnState as u}from"../lib/learn-state.js";import{buildStarterCells as d,dataFileUrls as y,parseKaggleTaskList as h,parseSubmitResponse as b,validateSubmissionText as p}from"../lib/kaggle.js";import{mediaHelpLines as m,sprintTaskForComp as f}from"../lib/sprint-meta.js";import{siteBase as k}from"../lib/site.js";const w=k();function $(){return process.env.ICOA_KAGGLE_SERVER?process.env.ICOA_KAGGLE_SERVER.replace(/\/$/,""):w}async function v(o){try{const e=await fetch($()+o,{signal:AbortSignal.timeout(3e4)});return await e.json()}catch{return null}}async function _(o,e){try{const n=await fetch($()+o,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),signal:AbortSignal.timeout(6e4)});return await n.json()}catch{return null}}async function S(o){try{const e=await fetch(o,{signal:AbortSignal.timeout(6e4)});return e.ok?await e.text():null}catch{return null}}function j(){return(a().language||"en").toLowerCase().startsWith("zh")}function A(o,e){return j()?o:e}function x(){console.log(),console.log(chalk.yellow(A(" 练习服务器现在连不上。"," The practice server is not reachable right now."))),console.log(chalk.gray(A(" 检查一下网络,或者过一会儿再试。"," Check your connection, or try again later."))),console.log()}function E(o){return s(process.cwd(),"icoa-kaggle",o)}function G(o,e){const n=[];let t="",l=0;const s=o=>/[ᄀ-ᅟ⺀-가-힣豈-︰-﹏-⦆¢-₩]/.test(o);for(const r of o){const o=s(r)?2:1;if(l+o>e){const o=t.lastIndexOf(" ");if(!s(r)&&" "!==r&&o>e/3){const e=t.slice(o+1);n.push(t.slice(0,o)),t=e,l=[...e].reduce((o,e)=>o+(s(e)?2:1),0)}else n.push(t),t="",l=0;if(" "===r&&""===t)continue}t+=r,l+=o}return""!==t.trim()&&n.push(t),n}function P(o){return"local-eval"===o?.judge?(console.log(chalk.bold.white(A(" 出分(就在 notebook 里):"," Scoring (inside the notebook):"))),void console.log(chalk.gray(A(" 跑最后一格 —— "," run the final cell — the "))+chalk.white("FINAL")+chalk.gray(A(" 那一行就是你的分数,没有东西要上传。"," line is your score. Nothing to upload.")))):"icoa-holdout"===o?.judge?(console.log(chalk.bold.white(A(" 交给 ICOA 判分(服务端评分):"," Submit to ICOA (scored on the server):"))),console.log(chalk.gray(" ")+chalk.bold.cyan(`kaggle submit ${o.comp_id}`)),console.log(chalk.gray(A(" 与秘密测试标签比对 → 实时排名看 "," scores vs the secret test labels → live rank on "))+chalk.cyan("board")),void console.log(chalk.gray(A(" 在 notebook 里可直接敲 "," inside the notebook just type "))+chalk.cyan("submit")+chalk.gray(A(" —— 不用退出来"," — no need to leave")))):"nitro"===o?.judge?(console.log(chalk.bold.white(A(" 怎么知道做得对不对:"," How to tell whether you got it right:"))),console.log(chalk.gray(A(" · 这道题来自 judge.nitro-ai.org,判分在他们服务器上"," · this task comes from judge.nitro-ai.org — the judging lives on their server"))),console.log(chalk.gray(A(" · 想要真分数:去 "," · for a real score: register at "))+chalk.cyan("judge.nitro-ai.org")+chalk.gray(A(" 注册后提交"," and submit there"))),void console.log(chalk.gray(A(" · 冲刺期更快的办法:做完对照官方解答,或问 "," · faster right now: compare against the official solution, or ask "))+chalk.cyan("ai4ioai"))):(console.log(chalk.bold.white(A(" 去 Kaggle 提交(手动,用你自己的账号):"," Submit on Kaggle (manual, your own account):"))),void(o?.kaggle_url?(console.log(chalk.gray(A(" 1. 打开 "," 1. open "))+chalk.cyan(o.kaggle_url)),o?.needs_gpu?console.log(chalk.gray(A(" 2. 交 notebook 产出的那个文件 —— 最后一格写了确切步骤"," 2. submit the file your notebook produced — its final cell has the exact steps"))):console.log(chalk.gray(A(" 2. Submit Predictions → 上传 "," 2. Submit Predictions → upload "))+chalk.white("submission.csv")),console.log(chalk.gray(A(" 3. 看分数,改 MODEL 那一格,重跑,再来一轮。"," 3. read your score, improve the MODEL cell, re-run, repeat.")))):(console.log(chalk.gray(A(" 在 kaggle.com 搜这个比赛名提交,或做完对照官方解答自查。"," Search kaggle.com for this competition, or check yourself against the official solution."))),console.log(chalk.gray(A(" 卡住了问 "," Stuck? ask "))+chalk.cyan("ai4ioai")+chalk.gray(A(" —— 它是弱模型,把问题问准是这次要练的本事。"," — a deliberately weak model; asking precisely is the skill you are training."))))))}export async function runKaggle(n){const t=r(),a=async o=>await t.ask(o)??"";try{console.log(),console.log(chalk.bold.cyan(" 📊 KAGGLE PRACTICE")+chalk.gray(A(" · 真实的 ML 比赛,配一本能直接跑的 starter notebook"," · real ML competitions, a runnable starter notebook"))),console.log(chalk.gray(A(" 小题在你自己的 CPU 上跑;"," Small tasks run on your CPU; "))+chalk.magenta(A("⚡ GPU 题","⚡ GPU tasks"))+chalk.gray(A("在 ICOA 的 GPU 箱上跑。"," run on the ICOA GPU box."))),console.log();const t=await v("/api/icoa/kaggle/tasks");if(!t)return x();const r=h(t);if(0===r.length)return console.log(chalk.yellow(A(" 现在还没有上线的练习赛 —— 过一会儿再来。"," No practice competitions are live yet — check back soon."))),void console.log();let c=n?r.find(o=>o.comp_id===n):void 0;if(n&&!c&&console.log(chalk.yellow(A(` 没有叫 "${n}" 的比赛。`,` No competition called "${n}".`))),!c){console.log(chalk.white(A(" 正在开放的练习赛:"," Live practice competitions:"))),r.forEach((o,e)=>{const n=o.needs_gpu?chalk.magenta(" ⚡GPU"):"";console.log(chalk.gray(` ${e+1}. `)+chalk.cyan(o.comp_id)+chalk.gray(` — ${o.title} · ${o.metric}`)+n)});for(let o=0;o<3&&!c;o++){const o=(await a(chalk.cyan(A(" 选一个(序号或 id): "," Pick one (number or id): ")))).trim();if(!o)return;const e=Number(o);c=!Number.isNaN(e)&&r[e-1]||r.find(e=>e.comp_id===o.toLowerCase())||void 0,c||console.log(chalk.gray(A(` 输 1-${r.length} 之间的序号,或者一个 id,比如 "${r[0].comp_id}"。`,` Try a number 1-${r.length} or an id like "${r[0].comp_id}".`)))}if(!c)return}const u=c.comp_id;if(i(`kaggle: download ${u}`),c.needs_gpu)return void await async function(n,t){const r=E(n);if(console.log(),console.log(chalk.white(` ${t.title}`)+chalk.gray(` · ${t.metric}`)+chalk.bold.magenta(" ⚡ GPU task")),t.description){console.log();for(const o of G(t.description,74))console.log(chalk.white(` ${o}`))}console.log(),console.log(chalk.gray(A(" 正在取 notebook…"," Fetching the notebook…"))+chalk.gray(A(" (数据留在 GPU 箱上,不下载)"," (the dataset stays on the GPU box — no download)")));const a=await v(`/api/icoa/kaggle/${n}/starter`),i=a?.data?.starter??a?.starter;if("string"!=typeof i||0===i.length)return x();const c=s(r,"starter.ipynb");let g=!1;try{e(r,{recursive:!0}),o(c)?g=!0:l(c,i),l(s(r,"meta.json"),`${JSON.stringify(t,null,2)}\n`)}catch{return void console.log(chalk.yellow(A(" 写不进 notebook —— 检查一下这个目录的写权限。"," Could not write the notebook here — check directory permissions.")))}console.log(chalk.green(" ✓ notebook: ")+chalk.white(c)+chalk.gray(g?A(" (保留了你原来那本)"," (kept your existing one)"):A(" (里面是真正的 IOAI 题)"," (the real IOAI task inside)"))),console.log(chalk.green(" ✓ dataset: ")+chalk.white(A("已经在 GPU 箱上","already on the GPU box"))+chalk.gray(A(" —— 不用下载"," — no download needed"))),console.log(),console.log(chalk.bold.white(A(" 这是 GPU 题 —— 三步:"," This is a GPU task — 3 steps:"))),console.log(chalk.gray(" 1. ")+chalk.bold.cyan(`icoa ipynb ${c}`)),console.log(chalk.gray(" 2. ")+chalk.bold.cyan("gpu key <your-key>")+chalk.gray(A(" 然后 "," then "))+chalk.bold.cyan("gpu on")+chalk.gray(A(" (连到数据所在的那台箱子)"," (connects you to the box where the data lives)"))),console.log(chalk.gray(" 3. ")+chalk.bold.cyan("run all")+chalk.gray(A(" → notebook 在 GPU 上训练。"," → the notebook trains on the GPU."))),console.log(),console.log(chalk.gray(A(" 做题时随时看: "," While you work: "))+chalk.cyan("gpu status")+chalk.gray(A(" 你还剩多少时间 · "," your time left · "))+chalk.cyan("gpu queue")+chalk.gray(A(" 谁在用,谁在等"," who is on, who is waiting"))),console.log(),t.kaggle_url&&(console.log(chalk.gray(A(" 去 Kaggle 提交: "," Submit on Kaggle: "))+chalk.cyan(t.kaggle_url)),console.log())}(u,c);if(console.log(),console.log(chalk.white(` ${c.title}`)+chalk.gray(` · ${c.metric}`)),c.description){console.log();for(const o of G(c.description,74))console.log(chalk.white(` ${o}`))}console.log(),console.log(chalk.gray(A(" 正在下载数据…"," Downloading data…")));let b=null;const p=y(c);if(p){const[o,e,n]=await Promise.all([S(p.train),S(p.test),S(p.sample_submission)]);o&&e&&n&&(b={train:o,test:e,sample_submission:n})}if(!b){const o=await v(`/api/icoa/kaggle/${u}/data`);b=o?.data??o}if(!b?.train||!b?.test||!b?.sample_submission)return x();const m=E(u),k={...c,...b.meta??{}};try{e(m,{recursive:!0}),l(s(m,"train.csv"),b.train),l(s(m,"test.csv"),b.test),l(s(m,"sample_submission.csv"),b.sample_submission),l(s(m,"meta.json"),`${JSON.stringify(k,null,2)}\n`)}catch{return void console.log(chalk.yellow(" Could not write the data files here — check directory permissions."))}const w=s(m,"starter.ipynb");let $=!1;if(o(w))$=!0;else{const o=await v(`/api/icoa/kaggle/${u}/starter`),e=o?.data?.starter??o?.starter;"string"==typeof e&&e.length>0?l(w,e):l(w,g(d(m,k)))}const _="number"==typeof k.n_train?`${k.n_train} rows`:"labelled",j="number"==typeof k.n_test?`${k.n_test} rows`:"predict these";console.log(chalk.green(" ✓ train: ")+chalk.white(s(m,"train.csv"))+chalk.gray(` (${_})`)),console.log(chalk.green(" ✓ test: ")+chalk.white(s(m,"test.csv"))+chalk.gray(` (${j})`)),console.log(chalk.green(" ✓ format: ")+chalk.white(s(m,"sample_submission.csv")));const P=f(u),I="none"===(P?.baseline||"").trim();console.log(chalk.green(" ✓ notebook: ")+chalk.white(w)+chalk.gray($?" (kept your existing one)":I?A(" (starter 在里面,FEATURES 那一格要你自己写)"," (starter inside — the FEATURES cell is yours to write)"):" (runnable baseline inside)")),(k.licence||k.attribution)&&console.log(chalk.gray(` licence: ${k.licence??"—"} · ${k.attribution??""}`)),console.log(),N(u,k,w),console.log()}finally{t.close()}}function I(o){const e=j();switch(o){case"audio":return e?"音频":"audio";case"image":return e?"图片":"images";case"weights":return e?"模型权重":"model weights";default:return e?"附件":"the extra files"}}function N(o,e,n){const t=f(o),l=t?m(t,j()):[];if(l.length>0){for(const o of l)for(const e of G(o,74))console.log(chalk.yellow(` ${e}`));console.log()}if(e?.needs_gpu)console.log(chalk.bold.white(" Your move — 4 steps (⚡ GPU task):")),console.log(chalk.gray(" 1. ")+chalk.bold.cyan(`icoa ipynb ${n}`)),console.log(chalk.gray(" 2. ")+chalk.bold.cyan("gpu key <your-key>")+chalk.gray(" then ")+chalk.bold.cyan("gpu on")+chalk.gray(" → ")+chalk.bold.cyan("run all")+chalk.gray(" (trains on the GPU box)")),console.log(chalk.gray(" 3. ")+chalk.bold.cyan("download <file>")+chalk.gray(" → pulls your results (submission.csv / predictions.zip) here")),console.log(chalk.gray(" 4. submit on kaggle.com — the notebook's final cell has the exact steps.")),console.log(),console.log(chalk.gray(" While you work: ")+chalk.cyan("gpu status")+chalk.gray(" your time left · ")+chalk.cyan("gpu queue")+chalk.gray(" who is on, who is waiting"));else{const l=t?.media?.pack,s="none"===(t?.baseline||"").trim();console.log(chalk.bold.white(l?A(" 接下来 4 步:"," Your move — 4 steps:"):A(" 接下来 3 步:"," Your move — 3 steps:")));let r=1;l&&console.log(chalk.gray(` ${r++}. `)+chalk.bold.cyan(`teleload ${l}`)+chalk.gray(A(` → 把${I(t?.media?.kind)}拉下来(CSV 里没有)`,` → fetch the ${I(t?.media?.kind)} (the CSVs do not carry it)`))),console.log(chalk.gray(` ${r++}. `)+chalk.bold.cyan(`icoa ipynb ${n}`)+chalk.gray(s?A(" → run all 到 FEATURES 那格会停,那格是你的活"," → run all stops at FEATURES — that cell is your job"):" → run all (baseline trains)")),console.log(chalk.gray(` ${r++}. it writes `)+chalk.white("submission.csv")+chalk.gray(" → verify with ")+chalk.cyan(`kaggle check ${o}`)),"icoa-holdout"===e?.judge?console.log(chalk.gray(` ${r}. `)+chalk.bold.cyan(`kaggle submit ${o}`)+chalk.gray(" → scored + ranked live.")):"nitro"===e?.judge?console.log(chalk.gray(` ${r}. scored at `)+chalk.cyan("judge.nitro-ai.org")+chalk.gray(" (not Kaggle) — or just work the val score. See below.")):console.log(chalk.gray(` ${r}. upload it on Kaggle yourself — then beat the baseline.`))}console.log(),console.log(chalk.bold.white(" 🤝 Want AI help? Set up your cockpit:")),console.log(chalk.gray(" 1. split the screen: ")+chalk.bold.cyan("Ctrl-b %")+chalk.gray(" (lesson/work left · AI right)")),console.log(chalk.gray(" 2. in the right pane: ")+chalk.bold.cyan("icoa")+chalk.gray(" → ↑↓ pick ")+chalk.bold.white("🧠 Learn")+chalk.gray(" → at ")+chalk.cyan("icoa learn>")+chalk.gray(" type ")+chalk.bold.cyan("ai4ioai")+chalk.gray(" → live Gemma 3")),console.log(chalk.gray(" 3. ask it for feature ideas or bug fixes — then ")+chalk.bold.white("VERIFY")+chalk.gray(" with your val score (a tip that lowers it, drop it).")),console.log(),P(e)}export async function runKaggleGuide(e){console.log();const t=C(e,"guide");if(!t)return;const l=E(t);let r=null;try{r=JSON.parse(n(s(l,"meta.json"),"utf-8"))}catch{r=null}i(`kaggle: guide ${t}`);const a=s(l,"starter.ipynb"),c=o(s(l,"submission.csv"));console.log(chalk.bold.white(` ═══ 📖 ${r?.title??t} — walkthrough ═══`)),console.log(),console.log(chalk.gray(" Where you are: data ")+chalk.green("downloaded ✓")+chalk.gray(" · submission.csv: ")+(c?chalk.green("written ✓"):chalk.yellow("not yet — run the notebook"))),console.log(),N(t,r,a),console.log(),console.log(chalk.gray(" Inside the notebook, ")+chalk.cyan("guide")+chalk.gray(" shows the notebook-level guide.")),console.log()}function C(e,n){if(e)return e;const l=s(process.cwd(),"icoa-kaggle"),r=o(l)?t(l,{withFileTypes:!0}).filter(e=>e.isDirectory()&&(o(s(l,e.name,"sample_submission.csv"))||o(s(l,e.name,"meta.json")))).map(o=>o.name):[];if(1===r.length)return r[0];if(0===r.length)return console.log(chalk.yellow(" No downloaded competition found here — run ")+chalk.cyan("kaggle")+chalk.yellow(" first.")),console.log(),null;console.log(chalk.white(" Several competitions downloaded — pick one:"));for(const o of r)console.log(chalk.gray(` kaggle ${n} `)+chalk.cyan(o));return console.log(),null}export async function runKaggleCheck(e){console.log();const t=C(e,"check");if(!t)return;const l=E(t),r=s(l,"submission.csv"),a=s(l,"sample_submission.csv");try{if(JSON.parse(n(s(l,"meta.json"),"utf-8")).needs_gpu&&!o(a))return console.log(chalk.gray(" ⚡ GPU task — results are produced and validated by the notebook itself.")),console.log(chalk.gray(" Pull them with ")+chalk.cyan("download <file>")+chalk.gray(" inside ")+chalk.cyan("icoa ipynb")+chalk.gray(", then submit on kaggle.com (final cell has the steps).")),void console.log()}catch{}if(!o(a))return console.log(chalk.yellow(` No data for "${t}" here — run `)+chalk.cyan(`kaggle ${t}`)+chalk.yellow(" first.")),void console.log();if(!o(r))return console.log(chalk.yellow(" No submission.csv yet — open the notebook and run all cells:")),console.log(chalk.gray(" ")+chalk.cyan(`icoa ipynb ${s(l,"starter.ipynb")}`)),void console.log();const g=p(n(r,"utf-8"),n(a,"utf-8"));i(`kaggle: check ${t} → ${g.ok?"ok":`rejected: ${g.problems[0]??"format"}`}`);let u=null;try{u=JSON.parse(n(s(l,"meta.json"),"utf-8"))}catch{u=null}if(g.ok)console.log(chalk.bold.green(" ✓ submission.csv has the right format")+chalk.gray(" (matches sample_submission.csv)")),console.log(chalk.gray(" ")+chalk.white(r)),console.log(),P(u),c(l);else{console.log(chalk.bold.red(" ✗ submission.csv would be rejected:"));for(const o of g.problems)console.log(chalk.yellow(` · ${o}`));console.log(),console.log(chalk.gray(" Fix the submission cell in the notebook and re-run it."))}console.log()}export async function runKaggleSubmit(e,t){console.log();const l=C(e,"submit");if(!l)return;const a=E(l),c=s(a,"submission.csv"),g=s(a,"sample_submission.csv");let d=null;try{d=JSON.parse(n(s(a,"meta.json"),"utf-8"))}catch{d=null}if("local-eval"===d?.judge)return console.log(chalk.yellow(" This task is scored inside the notebook — nothing to upload.")),P(d),void console.log();if(!d||!o(g))return console.log(chalk.yellow(` No data for "${l}" here — run `)+chalk.cyan(`kaggle ${l}`)+chalk.yellow(" first.")),void console.log();if("icoa-holdout"!==d.judge)return console.log(chalk.yellow(" This competition is judged on kaggle.com — nothing to submit here.")),console.log(),P(d),void console.log();if(!o(c))return console.log(chalk.yellow(" No submission.csv yet — open the notebook and run all cells:")),console.log(chalk.gray(" ")+chalk.cyan(`icoa ipynb ${s(a,"starter.ipynb")}`)),void console.log();const y=n(c,"utf-8"),h=p(y,n(g,"utf-8"));if(!h.ok){i(`kaggle: submit ${l} → local-reject: ${h.problems[0]??"format"}`),console.log(chalk.bold.red(" ✗ submission.csv would be rejected:"));for(const o of h.problems)console.log(chalk.yellow(` · ${o}`));return console.log(),console.log(chalk.gray(" Fix the submission cell in the notebook and re-run it.")),void console.log()}let m=u()?.token??"";if(!m){const o=r();try{m=(await o.ask(chalk.cyan(" Your learn token: "))??"").trim()}finally{o.close()}if(!m)return console.log(chalk.gray(" A learn token identifies you on the board — ask your coach for one.")),void console.log()}const f=Date.now();let k,w=null;process.stdout.isTTY?(process.stdout.write(chalk.gray(" ⚖ scoring on the server… 0.0s")),w=setInterval(()=>{process.stdout.write(chalk.gray(`\r ⚖ scoring on the server… ${((Date.now()-f)/1e3).toFixed(1)}s `))},200)):console.log(chalk.gray(" Scoring on the server…"));try{k=await _(`/api/icoa/kaggle/${l}/submit`,{token:m,submission_csv:y,...t?{job:t}:{}})}finally{w&&(clearInterval(w),process.stdout.write(`\r ${chalk.gray(`⚖ scored in ${((Date.now()-f)/1e3).toFixed(1)}s`)}${" ".repeat(12)}\n`))}const $=b(k);if(i(`kaggle: submit ${l} → ${$.ok?`${$.metric}=${$.score}${$.deduped?" (deduped)":""} rank=${$.rank??"-"}`:$.reason??"error"}`),console.log(),$.ok){const o=`${$.metric}: ${Number($.score).toFixed(4).replace(/\.?0+$/,"")}`,e="public"===$.leaderboard?chalk.gray(" (public leaderboard — the private split is revealed after the event)"):"";$.deduped&&$.sameCode?(console.log(chalk.bold.white(` ✓ same code as before — ${o}`)+e),console.log(chalk.gray(" The notebook code is unchanged, so this is the earlier result; it does not count as a submission."))):$.deduped?(console.log(chalk.bold.white(` ✓ already scored — ${o}`)+e),console.log(chalk.gray(" These exact predictions were scored before. Improve the MODEL cell and re-run."))):(console.log(chalk.bold.green(` ✓ scored — ${o}`)+e),"number"==typeof $.rank&&"number"==typeof $.fieldSize&&console.log(chalk.bold.white(` 🏁 board rank #${$.rank}`)+chalk.gray(` of ${$.fieldSize}`))),"number"==typeof $.used&&"number"==typeof $.cap&&console.log(chalk.gray(` ⚖ submissions used ${$.used}/${$.cap}`)),$.boardEvent&&console.log(chalk.gray(" standings: ")+chalk.cyan(`kaggle board ${l}`)+chalk.gray(" (or ")+chalk.cyan(`board ${$.boardEvent} ${l}`)+chalk.gray(")")),$.submissionId&&"public"===$.leaderboard&&console.log(chalk.gray(` id ${$.submissionId} · pick the two that count on the private board: `)+chalk.cyan(`kaggle final ${l} ${$.submissionId}`)+chalk.gray(" [<id2>] (unpicked = public best + last)"))}else switch($.reason){case"bad_token":console.log(chalk.bold.red(" ✗ that learn token was not accepted")+chalk.gray(" (invalid or expired)")),console.log(chalk.gray(" Check the token with your coach, then try again."));break;case"format":console.log(chalk.bold.red(" ✗ the server rejected submission.csv:"));for(const o of $.problems??[])console.log(chalk.yellow(` · ${o}`));break;case"event_closed":case"outside_window":console.log(chalk.bold.yellow(" ⏱ the submission window for this competition is closed."));break;case"cap_reached":console.log(chalk.bold.yellow(" ⏱ you have used all your submissions for this competition."));break;case"kaggle_judged":console.log(chalk.yellow(" This competition is judged on kaggle.com — nothing to submit here."));break;default:console.log(chalk.yellow(" The scoring server is not reachable right now — try again in a minute."))}console.log()}export async function runKaggleFinal(o,e=[]){console.log();const n=C(o,"final");if(!n)return;const t=u()?.token??"";if(!t)return console.log(chalk.gray(" A learn token identifies you on the board — open Learn once with your token first.")),void console.log();const l=await _(`/api/icoa/kaggle/${n}/final`,{token:t,ids:e.slice(0,2)})??{},s=l.data??l??{};if("string"==typeof s.reason){if("bad_id"===s.reason){console.log(chalk.bold.red(` ✗ ${String(s.error)}`));const o=Array.isArray(s.yours)?s.yours:[];o.length>0&&console.log(chalk.gray(` your scored ids: ${o.join(", ")}`))}else"bad_token"===s.reason?console.log(chalk.bold.red(" ✗ that learn token was not accepted")):console.log(chalk.yellow(` ${String(s.error??s.reason)}`));return void console.log()}if(!Array.isArray(s.final))return console.log(chalk.yellow(" The scoring server is not reachable right now — try again in a minute.")),void console.log();const r=s.final;if(s.chosen&&r.length>0){console.log(chalk.bold.green(" ✓ your final picks for the private board:"));for(const o of r)console.log(chalk.white(` ${o.id}`)+chalk.gray(` public ${Number(o.score).toFixed(4)}`))}else console.log(chalk.white(" No picks yet — the auto rule applies: ")+chalk.gray(String(s.rule_if_unchosen??"public best + last")));const a=Array.isArray(s.yours)?s.yours:[];a.length>0&&(console.log(chalk.gray(` your last ${a.length} scored: `)+a.map(o=>`${o.id} ${Number(o.score).toFixed(4)}`).join(chalk.gray(" · "))),console.log(chalk.gray(" change: ")+chalk.cyan(`kaggle final ${n} <id> [<id>]`))),i(`kaggle: final ${n} → ${r.map(o=>o.id).join(",")||"auto"}`),console.log()}const K=o=>"number"==typeof o?o.toFixed(4):" – ";export async function runKaggleBoard(o){console.log();const e=o?.trim()||C(void 0,"board");if(!e)return;const n=u()?.token??"",t=await _(`/api/icoa/kaggle/${e}/board`,n?{token:n}:{})??{},l=t.data??t??{};if(!Array.isArray(l.public))return"unknown_comp"===l.reason?console.log(chalk.bold.red(` ✗ no competition called ${e}`)):console.log(chalk.yellow(" The scoring server is not reachable right now — try again in a minute.")),void console.log();const s=l.public,r="public"===l.leaderboard,a="number"==typeof l.public_fraction?Math.round(100*l.public_fraction):null;if(console.log(chalk.bold.white(` 🏆 ${String(l.title??e)}`)+chalk.gray(` · ${r?`public leaderboard (${a??"?"}% of the test set)`:"leaderboard (whole test set)"} · ${String(l.metric??"")}`)),l.closed&&console.log(chalk.yellow(" ⏹ closed — no more submissions are accepted")),0===s.length)console.log(chalk.gray(" nobody has scored yet — be the first: ")+chalk.cyan(`kaggle submit ${e}`));else{console.log(chalk.gray(" # who score subs"));for(const o of s){const e=` ${String(o.rank).padStart(3)} ${o.who.padEnd(9)} ${K(o.score)} ${String(o.n).padStart(3)}`;console.log(o.you?chalk.bold.green(`${e} ◀ you`):chalk.white(e))}}if(Array.isArray(l.private)){const o=l.private;console.log(),console.log(chalk.bold.white(" 🔒 private leaderboard — FINAL")+chalk.gray(r?` (the other ${100-(a??0)}% of the test set)`:"")),console.log(chalk.gray(" # who private public sub id code seat picked"));for(const e of o){const o=` ${String(e.rank).padStart(3)} ${e.who.padEnd(9)} ${K(e.score)} ${K(e.public_score)} ${(e.sub_id??"-").padEnd(12)} ${(e.code_sha??"-").padEnd(12)} ${(e.job_sid??"-").padEnd(20)} ${e.how??""}`;console.log(e.you?chalk.bold.green(`${o} ◀ you`):chalk.white(o))}console.log(chalk.gray(" picked = chosen (kaggle final) or auto (public best + last) · code = sha of the notebook cells that made it"))}else r&&(console.log(chalk.gray(" (private split is revealed by the organiser after the event — then it shows here)")),n&&console.log(chalk.gray(" pick the two that count: ")+chalk.cyan(`kaggle final ${e} <id> [<id>]`)));i(`kaggle: board ${e} → ${s.length} rows${Array.isArray(l.private)?" + private":""}`),console.log()}export function registerKaggleCommand(o){o.command("kaggle").argument("[action]",'competition id, or "check" / "submit" / "board" / "final"').argument("[comp]",'competition id (with "check" / "submit" / "board" / "final")').argument("[ids...]",'submission ids (with "final")').description("Kaggle-style ML practice — dataset + starter notebook, check & submit & board & final").action(async(o,e,n=[])=>{"check"===o?await runKaggleCheck(e):"submit"===o?await runKaggleSubmit(e):"board"===o?await runKaggleBoard(e):"final"===o?await runKaggleFinal(e,n):"guide"===o?await runKaggleGuide(e):await runKaggle(o)}),o.command("guide").argument("[comp]","competition id").description("Show the competition walkthrough (3 steps + AI cockpit)").action(async o=>{await runKaggleGuide(o)})}
|
|
1
|
+
import{existsSync as o,mkdirSync as e,readFileSync as n,readdirSync as t,writeFileSync as l}from"node:fs";import{join as s}from"node:path";import chalk from"chalk";import{makeReplAsker as r}from"../lib/repl-asker.js";import{getConfig as a}from"../lib/config.js";import{logCommand as i}from"../lib/logger.js";import{openFile as c}from"../lib/open-file.js";import{serializeIpynb as g}from"../lib/notebook-doc.js";import{loadLearnState as d}from"../lib/learn-state.js";import{buildStarterCells as u,dataFileUrls as y,parseKaggleTaskList as h,parseSubmitResponse as b,validateSubmissionText as p}from"../lib/kaggle.js";import{mediaHelpLines as m,sprintTaskForComp as f}from"../lib/sprint-meta.js";import{siteBase as k}from"../lib/site.js";const w=k();function $(){return process.env.ICOA_KAGGLE_SERVER?process.env.ICOA_KAGGLE_SERVER.replace(/\/$/,""):w}async function v(o){try{const e=await fetch($()+o,{signal:AbortSignal.timeout(3e4)});return await e.json()}catch{return null}}async function _(o,e){try{const n=await fetch($()+o,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),signal:AbortSignal.timeout(6e4)});return await n.json()}catch{return null}}async function S(o){try{const e=await fetch(o,{signal:AbortSignal.timeout(6e4)});return e.ok?await e.text():null}catch{return null}}function j(){return(a().language||"en").toLowerCase().startsWith("zh")}function A(o,e){return j()?o:e}function x(){console.log(),console.log(chalk.yellow(A(" 练习服务器现在连不上。"," The practice server is not reachable right now."))),console.log(chalk.gray(A(" 检查一下网络,或者过一会儿再试。"," Check your connection, or try again later."))),console.log()}function E(o){return s(process.cwd(),"icoa-kaggle",o)}function G(o,e){const n=[];let t="",l=0;const s=o=>/[ᄀ-ᅟ⺀-가-힣豈-︰-﹏-⦆¢-₩]/.test(o);for(const r of o){const o=s(r)?2:1;if(l+o>e){const o=t.lastIndexOf(" ");if(!s(r)&&" "!==r&&o>e/3){const e=t.slice(o+1);n.push(t.slice(0,o)),t=e,l=[...e].reduce((o,e)=>o+(s(e)?2:1),0)}else n.push(t),t="",l=0;if(" "===r&&""===t)continue}t+=r,l+=o}return""!==t.trim()&&n.push(t),n}function P(o){return"local-eval"===o?.judge?(console.log(chalk.bold.white(A(" 出分(就在 notebook 里):"," Scoring (inside the notebook):"))),void console.log(chalk.gray(A(" 跑最后一格 —— "," run the final cell — the "))+chalk.white("FINAL")+chalk.gray(A(" 那一行就是你的分数,没有东西要上传。"," line is your score. Nothing to upload.")))):"icoa-holdout"===o?.judge?(console.log(chalk.bold.white(A(" 交给 ICOA 判分(服务端评分):"," Submit to ICOA (scored on the server):"))),console.log(chalk.gray(" ")+chalk.bold.cyan(`kaggle submit ${o.comp_id}`)),console.log(chalk.gray(A(" 与秘密测试标签比对 → 实时排名看 "," scores vs the secret test labels → live rank on "))+chalk.cyan("board")),void console.log(chalk.gray(A(" 在 notebook 里可直接敲 "," inside the notebook just type "))+chalk.cyan("submit")+chalk.gray(A(" —— 不用退出来"," — no need to leave")))):"nitro"===o?.judge?(console.log(chalk.bold.white(A(" 怎么知道做得对不对:"," How to tell whether you got it right:"))),console.log(chalk.gray(A(" · 这道题来自 judge.nitro-ai.org,判分在他们服务器上"," · this task comes from judge.nitro-ai.org — the judging lives on their server"))),console.log(chalk.gray(A(" · 想要真分数:去 "," · for a real score: register at "))+chalk.cyan("judge.nitro-ai.org")+chalk.gray(A(" 注册后提交"," and submit there"))),void console.log(chalk.gray(A(" · 冲刺期更快的办法:做完对照官方解答,或问 "," · faster right now: compare against the official solution, or ask "))+chalk.cyan("ai4ioai"))):(console.log(chalk.bold.white(A(" 去 Kaggle 提交(手动,用你自己的账号):"," Submit on Kaggle (manual, your own account):"))),void(o?.kaggle_url?(console.log(chalk.gray(A(" 1. 打开 "," 1. open "))+chalk.cyan(o.kaggle_url)),o?.needs_gpu?console.log(chalk.gray(A(" 2. 交 notebook 产出的那个文件 —— 最后一格写了确切步骤"," 2. submit the file your notebook produced — its final cell has the exact steps"))):console.log(chalk.gray(A(" 2. Submit Predictions → 上传 "," 2. Submit Predictions → upload "))+chalk.white("submission.csv")),console.log(chalk.gray(A(" 3. 看分数,改 MODEL 那一格,重跑,再来一轮。"," 3. read your score, improve the MODEL cell, re-run, repeat.")))):(console.log(chalk.gray(A(" 在 kaggle.com 搜这个比赛名提交,或做完对照官方解答自查。"," Search kaggle.com for this competition, or check yourself against the official solution."))),console.log(chalk.gray(A(" 卡住了问 "," Stuck? ask "))+chalk.cyan("ai4ioai")+chalk.gray(A(" —— 它是弱模型,把问题问准是这次要练的本事。"," — a deliberately weak model; asking precisely is the skill you are training."))))))}export async function runKaggle(n){const t=r(),a=async o=>await t.ask(o)??"";try{console.log(),console.log(chalk.bold.cyan(" 📊 KAGGLE PRACTICE")+chalk.gray(A(" · 真实的 ML 比赛,配一本能直接跑的 starter notebook"," · real ML competitions, a runnable starter notebook"))),console.log(chalk.gray(A(" 小题在你自己的 CPU 上跑;"," Small tasks run on your CPU; "))+chalk.magenta(A("⚡ GPU 题","⚡ GPU tasks"))+chalk.gray(A("在 ICOA 的 GPU 箱上跑。"," run on the ICOA GPU box."))),console.log();const t=await v("/api/icoa/kaggle/tasks");if(!t)return x();const r=h(t);if(0===r.length)return console.log(chalk.yellow(A(" 现在还没有上线的练习赛 —— 过一会儿再来。"," No practice competitions are live yet — check back soon."))),void console.log();let c=n?r.find(o=>o.comp_id===n):void 0;if(n&&!c&&console.log(chalk.yellow(A(` 没有叫 "${n}" 的比赛。`,` No competition called "${n}".`))),!c){console.log(chalk.white(A(" 正在开放的练习赛:"," Live practice competitions:"))),r.forEach((o,e)=>{const n=o.needs_gpu?chalk.magenta(" ⚡GPU"):"";console.log(chalk.gray(` ${e+1}. `)+chalk.cyan(o.comp_id)+chalk.gray(` — ${o.title} · ${o.metric}`)+n)});for(let o=0;o<3&&!c;o++){const o=(await a(chalk.cyan(A(" 选一个(序号或 id): "," Pick one (number or id): ")))).trim();if(!o)return;const e=Number(o);c=!Number.isNaN(e)&&r[e-1]||r.find(e=>e.comp_id===o.toLowerCase())||void 0,c||console.log(chalk.gray(A(` 输 1-${r.length} 之间的序号,或者一个 id,比如 "${r[0].comp_id}"。`,` Try a number 1-${r.length} or an id like "${r[0].comp_id}".`)))}if(!c)return}const d=c.comp_id;if(i(`kaggle: download ${d}`),c.needs_gpu)return void await async function(n,t){const r=E(n);if(console.log(),console.log(chalk.white(` ${t.title}`)+chalk.gray(` · ${t.metric}`)+chalk.bold.magenta(" ⚡ GPU task")),t.description){console.log();for(const o of G(t.description,74))console.log(chalk.white(` ${o}`))}console.log(),console.log(chalk.gray(A(" 正在取 notebook…"," Fetching the notebook…"))+chalk.gray(A(" (数据留在 GPU 箱上,不下载)"," (the dataset stays on the GPU box — no download)")));const a=await v(`/api/icoa/kaggle/${n}/starter`),i=a?.data?.starter??a?.starter;if("string"!=typeof i||0===i.length)return x();const c=s(r,"starter.ipynb");let g=!1;try{e(r,{recursive:!0}),o(c)?g=!0:l(c,i),l(s(r,"meta.json"),`${JSON.stringify(t,null,2)}\n`)}catch{return void console.log(chalk.yellow(A(" 写不进 notebook —— 检查一下这个目录的写权限。"," Could not write the notebook here — check directory permissions.")))}console.log(chalk.green(" ✓ notebook: ")+chalk.white(c)+chalk.gray(g?A(" (保留了你原来那本)"," (kept your existing one)"):A(" (里面是真正的 IOAI 题)"," (the real IOAI task inside)"))),console.log(chalk.green(" ✓ dataset: ")+chalk.white(A("已经在 GPU 箱上","already on the GPU box"))+chalk.gray(A(" —— 不用下载"," — no download needed"))),console.log(),console.log(chalk.bold.white(A(" 这是 GPU 题 —— 三步:"," This is a GPU task — 3 steps:"))),console.log(chalk.gray(" 1. ")+chalk.bold.cyan(`icoa ipynb ${c}`)),console.log(chalk.gray(" 2. ")+chalk.bold.cyan("gpu key <your-key>")+chalk.gray(A(" 然后 "," then "))+chalk.bold.cyan("gpu on")+chalk.gray(A(" (连到数据所在的那台箱子)"," (connects you to the box where the data lives)"))),console.log(chalk.gray(" 3. ")+chalk.bold.cyan("run all")+chalk.gray(A(" → notebook 在 GPU 上训练。"," → the notebook trains on the GPU."))),console.log(),console.log(chalk.gray(A(" 做题时随时看: "," While you work: "))+chalk.cyan("gpu status")+chalk.gray(A(" 你还剩多少时间 · "," your time left · "))+chalk.cyan("gpu queue")+chalk.gray(A(" 谁在用,谁在等"," who is on, who is waiting"))),console.log(),t.kaggle_url&&(console.log(chalk.gray(A(" 去 Kaggle 提交: "," Submit on Kaggle: "))+chalk.cyan(t.kaggle_url)),console.log())}(d,c);if(console.log(),console.log(chalk.white(` ${c.title}`)+chalk.gray(` · ${c.metric}`)),c.description){console.log();for(const o of G(c.description,74))console.log(chalk.white(` ${o}`))}console.log(),console.log(chalk.gray(A(" 正在下载数据…"," Downloading data…")));let b=null;const p=y(c);if(p){const[o,e,n]=await Promise.all([S(p.train),S(p.test),S(p.sample_submission)]);o&&e&&n&&(b={train:o,test:e,sample_submission:n})}if(!b){const o=await v(`/api/icoa/kaggle/${d}/data`);b=o?.data??o}if(!b?.train||!b?.test||!b?.sample_submission)return x();const m=E(d),k={...c,...b.meta??{}};try{e(m,{recursive:!0}),l(s(m,"train.csv"),b.train),l(s(m,"test.csv"),b.test),l(s(m,"sample_submission.csv"),b.sample_submission),l(s(m,"meta.json"),`${JSON.stringify(k,null,2)}\n`)}catch{return void console.log(chalk.yellow(" Could not write the data files here — check directory permissions."))}const w=s(m,"starter.ipynb");let $=!1;if(o(w))$=!0;else{const o=await v(`/api/icoa/kaggle/${d}/starter`),e=o?.data?.starter??o?.starter;"string"==typeof e&&e.length>0?l(w,e):l(w,g(u(m,k)))}const _="number"==typeof k.n_train?`${k.n_train} rows`:"labelled",j="number"==typeof k.n_test?`${k.n_test} rows`:"predict these";console.log(chalk.green(" ✓ train: ")+chalk.white(s(m,"train.csv"))+chalk.gray(` (${_})`)),console.log(chalk.green(" ✓ test: ")+chalk.white(s(m,"test.csv"))+chalk.gray(` (${j})`)),console.log(chalk.green(" ✓ format: ")+chalk.white(s(m,"sample_submission.csv")));const P=f(d),I="none"===(P?.baseline||"").trim();console.log(chalk.green(" ✓ notebook: ")+chalk.white(w)+chalk.gray($?" (kept your existing one)":I?A(" (starter 在里面,FEATURES 那一格要你自己写)"," (starter inside — the FEATURES cell is yours to write)"):" (runnable baseline inside)")),(k.licence||k.attribution)&&console.log(chalk.gray(` licence: ${k.licence??"—"} · ${k.attribution??""}`)),console.log(),N(d,k,w),console.log()}finally{t.close()}}function I(o){const e=j();switch(o){case"audio":return e?"音频":"audio";case"image":return e?"图片":"images";case"weights":return e?"模型权重":"model weights";default:return e?"附件":"the extra files"}}function N(o,e,n){const t=f(o),l=t?m(t,j()):[];if(l.length>0){for(const o of l)for(const e of G(o,74))console.log(chalk.yellow(` ${e}`));console.log()}if(e?.needs_gpu)console.log(chalk.bold.white(" Your move — 4 steps (⚡ GPU task):")),console.log(chalk.gray(" 1. ")+chalk.bold.cyan(`icoa ipynb ${n}`)),console.log(chalk.gray(" 2. ")+chalk.bold.cyan("gpu key <your-key>")+chalk.gray(" then ")+chalk.bold.cyan("gpu on")+chalk.gray(" → ")+chalk.bold.cyan("run all")+chalk.gray(" (trains on the GPU box)")),console.log(chalk.gray(" 3. ")+chalk.bold.cyan("download <file>")+chalk.gray(" → pulls your results (submission.csv / predictions.zip) here")),console.log(chalk.gray(" 4. submit on kaggle.com — the notebook's final cell has the exact steps.")),console.log(),console.log(chalk.gray(" While you work: ")+chalk.cyan("gpu status")+chalk.gray(" your time left · ")+chalk.cyan("gpu queue")+chalk.gray(" who is on, who is waiting"));else{const l=t?.media?.pack,s="none"===(t?.baseline||"").trim();console.log(chalk.bold.white(l?A(" 接下来 4 步:"," Your move — 4 steps:"):A(" 接下来 3 步:"," Your move — 3 steps:")));let r=1;l&&console.log(chalk.gray(` ${r++}. `)+chalk.bold.cyan(`teleload ${l}`)+chalk.gray(A(` → 把${I(t?.media?.kind)}拉下来(CSV 里没有)`,` → fetch the ${I(t?.media?.kind)} (the CSVs do not carry it)`))),console.log(chalk.gray(` ${r++}. `)+chalk.bold.cyan(`icoa ipynb ${n}`)+chalk.gray(s?A(" → run all 到 FEATURES 那格会停,那格是你的活"," → run all stops at FEATURES — that cell is your job"):" → run all (baseline trains)")),console.log(chalk.gray(` ${r++}. it writes `)+chalk.white("submission.csv")+chalk.gray(" → verify with ")+chalk.cyan(`kaggle check ${o}`)),"icoa-holdout"===e?.judge?console.log(chalk.gray(` ${r}. `)+chalk.bold.cyan(`kaggle submit ${o}`)+chalk.gray(" → scored + ranked live.")):"nitro"===e?.judge?console.log(chalk.gray(` ${r}. scored at `)+chalk.cyan("judge.nitro-ai.org")+chalk.gray(" (not Kaggle) — or just work the val score. See below.")):console.log(chalk.gray(` ${r}. upload it on Kaggle yourself — then beat the baseline.`))}console.log(),console.log(chalk.bold.white(" 🤝 Want AI help? Set up your cockpit:")),console.log(chalk.gray(" 1. split the screen: ")+chalk.bold.cyan("Ctrl-b %")+chalk.gray(" (lesson/work left · AI right)")),console.log(chalk.gray(" 2. in the right pane: ")+chalk.bold.cyan("icoa")+chalk.gray(" → ↑↓ pick ")+chalk.bold.white("🧠 Learn")+chalk.gray(" → at ")+chalk.cyan("icoa learn>")+chalk.gray(" type ")+chalk.bold.cyan("ai4ioai")+chalk.gray(" → live Gemma 3")),console.log(chalk.gray(" 3. ask it for feature ideas or bug fixes — then ")+chalk.bold.white("VERIFY")+chalk.gray(" with your val score (a tip that lowers it, drop it).")),console.log(),P(e)}export async function runKaggleGuide(e){console.log();const t=C(e,"guide");if(!t)return;const l=E(t);let r=null;try{r=JSON.parse(n(s(l,"meta.json"),"utf-8"))}catch{r=null}i(`kaggle: guide ${t}`);const a=s(l,"starter.ipynb"),c=o(s(l,"submission.csv"));console.log(chalk.bold.white(` ═══ 📖 ${r?.title??t} — walkthrough ═══`)),console.log(),console.log(chalk.gray(" Where you are: data ")+chalk.green("downloaded ✓")+chalk.gray(" · submission.csv: ")+(c?chalk.green("written ✓"):chalk.yellow("not yet — run the notebook"))),console.log(),N(t,r,a),console.log(),console.log(chalk.gray(" Inside the notebook, ")+chalk.cyan("guide")+chalk.gray(" shows the notebook-level guide.")),console.log()}function C(e,n){if(e)return e;const l=s(process.cwd(),"icoa-kaggle"),r=o(l)?t(l,{withFileTypes:!0}).filter(e=>e.isDirectory()&&(o(s(l,e.name,"sample_submission.csv"))||o(s(l,e.name,"meta.json")))).map(o=>o.name):[];if(1===r.length)return r[0];if(0===r.length)return console.log(chalk.yellow(" No downloaded competition found here — run ")+chalk.cyan("kaggle")+chalk.yellow(" first.")),console.log(),null;console.log(chalk.white(" Several competitions downloaded — pick one:"));for(const o of r)console.log(chalk.gray(` kaggle ${n} `)+chalk.cyan(o));return console.log(),null}export async function runKaggleCheck(e){console.log();const t=C(e,"check");if(!t)return;const l=E(t),r=s(l,"submission.csv"),a=s(l,"sample_submission.csv");try{if(JSON.parse(n(s(l,"meta.json"),"utf-8")).needs_gpu&&!o(a))return console.log(chalk.gray(" ⚡ GPU task — results are produced and validated by the notebook itself.")),console.log(chalk.gray(" Pull them with ")+chalk.cyan("download <file>")+chalk.gray(" inside ")+chalk.cyan("icoa ipynb")+chalk.gray(", then submit on kaggle.com (final cell has the steps).")),void console.log()}catch{}if(!o(a))return console.log(chalk.yellow(` No data for "${t}" here — run `)+chalk.cyan(`kaggle ${t}`)+chalk.yellow(" first.")),void console.log();if(!o(r))return console.log(chalk.yellow(" No submission.csv yet — open the notebook and run all cells:")),console.log(chalk.gray(" ")+chalk.cyan(`icoa ipynb ${s(l,"starter.ipynb")}`)),void console.log();const g=p(n(r,"utf-8"),n(a,"utf-8"));i(`kaggle: check ${t} → ${g.ok?"ok":`rejected: ${g.problems[0]??"format"}`}`);let d=null;try{d=JSON.parse(n(s(l,"meta.json"),"utf-8"))}catch{d=null}if(g.ok)console.log(chalk.bold.green(" ✓ submission.csv has the right format")+chalk.gray(" (matches sample_submission.csv)")),console.log(chalk.gray(" ")+chalk.white(r)),console.log(),P(d),c(l);else{console.log(chalk.bold.red(" ✗ submission.csv would be rejected:"));for(const o of g.problems)console.log(chalk.yellow(` · ${o}`));console.log(),console.log(chalk.gray(" Fix the submission cell in the notebook and re-run it."))}console.log()}export async function runKaggleSubmit(e,t){console.log();const l=C(e,"submit");if(!l)return;const a=E(l),c=s(a,"submission.csv"),g=s(a,"sample_submission.csv");let u=null;try{u=JSON.parse(n(s(a,"meta.json"),"utf-8"))}catch{u=null}if("local-eval"===u?.judge)return console.log(chalk.yellow(" This task is scored inside the notebook — nothing to upload.")),P(u),void console.log();if(!u||!o(g))return console.log(chalk.yellow(` No data for "${l}" here — run `)+chalk.cyan(`kaggle ${l}`)+chalk.yellow(" first.")),void console.log();if("icoa-holdout"!==u.judge)return console.log(chalk.yellow(" This competition is judged on kaggle.com — nothing to submit here.")),console.log(),P(u),void console.log();if(!o(c))return console.log(chalk.yellow(" No submission.csv yet — open the notebook and run all cells:")),console.log(chalk.gray(" ")+chalk.cyan(`icoa ipynb ${s(a,"starter.ipynb")}`)),void console.log();const y=n(c,"utf-8"),h=p(y,n(g,"utf-8"));if(!h.ok){i(`kaggle: submit ${l} → local-reject: ${h.problems[0]??"format"}`),console.log(chalk.bold.red(" ✗ submission.csv would be rejected:"));for(const o of h.problems)console.log(chalk.yellow(` · ${o}`));return console.log(),console.log(chalk.gray(" Fix the submission cell in the notebook and re-run it.")),void console.log()}let m=d()?.token??"";if(!m){const o=r();try{m=(await o.ask(chalk.cyan(" Your learn token: "))??"").trim()}finally{o.close()}if(!m)return console.log(chalk.gray(" A learn token identifies you on the board — ask your coach for one.")),void console.log()}const f=Date.now();let k,w=null;process.stdout.isTTY?(process.stdout.write(chalk.gray(" ⚖ scoring on the server… 0.0s")),w=setInterval(()=>{process.stdout.write(chalk.gray(`\r ⚖ scoring on the server… ${((Date.now()-f)/1e3).toFixed(1)}s `))},200)):console.log(chalk.gray(" Scoring on the server…"));try{k=await _(`/api/icoa/kaggle/${l}/submit`,{token:m,submission_csv:y,...t?{job:t}:{}})}finally{w&&(clearInterval(w),process.stdout.write(`\r ${chalk.gray(`⚖ scored in ${((Date.now()-f)/1e3).toFixed(1)}s`)}${" ".repeat(12)}\n`))}const $=b(k);if(i(`kaggle: submit ${l} → ${$.ok?`${$.metric}=${$.score}${$.deduped?" (deduped)":""} rank=${$.rank??"-"}`:$.reason??"error"}`),console.log(),$.ok){const o=`${$.metric}: ${Number($.score).toFixed(4).replace(/\.?0+$/,"")}`,e="public"===$.leaderboard?chalk.gray(" (public leaderboard — the private split is revealed after the event)"):"";if($.deduped&&$.sameCode?(console.log(chalk.bold.white(` ✓ same code as before — ${o}`)+e),console.log(chalk.gray(" The notebook code is unchanged, so this is the earlier result; it does not count as a submission."))):$.deduped?(console.log(chalk.bold.white(` ✓ already scored — ${o}`)+e),console.log(chalk.gray(" These exact predictions were scored before. Improve the MODEL cell and re-run."))):(console.log(chalk.bold.green(` ✓ scored — ${o}`)+e),"number"==typeof $.rank&&"number"==typeof $.fieldSize&&console.log(chalk.bold.white(` 🏁 board rank #${$.rank}`)+chalk.gray(` of ${$.fieldSize}`))),$.parts&&$.parts.length>0){const o=Math.max(...$.parts.map(([o])=>o.length));for(const[e,n]of $.parts)console.log(chalk.gray(` ${e.padEnd(o)} `)+chalk.white(n.toFixed(4)))}"number"==typeof $.used&&"number"==typeof $.cap&&console.log(chalk.gray(` ⚖ submissions used ${$.used}/${$.cap}`)),$.boardEvent&&console.log(chalk.gray(" standings: ")+chalk.cyan(`kaggle board ${l}`)+chalk.gray(" (or ")+chalk.cyan(`board ${$.boardEvent} ${l}`)+chalk.gray(")")),$.submissionId&&"public"===$.leaderboard&&console.log(chalk.gray(` id ${$.submissionId} · pick the two that count on the private board: `)+chalk.cyan(`kaggle final ${l} ${$.submissionId}`)+chalk.gray(" [<id2>] (unpicked = public best + last)"))}else switch($.reason){case"bad_token":console.log(chalk.bold.red(" ✗ that learn token was not accepted")+chalk.gray(" (invalid or expired)")),console.log(chalk.gray(" Check the token with your coach, then try again."));break;case"format":console.log(chalk.bold.red(" ✗ the server rejected submission.csv:"));for(const o of $.problems??[])console.log(chalk.yellow(` · ${o}`));break;case"event_closed":case"outside_window":console.log(chalk.bold.yellow(" ⏱ the submission window for this competition is closed."));break;case"cap_reached":console.log(chalk.bold.yellow(" ⏱ you have used all your submissions for this competition."));break;case"kaggle_judged":console.log(chalk.yellow(" This competition is judged on kaggle.com — nothing to submit here."));break;default:console.log(chalk.yellow(" The scoring server is not reachable right now — try again in a minute."))}console.log()}export async function runKaggleFinal(o,e=[]){console.log();const n=C(o,"final");if(!n)return;const t=d()?.token??"";if(!t)return console.log(chalk.gray(" A learn token identifies you on the board — open Learn once with your token first.")),void console.log();const l=await _(`/api/icoa/kaggle/${n}/final`,{token:t,ids:e.slice(0,2)})??{},s=l.data??l??{};if("string"==typeof s.reason){if("bad_id"===s.reason){console.log(chalk.bold.red(` ✗ ${String(s.error)}`));const o=Array.isArray(s.yours)?s.yours:[];o.length>0&&console.log(chalk.gray(` your scored ids: ${o.join(", ")}`))}else"bad_token"===s.reason?console.log(chalk.bold.red(" ✗ that learn token was not accepted")):console.log(chalk.yellow(` ${String(s.error??s.reason)}`));return void console.log()}if(!Array.isArray(s.final))return console.log(chalk.yellow(" The scoring server is not reachable right now — try again in a minute.")),void console.log();const r=s.final;if(s.chosen&&r.length>0){console.log(chalk.bold.green(" ✓ your final picks for the private board:"));for(const o of r)console.log(chalk.white(` ${o.id}`)+chalk.gray(` public ${Number(o.score).toFixed(4)}`))}else console.log(chalk.white(" No picks yet — the auto rule applies: ")+chalk.gray(String(s.rule_if_unchosen??"public best + last")));const a=Array.isArray(s.yours)?s.yours:[];a.length>0&&(console.log(chalk.gray(` your last ${a.length} scored: `)+a.map(o=>`${o.id} ${Number(o.score).toFixed(4)}`).join(chalk.gray(" · "))),console.log(chalk.gray(" change: ")+chalk.cyan(`kaggle final ${n} <id> [<id>]`))),i(`kaggle: final ${n} → ${r.map(o=>o.id).join(",")||"auto"}`),console.log()}const K=o=>"number"==typeof o?o.toFixed(4):" – ";export async function runKaggleBoard(o){console.log();const e=o?.trim()||C(void 0,"board");if(!e)return;const n=d()?.token??"",t=await _(`/api/icoa/kaggle/${e}/board`,n?{token:n}:{})??{},l=t.data??t??{};if(!Array.isArray(l.public))return"unknown_comp"===l.reason?console.log(chalk.bold.red(` ✗ no competition called ${e}`)):console.log(chalk.yellow(" The scoring server is not reachable right now — try again in a minute.")),void console.log();const s=l.public,r="public"===l.leaderboard,a="number"==typeof l.public_fraction?Math.round(100*l.public_fraction):null;if(console.log(chalk.bold.white(` 🏆 ${String(l.title??e)}`)+chalk.gray(` · ${r?`public leaderboard (${a??"?"}% of the test set)`:"leaderboard (whole test set)"} · ${String(l.metric??"")}`)),l.closed&&console.log(chalk.yellow(" ⏹ closed — no more submissions are accepted")),0===s.length)console.log(chalk.gray(" nobody has scored yet — be the first: ")+chalk.cyan(`kaggle submit ${e}`));else{console.log(chalk.gray(" # who score subs"));for(const o of s){const e=` ${String(o.rank).padStart(3)} ${o.who.padEnd(9)} ${K(o.score)} ${String(o.n).padStart(3)}`;console.log(o.you?chalk.bold.green(`${e} ◀ you`):chalk.white(e))}}if(Array.isArray(l.private)){const o=l.private;console.log(),console.log(chalk.bold.white(" 🔒 private leaderboard — FINAL")+chalk.gray(r?` (the other ${100-(a??0)}% of the test set)`:"")),console.log(chalk.gray(" # who private public sub id code seat picked"));for(const e of o){const o=` ${String(e.rank).padStart(3)} ${e.who.padEnd(9)} ${K(e.score)} ${K(e.public_score)} ${(e.sub_id??"-").padEnd(12)} ${(e.code_sha??"-").padEnd(12)} ${(e.job_sid??"-").padEnd(20)} ${e.how??""}`;console.log(e.you?chalk.bold.green(`${o} ◀ you`):chalk.white(o))}console.log(chalk.gray(" picked = chosen (kaggle final) or auto (public best + last) · code = sha of the notebook cells that made it"))}else r&&(console.log(chalk.gray(" (private split is revealed by the organiser after the event — then it shows here)")),n&&console.log(chalk.gray(" pick the two that count: ")+chalk.cyan(`kaggle final ${e} <id> [<id>]`)));i(`kaggle: board ${e} → ${s.length} rows${Array.isArray(l.private)?" + private":""}`),console.log()}export function registerKaggleCommand(o){o.command("kaggle").argument("[action]",'competition id, or "check" / "submit" / "board" / "final"').argument("[comp]",'competition id (with "check" / "submit" / "board" / "final")').argument("[ids...]",'submission ids (with "final")').description("Kaggle-style ML practice — dataset + starter notebook, check & submit & board & final").action(async(o,e,n=[])=>{"check"===o?await runKaggleCheck(e):"submit"===o?await runKaggleSubmit(e):"board"===o?await runKaggleBoard(e):"final"===o?await runKaggleFinal(e,n):"guide"===o?await runKaggleGuide(e):await runKaggle(o)}),o.command("guide").argument("[comp]","competition id").description("Show the competition walkthrough (3 steps + AI cockpit)").action(async o=>{await runKaggleGuide(o)})}
|
package/dist/commands/setter.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{createHash as e}from"node:crypto";import{appendFileSync as t,existsSync as s,mkdirSync as n,readdirSync as r,readFileSync as a,statSync as o,writeFileSync as i}from"node:fs";import{basename as c,join as l,resolve as u}from"node:path";import{gzipSync as d}from"node:zlib";import chalk from"chalk";import{renderPanel as h}from"../lib/guide-panel.js";import{logCommand as f}from"../lib/logger.js";import{parseIpynb as m,serializeIpynb as b}from"../lib/notebook-doc.js";import{BASELINE_FILE as p,INLINE_LIMIT_BYTES as g,LOWER_IS_BETTER as w,METRICS as v,metricGroups as y,TASK_FILES as k,baselineCells as $,capsulePlan as S,checkTask as _,parseTaskMeta as B,scoreSubmission as j,studentBundle as z,tarArchive as A,validName as x,verdictLines as G}from"../lib/setter.js";import{getCliVersion as N}from"../lib/version.js";const O=(e="")=>console.log(e),C=e=>console.log(chalk.red(` ✗ ${e}`)),F=e=>console.log(chalk.green(` ✓ ${e}`)),J=e=>console.log(chalk.gray(` ${e}`)),q=(e,t)=>console.log(chalk.gray(" ▸ next, type: ")+chalk.bold.green(e)+chalk.gray(` ${t}`));function P(e){if(e){const t=u(e);return s(l(t,"task.json"))?t:null}const t=process.cwd();if(s(l(t,"task.json")))return t;try{const e=r(t).filter(e=>!e.startsWith(".")&&s(l(t,e,"task.json")));if(1===e.length)return l(t,e[0])}catch{}return null}function M(e,t){const n=l(e,t);if(s(n)){if(o(n).size>419430400)throw new Error(`${t} is over 400 MB — too large to check here; tell the organiser, this one goes through the dataset bucket`);return a(n,"utf-8")}}function L(e){const{meta:t,problems:s}=B(M(e,"task.json")??"");for(const e of s)C(e);return t}function T(e,t){return _(t,{train:M(e,"train.csv"),testA:M(e,"test_a.csv"),labelsA:M(e,"labels_a.csv"),testB:M(e,"test_b.csv"),labelsB:M(e,"labels_b.csv")})}function U(e,s,r){try{n(l(e,".setter"),{recursive:!0}),t(l(e,".setter","ledger.jsonl"),`${JSON.stringify({at:(new Date).toISOString(),cli:N(),action:s,...r})}\n`)}catch{}}function D(e){try{return a(l(e,".setter","ledger.jsonl"),"utf-8").split("\n").filter(e=>e.trim()).map(e=>JSON.parse(e))}catch{return[]}}const I=e=>Number.isFinite(e)?e.toFixed(4):String(e),E=e=>e>=1048576?`${(e/1024/1024).toFixed(1)} MB`:`${Math.max(1,Math.round(e/1024))} KB`;function V(){const e={classes:"label = a class · submit the predicted class","probability of 1":"label = 0 or 1 · submit the probability of 1",numbers:"label = a number · submit a number"};O();for(const t of y()){O(chalk.bold.white(` ${t.kind}`)+chalk.gray(` · ${e[t.kind]}`));for(const e of t.names){const t=w.has(e)?"lower is better ":"higher is better";O(chalk.cyan(` ${e.padEnd(18)}`)+chalk.gray(`${t} ${W[e]??""}`))}}O(),J('use one: setter new <name> <metric> · or edit "metric" in task.json'),J("a metric that is not here (several values per row, images, text): tell the organiser"),O()}const W={balanced_accuracy:"mean recall per class",macro_f1:"F1 per class, plain mean",weighted_f1:"F1 per class, weighted by class size",mcc:"Matthews correlation",f1:"labels 0 / 1, F1 of class 1",qwk:"whole-number grades, quadratic weighted kappa",auc:"area under the ROC curve",ap:"average precision (area under precision–recall)",logloss:"binary cross-entropy",brier:"mean squared error of the probability",rmsle:"labels 0 or more",mape:"avoid when labels can be 0",smape:"symmetric percentage error, 0 to 2",r2:"coefficient of determination",pearson:"linear correlation",spearman:"rank correlation"};function H(e){try{return JSON.parse(a(e,"utf-8"))}catch{return null}}export async function runSetter(t=[]){const[y,..._]=t,B=(y||"").toLowerCase();f(`setter ${B}`.trim());try{""===B||"help"===B||"?"===B||"guide"===B||"status"===B?function(e){O();for(const t of h(function(e){const t=t=>!!e&&s(l(e,t));return{id:"setter",title:{en:"Problem setter",zh:"命题工具"},what:{en:"Build one task folder and test it yourself, start to finish, on this machine: the data checks, the files a contestant gets, the A and B scores of your own solutions. Nothing leaves your computer until you hand the archive over.",zh:"在这台机器上把一道题从头到尾自己测完:数据检查、学生拿到的文件、你自己的解在 A 榜和 B 榜上的分数。在你亲手把压缩包交出去之前,没有任何东西离开你的电脑。"},steps:[{verb:"setter new my-task",what:{en:"make the folder: task.json and a baseline notebook that already runs",zh:"建文件夹:task.json 和一份能直接跑的 baseline notebook"},done:()=>t("task.json")},{verb:"setter check",what:{en:"after you put train.csv, test_a.csv + labels_a.csv, test_b.csv + labels_b.csv in it: the builder's checks",zh:"把 train.csv、test_a.csv + labels_a.csv、test_b.csv + labels_b.csv 放进去之后:跑建赛脚本的那套检查"},done:()=>t(".setter/check.json")},{verb:"setter student",what:{en:"write student/ — exactly what a contestant downloads",zh:"生成 student/ —— 学生下载到的就是这些"},done:()=>t("student/test.csv")},{verb:"ipynb",what:{en:"open student/baseline.ipynb and run all, as a contestant would — it writes submission.csv",zh:"打开 student/baseline.ipynb 敲 run all,像学生一样跑 —— 它会写出 submission.csv"},done:()=>t("student/submission.csv")},{verb:"setter score",what:{en:"A score, B score and the difference (setter score ref.csv for another solution)",zh:"A 榜分、B 榜分和差值(setter score ref.csv 给另一份解算分)"},done:()=>t(".setter/scores.json")},{verb:"setter plan 3 1.8",what:{en:"GPU tasks: minutes and peak GiB of your reference run → capsule size, seats per card",zh:"GPU 题:参考解的分钟数和峰值显存 → 胶囊档位、每张卡坐几人"},optional:!0,done:()=>t(".setter/plan.json")},{verb:"setter pack",what:{en:"one archive + report to send to the organiser",zh:"打成一个压缩包 + 报告,发给组织者"},done:()=>!!e&&r(e).some(e=>e.endsWith("-handin.tar.gz"))}],notes:[{en:`${v.length} metrics — setter metrics lists them. Files are plain CSV: commas, no quotes. Publishing stays with the organiser.`,zh:`${v.length} 个指标,敲 setter metrics 看全表。文件是普通 CSV:逗号分隔,不带引号。发布由组织者来做。`}],leave:{verb:"menu",what:{en:"main menu",zh:"主菜单"}}}}(e),"full"))O(t);J(e?`task folder: ${e}`:"no task folder here yet — setter new <name> makes one in the current directory"),O()}(P(_[0])):"new"===B||"init"===B||"create"===B?function(e,t){if(!e||!x(e))return C("give the task a short name: 3–40 characters of a–z, 0–9 and -"),void J("example: setter new sydney-rain (add a metric: setter new sydney-rain rmse)");const r=(t||"accuracy").toLowerCase();if(!v.includes(r))return C(`metric "${t}" is not one the judge has`),void V();const a=u(e);if(s(a))return C(`${e}/ already exists — nothing was changed`),void q("setter check",`(inside ${e}/, or: setter check ${e})`);n(a,{recursive:!0});const o={name:e,title:e,metric:r,label:"label",id:"id",per_day:5};i(l(a,"task.json"),`${JSON.stringify({...o,description:""},null,2)}\n`),i(l(a,p),b($(o).map(e=>({...e,result:null})))),F(`made ${e}/ with task.json and ${p}`),J('now put your data in it (plain CSV, an "id" column in every file):'),J(" train.csv id + features + label"),J(" test_a.csv labels_a.csv the public-board rows, and id,label for them"),J(" test_b.csv labels_b.csv the private-board rows, and id,label for them"),J('edit task.json if your label column is not called "label" or the title should read better.'),q("setter check",`(inside ${e}/, or: setter check ${e})`)}(_[0],_[1]):"metrics"===B||"metric"===B?V():"check"===B||"verify"===B||"validate"===B?function(e){const t=P(e);if(!t)return C("no task folder found (a folder with task.json)"),q("setter new my-task","makes one here"),null;const r=L(t);if(!r)return null;let a;try{a=T(t,r)}catch(e){return C(e.message),null}if(O(),O(chalk.bold.white(` ${r.title}`)+chalk.gray(` · ${r.name} · metric ${r.metric} · predict "${r.label}"`)),!a.ok){for(const e of a.problems)C(e);return J("fix the first line above and run setter check again — nothing was written."),null}if(F(`train ${a.nTrain} rows · A ${a.nA} rows · B ${a.nB} rows · ${a.features.length} feature columns`),a.classes.length){const e=a.classes.reduce((e,[,t])=>e+t,0);J(`classes in train: ${a.classes.map(([t,s])=>`${t} ${(100*s/e).toFixed(1)}%`).join(" · ")}`)}for(const e of a.warnings)O(chalk.yellow(` ⚠ ${e}`));const c=["train.csv","test_a.csv","test_b.csv"].reduce((e,s)=>e+o(l(t,s)).size,0);c>g?O(chalk.yellow(` ⚠ data is ${E(c)} — over the ${E(g)} a competition carries inline; the organiser will host it in the dataset bucket`)):J(`data size ${E(c)} — fits the inline bundle`),s(l(t,p))||O(chalk.yellow(` ⚠ no ${p} yet — every task ships a baseline the contestants start from`)),n(l(t,".setter"),{recursive:!0}),i(l(t,".setter","check.json"),`${JSON.stringify({at:(new Date).toISOString(),cli:N(),n_train:a.nTrain,n_a:a.nA,n_b:a.nB,features:a.features.length,warnings:a.warnings,data_bytes:c},null,2)}\n`),U(t,"check",{n_train:a.nTrain,n_a:a.nA,n_b:a.nB,features:a.features.length,data_bytes:c,warnings:a.warnings.length}),q("setter student","writes student/ — what a contestant downloads")}(_[0]):"student"===B||"bundle"===B||"build"===B?function(e){const t=P(e);if(!t)return void C("no task folder found — setter new my-task makes one");const r=L(t);if(!r)return;let o;try{o=T(t,r)}catch(e){return void C(e.message)}if(!o.ok){for(const e of o.problems)C(e);return void q("setter check","shows the same problems with the fix")}const u=l(t,"student");n(u,{recursive:!0});const d=z(r,o);i(l(u,"train.csv"),d.train),i(l(u,"test.csv"),d.test),i(l(u,"sample_submission.csv"),d.sample);const h=l(t,p);s(h)&&i(l(u,p),a(h)),U(t,"student",{rows:o.nA+o.nB,baseline:s(h)}),F(`student/ written: train.csv · test.csv (${o.nA+o.nB} rows, A and B mixed) · sample_submission.csv${s(h)?` · ${p}`:""}`),J("no labels and no A/B list are in student/ — it is safe to show to a tester."),s(h)?q(`ipynb ${l(c(t),"student",p)}`,"then run all — it writes student/submission.csv"):O(chalk.yellow(` ⚠ put your ${p} in the task folder and run setter student again`))}(_[0]):"score"===B||"grade"===B?function(t){let r,o;for(const e of t)s(l(u(e),"task.json"))?r=e:o=e;const d=P(r);if(!d)return void C("no task folder found — setter new my-task makes one");const h=L(d);if(!h)return;const f=o?s(u(o))?u(o):l(d,o):l(d,"student","submission.csv");if(!s(f))return C(`no submission at ${f}`),void q(`ipynb ${l(c(d),"student",p)}`,"run all writes student/submission.csv");let m;try{m=T(d,h)}catch(e){return void C(e.message)}if(!m.ok){for(const e of m.problems)C(e);return}const b=a(f,"utf-8"),g=j(h,m,b);if(!g.ok){for(const e of g.problems)C(e);return}const v=w.has(h.metric)?"lower is better":"higher is better";O(),O(chalk.bold.white(` ${c(f)}`)+chalk.gray(` · ${h.metric} (${v})`)),O(chalk.white(` A public board ${String(m.nA).padStart(7)} rows `)+chalk.bold.white(I(g.a))),O(chalk.white(` B private board ${String(m.nB).padStart(7)} rows `)+chalk.bold.white(I(g.b))),O(chalk.gray(` all ${String(m.nA+m.nB).padStart(7)} rows ${I(g.all)} A looks ${g.gap>=0?"better":"worse"} than B by ${I(Math.abs(g.gap))}`));const y=l(d,".setter","scores.json");let k=[];try{k=JSON.parse(a(y,"utf-8"))}catch{k=[]}const $=f.startsWith(d)?f.slice(d.length+1):f,S={file:$,sha:e("sha256").update(b).digest("hex").slice(0,16),a:g.a,b:g.b,all:g.all,gap:g.gap,at:(new Date).toISOString()};k=[...k.filter(e=>e.file!==$),S],U(d,"score",{file:$,sha:S.sha,metric:h.metric,a:g.a,b:g.b,all:g.all}),n(l(d,".setter"),{recursive:!0}),i(y,`${JSON.stringify(k,null,2)}\n`);const _=k.find(e=>e.file===l("student","submission.csv")),B=k.filter(e=>e!==_),z=w.has(h.metric),A=B.sort((e,t)=>z?e.b-t.b:t.b-e.b)[0];if(_){const e=e=>({ok:!0,problems:[],a:e.a,b:e.b,all:e.all,gap:e.gap,nMissing:0});O();for(const t of G(h,e(_),A?e(A):null))J(`· ${t}`);return A?void q("setter pack","one archive + report for the organiser"):void q("setter score ref.csv","score your own better solution too — that shows how much room contestants have")}J("· this is not the baseline run (student/submission.csv) — score that one too to see the room between them"),q(`ipynb ${l(c(d),"student",p)}`,"then run all, then setter score")}(_):"plan"===B||"capsule"===B||"capacity"===B?function(e){const t=P(),s="cpu"===e[0]?.toLowerCase(),r=e.filter(e=>"cpu"!==e.toLowerCase()).map(Number),a=r[0],o=s?0:r[1],c=s?r[1]:r[2];if(!Number.isFinite(a)||a<=0||!s&&(!Number.isFinite(o)||o<=0))return C("give the two numbers your reference run measured: minutes and peak GPU memory in GiB"),J("example: setter plan 3 1.8 200 (3 minutes, 1.8 GiB peak, 200 contestants at once)"),J(" setter plan cpu 2 200 (a task that needs no GPU)"),void J("where the numbers come from: run the reference on a seat (gpufortask → ipynb → gpu on) and type selfcheck.");const u=S(o,a,!s,Number.isFinite(c)?c:void 0);if(O(),"cpu"===u.tier)F(`CPU task · ${a} min per run — contestants run it on their own machine, no seat needed`);else{if(!u.fits)return void C(`peak ${o} GiB does not fit the largest capsule (cap ${u.capGib} GiB) — shrink the batch or the model; a task every card can run beats one that only fits the biggest`);F(`${u.squares}-square capsule · peak ${o} GiB under the ${u.capGib} GiB cap · ${a} min per run`),O(chalk.white(" card seats at once runs per card-hour")+(u.cards?chalk.white(` cards for ${c} at once`):""));for(const[e,t]of[["l4","L4 24 GB"],["ada","Ada 20 GB"],["t4","T4 16 GB"]])u.perCard[e]?O(chalk.white(` ${t} ${String(u.perCard[e]).padStart(6)} ${String(u.runsPerCardHour?.[e]??"-").padStart(6)}`)+(u.cards?chalk.white(` ${String(u.cards[e]).padStart(6)}`):"")):O(chalk.gray(` ${t} — (a 3-square task runs on L4 only, so every contestant gets the same card)`));J("every contestant of one task gets the same capsule size on the same card type — that is what makes the times comparable."),J("a planning estimate from fixed capsule sizes; the organiser calibrates the real number on the fleet before the event.")}t&&(n(l(t,".setter"),{recursive:!0}),i(l(t,".setter","plan.json"),`${JSON.stringify({minutes:a,peak_gib:o,contestants:c??null,...u},null,2)}\n`),U(t,"plan",{minutes:a,peak_gib:o,squares:u.squares,tier:u.tier}),J("saved with the task — setter pack carries it to the organiser."))}(_):"pack"===B||"handin"===B||"hand-in"===B||"package"===B?function(t){const n=P(t);if(!n)return void C("no task folder found — setter new my-task makes one");const r=L(n);if(!r)return;let o;try{o=T(n,r)}catch(e){return void C(e.message)}if(!o.ok){for(const e of o.problems)C(e);return void q("setter check","the archive is only made from a folder that passes")}if(!s(l(n,p)))return void C(`${p} is missing — every task ships a baseline`);try{m(a(l(n,p),"utf-8"))}catch(e){return void C(`${p} does not open as a notebook: ${e.message}`)}let u=[];try{u=JSON.parse(a(l(n,".setter","scores.json"),"utf-8"))}catch{u=[]}u.some(e=>e.file===l("student","submission.csv"))||O(chalk.yellow(" ⚠ the baseline has not been scored — the organiser will ask for it; run it and setter score first if you can"));const h=[...k];for(const e of["description.md","ref.csv","reference.ipynb"])s(l(n,e))&&h.push(e);const f=h.map(e=>({name:`${r.name}/${e}`,data:new Uint8Array(a(l(n,e)))})),b={task:r,packed_at:(new Date).toISOString(),cli:N(),rows:{train:o.nTrain,a:o.nA,b:o.nB},features:o.features.length,warnings:o.warnings,scores:u,capsule:H(l(n,".setter","plan.json")),ledger:D(n),files:f.map(t=>({name:t.name,bytes:t.data.length,sha256:e("sha256").update(t.data).digest("hex")}))};f.push({name:`${r.name}/setter-report.json`,data:(new TextEncoder).encode(`${JSON.stringify(b,null,2)}\n`)});const g=l(n,`${r.name}-handin.tar.gz`),w=d(A(f));i(g,w),F(`${c(g)} · ${E(w.length)} · ${f.length} files`),J(g),O(chalk.yellow(" ⚠ this archive contains the B labels — send it only to the organiser, never to a shared channel.")),J("the organiser runs the same checks again, builds the competition, and you then walk it as a contestant: kaggle <name>.")}(_[0]):(C(`setter does not know "${y}"`),J("it knows: new · check · student · score · plan · pack (bare setter shows the steps)"))}catch(e){C(e.message)}}export function registerSetterCommand(e){e.command("setter [args...]").description("Problem setter: build a task folder and test it end to end locally (new · check · student · score · pack)").action(async(e=[])=>{await runSetter(e)})}
|
|
1
|
+
import{createHash as e}from"node:crypto";import{appendFileSync as t,existsSync as s,mkdirSync as r,readdirSync as n,readFileSync as a,statSync as o,writeFileSync as i}from"node:fs";import{basename as c,join as l,resolve as u}from"node:path";import{gzipSync as h}from"node:zlib";import chalk from"chalk";import{renderPanel as d}from"../lib/guide-panel.js";import{logCommand as m}from"../lib/logger.js";import{parseIpynb as f,serializeIpynb as b}from"../lib/notebook-doc.js";import{printNoPython312 as p,probePython as g}from"../lib/python-setup.js";import{BASELINE_FILE as w,CUSTOM as y,CUSTOM_WORDS as k,HELPER_FILE as $,INLINE_LIMIT_BYTES as v,LOWER_IS_BETTER as _,METRICS as S,metricGroups as j,SCORER_FILE as B,TASK_FILES as z,baselineCells as A,capsulePlan as x,checkTask as O,parseTaskMeta as N,perfectSubmission as G,predColumns as C,readSubmission as F,scoreSubmission as M,scorerTemplate as J,studentBundle as P,tarArchive as q,validName as E,verdictLines as L,withScorerColumns as T}from"../lib/setter.js";import{helperSource as U,runScorer as I}from"../lib/setter-scorer.js";import{getCliVersion as D}from"../lib/version.js";const H=(e="")=>console.log(e),R=e=>console.log(chalk.red(` ✗ ${e}`)),V=e=>console.log(chalk.green(` ✓ ${e}`)),W=e=>console.log(chalk.gray(` ${e}`)),K=(e,t)=>console.log(chalk.gray(" ▸ next, type: ")+chalk.bold.green(e)+chalk.gray(` ${t}`));function Q(e){if(e){const t=u(e);return s(l(t,"task.json"))?t:null}const t=process.cwd();if(s(l(t,"task.json")))return t;try{const e=n(t).filter(e=>!e.startsWith(".")&&s(l(t,e,"task.json")));if(1===e.length)return l(t,e[0])}catch{}return null}function X(e,t){const r=l(e,t);if(s(r)){if(o(r).size>419430400)throw new Error(`${t} is over 400 MB — too large to check here; tell the organiser, this one goes through the dataset bucket`);return a(r,"utf-8")}}function Y(e){const{meta:t,problems:r}=N(X(e,"task.json")??"");for(const e of r)R(e);if(!t)return null;if(t.metric===y&&!function(e){let t=!0;return s(l(e,B))||(R(`${B} is missing — a weighted task is scored by it`),W("setter new <another-name> weighted makes a fresh one you can copy over"),t=!1),s(l(e,$))||(R(`${$} is missing — ${B} imports its metrics from it`),W(`setter new <another-name> weighted makes a fresh one you can copy over (do not edit ${$})`),t=!1),t}(e))return null;const n=T(t,X(e,B));for(const e of n.problems)R(e);return n.problems.length?null:n.meta}const Z=e=>e.metric===y,ee=e=>Z(e)?`weighted score (${B})`:e.metric;function te(e,t,s){const r=g();if(!r.ok||!r.cmd)return p(!1),W(`${B} runs on Python 3.12 on this machine — after python setup, run this again.`),null;const n=I(e,t,s,r.cmd);if(!n.ok){R(n.error??`${B} failed`);for(const e of n.detail??[])H(chalk.gray(` ${e}`));return null}if(n.columns&&n.columns.join(",")!==C(t).join(","))return R(`${B} COLUMNS is ${n.columns.join(", ")} but this tool read ${C(t).join(", ")} — write it as one plain line: COLUMNS = ["a", "b"]`),null;for(const[e,t]of[["A",n.a],["B",n.b],["all rows",n.all]])if(!Number.isFinite(t.score))return R(`${B} returned a score that is not a finite number (${e}) — check for a division by zero`),null;return n}function se(e,t){return O(t,{train:X(e,"train.csv"),testA:X(e,"test_a.csv"),labelsA:X(e,"labels_a.csv"),testB:X(e,"test_b.csv"),labelsB:X(e,"labels_b.csv")})}function re(e,s,n){try{r(l(e,".setter"),{recursive:!0}),t(l(e,".setter","ledger.jsonl"),`${JSON.stringify({at:(new Date).toISOString(),cli:D(),action:s,...n})}\n`)}catch{}}function ne(e){try{return a(l(e,".setter","ledger.jsonl"),"utf-8").split("\n").filter(e=>e.trim()).map(e=>JSON.parse(e))}catch{return[]}}const ae=e=>Number.isFinite(e)?e.toFixed(4):String(e),oe=e=>e>=1048576?`${(e/1024/1024).toFixed(1)} MB`:`${Math.max(1,Math.round(e/1024))} KB`;function ie(){const e={classes:"label = a class · submit the predicted class","probability of 1":"label = 0 or 1 · submit the probability of 1",numbers:"label = a number · submit a number"};H();for(const t of j()){H(chalk.bold.white(` ${t.kind}`)+chalk.gray(` · ${e[t.kind]}`));for(const e of t.names){const t=_.has(e)?"lower is better ":"higher is better";H(chalk.cyan(` ${e.padEnd(18)}`)+chalk.gray(`${t} ${ce[e]??""}`))}}H(),W('use one: setter new <name> <metric> · or edit "metric" in task.json'),W("a metric that is not here (images, text, ranked lists): tell the organiser"),H(),H(chalk.bold.white(" Several metrics with weights?")),H(chalk.bold.cyan(" setter new <name> weighted")),H()}const ce={balanced_accuracy:"mean recall per class",macro_f1:"F1 per class, plain mean",weighted_f1:"F1 per class, weighted by class size",mcc:"Matthews correlation",f1:"labels 0 / 1, F1 of class 1",qwk:"whole-number grades, quadratic weighted kappa",auc:"area under the ROC curve",ap:"average precision (area under precision–recall)",logloss:"binary cross-entropy",brier:"mean squared error of the probability",rmsle:"labels 0 or more",mape:"avoid when labels can be 0",smape:"symmetric percentage error, 0 to 2",r2:"coefficient of determination",pearson:"linear correlation",spearman:"rank correlation"};function le(e){try{return JSON.parse(a(e,"utf-8"))}catch{return null}}export async function runSetter(t=[]){const[p,...g]=t,j=(p||"").toLowerCase();m(`setter ${j}`.trim());try{""===j||"help"===j||"?"===j||"guide"===j||"status"===j?function(e){H();for(const t of d(function(e){const t=t=>!!e&&s(l(e,t));return{id:"setter",title:{en:"Problem setter",zh:"命题工具"},what:{en:"Build one task folder and test it yourself, start to finish, on this machine: the data checks, the files a contestant gets, the A and B scores of your own solutions. Nothing leaves your computer until you hand the archive over.",zh:"在这台机器上把一道题从头到尾自己测完:数据检查、学生拿到的文件、你自己的解在 A 榜和 B 榜上的分数。在你亲手把压缩包交出去之前,没有任何东西离开你的电脑。"},steps:[{verb:"setter new my-task",what:{en:"make the folder: task.json and a baseline notebook that already runs",zh:"建文件夹:task.json 和一份能直接跑的 baseline notebook"},done:()=>t("task.json")},{verb:"setter check",what:{en:"after you put train.csv, test_a.csv + labels_a.csv, test_b.csv + labels_b.csv in it: the builder's checks",zh:"把 train.csv、test_a.csv + labels_a.csv、test_b.csv + labels_b.csv 放进去之后:跑建赛脚本的那套检查"},done:()=>t(".setter/check.json")},{verb:"setter student",what:{en:"write student/ — exactly what a contestant downloads",zh:"生成 student/ —— 学生下载到的就是这些"},done:()=>t("student/test.csv")},{verb:"ipynb",what:{en:"open student/baseline.ipynb and run all, as a contestant would — it writes submission.csv",zh:"打开 student/baseline.ipynb 敲 run all,像学生一样跑 —— 它会写出 submission.csv"},done:()=>t("student/submission.csv")},{verb:"setter score",what:{en:"A score, B score and the difference (setter score ref.csv for another solution)",zh:"A 榜分、B 榜分和差值(setter score ref.csv 给另一份解算分)"},done:()=>t(".setter/scores.json")},{verb:"setter plan 3 1.8",what:{en:"GPU tasks: minutes and peak GiB of your reference run → capsule size, seats per card",zh:"GPU 题:参考解的分钟数和峰值显存 → 胶囊档位、每张卡坐几人"},optional:!0,done:()=>t(".setter/plan.json")},{verb:"setter pack",what:{en:"one archive + report to send to the organiser",zh:"打成一个压缩包 + 报告,发给组织者"},done:()=>!!e&&n(e).some(e=>e.endsWith("-handin.tar.gz"))}],notes:[{en:`${S.length} metrics — setter metrics lists them. Several metrics with weights: setter new my-task weighted. Files are plain CSV: commas, no quotes. Publishing stays with the organiser.`,zh:`${S.length} 个指标,敲 setter metrics 看全表。多个指标加权:setter new my-task weighted。文件是普通 CSV:逗号分隔,不带引号。发布由组织者来做。`}],leave:{verb:"menu",what:{en:"main menu",zh:"主菜单"}}}}(e),"full"))H(t);W(e?`task folder: ${e}`:"no task folder here yet — setter new <name> makes one in the current directory"),H()}(Q(g[0])):"new"===j||"init"===j||"create"===j?function(e,t){if(!e||!E(e))return R("give the task a short name: 3–40 characters of a–z, 0–9 and -"),void W("example: setter new sydney-rain (add a metric: setter new sydney-rain rmse)");const n=(t||"accuracy").toLowerCase(),a=k.includes(n)?y:n;if(a!==y&&!S.includes(a))return R(`metric "${t}" is not one the judge has`),void ie();const o=u(e);if(s(o))return R(`${e}/ already exists — nothing was changed`),void K("setter check",`(inside ${e}/, or: setter check ${e})`);const c={name:e,title:e,metric:a,label:"label",id:"id",per_day:5},h=a===y?U():"";if(r(o,{recursive:!0}),i(l(o,"task.json"),`${JSON.stringify({...c,description:""},null,2)}\n`),i(l(o,w),b(A(c).map(e=>({...e,result:null})))),a===y)return i(l(o,B),J()),i(l(o,$),h),V(`made ${e}/ with task.json, ${B}, ${$} and ${w}`),W(`1. put your data in ${e}/ : train.csv · test_a.csv + labels_a.csv · test_b.csv + labels_b.csv`),W(`2. open ${e}/${B} → list your metrics and weights (three marked places, nothing else)`),void K("setter check",`(inside ${e}/, or: setter check ${e}) — it runs your scorer`);V(`made ${e}/ with task.json and ${w}`),W('now put your data in it (plain CSV, an "id" column in every file):'),W(" train.csv id + features + label"),W(" test_a.csv labels_a.csv the public-board rows, and id,label for them"),W(" test_b.csv labels_b.csv the private-board rows, and id,label for them"),W('edit task.json if your label column is not called "label" or the title should read better.'),K("setter check",`(inside ${e}/, or: setter check ${e})`)}(g[0],g[1]):"metrics"===j||"metric"===j?ie():"check"===j||"verify"===j||"validate"===j?function(t){const n=Q(t);if(!n)return R("no task folder found (a folder with task.json)"),K("setter new my-task","makes one here"),null;const c=Y(n);if(!c)return null;let u,h;try{u=se(n,c)}catch(e){return R(e.message),null}if(H(),H(chalk.bold.white(` ${c.title}`)+chalk.gray(` · ${c.name} · metric ${ee(c)} · predict ${C(c).map(e=>`"${e}"`).join(", ")}`)),!u.ok){for(const e of u.problems)R(e);return W("fix the first line above and run setter check again — nothing was written."),null}if(V(`train ${u.nTrain} rows · A ${u.nA} rows · B ${u.nB} rows · ${u.features.length} feature columns`),u.classes.length){const e=u.classes.reduce((e,[,t])=>e+t,0);W(`classes in train: ${u.classes.map(([t,s])=>`${t} ${(100*s/e).toFixed(1)}%`).join(" · ")}`)}if(Z(c)){const t=te(n,c,G(c,u));if(!t)return null;const s=te(n,c,P(c,u).sample);if(!s)return null;const r=t.higher?"higher is better":"lower is better",o=`perfect ${ae(t.all.score)} · sample fill ${ae(s.all.score)}`;if(!(t.higher?t.all.score>s.all.score:t.all.score<s.all.score))return R(`${B} runs, but ${o} (${r}) — a perfect submission must score strictly better`),W("check the weights, the columns each metric reads, and HIGHER_IS_BETTER — nothing was written."),null;V(`${B} runs · ${o} (${r})`);const i=Object.keys(t.all.parts);i.length&&W(`parts: ${i.join(" · ")}`),h={sha:e("sha256").update(a(l(n,B))).digest("hex").slice(0,16),higher_is_better:t.higher,perfect:t.all.score,sample_fill:s.all.score,parts:i}}for(const e of u.warnings)H(chalk.yellow(` ⚠ ${e}`));const d=["train.csv","test_a.csv","test_b.csv"].reduce((e,t)=>e+o(l(n,t)).size,0);d>v?H(chalk.yellow(` ⚠ data is ${oe(d)} — over the ${oe(v)} a competition carries inline; the organiser will host it in the dataset bucket`)):W(`data size ${oe(d)} — fits the inline bundle`),s(l(n,w))||H(chalk.yellow(` ⚠ no ${w} yet — every task ships a baseline the contestants start from`)),r(l(n,".setter"),{recursive:!0}),i(l(n,".setter","check.json"),`${JSON.stringify({at:(new Date).toISOString(),cli:D(),n_train:u.nTrain,n_a:u.nA,n_b:u.nB,features:u.features.length,warnings:u.warnings,data_bytes:d,...h?{scorer:h}:{}},null,2)}\n`),re(n,"check",{n_train:u.nTrain,n_a:u.nA,n_b:u.nB,features:u.features.length,data_bytes:d,warnings:u.warnings.length,...h?{scorer:h}:{}}),K("setter student","writes student/ — what a contestant downloads")}(g[0]):"student"===j||"bundle"===j||"build"===j?function(e){const t=Q(e);if(!t)return void R("no task folder found — setter new my-task makes one");const n=Y(t);if(!n)return;let o;try{o=se(t,n)}catch(e){return void R(e.message)}if(!o.ok){for(const e of o.problems)R(e);return void K("setter check","shows the same problems with the fix")}const u=l(t,"student");r(u,{recursive:!0});const h=P(n,o);i(l(u,"train.csv"),h.train),i(l(u,"test.csv"),h.test),i(l(u,"sample_submission.csv"),h.sample);const d=l(t,w);s(d)&&i(l(u,w),a(d)),re(t,"student",{rows:o.nA+o.nB,baseline:s(d)}),V(`student/ written: train.csv · test.csv (${o.nA+o.nB} rows, A and B mixed) · sample_submission.csv${s(d)?` · ${w}`:""}`),W("no labels and no A/B list are in student/ — it is safe to show to a tester."),s(d)?K(`ipynb ${l(c(t),"student",w)}`,"then run all — it writes student/submission.csv"):H(chalk.yellow(` ⚠ put your ${w} in the task folder and run setter student again`))}(g[0]):"score"===j||"grade"===j?function(t){let n,o;for(const e of t)s(l(u(e),"task.json"))?n=e:o=e;const h=Q(n);if(!h)return void R("no task folder found — setter new my-task makes one");const d=Y(h);if(!d)return;const m=o?s(u(o))?u(o):l(h,o):l(h,"student","submission.csv");if(!s(m))return R(`no submission at ${m}`),void K(`ipynb ${l(c(h),"student",w)}`,"run all writes student/submission.csv");let f;try{f=se(h,d)}catch(e){return void R(e.message)}if(!f.ok){for(const e of f.problems)R(e);return}const b=a(m,"utf-8");let p,g=null;if(Z(d)){const e=F(d,f,b);if(!e.ok){for(const t of e.problems)R(t);return}if(g=te(h,d,b),!g)return;const[t,s]=[g.a.score,g.b.score];p={ok:!0,problems:[],a:t,b:s,all:g.all.score,gap:g.higher?t-s:s-t,nMissing:0}}else if(p=M(d,f,b),!p.ok){for(const e of p.problems)R(e);return}const y=g?!g.higher:_.has(d.metric),k=y?"lower is better":"higher is better";let $;if(H(),H(chalk.bold.white(` ${c(m)}`)+chalk.gray(` · ${ee(d)} (${k})`)),H(chalk.white(` A public board ${String(f.nA).padStart(7)} rows `)+chalk.bold.white(ae(p.a))),H(chalk.white(` B private board ${String(f.nB).padStart(7)} rows `)+chalk.bold.white(ae(p.b))),H(chalk.gray(` all ${String(f.nA+f.nB).padStart(7)} rows ${ae(p.all)} A looks ${p.gap>=0?"better":"worse"} than B by ${ae(Math.abs(p.gap))}`)),g&&Object.keys(g.all.parts).length){$={};const e=Object.keys(g.all.parts),t=Math.max(4,...e.map(e=>e.length));H(),H(chalk.gray(` ${"part".padEnd(t)} A B`));for(const s of e){const e={a:g.a.parts[s],b:g.b.parts[s],all:g.all.parts[s]};$[s]=e,H(chalk.white(` ${s.padEnd(t)} ${ae(e.a).padStart(8)} ${ae(e.b).padStart(8)}`))}}const v=l(h,".setter","scores.json");let S=[];try{S=JSON.parse(a(v,"utf-8"))}catch{S=[]}const j=m.startsWith(h)?m.slice(h.length+1):m,B={file:j,sha:e("sha256").update(b).digest("hex").slice(0,16),a:p.a,b:p.b,all:p.all,gap:p.gap,at:(new Date).toISOString(),...g?{higher_is_better:g.higher}:{},...$?{parts:$}:{}};S=[...S.filter(e=>e.file!==j),B],re(h,"score",{file:j,sha:B.sha,metric:d.metric,a:p.a,b:p.b,all:p.all,...g?{higher_is_better:g.higher}:{},...$?{parts:$}:{}}),r(l(h,".setter"),{recursive:!0}),i(v,`${JSON.stringify(S,null,2)}\n`);const z=S.find(e=>e.file===l("student","submission.csv")),A=S.filter(e=>e!==z).sort((e,t)=>y?e.b-t.b:t.b-e.b)[0];if(z){const e=e=>({ok:!0,problems:[],a:e.a,b:e.b,all:e.all,gap:e.gap,nMissing:0});H();for(const t of L(d,e(z),A?e(A):null,g?g.higher:void 0))W(`· ${t}`);return A?void K("setter pack","one archive + report for the organiser"):void K("setter score ref.csv","score your own better solution too — that shows how much room contestants have")}W("· this is not the baseline run (student/submission.csv) — score that one too to see the room between them"),K(`ipynb ${l(c(h),"student",w)}`,"then run all, then setter score")}(g):"plan"===j||"capsule"===j||"capacity"===j?function(e){const t=Q(),s="cpu"===e[0]?.toLowerCase(),n=e.filter(e=>"cpu"!==e.toLowerCase()).map(Number),a=n[0],o=s?0:n[1],c=s?n[1]:n[2];if(!Number.isFinite(a)||a<=0||!s&&(!Number.isFinite(o)||o<=0))return R("give the two numbers your reference run measured: minutes and peak GPU memory in GiB"),W("example: setter plan 3 1.8 200 (3 minutes, 1.8 GiB peak, 200 contestants at once)"),W(" setter plan cpu 2 200 (a task that needs no GPU)"),void W("where the numbers come from: run the reference on a seat (gpufortask → ipynb → gpu on) and type selfcheck.");const u=x(o,a,!s,Number.isFinite(c)?c:void 0);if(H(),"cpu"===u.tier)V(`CPU task · ${a} min per run — contestants run it on their own machine, no seat needed`);else{if(!u.fits)return void R(`peak ${o} GiB does not fit the largest capsule (cap ${u.capGib} GiB) — shrink the batch or the model; a task every card can run beats one that only fits the biggest`);V(`${u.squares}-square capsule · peak ${o} GiB under the ${u.capGib} GiB cap · ${a} min per run`),H(chalk.white(" card seats at once runs per card-hour")+(u.cards?chalk.white(` cards for ${c} at once`):""));for(const[e,t]of[["l4","L4 24 GB"],["ada","Ada 20 GB"],["t4","T4 16 GB"]])u.perCard[e]?H(chalk.white(` ${t} ${String(u.perCard[e]).padStart(6)} ${String(u.runsPerCardHour?.[e]??"-").padStart(6)}`)+(u.cards?chalk.white(` ${String(u.cards[e]).padStart(6)}`):"")):H(chalk.gray(` ${t} — (a 3-square task runs on L4 only, so every contestant gets the same card)`));W("every contestant of one task gets the same capsule size on the same card type — that is what makes the times comparable."),W("a planning estimate from fixed capsule sizes; the organiser calibrates the real number on the fleet before the event.")}t&&(r(l(t,".setter"),{recursive:!0}),i(l(t,".setter","plan.json"),`${JSON.stringify({minutes:a,peak_gib:o,contestants:c??null,...u},null,2)}\n`),re(t,"plan",{minutes:a,peak_gib:o,squares:u.squares,tier:u.tier}),W("saved with the task — setter pack carries it to the organiser."))}(g):"pack"===j||"handin"===j||"hand-in"===j||"package"===j?function(t){const r=Q(t);if(!r)return void R("no task folder found — setter new my-task makes one");const n=Y(r);if(!n)return;let o;try{o=se(r,n)}catch(e){return void R(e.message)}if(!o.ok){for(const e of o.problems)R(e);return void K("setter check","the archive is only made from a folder that passes")}if(!s(l(r,w)))return void R(`${w} is missing — every task ships a baseline`);try{f(a(l(r,w),"utf-8"))}catch(e){return void R(`${w} does not open as a notebook: ${e.message}`)}let u=[];try{u=JSON.parse(a(l(r,".setter","scores.json"),"utf-8"))}catch{u=[]}u.some(e=>e.file===l("student","submission.csv"))||H(chalk.yellow(" ⚠ the baseline has not been scored — the organiser will ask for it; run it and setter score first if you can"));const d=[...z];Z(n)&&d.push(B,$);for(const e of["description.md","ref.csv","reference.ipynb"])s(l(r,e))&&d.push(e);const m=d.map(e=>({name:`${n.name}/${e}`,data:new Uint8Array(a(l(r,e)))})),b={task:n,packed_at:(new Date).toISOString(),cli:D(),rows:{train:o.nTrain,a:o.nA,b:o.nB},features:o.features.length,warnings:o.warnings,scores:u,capsule:le(l(r,".setter","plan.json")),ledger:ne(r),files:m.map(t=>({name:t.name,bytes:t.data.length,sha256:e("sha256").update(t.data).digest("hex")}))};m.push({name:`${n.name}/setter-report.json`,data:(new TextEncoder).encode(`${JSON.stringify(b,null,2)}\n`)});const p=l(r,`${n.name}-handin.tar.gz`),g=h(q(m));i(p,g),V(`${c(p)} · ${oe(g.length)} · ${m.length} files`),W(p),H(chalk.yellow(" ⚠ this archive contains the B labels — send it only to the organiser, never to a shared channel.")),W("the organiser runs the same checks again, builds the competition, and you then walk it as a contestant: kaggle <name>.")}(g[0]):(R(`setter does not know "${p}"`),W("it knows: new · metrics · check · student · score · plan · pack (bare setter shows the steps)"))}catch(e){R(e.message)}}export function registerSetterCommand(e){e.command("setter [args...]").description("Problem setter: build a task folder and test it end to end locally (new · check · student · score · pack)").action(async(e=[])=>{await runSetter(e)})}
|
package/dist/lib/kaggle.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{sprintTaskForComp as e}from"./sprint-meta.js";export function parseKaggleTaskList(e){const t=e??{},s=(t.data??t).tasks;if(!Array.isArray(s))return[];const i=[];for(const e of s){if(!e||"object"!=typeof e)continue;const t=e;"string"==typeof t.comp_id&&""!==t.comp_id&&i.push({comp_id:t.comp_id,title:"string"==typeof t.title?t.title:t.comp_id,metric:"string"==typeof t.metric?t.metric:"accuracy",description:"string"==typeof t.description&&""!==t.description?t.description:void 0,kaggle_url:"string"==typeof t.kaggle_url?t.kaggle_url:null,licence:"string"==typeof t.licence?t.licence:void 0,attribution:"string"==typeof t.attribution?t.attribution:void 0,data_base:"string"==typeof t.data_base&&""!==t.data_base?t.data_base:null,id_col:"string"==typeof t.id_col?t.id_col:void 0,target_col:"string"==typeof t.target_col?t.target_col:void 0,n_train:"number"==typeof t.n_train?t.n_train:void 0,n_test:"number"==typeof t.n_test?t.n_test:void 0,judge:"string"==typeof t.judge&&""!==t.judge?t.judge:void 0,board_event:"string"==typeof t.board_event&&""!==t.board_event?t.board_event:null,needs_gpu:!0===t.needs_gpu})}return i}export function dataFileUrls(e){const t=e.data_base;if("string"!=typeof t||""===t)return null;const s=t.replace(/\/$/,"");return{train:`${s}/train.csv`,test:`${s}/test.csv`,sample_submission:`${s}/sample_submission.csv`}}export function buildStarterCells(t,s){const i=s.id_col??"id",a=s.target_col??"target";var n;n=s.metric,/rmse|mse|mae|error/i.test(n);const o="icoa-holdout"===s.judge,r=e=>({kind:"markdown",source:e,result:null}),l=e=>({kind:"code",source:e.join("\n"),result:null}),c=[`# ${s.title}`,"",...s.description?[s.description,""]:[],`Kaggle-style practice — metric: **${s.metric}**.`,"","- `train.csv` — labelled training data",`- \`test.csv\` — rows to predict (no \`${a}\` column)`,"- `sample_submission.csv` — the exact format Kaggle expects","","Run all cells top to bottom: a baseline model trains on your CPU and","`submission.csv` appears next to the data. Then improve the model cell",...o?["and re-run — `icoa kaggle submit` scores it and ranks you on the board."]:["and re-run — your score is whatever Kaggle says it is."],"","**AI help:** split your terminal (`Ctrl-b %`), run `icoa` in the right pane","and type `ai4ioai` for a live Gemma 3n E4B. Ask it for feature ideas or bug fixes —","then verify every suggestion with the val score. A tip that lowers it, drop it.",...s.licence?["",`**Data licence:** ${s.licence}`]:[],...s.attribution?[`**Attribution:** ${s.attribution}`]:[]].join("\n"),p=l(["# setup — load the data (leave as is)","import os","import numpy as np","import pandas as pd","from pandas.api.types import is_numeric_dtype","from sklearn.feature_extraction.text import TfidfVectorizer","from sklearn.model_selection import train_test_split","from sklearn.metrics import accuracy_score, mean_squared_error","","# same code everywhere: in a GPU session these env vars point at the","# box's own dataset copy + workspace — locally they are unset.",`DIR = os.environ.get("ICOA_KAGGLE_DATA") or r"${t}"`,'OUT = os.environ.get("ICOA_KAGGLE_OUT") or DIR',...s.data_base?[`DATA_BASE = "${s.data_base.replace(/\/$/,"")}"`,'if not os.path.exists(DIR + "/train.csv"):'," import urllib.request"," os.makedirs(DIR, exist_ok=True)",' for f in ("train.csv", "test.csv", "sample_submission.csv"):',' urllib.request.urlretrieve(DATA_BASE + "/" + f, DIR + "/" + f)']:[],'train = pd.read_csv(DIR + "/train.csv")','test = pd.read_csv(DIR + "/test.csv")','sub_fmt = pd.read_csv(DIR + "/sample_submission.csv")',"","# 🔴 The submission column names are NOT always the training column names.","# On tasks imported from other platforms the two genuinely differ: train.csv",'# labels the answer "class" or "result" while sample_submission.csv calls it','# "answer". Hard-coding the submission name here used to raise KeyError on the',"# very first cell. So trust the DATA: whatever column train has and test lacks","# is the thing to predict.",`SUB_ID, SUB_TARGET = "${i}", "${a}" # names the submission file wants`,"only_train = [c for c in train.columns if c not in test.columns]","TARGET = only_train[-1] if only_train else SUB_TARGET","shared = [c for c in train.columns if c in test.columns]","# Only treat a column as the row id when the submission actually names it AND",'# both files carry it. Falling back to "first shared column" silently spent a',"# real feature as an id on tasks whose id lives only in test.csv.","ID = SUB_ID if SUB_ID in shared else None",'assert TARGET in train.columns, f"cannot find the label column in {list(train.columns)}"',"","# Regression or classification? Decided from the labels, not from metadata —","# several tasks ship continuous labels, and handing those to a classifier",'# raises "Unknown label type: continuous".',"REG = is_numeric_dtype(train[TARGET]) and train[TARGET].nunique() > 20",'print(f"train {train.shape} · test {test.shape} · predicting {TARGET!r} "'," f\"({'regression' if REG else 'classification'})\")","# 前几行 —— 看一眼真实数据长什么样,比看列名有用得多","print(train.head(3).to_string(max_colwidth=24)[:700])"]),u=e(s.comp_id),d=u?.media,m="audio"===d?.kind?"scipy.io.wavfile.read(path) -> numpy, then scipy.signal for a spectrogram":"image"===d?.kind?"PIL.Image.open(path) -> numpy":"weights"===d?.kind?"torch.load(path, weights_only=True) — the model itself is the input here":"unpack the archive inside the pack, then read what is in it",h=l(["# FEATURES — turn raw columns into numbers a model can use (improve THIS cell).","# Numbers: fill gaps with the median. Category (<= 20 distinct): one-hot.","# Free text: TF-IDF — on a text task the sentences ARE the signal, and dropping",'# them as "high-cardinality" leaves nothing to train on (that used to crash the','# whole notebook with "No objects to concatenate").',"# only columns present in BOTH files — a train-only column cannot be a feature","# at prediction time, and indexing test with it raises KeyError.","feat = [c for c in shared if c not in (ID, TARGET)]","Xtr_raw, Xte_raw = train[feat].copy(), test[feat].copy()","num = [c for c in feat if is_numeric_dtype(Xtr_raw[c])]","cat = [c for c in feat if c not in num and Xtr_raw[c].nunique() <= 20]","# Everything else is vectorised as text. Requiring a long average length here","# used to throw away high-cardinality SHORT strings — ingredient names, single","# words, category codes — which on several tasks are the entire signal, leaving","# the notebook with zero features.","txt = [c for c in feat if c not in num and c not in cat]","for c in num:"," med = Xtr_raw[c].median()"," Xtr_raw[c] = Xtr_raw[c].fillna(med)"," Xte_raw[c] = Xte_raw[c].fillna(med)","blocks_tr, blocks_te = [], []","if num or cat:"," d_tr = pd.get_dummies(Xtr_raw[num + cat], columns=cat, dummy_na=True)"," d_te = pd.get_dummies(Xte_raw[num + cat], columns=cat, dummy_na=True)",' d_tr, d_te = d_tr.align(d_te, join="left", axis=1, fill_value=0)'," blocks_tr.append(d_tr.to_numpy(dtype=float))"," blocks_te.append(d_te.to_numpy(dtype=float))","for c in txt:"," # fillna BEFORE astype(str): a real NaN reaching the vectoriser raises",' # "np.nan is an invalid document", and astype alone does not catch it.',' tr_txt = Xtr_raw[c].fillna("").astype(str)',' te_txt = Xte_raw[c].fillna("").astype(str)'," # min_df=1: on a short-token column (one ingredient per row) min_df=2 can"," # filter the vocabulary down to nothing."," long_col = tr_txt.str.len().mean() >= 12"," # Try the good settings, then a permissive fallback, then give up on THIS"," # column only. A column whose vocabulary prunes to nothing used to abort the",' # whole run with "After pruning, no terms remain".',' for kw in ({"min_df": 2 if long_col else 1,',' "analyzer": "word" if long_col else "char_wb"},',' {"min_df": 1, "analyzer": "char_wb"}):'," try:"," vec = TfidfVectorizer(max_features=3000, ngram_range=(1, 2), **kw)"," b_tr = vec.fit_transform(tr_txt).toarray()"," blocks_tr.append(b_tr)"," blocks_te.append(vec.transform(te_txt).toarray())"," break"," except ValueError as e:"," last = e"," else:",' print(f" skipped text column {c!r}: {last}")',"if not blocks_tr:"," raise SystemExit(",' f"No usable feature columns. The shared columns are {feat} — none of them "',' f"is numeric, categorical or text.\\n"',...d?[` "The signal for this task is in ${d.kind} files, not in the CSVs.\\n"`,` " 1. fetch them (${d.mb}MB): teleload ${d.pack}\\n"`,...d.glob?[` " files land at <pack dir>/${d.glob}\\n"`]:[],` " 2. replace this cell: ${m}\\n"`,' " build one feature row per file, then keep the rest of the notebook.\\n"']:[' f"Two kinds of task land here, and both are real work rather than a bug:\\n"',' f" 1. audio/image — train.csv only lists sample ids and labels and the "',' f"signal is in media files. Read them with scipy.io.wavfile (audio) or "',' f"PIL (images) and turn each file into a feature row.\\n"',' f" 2. train.csv and test.csv describe different things (a retrieval or "',' f"matching task). Read both files, work out what links a test row to a "',' f"train row, and build that mapping yourself.\\n"'],...d?[' f"columns — train={list(train.columns)} test={list(test.columns)}"']:[' f"Either way: replace this cell. train={list(train.columns)} "',' f"test={list(test.columns)}"']," )","X = np.hstack(blocks_tr)","X_test = np.hstack(blocks_te)","y = train[TARGET]","X_tr, X_val, y_tr, y_val = train_test_split(X, y, test_size=0.2, random_state=0)",'print(f"{X.shape[1]} model-ready features "',' f"({len(num)} numeric, {len(cat)} one-hot, {len(txt)} TF-IDF text)")']),f=l(["# EDA — look at the labels before you model. Produces ONE figure:","# left = what you are predicting (class balance / value spread)","# right = what the FEATURES cell will have to work with","# icoa ipynb saves it as a .png and opens your image viewer (type `open` to reopen).","import matplotlib.pyplot as plt","try:"," fig, ax = plt.subplots(1, 2, figsize=(11, 4))"," if REG:",' vals = pd.to_numeric(train[TARGET], errors="coerce").dropna()'," ax[0].hist(vals, bins=30)",' ax[0].set_title(f"{TARGET} — {len(vals)} values, mean {vals.mean():.3g}")'," else:"," vc = train[TARGET].astype(str).value_counts().head(12)"," ax[0].bar(range(len(vc)), vc.values)"," ax[0].set_xticks(range(len(vc)))",' ax[0].set_xticklabels([s[:14] for s in vc.index], rotation=45, ha="right")',' # An imbalanced target is the single most common reason a "good"'," # accuracy means nothing — show the majority share right in the title."," top = vc.iloc[0] / len(train)",' ax[0].set_title(f"{TARGET} — {train[TARGET].nunique()} classes · biggest {top:.0%}")',' kinds = {"numeric": 0, "category": 0, "text": 0}'," for c in [c for c in shared if c not in (ID, TARGET)]:"," if is_numeric_dtype(train[c]):",' kinds["numeric"] += 1'," elif train[c].nunique() <= 20:",' kinds["category"] += 1'," else:",' kinds["text"] += 1'," miss = train.isna().mean()"," miss = miss[miss > 0].sort_values(ascending=False).head(10)"," if len(miss) > 0:"," ax[1].barh(range(len(miss)), miss.values)"," ax[1].set_yticks(range(len(miss)))"," ax[1].set_yticklabels([s[:16] for s in miss.index])",' ax[1].set_title("missing rate (top columns)")'," else:"," ax[1].bar(list(kinds), list(kinds.values()))",' ax[1].set_title(f"{sum(kinds.values())} usable columns · no missing values")'," plt.tight_layout()"," plt.show()",' print("columns by kind:", kinds)',"except Exception as e:"," # EDA is never allowed to stop the run — `run all` halts on the first"," # error, and a plotting hiccup must not cost the student their baseline.",' print("EDA skipped:", type(e).__name__, e)']),g=l(["# MODEL — a working baseline. Improve THIS cell to climb the board.","from sklearn.linear_model import LogisticRegression, Ridge","# REG was decided from the labels back in setup.","model = Ridge() if REG else LogisticRegression(max_iter=2000)"]),_=l(["# validate — quick local score on a held-out fifth of train","model.fit(X_tr, y_tr)","p_val = model.predict(X_val)","if REG:",' print("val RMSE:", round(mean_squared_error(y_val, p_val) ** 0.5, 4))',"else:"," acc = accuracy_score(y_val, p_val)",' print("val accuracy:", round(acc, 4))'," if acc < 0.3:"," # A near-zero baseline is the expected result on several of these tasks,"," # not a broken notebook — say so, or the student reads it as a bug.",' print(" A score this low usually means the linear baseline cannot "',' "represent the pattern at all (combination rules, sequences, "',' "spelling). That IS the task: build features that expose it, "',' "or switch to a model that can. The chain itself works — "',' "submission.csv is written either way.")']),b=l(["# submission — retrain on ALL data, write submission.csv (Kaggle format)","model.fit(X, y)","pred = model.predict(X_test)",'out = OUT + "/submission.csv"',"# Copy the layout of sample_submission.csv and swap in the predictions, rather","# than rebuilding the file from column names. Some tasks want more than two","# columns (e.g. subtaskID,datapointID,answer) and a hand-built 2-column frame","# is rejected by the grader even when every prediction is right.","if len(sub_fmt) == len(pred):"," sub = sub_fmt.copy()"," sub[sub.columns[-1]] = pred","else:"," keys = {}"," if ID is not None and ID in test.columns:"," keys[SUB_ID] = test[ID]"," keys[SUB_TARGET] = pred"," sub = pd.DataFrame(keys)",' print(f"NOTE: sample_submission has {len(sub_fmt)} rows but test has "',' f"{len(pred)} — built the file from column names instead; "',' f"check it against the task page before submitting.")',"sub.to_csv(out, index=False)",'print("wrote", out, "·", list(sub.columns))',o?`print("score it: icoa kaggle submit ${s.comp_id}")`:'print("format check: icoa kaggle check")']),y=o?["## Submit to ICOA (scored on the server)","",`1. In the ICOA CLI run \`icoa kaggle submit ${s.comp_id}\``,"2. The server scores `submission.csv` against the secret test labels"," and your rank lands on the live board (`icoa board`).","3. Improve the MODEL cell, re-run, submit again — best score counts."]:"nitro"===s.judge?["## Where this one gets scored","","This task comes from **judge.nitro-ai.org** and the answer key stays on","their server, so ICOA cannot score it for you. That does not make it a","practice stub — it is a real task, just judged elsewhere.","",`1. \`icoa kaggle check ${s.comp_id}\` — checks the FORMAT of`," `submission.csv` (headers, row count, ids). Format errors are the most"," common way a correct answer still scores zero.","2. Watch the **val score** printed above — that is your real feedback loop"," during the sprint, and it needs no account anywhere.","3. Want the official number? Register at judge.nitro-ai.org and submit"," there. Otherwise compare against the official solution, or ask"," `ai4ioai` to review your approach."]:s.kaggle_url?["## Submit on Kaggle (manual, your own account)","",`1. Open ${s.kaggle_url}`,"2. **Submit Predictions** → upload `submission.csv`","3. Read your leaderboard score, come back, improve the MODEL cell, repeat."]:["## Submit on Kaggle (manual, your own account)","","_This dataset has no live Kaggle page linked yet._","The loop is identical: upload `submission.csv`, read the score, improve","the MODEL cell, repeat. Meanwhile the **val score** printed above is a","real signal you can work against with no account at all."];return[r(c),p,f,h,g,_,b,r(y.join("\n"))]}export function parseSubmitResponse(e){const t=e&&"object"==typeof e?e:{},s=t.data??t,i=s&&"object"==typeof s?s:{};return"number"==typeof i.score&&"string"==typeof i.metric?{ok:!0,score:i.score,metric:i.metric,rank:"number"==typeof i.rank_at_time?i.rank_at_time:null,fieldSize:"number"==typeof i.field_size?i.field_size:null,deduped:!0===i.deduped,sameCode:!0===i.same_code,submissionId:"string"==typeof i.submission_id?i.submission_id:null,leaderboard:"string"==typeof i.leaderboard?i.leaderboard:null,boardEvent:"string"==typeof i.board_event?i.board_event:null,used:"number"==typeof i.submissions_used?i.submissions_used:null,cap:"number"==typeof i.submissions_cap?i.submissions_cap:null}:"string"==typeof i.reason&&""!==i.reason?{ok:!1,reason:i.reason,error:"string"==typeof i.error?i.error:void 0,problems:Array.isArray(i.problems)?i.problems.filter(e=>"string"==typeof e):void 0}:{ok:!1,reason:"network"}}export function validateSubmissionText(e,t){const s=[],i=e.trim().split(/\r?\n/),a=t.trim().split(/\r?\n/),n=i[0]??"",o=a[0]??"";n.trim()!==o.trim()&&s.push(`header mismatch: expected "${o}", got "${n}"`);const r=i.slice(1).filter(e=>""!==e.trim()),l=a.slice(1).filter(e=>""!==e.trim());r.length!==l.length&&s.push(`row count mismatch: expected ${l.length}, got ${r.length}`);const c=e=>new Set(e.map(e=>(e.split(",")[0]??"").trim())),p=c(r),u=c(l),d=[...u].filter(e=>!p.has(e)),m=[...p].filter(e=>!u.has(e));if(d.length>0||m.length>0){const e=[];d.length>0&&e.push(`${d.length} missing (e.g. ${d[0]})`),m.length>0&&e.push(`${m.length} unknown (e.g. ${m[0]})`),s.push(`id mismatch vs sample_submission: ${e.join(", ")}`)}for(let e=0;e<r.length;e++)if(r[e].split(",").some(e=>""===e.trim())){s.push(`empty cell in data row ${e+1}`);break}return{ok:0===s.length,problems:s}}export function buildMediaStarterCells(e,t){const s=e=>({kind:"markdown",source:e,result:null}),i=e=>({kind:"code",source:e.join("\n"),result:null}),a=t.media?.kind??"archive",n=t.title??"ICOA sprint task",o=[`# ${t.seq?`${t.seq} · `:""}${n}`,"",...t.topic_cn?[t.topic_cn,""]:[],"This task ships **data, not a baseline** — the CSV starter cannot help here",`because the signal lives in the ${a} files you just pulled.`,"","Run all cells: they inventory the pack and draw one picture of what is inside,","so you start from evidence instead of from the folder name. The last cell says","what you have to build — that part is the task.","","**Reading the files:** on your laptop the notebook env has numpy / scipy /","pandas / scikit-learn / torch / transformers / Pillow — but NOT librosa,","torchaudio or soundfile. The GPU contest box has librosa and torchaudio too."].join("\n"),r=i(["# inventory — what is actually in this pack (counts, sizes, a few real names)","import os","import numpy as np",`DIR = os.environ.get("ICOA_PACK_DIR") or r"${e}"`,"by_ext, total, sample = {}, 0, {}","for root, _dirs, files in os.walk(DIR):"," for f in files:",' ext = os.path.splitext(f)[1].lower() or "(no ext)"'," p = os.path.join(root, f)"," by_ext[ext] = by_ext.get(ext, 0) + 1"," total += os.path.getsize(p)"," sample.setdefault(ext, p)",'print(f"{DIR}\\n{sum(by_ext.values())} files · {total/1048576:.0f} MB")',"for ext, n in sorted(by_ext.items(), key=lambda kv: -kv[1])[:8]:",' print(f" {ext:>10} {n:>6} e.g. {os.path.relpath(sample[ext], DIR)}")',"# 🔴 a leftover .zip means the pack still needs unpacking before you can read it",'zips = [p for e, p in sample.items() if e == ".zip"]',"if zips:",' print("\\nNOTE: there is still a .zip in here — unpack it first:")',' print(f" import zipfile; zipfile.ZipFile(r\\"{zips[0]}\\").extractall(DIR)")']),l=["# EDA — one picture of what is inside. icoa ipynb saves it as a .png and","# opens your image viewer (`open` reopens it).","import matplotlib.pyplot as plt","try:"];"image"===a?l.push(" from PIL import Image"," imgs = []"," for root, _d, files in os.walk(DIR):"," for f in sorted(files):",' if os.path.splitext(f)[1].lower() in (".jpg", ".jpeg", ".png"):'," imgs.append(os.path.join(root, f))"," if len(imgs) >= 9:"," break"," fig, axes = plt.subplots(3, 3, figsize=(7, 7))"," for ax, p in zip(axes.ravel(), imgs[:9]):"," ax.imshow(Image.open(p))"," ax.set_title(os.path.basename(p)[:18], fontsize=8)"," for ax in axes.ravel():",' ax.axis("off")',' plt.suptitle(f"{len(imgs)}+ images · first 9")'):"weights"===a?l.push(" import torch"," wfiles = []"," for root, _d, files in os.walk(DIR):"," for f in sorted(files):",' if os.path.splitext(f)[1].lower() in (".pt", ".pth"):'," wfiles.append(os.path.join(root, f))"," fig, ax = plt.subplots(1, max(1, min(2, len(wfiles))), figsize=(10, 4), squeeze=False)"," for i, p in enumerate(wfiles[:2]):"," # weights_only=True: never unpickle arbitrary objects from a data file",' obj = torch.load(p, map_location="cpu", weights_only=True)',' t = obj if hasattr(obj, "flatten") else list(obj.values())[0]'," v = np.asarray(t, dtype=float).ravel()"," ax[0][i].hist(v, bins=60)",' ax[0][i].set_title(f"{os.path.basename(p)} · shape {tuple(np.shape(t))}", fontsize=9)',' print(os.path.basename(p), "shape", tuple(np.shape(t)))'):l.push(" import pandas as pd"," csvs = []"," for root, _d, files in os.walk(DIR):"," for f in files:",' if f.lower().endswith(".csv"):'," csvs.append(os.path.join(root, f))"," csvs.sort(key=os.path.getsize, reverse=True)"," df = pd.read_csv(csvs[0])"," print(os.path.basename(csvs[0]), df.shape)"," print(df.head(3).to_string(max_colwidth=24)[:600])"," # the last low-cardinality column is the usual label; if there is none,"," # plot the row-length spread so the picture still says something true"," cand = [c for c in df.columns if df[c].nunique() <= 30]"," fig, ax = plt.subplots(figsize=(8, 4))"," if cand:"," vc = df[cand[-1]].astype(str).value_counts().head(12)"," ax.bar(range(len(vc)), vc.values)"," ax.set_xticks(range(len(vc)))",' ax.set_xticklabels([s[:14] for s in vc.index], rotation=45, ha="right")',' ax.set_title(f"{cand[-1]} — {df[cand[-1]].nunique()} values")'," else:"," ax.hist([len(str(x)) for x in df[df.columns[-1]]], bins=30)",' ax.set_title(f"length of {df.columns[-1]}")'),l.push(" plt.tight_layout()"," plt.show()","except Exception as e:"," # never stop the run on a plotting problem — `run all` halts on first error",' print("EDA skipped:", type(e).__name__, e)');const c=["## Your job","","There is no baseline to beat here — you build the first one. The shape of it:","","1. **turn each file into a row of numbers** — that is the whole trick","image"===a?" (`PIL.Image.open` → resize → `np.asarray` → flatten, or a pretrained encoder)":"weights"===a?" (`torch.load(..., weights_only=True)` → the tensors themselves ARE the features)":" (read the CSVs / unpack the archive, then vectorise the text columns)","2. keep a held-out fifth of the data and print a val score every time","3. write your predictions in exactly the format the task page asks for","","`add` makes a new code cell · `add md` a note cell · `run all` runs everything ·","a cell that raises offers `ai N`, which sends that cell and its traceback to the AI."].join("\n");return[s(o),r,i(l),s(c)]}
|
|
1
|
+
import{sprintTaskForComp as e}from"./sprint-meta.js";export function parseKaggleTaskList(e){const t=e??{},s=(t.data??t).tasks;if(!Array.isArray(s))return[];const i=[];for(const e of s){if(!e||"object"!=typeof e)continue;const t=e;"string"==typeof t.comp_id&&""!==t.comp_id&&i.push({comp_id:t.comp_id,title:"string"==typeof t.title?t.title:t.comp_id,metric:"string"==typeof t.metric?t.metric:"accuracy",description:"string"==typeof t.description&&""!==t.description?t.description:void 0,kaggle_url:"string"==typeof t.kaggle_url?t.kaggle_url:null,licence:"string"==typeof t.licence?t.licence:void 0,attribution:"string"==typeof t.attribution?t.attribution:void 0,data_base:"string"==typeof t.data_base&&""!==t.data_base?t.data_base:null,id_col:"string"==typeof t.id_col?t.id_col:void 0,target_col:"string"==typeof t.target_col?t.target_col:void 0,n_train:"number"==typeof t.n_train?t.n_train:void 0,n_test:"number"==typeof t.n_test?t.n_test:void 0,judge:"string"==typeof t.judge&&""!==t.judge?t.judge:void 0,board_event:"string"==typeof t.board_event&&""!==t.board_event?t.board_event:null,needs_gpu:!0===t.needs_gpu})}return i}export function dataFileUrls(e){const t=e.data_base;if("string"!=typeof t||""===t)return null;const s=t.replace(/\/$/,"");return{train:`${s}/train.csv`,test:`${s}/test.csv`,sample_submission:`${s}/sample_submission.csv`}}export function buildStarterCells(t,s){const i=s.id_col??"id",a=s.target_col??"target";var n;n=s.metric,/rmse|mse|mae|error/i.test(n);const o="icoa-holdout"===s.judge,r=e=>({kind:"markdown",source:e,result:null}),l=e=>({kind:"code",source:e.join("\n"),result:null}),c=[`# ${s.title}`,"",...s.description?[s.description,""]:[],`Kaggle-style practice — metric: **${s.metric}**.`,"","- `train.csv` — labelled training data",`- \`test.csv\` — rows to predict (no \`${a}\` column)`,"- `sample_submission.csv` — the exact format Kaggle expects","","Run all cells top to bottom: a baseline model trains on your CPU and","`submission.csv` appears next to the data. Then improve the model cell",...o?["and re-run — `icoa kaggle submit` scores it and ranks you on the board."]:["and re-run — your score is whatever Kaggle says it is."],"","**AI help:** split your terminal (`Ctrl-b %`), run `icoa` in the right pane","and type `ai4ioai` for a live Gemma 3n E4B. Ask it for feature ideas or bug fixes —","then verify every suggestion with the val score. A tip that lowers it, drop it.",...s.licence?["",`**Data licence:** ${s.licence}`]:[],...s.attribution?[`**Attribution:** ${s.attribution}`]:[]].join("\n"),p=l(["# setup — load the data (leave as is)","import os","import numpy as np","import pandas as pd","from pandas.api.types import is_numeric_dtype","from sklearn.feature_extraction.text import TfidfVectorizer","from sklearn.model_selection import train_test_split","from sklearn.metrics import accuracy_score, mean_squared_error","","# same code everywhere: in a GPU session these env vars point at the","# box's own dataset copy + workspace — locally they are unset.",`DIR = os.environ.get("ICOA_KAGGLE_DATA") or r"${t}"`,'OUT = os.environ.get("ICOA_KAGGLE_OUT") or DIR',...s.data_base?[`DATA_BASE = "${s.data_base.replace(/\/$/,"")}"`,'if not os.path.exists(DIR + "/train.csv"):'," import urllib.request"," os.makedirs(DIR, exist_ok=True)",' for f in ("train.csv", "test.csv", "sample_submission.csv"):',' urllib.request.urlretrieve(DATA_BASE + "/" + f, DIR + "/" + f)']:[],'train = pd.read_csv(DIR + "/train.csv")','test = pd.read_csv(DIR + "/test.csv")','sub_fmt = pd.read_csv(DIR + "/sample_submission.csv")',"","# 🔴 The submission column names are NOT always the training column names.","# On tasks imported from other platforms the two genuinely differ: train.csv",'# labels the answer "class" or "result" while sample_submission.csv calls it','# "answer". Hard-coding the submission name here used to raise KeyError on the',"# very first cell. So trust the DATA: whatever column train has and test lacks","# is the thing to predict.",`SUB_ID, SUB_TARGET = "${i}", "${a}" # names the submission file wants`,"only_train = [c for c in train.columns if c not in test.columns]","TARGET = only_train[-1] if only_train else SUB_TARGET","shared = [c for c in train.columns if c in test.columns]","# Only treat a column as the row id when the submission actually names it AND",'# both files carry it. Falling back to "first shared column" silently spent a',"# real feature as an id on tasks whose id lives only in test.csv.","ID = SUB_ID if SUB_ID in shared else None",'assert TARGET in train.columns, f"cannot find the label column in {list(train.columns)}"',"","# Regression or classification? Decided from the labels, not from metadata —","# several tasks ship continuous labels, and handing those to a classifier",'# raises "Unknown label type: continuous".',"REG = is_numeric_dtype(train[TARGET]) and train[TARGET].nunique() > 20",'print(f"train {train.shape} · test {test.shape} · predicting {TARGET!r} "'," f\"({'regression' if REG else 'classification'})\")","# 前几行 —— 看一眼真实数据长什么样,比看列名有用得多","print(train.head(3).to_string(max_colwidth=24)[:700])"]),u=e(s.comp_id),d=u?.media,m="audio"===d?.kind?"scipy.io.wavfile.read(path) -> numpy, then scipy.signal for a spectrogram":"image"===d?.kind?"PIL.Image.open(path) -> numpy":"weights"===d?.kind?"torch.load(path, weights_only=True) — the model itself is the input here":"unpack the archive inside the pack, then read what is in it",h=l(["# FEATURES — turn raw columns into numbers a model can use (improve THIS cell).","# Numbers: fill gaps with the median. Category (<= 20 distinct): one-hot.","# Free text: TF-IDF — on a text task the sentences ARE the signal, and dropping",'# them as "high-cardinality" leaves nothing to train on (that used to crash the','# whole notebook with "No objects to concatenate").',"# only columns present in BOTH files — a train-only column cannot be a feature","# at prediction time, and indexing test with it raises KeyError.","feat = [c for c in shared if c not in (ID, TARGET)]","Xtr_raw, Xte_raw = train[feat].copy(), test[feat].copy()","num = [c for c in feat if is_numeric_dtype(Xtr_raw[c])]","cat = [c for c in feat if c not in num and Xtr_raw[c].nunique() <= 20]","# Everything else is vectorised as text. Requiring a long average length here","# used to throw away high-cardinality SHORT strings — ingredient names, single","# words, category codes — which on several tasks are the entire signal, leaving","# the notebook with zero features.","txt = [c for c in feat if c not in num and c not in cat]","for c in num:"," med = Xtr_raw[c].median()"," Xtr_raw[c] = Xtr_raw[c].fillna(med)"," Xte_raw[c] = Xte_raw[c].fillna(med)","blocks_tr, blocks_te = [], []","if num or cat:"," d_tr = pd.get_dummies(Xtr_raw[num + cat], columns=cat, dummy_na=True)"," d_te = pd.get_dummies(Xte_raw[num + cat], columns=cat, dummy_na=True)",' d_tr, d_te = d_tr.align(d_te, join="left", axis=1, fill_value=0)'," blocks_tr.append(d_tr.to_numpy(dtype=float))"," blocks_te.append(d_te.to_numpy(dtype=float))","for c in txt:"," # fillna BEFORE astype(str): a real NaN reaching the vectoriser raises",' # "np.nan is an invalid document", and astype alone does not catch it.',' tr_txt = Xtr_raw[c].fillna("").astype(str)',' te_txt = Xte_raw[c].fillna("").astype(str)'," # min_df=1: on a short-token column (one ingredient per row) min_df=2 can"," # filter the vocabulary down to nothing."," long_col = tr_txt.str.len().mean() >= 12"," # Try the good settings, then a permissive fallback, then give up on THIS"," # column only. A column whose vocabulary prunes to nothing used to abort the",' # whole run with "After pruning, no terms remain".',' for kw in ({"min_df": 2 if long_col else 1,',' "analyzer": "word" if long_col else "char_wb"},',' {"min_df": 1, "analyzer": "char_wb"}):'," try:"," vec = TfidfVectorizer(max_features=3000, ngram_range=(1, 2), **kw)"," b_tr = vec.fit_transform(tr_txt).toarray()"," blocks_tr.append(b_tr)"," blocks_te.append(vec.transform(te_txt).toarray())"," break"," except ValueError as e:"," last = e"," else:",' print(f" skipped text column {c!r}: {last}")',"if not blocks_tr:"," raise SystemExit(",' f"No usable feature columns. The shared columns are {feat} — none of them "',' f"is numeric, categorical or text.\\n"',...d?[` "The signal for this task is in ${d.kind} files, not in the CSVs.\\n"`,` " 1. fetch them (${d.mb}MB): teleload ${d.pack}\\n"`,...d.glob?[` " files land at <pack dir>/${d.glob}\\n"`]:[],` " 2. replace this cell: ${m}\\n"`,' " build one feature row per file, then keep the rest of the notebook.\\n"']:[' f"Two kinds of task land here, and both are real work rather than a bug:\\n"',' f" 1. audio/image — train.csv only lists sample ids and labels and the "',' f"signal is in media files. Read them with scipy.io.wavfile (audio) or "',' f"PIL (images) and turn each file into a feature row.\\n"',' f" 2. train.csv and test.csv describe different things (a retrieval or "',' f"matching task). Read both files, work out what links a test row to a "',' f"train row, and build that mapping yourself.\\n"'],...d?[' f"columns — train={list(train.columns)} test={list(test.columns)}"']:[' f"Either way: replace this cell. train={list(train.columns)} "',' f"test={list(test.columns)}"']," )","X = np.hstack(blocks_tr)","X_test = np.hstack(blocks_te)","y = train[TARGET]","X_tr, X_val, y_tr, y_val = train_test_split(X, y, test_size=0.2, random_state=0)",'print(f"{X.shape[1]} model-ready features "',' f"({len(num)} numeric, {len(cat)} one-hot, {len(txt)} TF-IDF text)")']),f=l(["# EDA — look at the labels before you model. Produces ONE figure:","# left = what you are predicting (class balance / value spread)","# right = what the FEATURES cell will have to work with","# icoa ipynb saves it as a .png and opens your image viewer (type `open` to reopen).","import matplotlib.pyplot as plt","try:"," fig, ax = plt.subplots(1, 2, figsize=(11, 4))"," if REG:",' vals = pd.to_numeric(train[TARGET], errors="coerce").dropna()'," ax[0].hist(vals, bins=30)",' ax[0].set_title(f"{TARGET} — {len(vals)} values, mean {vals.mean():.3g}")'," else:"," vc = train[TARGET].astype(str).value_counts().head(12)"," ax[0].bar(range(len(vc)), vc.values)"," ax[0].set_xticks(range(len(vc)))",' ax[0].set_xticklabels([s[:14] for s in vc.index], rotation=45, ha="right")',' # An imbalanced target is the single most common reason a "good"'," # accuracy means nothing — show the majority share right in the title."," top = vc.iloc[0] / len(train)",' ax[0].set_title(f"{TARGET} — {train[TARGET].nunique()} classes · biggest {top:.0%}")',' kinds = {"numeric": 0, "category": 0, "text": 0}'," for c in [c for c in shared if c not in (ID, TARGET)]:"," if is_numeric_dtype(train[c]):",' kinds["numeric"] += 1'," elif train[c].nunique() <= 20:",' kinds["category"] += 1'," else:",' kinds["text"] += 1'," miss = train.isna().mean()"," miss = miss[miss > 0].sort_values(ascending=False).head(10)"," if len(miss) > 0:"," ax[1].barh(range(len(miss)), miss.values)"," ax[1].set_yticks(range(len(miss)))"," ax[1].set_yticklabels([s[:16] for s in miss.index])",' ax[1].set_title("missing rate (top columns)")'," else:"," ax[1].bar(list(kinds), list(kinds.values()))",' ax[1].set_title(f"{sum(kinds.values())} usable columns · no missing values")'," plt.tight_layout()"," plt.show()",' print("columns by kind:", kinds)',"except Exception as e:"," # EDA is never allowed to stop the run — `run all` halts on the first"," # error, and a plotting hiccup must not cost the student their baseline.",' print("EDA skipped:", type(e).__name__, e)']),g=l(["# MODEL — a working baseline. Improve THIS cell to climb the board.","from sklearn.linear_model import LogisticRegression, Ridge","# REG was decided from the labels back in setup.","model = Ridge() if REG else LogisticRegression(max_iter=2000)"]),_=l(["# validate — quick local score on a held-out fifth of train","model.fit(X_tr, y_tr)","p_val = model.predict(X_val)","if REG:",' print("val RMSE:", round(mean_squared_error(y_val, p_val) ** 0.5, 4))',"else:"," acc = accuracy_score(y_val, p_val)",' print("val accuracy:", round(acc, 4))'," if acc < 0.3:"," # A near-zero baseline is the expected result on several of these tasks,"," # not a broken notebook — say so, or the student reads it as a bug.",' print(" A score this low usually means the linear baseline cannot "',' "represent the pattern at all (combination rules, sequences, "',' "spelling). That IS the task: build features that expose it, "',' "or switch to a model that can. The chain itself works — "',' "submission.csv is written either way.")']),b=l(["# submission — retrain on ALL data, write submission.csv (Kaggle format)","model.fit(X, y)","pred = model.predict(X_test)",'out = OUT + "/submission.csv"',"# Copy the layout of sample_submission.csv and swap in the predictions, rather","# than rebuilding the file from column names. Some tasks want more than two","# columns (e.g. subtaskID,datapointID,answer) and a hand-built 2-column frame","# is rejected by the grader even when every prediction is right.","if len(sub_fmt) == len(pred):"," sub = sub_fmt.copy()"," sub[sub.columns[-1]] = pred","else:"," keys = {}"," if ID is not None and ID in test.columns:"," keys[SUB_ID] = test[ID]"," keys[SUB_TARGET] = pred"," sub = pd.DataFrame(keys)",' print(f"NOTE: sample_submission has {len(sub_fmt)} rows but test has "',' f"{len(pred)} — built the file from column names instead; "',' f"check it against the task page before submitting.")',"sub.to_csv(out, index=False)",'print("wrote", out, "·", list(sub.columns))',o?`print("score it: icoa kaggle submit ${s.comp_id}")`:'print("format check: icoa kaggle check")']),y=o?["## Submit to ICOA (scored on the server)","",`1. In the ICOA CLI run \`icoa kaggle submit ${s.comp_id}\``,"2. The server scores `submission.csv` against the secret test labels"," and your rank lands on the live board (`icoa board`).","3. Improve the MODEL cell, re-run, submit again — best score counts."]:"nitro"===s.judge?["## Where this one gets scored","","This task comes from **judge.nitro-ai.org** and the answer key stays on","their server, so ICOA cannot score it for you. That does not make it a","practice stub — it is a real task, just judged elsewhere.","",`1. \`icoa kaggle check ${s.comp_id}\` — checks the FORMAT of`," `submission.csv` (headers, row count, ids). Format errors are the most"," common way a correct answer still scores zero.","2. Watch the **val score** printed above — that is your real feedback loop"," during the sprint, and it needs no account anywhere.","3. Want the official number? Register at judge.nitro-ai.org and submit"," there. Otherwise compare against the official solution, or ask"," `ai4ioai` to review your approach."]:s.kaggle_url?["## Submit on Kaggle (manual, your own account)","",`1. Open ${s.kaggle_url}`,"2. **Submit Predictions** → upload `submission.csv`","3. Read your leaderboard score, come back, improve the MODEL cell, repeat."]:["## Submit on Kaggle (manual, your own account)","","_This dataset has no live Kaggle page linked yet._","The loop is identical: upload `submission.csv`, read the score, improve","the MODEL cell, repeat. Meanwhile the **val score** printed above is a","real signal you can work against with no account at all."];return[r(c),p,f,h,g,_,b,r(y.join("\n"))]}export function parseSubmitResponse(e){const t=e&&"object"==typeof e?e:{},s=t.data??t,i=s&&"object"==typeof s?s:{};return"number"==typeof i.score&&"string"==typeof i.metric?{ok:!0,score:i.score,metric:i.metric,rank:"number"==typeof i.rank_at_time?i.rank_at_time:null,fieldSize:"number"==typeof i.field_size?i.field_size:null,deduped:!0===i.deduped,sameCode:!0===i.same_code,submissionId:"string"==typeof i.submission_id?i.submission_id:null,leaderboard:"string"==typeof i.leaderboard?i.leaderboard:null,boardEvent:"string"==typeof i.board_event?i.board_event:null,used:"number"==typeof i.submissions_used?i.submissions_used:null,cap:"number"==typeof i.submissions_cap?i.submissions_cap:null,parts:i.parts&&"object"==typeof i.parts?Object.entries(i.parts).filter(e=>"number"==typeof e[1]).slice(0,24):void 0}:"string"==typeof i.reason&&""!==i.reason?{ok:!1,reason:i.reason,error:"string"==typeof i.error?i.error:void 0,problems:Array.isArray(i.problems)?i.problems.filter(e=>"string"==typeof e):void 0}:{ok:!1,reason:"network"}}export function validateSubmissionText(e,t){const s=[],i=e.trim().split(/\r?\n/),a=t.trim().split(/\r?\n/),n=i[0]??"",o=a[0]??"";n.trim()!==o.trim()&&s.push(`header mismatch: expected "${o}", got "${n}"`);const r=i.slice(1).filter(e=>""!==e.trim()),l=a.slice(1).filter(e=>""!==e.trim());r.length!==l.length&&s.push(`row count mismatch: expected ${l.length}, got ${r.length}`);const c=e=>new Set(e.map(e=>(e.split(",")[0]??"").trim())),p=c(r),u=c(l),d=[...u].filter(e=>!p.has(e)),m=[...p].filter(e=>!u.has(e));if(d.length>0||m.length>0){const e=[];d.length>0&&e.push(`${d.length} missing (e.g. ${d[0]})`),m.length>0&&e.push(`${m.length} unknown (e.g. ${m[0]})`),s.push(`id mismatch vs sample_submission: ${e.join(", ")}`)}for(let e=0;e<r.length;e++)if(r[e].split(",").some(e=>""===e.trim())){s.push(`empty cell in data row ${e+1}`);break}return{ok:0===s.length,problems:s}}export function buildMediaStarterCells(e,t){const s=e=>({kind:"markdown",source:e,result:null}),i=e=>({kind:"code",source:e.join("\n"),result:null}),a=t.media?.kind??"archive",n=t.title??"ICOA sprint task",o=[`# ${t.seq?`${t.seq} · `:""}${n}`,"",...t.topic_cn?[t.topic_cn,""]:[],"This task ships **data, not a baseline** — the CSV starter cannot help here",`because the signal lives in the ${a} files you just pulled.`,"","Run all cells: they inventory the pack and draw one picture of what is inside,","so you start from evidence instead of from the folder name. The last cell says","what you have to build — that part is the task.","","**Reading the files:** on your laptop the notebook env has numpy / scipy /","pandas / scikit-learn / torch / transformers / Pillow — but NOT librosa,","torchaudio or soundfile. The GPU contest box has librosa and torchaudio too."].join("\n"),r=i(["# inventory — what is actually in this pack (counts, sizes, a few real names)","import os","import numpy as np",`DIR = os.environ.get("ICOA_PACK_DIR") or r"${e}"`,"by_ext, total, sample = {}, 0, {}","for root, _dirs, files in os.walk(DIR):"," for f in files:",' ext = os.path.splitext(f)[1].lower() or "(no ext)"'," p = os.path.join(root, f)"," by_ext[ext] = by_ext.get(ext, 0) + 1"," total += os.path.getsize(p)"," sample.setdefault(ext, p)",'print(f"{DIR}\\n{sum(by_ext.values())} files · {total/1048576:.0f} MB")',"for ext, n in sorted(by_ext.items(), key=lambda kv: -kv[1])[:8]:",' print(f" {ext:>10} {n:>6} e.g. {os.path.relpath(sample[ext], DIR)}")',"# 🔴 a leftover .zip means the pack still needs unpacking before you can read it",'zips = [p for e, p in sample.items() if e == ".zip"]',"if zips:",' print("\\nNOTE: there is still a .zip in here — unpack it first:")',' print(f" import zipfile; zipfile.ZipFile(r\\"{zips[0]}\\").extractall(DIR)")']),l=["# EDA — one picture of what is inside. icoa ipynb saves it as a .png and","# opens your image viewer (`open` reopens it).","import matplotlib.pyplot as plt","try:"];"image"===a?l.push(" from PIL import Image"," imgs = []"," for root, _d, files in os.walk(DIR):"," for f in sorted(files):",' if os.path.splitext(f)[1].lower() in (".jpg", ".jpeg", ".png"):'," imgs.append(os.path.join(root, f))"," if len(imgs) >= 9:"," break"," fig, axes = plt.subplots(3, 3, figsize=(7, 7))"," for ax, p in zip(axes.ravel(), imgs[:9]):"," ax.imshow(Image.open(p))"," ax.set_title(os.path.basename(p)[:18], fontsize=8)"," for ax in axes.ravel():",' ax.axis("off")',' plt.suptitle(f"{len(imgs)}+ images · first 9")'):"weights"===a?l.push(" import torch"," wfiles = []"," for root, _d, files in os.walk(DIR):"," for f in sorted(files):",' if os.path.splitext(f)[1].lower() in (".pt", ".pth"):'," wfiles.append(os.path.join(root, f))"," fig, ax = plt.subplots(1, max(1, min(2, len(wfiles))), figsize=(10, 4), squeeze=False)"," for i, p in enumerate(wfiles[:2]):"," # weights_only=True: never unpickle arbitrary objects from a data file",' obj = torch.load(p, map_location="cpu", weights_only=True)',' t = obj if hasattr(obj, "flatten") else list(obj.values())[0]'," v = np.asarray(t, dtype=float).ravel()"," ax[0][i].hist(v, bins=60)",' ax[0][i].set_title(f"{os.path.basename(p)} · shape {tuple(np.shape(t))}", fontsize=9)',' print(os.path.basename(p), "shape", tuple(np.shape(t)))'):l.push(" import pandas as pd"," csvs = []"," for root, _d, files in os.walk(DIR):"," for f in files:",' if f.lower().endswith(".csv"):'," csvs.append(os.path.join(root, f))"," csvs.sort(key=os.path.getsize, reverse=True)"," df = pd.read_csv(csvs[0])"," print(os.path.basename(csvs[0]), df.shape)"," print(df.head(3).to_string(max_colwidth=24)[:600])"," # the last low-cardinality column is the usual label; if there is none,"," # plot the row-length spread so the picture still says something true"," cand = [c for c in df.columns if df[c].nunique() <= 30]"," fig, ax = plt.subplots(figsize=(8, 4))"," if cand:"," vc = df[cand[-1]].astype(str).value_counts().head(12)"," ax.bar(range(len(vc)), vc.values)"," ax.set_xticks(range(len(vc)))",' ax.set_xticklabels([s[:14] for s in vc.index], rotation=45, ha="right")',' ax.set_title(f"{cand[-1]} — {df[cand[-1]].nunique()} values")'," else:"," ax.hist([len(str(x)) for x in df[df.columns[-1]]], bins=30)",' ax.set_title(f"length of {df.columns[-1]}")'),l.push(" plt.tight_layout()"," plt.show()","except Exception as e:"," # never stop the run on a plotting problem — `run all` halts on first error",' print("EDA skipped:", type(e).__name__, e)');const c=["## Your job","","There is no baseline to beat here — you build the first one. The shape of it:","","1. **turn each file into a row of numbers** — that is the whole trick","image"===a?" (`PIL.Image.open` → resize → `np.asarray` → flatten, or a pretrained encoder)":"weights"===a?" (`torch.load(..., weights_only=True)` → the tensors themselves ARE the features)":" (read the CSVs / unpack the archive, then vectorise the text columns)","2. keep a held-out fifth of the data and print a val score every time","3. write your predictions in exactly the format the task page asks for","","`add` makes a new code cell · `add md` a note cell · `run all` runs everything ·","a cell that raises offers `ai N`, which sends that cell and its traceback to the AI."].join("\n");return[s(o),r,i(l),s(c)]}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import{spawnSync as e}from"node:child_process";import{existsSync as r,readFileSync as s}from"node:fs";import{fileURLToPath as t}from"node:url";import{predColumns as o}from"./setter.js";export function helperSourcePath(){return t(new URL("../../assets/setter/icoa_metrics.py",import.meta.url))}export function helperSource(){return s(helperSourcePath(),"utf-8")}const n=String.raw`
|
|
2
|
+
import csv, io, json, math, os, sys
|
|
3
|
+
sys.dont_write_bytecode = True
|
|
4
|
+
task, idc, cols = sys.argv[1], sys.argv[2], json.loads(sys.argv[3])
|
|
5
|
+
out, sys.stdout = sys.stdout, sys.stderr # anything the scorer prints must not mix with the result
|
|
6
|
+
|
|
7
|
+
def table(text):
|
|
8
|
+
rows = [[c.strip() for c in r] for r in csv.reader(io.StringIO(text)) if any(c.strip() for c in r)]
|
|
9
|
+
return (rows[0], rows[1:]) if rows else ([], [])
|
|
10
|
+
|
|
11
|
+
def labels(name):
|
|
12
|
+
with open(os.path.join(task, name), encoding="utf-8-sig", newline="") as f:
|
|
13
|
+
return table(f.read())
|
|
14
|
+
|
|
15
|
+
head, rows_a = labels("labels_a.csv")
|
|
16
|
+
head_b, rows_b = labels("labels_b.csv")
|
|
17
|
+
if head_b != head:
|
|
18
|
+
sys.exit("labels_a.csv and labels_b.csv have different columns")
|
|
19
|
+
ph, pr = table(sys.stdin.read().lstrip(""))
|
|
20
|
+
pat = ph.index(idc)
|
|
21
|
+
given = {r[pat]: r for r in pr}
|
|
22
|
+
id_at = head.index(idc)
|
|
23
|
+
|
|
24
|
+
# what a scorer may contain — the same rules the judge applies before it runs one
|
|
25
|
+
import ast
|
|
26
|
+
ALLOWED = {"math", "statistics", "collections", "itertools", "functools", "icoa_metrics"}
|
|
27
|
+
BANNED = {"open", "eval", "exec", "getattr", "type"}
|
|
28
|
+
RULE = " — scorer.py may only use Python's maths modules and icoa_metrics"
|
|
29
|
+
def refuse(node, what):
|
|
30
|
+
sys.exit("scorer.py line %d: %s%s" % (getattr(node, "lineno", 0), what, RULE))
|
|
31
|
+
try:
|
|
32
|
+
with open(os.path.join(task, "scorer.py"), encoding="utf-8-sig") as f:
|
|
33
|
+
tree = ast.parse(f.read(), "scorer.py")
|
|
34
|
+
except FileNotFoundError:
|
|
35
|
+
sys.exit("scorer.py is missing")
|
|
36
|
+
for node in ast.walk(tree):
|
|
37
|
+
names = []
|
|
38
|
+
if isinstance(node, ast.Import):
|
|
39
|
+
mods = [a.name for a in node.names]
|
|
40
|
+
elif isinstance(node, ast.ImportFrom):
|
|
41
|
+
mods = [node.module or "."] if not node.level else ["."]
|
|
42
|
+
else:
|
|
43
|
+
mods = []
|
|
44
|
+
for mod in mods:
|
|
45
|
+
if mod.split(".")[0] not in ALLOWED:
|
|
46
|
+
refuse(node, "import of %s" % mod)
|
|
47
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
48
|
+
names = [a.asname or a.name for a in node.names]
|
|
49
|
+
elif isinstance(node, ast.ClassDef):
|
|
50
|
+
refuse(node, "a class")
|
|
51
|
+
elif isinstance(node, (ast.With, ast.AsyncWith)):
|
|
52
|
+
refuse(node, "a with block")
|
|
53
|
+
elif isinstance(node, ast.Name):
|
|
54
|
+
names = [node.id]
|
|
55
|
+
if node.id in BANNED:
|
|
56
|
+
refuse(node, "%s()" % node.id)
|
|
57
|
+
elif isinstance(node, ast.Attribute):
|
|
58
|
+
names = [node.attr]
|
|
59
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
60
|
+
names = [node.name]
|
|
61
|
+
elif isinstance(node, ast.arg):
|
|
62
|
+
names = [node.arg]
|
|
63
|
+
for name in names:
|
|
64
|
+
if name.split(".")[-1].startswith("_"):
|
|
65
|
+
refuse(node, 'the name "%s" starts with an underscore' % name)
|
|
66
|
+
|
|
67
|
+
sys.path.insert(0, task)
|
|
68
|
+
import scorer
|
|
69
|
+
if not callable(getattr(scorer, "score", None)):
|
|
70
|
+
sys.exit("scorer.py has no score(pred, truth) function")
|
|
71
|
+
higher = getattr(scorer, "HIGHER_IS_BETTER", True)
|
|
72
|
+
if not isinstance(higher, bool):
|
|
73
|
+
sys.exit("scorer.py: HIGHER_IS_BETTER must be True or False")
|
|
74
|
+
|
|
75
|
+
def number(v, what):
|
|
76
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
77
|
+
raise TypeError("scorer.py: %s must be a number, got %r" % (what, v))
|
|
78
|
+
v = float(v)
|
|
79
|
+
return v if math.isfinite(v) else repr(v)
|
|
80
|
+
|
|
81
|
+
def run(rows):
|
|
82
|
+
truth = {h: [r[i] for r in rows] for i, h in enumerate(head)}
|
|
83
|
+
pred = {}
|
|
84
|
+
for c in cols:
|
|
85
|
+
at = ph.index(c)
|
|
86
|
+
pred[c] = [given[r[id_at]][at] if r[id_at] in given else "" for r in rows]
|
|
87
|
+
res = scorer.score(pred, truth)
|
|
88
|
+
if isinstance(res, dict):
|
|
89
|
+
if "score" not in res:
|
|
90
|
+
raise TypeError('scorer.py: score() returned a dict without "score"')
|
|
91
|
+
parts = res.get("parts") or {}
|
|
92
|
+
if not isinstance(parts, dict):
|
|
93
|
+
raise TypeError('scorer.py: "parts" must be a dict of name -> number')
|
|
94
|
+
return {"score": number(res["score"], '"score"'),
|
|
95
|
+
"parts": {str(k): number(v, 'part "%s"' % k) for k, v in parts.items()}}
|
|
96
|
+
return {"score": number(res, "the value score() returns"), "parts": {}}
|
|
97
|
+
|
|
98
|
+
result = {"higher": higher, "columns": getattr(scorer, "COLUMNS", None),
|
|
99
|
+
"a": run(rows_a), "b": run(rows_b), "all": run(rows_a + rows_b)}
|
|
100
|
+
out.write("ICOA_SCORER_RESULT " + json.dumps(result) + "\n")
|
|
101
|
+
`,i={score:Number.NaN,parts:{}},a=(e,r)=>({ok:!1,error:e,detail:r,higher:!0,columns:null,a:i,b:i,all:i});function c(e){const r=e??{},s=e=>"number"==typeof e?e:Number.NaN,t={};for(const[e,o]of Object.entries(r.parts??{}))t[e]=s(o);return{score:s(r.score),parts:t}}export function runScorer(s,t,i,d){const[l,...u]=r(d)?[d]:d.split(" ").filter(Boolean),f=e(l,[...u,"-c",n,s,t.id,JSON.stringify(o(t))],{input:i,encoding:"utf-8",timeout:6e4,killSignal:"SIGKILL",maxBuffer:16777216,shell:!1,cwd:s,windowsHide:!0,env:{...process.env,PYTHONDONTWRITEBYTECODE:"1",PYTHONIOENCODING:"utf-8"}}),p=(f.stderr||"").split(/\r?\n/).filter(e=>""!==e.trim()),m=p.findIndex(e=>/File ".*scorer\.py"/.test(e)),h=(m>=0?p.slice(m):p).slice(-8);if(f.error){const e=f.error.code;return"ETIMEDOUT"===e?a("scorer.py did not finish in 60 seconds — it was stopped",h):a(`could not start Python (${e??f.error.message})`)}if(f.signal)return a(`scorer.py was stopped (${f.signal}) — it has 60 seconds`,h);const y=(f.stdout||"").split(/\r?\n/).find(e=>e.startsWith("ICOA_SCORER_RESULT "));if(0!==f.status||!y)return a("scorer.py failed — the last lines Python printed:",h);try{const e=JSON.parse(y.slice(19)),r=Array.isArray(e.columns)?e.columns.map(String):null;return{ok:!0,higher:!1!==e.higher,columns:r,a:c(e.a),b:c(e.b),all:c(e.all)}}catch{return a("scorer.py printed a result this tool could not read",h)}}
|
package/dist/lib/setter.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{createHash as e}from"node:crypto";export const METRIC_KIND={accuracy:"class",balanced_accuracy:"class",macro_f1:"class",weighted_f1:"class",mcc:"class",f1:"binary",qwk:"grade",auc:"prob",ap:"prob",logloss:"prob",brier:"prob",mse:"reg",rmse:"reg",mae:"reg",rmsle:"reg",mape:"reg",smape:"reg",r2:"reg",pearson:"reg",spearman:"reg"};export const METRICS=Object.keys(METRIC_KIND);export const LOWER_IS_BETTER=new Set(["mse","rmse","mae","rmsle","mape","smape","logloss","brier"]);export function metricGroups(){const e=(...e)=>METRICS.filter(t=>e.includes(METRIC_KIND[t]));return[{kind:"classes",names:e("class","binary","grade")},{kind:"probability of 1",names:e("prob")},{kind:"numbers",names:e("reg")}]}export function parseCsv(e){const t=[],r=e.replace(/^/,"").split(/\r?\n/).filter(e=>""!==e.trim());if(0===r.length)return{table:{header:[],rows:[]},problems:["file is empty"]};e.includes('"')&&t.push('quoted fields (") are not supported — write plain comma-separated values');const s=r[0].split(",").map(e=>e.trim()),n=[];for(let e=1;e<r.length;e++){const a=r[e].split(",").map(e=>e.trim());if(a.length!==s.length){t.push(`row ${e+1} has ${a.length} fields, the header has ${s.length}`);break}n.push(a)}return{table:{header:s,rows:n},problems:t}}export function validName(e){return/^[a-z0-9][a-z0-9-]{1,38}[a-z0-9]$/.test(e)}export function parseTaskMeta(e){let t;try{t=JSON.parse(e)}catch{return{meta:null,problems:["task.json is not valid JSON"]}}const r=[],s=String(t.name??""),n=String(t.metric??""),a=String(t.label??"");if(validName(s)||r.push("task.json name: 3–40 characters of a–z, 0–9 and -"),METRICS.includes(n)||r.push(`task.json metric: one of ${METRICS.join(" / ")}`),a||r.push("task.json label: the name of the column to predict"),r.length)return{meta:null,problems:r};const o=Number(t.per_day);return{meta:{name:s,title:String(t.title||s),metric:n,label:a,id:String(t.id||"id"),per_day:Number.isFinite(o)&&o>=1?Math.floor(o):5,description:"string"==typeof t.description?t.description:void 0},problems:[]}}const t={train:"train.csv",testA:"test_a.csv",labelsA:"labels_a.csv",testB:"test_b.csv",labelsB:"labels_b.csv"};export function checkTask(e,r){const s={ok:!1,problems:[],warnings:[],nTrain:0,nA:0,nB:0,features:[],classes:[]},n={};for(const e of Object.keys(t)){const a=r[e];if(void 0===a){s.problems.push(`${t[e]} is missing`);continue}const{table:o,problems:i}=parseCsv(a);for(const r of i)s.problems.push(`${t[e]}: ${r}`);n[e]=o}if(s.problems.length)return s;const a=n.train,o=e.id;if(!a.header.includes(o)||!a.header.includes(e.label))return s.problems.push(`train.csv needs both "${o}" and "${e.label}" columns — its header is ${a.header.join(", ")}`),s;const i=a.header.filter(t=>t!==e.label);s.features=i.filter(e=>e!==o);const l=a.header.indexOf(o),c=a.header.indexOf(e.label),u=new Set;for(const e of a.rows){if(u.has(e[l]))return s.problems.push(`train.csv: duplicate id ${e[l]}`),s;u.add(e[l])}const m=new Map,p=new Map,h={A:[],B:[]};for(const[r,a,l]of[["A","testA","labelsA"],["B","testB","labelsB"]]){const c=n[a],u=n[l],d=t[a],f=t[l];if(c.header.includes(e.label))return s.problems.push(`${d} carries the label column "${e.label}" — remove it, labels go in ${f}`),s;if(c.header.join(",")!==i.join(","))return s.problems.push(`${d} columns differ from train.csv minus the label — test: ${c.header.join(", ")} · train: ${i.join(", ")}`),s;if(2!==u.header.length||u.header[0]!==o||u.header[1]!==e.label)return s.problems.push(`${f} must have exactly two columns (${o}, ${e.label}) — got ${u.header.join(", ")}`),s;const b=new Set,g=c.header.indexOf(o);for(const e of c.rows){if(b.has(e[g]))return s.problems.push(`${d} has duplicate ids (${e[g]})`),s;b.add(e[g])}const $=new Map;for(const e of u.rows){if(""===e[0]||""===e[1])return s.problems.push(`${f}: empty id or label near id ${e[0]||"(blank)"}`),s;if($.has(e[0]))return s.problems.push(`${f}: duplicate id ${e[0]}`),s;$.set(e[0],e[1])}const w=[...b].filter(e=>$.has(e)).length;if(w!==b.size||w!==$.size)return s.problems.push(`${d}: ${b.size} ids in the test file, ${$.size} in ${f}, ${w} in both — they must be the same set`),s;for(const e of b){const t=p.get(e);if(t)return s.problems.push(`id ${e} is in both ${t} and ${d}`),s;p.set(e,d),m.set(e,$.get(e)),h[r].push(e)}}const d=[...p.keys()].filter(e=>u.has(e));if(d.length)return s.problems.push(`${d.length} train ids are reused in test (e.g. ${d.slice(0,3).join(", ")}) — ids must be unique across files`),s;if(s.nTrain=a.rows.length,s.nA=h.A.length,s.nB=h.B.length,0===s.nA||0===s.nB)return s.problems.push("one board is empty — both test_a.csv and test_b.csv need rows"),s;const f=METRIC_KIND[e.metric];if("reg"!==f){const t=new Map;for(const e of a.rows)t.set(e[c],(t.get(e[c])||0)+1);s.classes=[...t.entries()].sort((e,t)=>t[1]-e[1]).slice(0,12);const r=new Set(t.keys()),n=[...new Set(m.values())].filter(e=>!r.has(e));n.length&&s.warnings.push(`test labels contain ${n.length} value(s) never seen in train (e.g. ${n.slice(0,3).join(", ")})`),t.size>Math.max(50,s.nTrain/2)&&s.warnings.push(`${t.size} distinct label values with a classification metric — is "${e.label}" really a class?`);const o=[...t.keys(),...m.values()];if("binary"===f||"prob"===f){const t=o.find(e=>""===e.trim()||0!==Number(e)&&1!==Number(e));if(void 0!==t)return s.problems.push(`metric ${e.metric} needs labels that are 0 or 1 — found "${t}"`),s}if("grade"===f){const t=o.find(e=>""===e.trim()||!Number.isInteger(Number(e)));if(void 0!==t)return s.problems.push(`metric ${e.metric} needs whole-number grades as labels — found "${t}"`),s}}else{const t=[...m.values()].find(e=>!Number.isFinite(Number(e)));if(void 0!==t)return s.problems.push(`metric ${e.metric} needs numeric labels — found "${t}"`),s;const r=[...m.values()].map(Number);if("rmsle"===e.metric&&r.some(e=>e<0))return s.problems.push("metric rmsle needs labels that are 0 or more — found a negative one"),s;"mape"===e.metric&&r.some(e=>0===e)&&s.warnings.push("some labels are exactly 0 — mape divides by the label, one such row dominates the score (smape or mae are safer)")}return(s.nA<100||s.nB<100)&&s.warnings.push(`a board with under 100 rows (A ${s.nA}, B ${s.nB}) moves a lot on a single row`),s.nB<s.nA&&s.warnings.push(`B (${s.nB}) is smaller than A (${s.nA}) — the final ranking usually rests on the larger set`),s.ok=!0,s.parsed={train:a,testA:n.testA,testB:n.testB,labels:m,aIds:h.A,bIds:h.B},s}const r=(e,t)=>`${[e,...t].map(e=>e.join(",")).join("\n")}\n`;export function studentBundle(t,s){if(!s.ok||!s.parsed)throw new Error("check the task first");const{train:n,testA:a,testB:o}=s.parsed,i=a.header.indexOf(t.id),l=r=>e("sha256").update(`${t.name}:order:${r}`).digest("hex"),c=[...a.rows,...o.rows].sort((e,t)=>l(e[i])<l(t[i])?-1:1),u=n.header.indexOf(t.label),m=METRIC_KIND[t.metric],p="prob"===m?"0.5":"reg"===m||"binary"===m?"0":s.classes[0]?.[0]??n.rows[0]?.[u]??"0";return{train:r(n.header,n.rows),test:r(a.header,c),sample:r([t.id,t.label],c.map(e=>[e[i],p]))}}function s(e,t){if(0===t.size)return Number.POSITIVE_INFINITY;let r=0;for(const[s,n]of t){const t=e.get(s);r+=((void 0!==t&&""!==t.trim()&&Number.isFinite(Number(t))?Number(t):0)-Number(n))**2}return r/t.size}function n(e){if(void 0===e||""===e.trim())return 0;const t=Number(e);return Number.isFinite(t)?t:0}function a(e){return void 0!==e&&""!==e.trim()&&1===Number(e)}function o(...[e,t]){return[...t].map(([t,r])=>[n(e.get(t)),Number(r)])}function i(...[e,t]){return[...t].map(([t,r])=>[n(e.get(t)),a(r)?1:0])}function l(...[e,t]){return[...t].map(([t,r])=>[(e.get(t)??"").trim(),r.trim()])}const c=e=>e.reduce((e,t)=>e+t,0);function u(e,t){return 0===e.length?Number.POSITIVE_INFINITY:c(e.map(([e,r])=>t(e,r)))/e.length}function m(e,t){const r=e.length;if(0===r)return 0;const s=c(e)/r,n=c(t)/r,a=c(e.map((e,r)=>(e-s)*(t[r]-n))),o=c(e.map(e=>(e-s)**2)),i=c(t.map(e=>(e-n)**2));return 0===o||0===i?0:a/Math.sqrt(o*i)}function p(e){const t=e.map((e,t)=>t).sort((t,r)=>e[t]-e[r]),r=new Array(e.length).fill(0);let s=0;for(;s<t.length;){let n=s;for(;n+1<t.length&&e[t[n+1]]===e[t[s]];)n++;for(let e=s;e<=n;e++)r[t[e]]=(s+n)/2+1;s=n+1}return r}function h(e){return[...new Set(e.map(([,e])=>e))].sort().map(t=>({lab:t,tp:e.filter(([e,r])=>e===t&&r===t).length,fp:e.filter(([e,r])=>e===t&&r!==t).length,fn:e.filter(([e,r])=>e!==t&&r===t).length}))}export function metricValue(e,t,r){switch(e){case"accuracy":return function(e,t){if(0===t.size)return 0;let r=0;for(const[s,n]of t)(e.get(s)??"").trim()===n.trim()&&r++;return r/t.size}(t,r);case"macro_f1":return function(e,t){const r=[...new Set([...t.values()].map(e=>e.trim()))].sort();if(0===r.length)return 0;let s=0;for(const n of r){let r=0,a=0,o=0;for(const[s,i]of t){const t=(e.get(s)??"").trim(),l=i.trim();t===n&&l===n?r++:t===n&&l!==n?a++:t!==n&&l===n&&o++}const i=2*r+a+o;s+=i?2*r/i:0}return s/r.length}(t,r);case"mse":return s(t,r);case"rmse":return Math.sqrt(s(t,r));case"balanced_accuracy":return function(e){const t=h(e);return 0===t.length?0:c(t.map(e=>e.tp/(e.tp+e.fn)))/t.length}(l(t,r));case"weighted_f1":return function(e){const t=h(e);return 0===t.length?0:c(t.map(e=>{const t=2*e.tp+e.fp+e.fn;return(e.tp+e.fn)*(t?2*e.tp/t:0)}))/e.length}(l(t,r));case"mcc":return function(e){const t=e.length;if(0===t)return 0;const r=new Map,s=new Map;let n=0;for(const[t,a]of e)r.set(a,(r.get(a)||0)+1),s.set(t,(s.get(t)||0)+1),t===a&&n++;const a=n*t-c([...r].map(([e,t])=>t*(s.get(e)||0))),o=t*t-c([...r.values()].map(e=>e*e)),i=t*t-c([...s.values()].map(e=>e*e));return 0===o||0===i?0:a/Math.sqrt(o*i)}(l(t,r));case"f1":return function(...[e,t]){let r=0,s=0,n=0;for(const[o,i]of t){const t=a(e.get(o)),l=a(i);t&&l?r++:t&&!l?s++:l&&!t&&n++}const o=2*r+s+n;return o?2*r/o:0}(t,r);case"qwk":return function(...[e,t]){const r=[...t.values()].map(e=>Math.floor(Number(e)+.5));if(0===r.length)return 0;const s=Math.min(...r),a=Math.max(...r),o=a-s+1;if(o<2)return 0;const i=[...t.keys()].map(t=>Math.min(Math.max(Math.floor(n(e.get(t))+.5),s),a)),l=new Array(o).fill(0),c=new Array(o).fill(0);let u=0;r.forEach((e,t)=>{l[e-s]++,c[i[t]-s]++,u+=(i[t]-e)**2});let m=0;for(let e=0;e<o;e++)for(let t=0;t<o;t++)m+=(e-t)**2*l[e]*c[t];return m/=r.length,0===m?0:1-u/m}(t,r);case"auc":return function(e){const t=c(e.map(([,e])=>e)),r=e.length-t;if(0===t||0===r)return.5;const s=p(e.map(([e])=>e));return(c(s.filter((t,r)=>1===e[r][1]))-t*(t+1)/2)/(t*r)}(i(t,r));case"ap":return function(e){const t=[...e].sort((e,t)=>t[0]-e[0]),r=c(t.map(([,e])=>e));if(0===r)return 0;let s=0,n=0,a=0,o=0,i=0;for(;i<t.length;){let e=i;for(;e<t.length&&t[e][0]===t[i][0];)n+=t[e][1],a++,e++;const l=n/r;s+=n/a*(l-o),o=l,i=e}return s}(i(t,r));case"logloss":return u(i(t,r),(e,t)=>{const r=Math.min(Math.max(e,1e-15),1-1e-15);return 1===t?-Math.log(r):-Math.log(1-r)});case"brier":return u(i(t,r),(e,t)=>(e-t)**2);case"mae":return u(o(t,r),(e,t)=>Math.abs(e-t));case"rmsle":return Math.sqrt(u(o(t,r),(e,t)=>(Math.log1p(Math.max(e,0))-Math.log1p(Math.max(t,0)))**2));case"mape":return u(o(t,r),(e,t)=>Math.abs(e-t)/Math.max(Math.abs(t),2220446049250313e-31));case"smape":return u(o(t,r),(e,t)=>{const r=Math.abs(e)+Math.abs(t);return r?2*Math.abs(e-t)/r:0});case"r2":return function(e){if(0===e.length)return 0;const t=c(e.map(([,e])=>e))/e.length,r=c(e.map(([,e])=>(e-t)**2));return 0===r?0:1-c(e.map(([e,t])=>(e-t)**2))/r}(o(t,r));case"pearson":{const e=o(t,r);return m(e.map(([e])=>e),e.map(([,e])=>e))}case"spearman":{const e=o(t,r);return m(p(e.map(([e])=>e)),p(e.map(([,e])=>e)))}}}export function scoreSubmission(e,t,r){const s={ok:!1,problems:[],a:0,b:0,all:0,gap:0,nMissing:0};if(!t.ok||!t.parsed)return s.problems.push("check the task first"),s;const{table:n,problems:a}=parseCsv(r);if(a.length)return s.problems.push(...a),s;if(2!==n.header.length||n.header[0]!==e.id||n.header[1]!==e.label)return s.problems.push(`the submission needs exactly two columns: ${e.id},${e.label} — got ${n.header.join(",")}`),s;const o=new Map;for(const e of n.rows){if(o.has(e[0]))return s.problems.push(`duplicate id ${e[0]} in the submission`),s;o.set(e[0],e[1])}const{labels:i,aIds:l,bIds:c}=t.parsed,u=[...o.keys()].filter(e=>!i.has(e));if(u.length)return s.problems.push(`${u.length} ids are not in the test set (e.g. ${u.slice(0,3).join(", ")})`),s;if(s.nMissing=i.size-o.size,s.nMissing>0)return s.problems.push(`${s.nMissing} test rows have no prediction — the judge refuses an incomplete submission`),s;const m=e=>new Map(e.map(e=>[e,i.get(e)]));return s.a=metricValue(e.metric,o,m(l)),s.b=metricValue(e.metric,o,m(c)),s.all=metricValue(e.metric,o,i),s.gap=LOWER_IS_BETTER.has(e.metric)?s.b-s.a:s.a-s.b,s.ok=!0,s}export function verdictLines(e,t,r){const s=[],n=LOWER_IS_BETTER.has(e.metric),a=(e,t)=>n?e<t:e>t,o=Math.abs(t.gap);if(o/Math.max(Math.abs(t.all),1e-9)>.05?s.push(`A and B disagree by ${o.toFixed(4)} on the baseline — check that the two test sets come from the same distribution`):s.push(`A and B agree on the baseline (difference ${o.toFixed(4)})`),r){if(a(r.b,t.b)){const e=Math.abs(r.b-t.b);s.push(`room to climb on B: ${e.toFixed(4)} (baseline ${t.b.toFixed(4)} → reference ${r.b.toFixed(4)})`),e<2*o&&s.push("that room is smaller than twice the A/B difference — the ranking may come down to noise")}else s.push("the reference does NOT beat the baseline on B — contestants have nothing to climb");a(r.a,t.a)!==a(r.b,t.b)&&s.push("baseline and reference rank differently on A and on B — the public board would mislead")}return s}export function tarArchive(e){const t=[],r=new TextEncoder;for(const s of e){const e=new Uint8Array(512),n=(t,s,n)=>{e.set(r.encode(t).subarray(0,n),s)};if(r.encode(s.name).length>99)throw new Error(`file name too long for the archive: ${s.name}`);n(s.name,0,100),n("0000644\0",100,8),n("0000000\0",108,8),n("0000000\0",116,8),n(`${s.data.length.toString(8).padStart(11,"0")}\0`,124,12),n("00000000000\0",136,12),n(" ",148,8),n("0",156,1),n("ustar\0",257,6),n("00",263,2);let a=0;for(const t of e)a+=t;n(`${a.toString(8).padStart(6,"0")}\0 `,148,8),t.push(e,s.data);const o=(512-s.data.length%512)%512;o&&t.push(new Uint8Array(o))}t.push(new Uint8Array(1024));const s=t.reduce((e,t)=>e+t.length,0),n=new Uint8Array(s);let a=0;for(const e of t)n.set(e,a),a+=e.length;return n}export const BASELINE_FILE="baseline.ipynb";export const TASK_FILES=["task.json","train.csv","test_a.csv","labels_a.csv","test_b.csv","labels_b.csv",BASELINE_FILE];export const INLINE_LIMIT_BYTES=5242880;export function baselineCells(e){const t=METRIC_KIND[e.metric],r="reg"===t||"prob"===t;return[{kind:"markdown",source:`# ${e.title}\n\nDescribe the task here: what one row is, what "${e.label}" means, and how a submission is scored (**${e.metric}**).\n\nThis notebook is the baseline every contestant starts from. Keep it short and make sure it runs top to bottom.`},{kind:"code",source:'import pandas as pd\n\ntrain = pd.read_csv("train.csv")\ntest = pd.read_csv("test.csv")\nprint(train.shape, test.shape)\ntrain.head()'},{kind:"code",source:r?`# Baseline: predict the training mean for every row${"prob"===t?" (the share of 1s — a probability)":""}.\npred = [train["${e.label}"].mean()] * len(test)`:`# Baseline: predict the most common class for every row.\npred = [train["${e.label}"].mode()[0]] * len(test)`},{kind:"code",source:`pd.DataFrame({"${e.id}": test["${e.id}"], "${e.label}": pred}).to_csv("submission.csv", index=False)\nprint("wrote submission.csv —", len(pred), "rows")`}]}export const SQUARE_GIB=3.5;export const CARD_UNITS={l4:6,ada:5,t4:4};export function capsulePlan(e,t,r,s){if(!r)return{tier:"cpu",squares:0,fits:!0,capGib:0,perCard:{l4:0,ada:0,t4:0}};const n=[1,2,3].find(t=>e<3.5*t-.5);if(!n)return{tier:"gpu",squares:3,fits:!1,capGib:10,perCard:{l4:0,ada:0,t4:0}};const a=(e,t)=>3===n&&"l4"!==t?0:Math.floor(e/n),o={l4:a(CARD_UNITS.l4,"l4"),ada:a(CARD_UNITS.ada,"ada"),t4:a(CARD_UNITS.t4,"t4")},i={tier:"gpu",squares:n,fits:!0,capGib:3.5*n-.5,perCard:o},l=e=>({l4:e(o.l4),ada:e(o.ada),t4:e(o.t4)});return s&&s>0&&(i.cards=l(e=>e?Math.ceil(s/e):0)),t>0&&(i.runsPerCardHour=l(e=>Math.floor(60*e/t))),i}
|
|
1
|
+
import{createHash as e}from"node:crypto";export const METRIC_KIND={accuracy:"class",balanced_accuracy:"class",macro_f1:"class",weighted_f1:"class",mcc:"class",f1:"binary",qwk:"grade",auc:"prob",ap:"prob",logloss:"prob",brier:"prob",mse:"reg",rmse:"reg",mae:"reg",rmsle:"reg",mape:"reg",smape:"reg",r2:"reg",pearson:"reg",spearman:"reg"};export const METRICS=Object.keys(METRIC_KIND);export const LOWER_IS_BETTER=new Set(["mse","rmse","mae","rmsle","mape","smape","logloss","brier"]);export function metricGroups(){const e=(...e)=>METRICS.filter(t=>e.includes(METRIC_KIND[t]));return[{kind:"classes",names:e("class","binary","grade")},{kind:"probability of 1",names:e("prob")},{kind:"numbers",names:e("reg")}]}export const CUSTOM="custom";export const CUSTOM_WORDS=["weighted","custom","multi"];export const SCORER_FILE="scorer.py";export const HELPER_FILE="icoa_metrics.py";export function predColumns(e){return e.columns?.length?e.columns:[e.label]}function t(e,t,s){if(0===e.length)return`${s}: list at least one column`;const n=e.find(e=>""===e||/[,"\s]/.test(e));return void 0!==n?`${s}: "${n}" is not a usable column name (no spaces, commas or quotes)`:e.includes(t)?`${s}: "${t}" is the id column — it cannot also be predicted`:new Set(e).size!==e.length?`${s}: a column is listed twice`:null}export function scorerColumns(e){const t=/^COLUMNS[ \t]*=[ \t]*\[([^\]\n]*)\]/m.exec(e);return t?[...t[1].matchAll(/"([^"]*)"|'([^']*)'/g)].map(e=>(e[1]??e[2]).trim()):null}export function withScorerColumns(e,s){if(e.metric!==CUSTOM||void 0===s)return{meta:e,problems:[]};const n=scorerColumns(s);if(!n)return{meta:e,problems:[]};const r=t(n,e.id,"scorer.py COLUMNS");return r?{meta:e,problems:[r]}:e.columns&&e.columns.join(",")!==n.join(",")?{meta:e,problems:[`scorer.py says COLUMNS = ${n.join(", ")} but task.json says columns = ${e.columns.join(", ")} — keep one (remove "columns" from task.json)`]}:{meta:{...e,columns:n,label:n[0]},problems:[]}}export function parseCsv(e){const t=[],s=e.replace(/^/,"").split(/\r?\n/).filter(e=>""!==e.trim());if(0===s.length)return{table:{header:[],rows:[]},problems:["file is empty"]};e.includes('"')&&t.push('quoted fields (") are not supported — write plain comma-separated values');const n=s[0].split(",").map(e=>e.trim()),r=[];for(let e=1;e<s.length;e++){const o=s[e].split(",").map(e=>e.trim());if(o.length!==n.length){t.push(`row ${e+1} has ${o.length} fields, the header has ${n.length}`);break}r.push(o)}return{table:{header:n,rows:r},problems:t}}export function validName(e){return/^[a-z0-9][a-z0-9-]{1,38}[a-z0-9]$/.test(e)}export function parseTaskMeta(e){let s;try{s=JSON.parse(e)}catch{return{meta:null,problems:["task.json is not valid JSON"]}}const n=[],r=String(s.name??""),o=String(s.metric??""),a=o===CUSTOM,i=String(s.id||"id");let c;if(void 0!==s.columns)if(!Array.isArray(s.columns)||s.columns.some(e=>"string"!=typeof e))n.push('task.json columns: a list of column names, e.g. ["rain", "mm"]');else{c=s.columns.map(e=>e.trim());const e=t(c,i,"task.json columns");e?n.push(e):a||1===c.length||n.push(`task.json columns: metric ${o} scores one column — several columns need "metric": "${CUSTOM}" (setter new <name> weighted)`)}const l=String(s.label??"")||c?.[0]||(a?"label":"");if(validName(r)||n.push("task.json name: 3–40 characters of a–z, 0–9 and -"),a||METRICS.includes(o)||n.push(`task.json metric: one of ${METRICS.join(" / ")} / ${CUSTOM}`),l||n.push("task.json label: the name of the column to predict"),n.length)return{meta:null,problems:n};const u=Number(s.per_day),m={name:r,title:String(s.title||r),metric:o,label:l,id:i,per_day:Number.isFinite(u)&&u>=1?Math.floor(u):5,description:"string"==typeof s.description?s.description:void 0};return c&&(m.columns=c,a||(m.label=c[0])),{meta:m,problems:[]}}const s={train:"train.csv",testA:"test_a.csv",labelsA:"labels_a.csv",testB:"test_b.csv",labelsB:"labels_b.csv"};export function checkTask(e,t){const n={ok:!1,problems:[],warnings:[],nTrain:0,nA:0,nB:0,features:[],classes:[]},r={};for(const e of Object.keys(s)){const o=t[e];if(void 0===o){n.problems.push(`${s[e]} is missing`);continue}const{table:a,problems:i}=parseCsv(o);for(const t of i)n.problems.push(`${s[e]}: ${t}`);r[e]=a}if(n.problems.length)return n;const o=r.train,a=e.id,i=predColumns(e),c=e.metric===CUSTOM;if(!o.header.includes(a)||i.some(e=>!o.header.includes(e)))return n.problems.push(1===i.length?`train.csv needs both "${a}" and "${i[0]}" columns — its header is ${o.header.join(", ")}`:`train.csv needs "${a}" and every prediction column (${i.join(", ")}) — its header is ${o.header.join(", ")}`),n;const l=o.header.filter(e=>!i.includes(e));n.features=l.filter(e=>e!==a);const u=o.header.indexOf(a),m=o.header.indexOf(e.label),p=new Set;for(const e of o.rows){if(p.has(e[u]))return n.problems.push(`train.csv: duplicate id ${e[u]}`),n;p.add(e[u])}const h=new Map,d=new Map;let f=null;const b=new Map,g={A:[],B:[]};for(const[e,t,o]of[["A","testA","labelsA"],["B","testB","labelsB"]]){const u=r[t],m=r[o],p=s[t],$=s[o],w=i.find(e=>u.header.includes(e));if(void 0!==w)return n.problems.push(`${p} carries the label column "${w}" — remove it, labels go in ${$}`),n;if(u.header.join(",")!==l.join(","))return n.problems.push(`${p} columns differ from train.csv minus the label — test: ${u.header.join(", ")} · train: ${l.join(", ")}`),n;if(c){const e=[a,...i].find(e=>!m.header.includes(e));if(void 0!==e)return n.problems.push(`${$} has no "${e}" column — it needs ${[a,...i].join(", ")} (extra columns for the scorer are fine); got ${m.header.join(", ")}`),n;if(new Set(m.header).size!==m.header.length)return n.problems.push(`${$} has a column name twice — got ${m.header.join(", ")}`),n;if(null!==f&&f!==m.header.join(","))return n.problems.push(`labels_a.csv and labels_b.csv must have the same columns — A: ${f.split(",").join(", ")} · B: ${m.header.join(", ")}`),n;f=m.header.join(",")}else if(2!==m.header.length||m.header[0]!==a||m.header[1]!==i[0])return n.problems.push(`${$} must have exactly two columns (${a}, ${i[0]}) — got ${m.header.join(", ")}`),n;const v=m.header.indexOf(a),M=i.map(e=>m.header.indexOf(e)),y=new Set,k=u.header.indexOf(a);for(const e of u.rows){if(y.has(e[k]))return n.problems.push(`${p} has duplicate ids (${e[k]})`),n;y.add(e[k])}const S=new Map;for(const e of m.rows){const t=e[v],s=M.map(t=>e[t]);if(""===t||s.includes(""))return n.problems.push(`${$}: empty id or label near id ${t||"(blank)"}`),n;if(S.has(t))return n.problems.push(`${$}: duplicate id ${t}`),n;S.set(t,s)}const x=[...y].filter(e=>S.has(e)).length;if(x!==y.size||x!==S.size)return n.problems.push(`${p}: ${y.size} ids in the test file, ${S.size} in ${$}, ${x} in both — they must be the same set`),n;for(const t of y){const s=b.get(t);if(s)return n.problems.push(`id ${t} is in both ${s} and ${p}`),n;b.set(t,p),d.set(t,S.get(t)),h.set(t,S.get(t)[0]),g[e].push(t)}}const $=[...b.keys()].filter(e=>p.has(e));if($.length)return n.problems.push(`${$.length} train ids are reused in test (e.g. ${$.slice(0,3).join(", ")}) — ids must be unique across files`),n;if(n.nTrain=o.rows.length,n.nA=g.A.length,n.nB=g.B.length,0===n.nA||0===n.nB)return n.problems.push("one board is empty — both test_a.csv and test_b.csv need rows"),n;const w=e.metric===CUSTOM?CUSTOM:METRIC_KIND[e.metric];if(w===CUSTOM);else if("reg"!==w){const t=new Map;for(const e of o.rows)t.set(e[m],(t.get(e[m])||0)+1);n.classes=[...t.entries()].sort((e,t)=>t[1]-e[1]).slice(0,12);const s=new Set(t.keys()),r=[...new Set(h.values())].filter(e=>!s.has(e));r.length&&n.warnings.push(`test labels contain ${r.length} value(s) never seen in train (e.g. ${r.slice(0,3).join(", ")})`),t.size>Math.max(50,n.nTrain/2)&&n.warnings.push(`${t.size} distinct label values with a classification metric — is "${e.label}" really a class?`);const a=[...t.keys(),...h.values()];if("binary"===w||"prob"===w){const t=a.find(e=>""===e.trim()||0!==Number(e)&&1!==Number(e));if(void 0!==t)return n.problems.push(`metric ${e.metric} needs labels that are 0 or 1 — found "${t}"`),n}if("grade"===w){const t=a.find(e=>""===e.trim()||!Number.isInteger(Number(e)));if(void 0!==t)return n.problems.push(`metric ${e.metric} needs whole-number grades as labels — found "${t}"`),n}}else{const t=[...h.values()].find(e=>!Number.isFinite(Number(e)));if(void 0!==t)return n.problems.push(`metric ${e.metric} needs numeric labels — found "${t}"`),n;const s=[...h.values()].map(Number);if("rmsle"===e.metric&&s.some(e=>e<0))return n.problems.push("metric rmsle needs labels that are 0 or more — found a negative one"),n;"mape"===e.metric&&s.some(e=>0===e)&&n.warnings.push("some labels are exactly 0 — mape divides by the label, one such row dominates the score (smape or mae are safer)")}return(n.nA<100||n.nB<100)&&n.warnings.push(`a board with under 100 rows (A ${n.nA}, B ${n.nB}) moves a lot on a single row`),n.nB<n.nA&&n.warnings.push(`B (${n.nB}) is smaller than A (${n.nA}) — the final ranking usually rests on the larger set`),n.ok=!0,n.parsed={train:o,testA:r.testA,testB:r.testB,labels:h,truth:d,aIds:g.A,bIds:g.B},n}const n=(e,t)=>`${[e,...t].map(e=>e.join(",")).join("\n")}\n`;export function studentBundle(t,s){if(!s.ok||!s.parsed)throw new Error("check the task first");const{train:r,testA:o,testB:a}=s.parsed,i=o.header.indexOf(t.id),c=s=>e("sha256").update(`${t.name}:order:${s}`).digest("hex"),l=[...o.rows,...a.rows].sort((e,t)=>c(e[i])<c(t[i])?-1:1),u=predColumns(t);let m;if(t.metric===CUSTOM)m=u.map(e=>{const t=r.header.indexOf(e),s=new Map;for(const e of r.rows)s.set(e[t],(s.get(e[t])||0)+1);return[...s.entries()].sort((e,t)=>t[1]-e[1])[0]?.[0]??"0"});else{const e=r.header.indexOf(u[0]),n=METRIC_KIND[t.metric];m=["prob"===n?"0.5":"reg"===n||"binary"===n?"0":s.classes[0]?.[0]??r.rows[0]?.[e]??"0"]}return{train:n(r.header,r.rows),test:n(o.header,l),sample:n([t.id,...u],l.map(e=>[e[i],...m]))}}export function perfectSubmission(e,t){if(!t.ok||!t.parsed)throw new Error("check the task first");const{truth:s,aIds:r,bIds:o}=t.parsed;return n([e.id,...predColumns(e)],[...r,...o].map(e=>[e,...s.get(e)]))}function r(e,t){if(0===t.size)return Number.POSITIVE_INFINITY;let s=0;for(const[n,r]of t){const t=e.get(n);s+=((void 0!==t&&""!==t.trim()&&Number.isFinite(Number(t))?Number(t):0)-Number(r))**2}return s/t.size}function o(e){if(void 0===e||""===e.trim())return 0;const t=Number(e);return Number.isFinite(t)?t:0}function a(e){return void 0!==e&&""!==e.trim()&&1===Number(e)}function i(...[e,t]){return[...t].map(([t,s])=>[o(e.get(t)),Number(s)])}function c(...[e,t]){return[...t].map(([t,s])=>[o(e.get(t)),a(s)?1:0])}function l(...[e,t]){return[...t].map(([t,s])=>[(e.get(t)??"").trim(),s.trim()])}const u=e=>e.reduce((e,t)=>e+t,0);function m(e,t){return 0===e.length?Number.POSITIVE_INFINITY:u(e.map(([e,s])=>t(e,s)))/e.length}function p(e,t){const s=e.length;if(0===s)return 0;const n=u(e)/s,r=u(t)/s,o=u(e.map((e,s)=>(e-n)*(t[s]-r))),a=u(e.map(e=>(e-n)**2)),i=u(t.map(e=>(e-r)**2));return 0===a||0===i?0:o/Math.sqrt(a*i)}function h(e){const t=e.map((e,t)=>t).sort((t,s)=>e[t]-e[s]),s=new Array(e.length).fill(0);let n=0;for(;n<t.length;){let r=n;for(;r+1<t.length&&e[t[r+1]]===e[t[n]];)r++;for(let e=n;e<=r;e++)s[t[e]]=(n+r)/2+1;n=r+1}return s}function d(e){return[...new Set(e.map(([,e])=>e))].sort().map(t=>({lab:t,tp:e.filter(([e,s])=>e===t&&s===t).length,fp:e.filter(([e,s])=>e===t&&s!==t).length,fn:e.filter(([e,s])=>e!==t&&s===t).length}))}export function metricValue(e,t,s){switch(e){case CUSTOM:throw new Error("a custom task is scored by its scorer.py, not by a built-in metric");case"accuracy":return function(e,t){if(0===t.size)return 0;let s=0;for(const[n,r]of t)(e.get(n)??"").trim()===r.trim()&&s++;return s/t.size}(t,s);case"macro_f1":return function(e,t){const s=[...new Set([...t.values()].map(e=>e.trim()))].sort();if(0===s.length)return 0;let n=0;for(const r of s){let s=0,o=0,a=0;for(const[n,i]of t){const t=(e.get(n)??"").trim(),c=i.trim();t===r&&c===r?s++:t===r&&c!==r?o++:t!==r&&c===r&&a++}const i=2*s+o+a;n+=i?2*s/i:0}return n/s.length}(t,s);case"mse":return r(t,s);case"rmse":return Math.sqrt(r(t,s));case"balanced_accuracy":return function(e){const t=d(e);return 0===t.length?0:u(t.map(e=>e.tp/(e.tp+e.fn)))/t.length}(l(t,s));case"weighted_f1":return function(e){const t=d(e);return 0===t.length?0:u(t.map(e=>{const t=2*e.tp+e.fp+e.fn;return(e.tp+e.fn)*(t?2*e.tp/t:0)}))/e.length}(l(t,s));case"mcc":return function(e){const t=e.length;if(0===t)return 0;const s=new Map,n=new Map;let r=0;for(const[t,o]of e)s.set(o,(s.get(o)||0)+1),n.set(t,(n.get(t)||0)+1),t===o&&r++;const o=r*t-u([...s].map(([e,t])=>t*(n.get(e)||0))),a=t*t-u([...s.values()].map(e=>e*e)),i=t*t-u([...n.values()].map(e=>e*e));return 0===a||0===i?0:o/Math.sqrt(a*i)}(l(t,s));case"f1":return function(...[e,t]){let s=0,n=0,r=0;for(const[o,i]of t){const t=a(e.get(o)),c=a(i);t&&c?s++:t&&!c?n++:c&&!t&&r++}const o=2*s+n+r;return o?2*s/o:0}(t,s);case"qwk":return function(...[e,t]){const s=[...t.values()].map(e=>Math.floor(Number(e)+.5));if(0===s.length)return 0;const n=Math.min(...s),r=Math.max(...s),a=r-n+1;if(a<2)return 0;const i=[...t.keys()].map(t=>Math.min(Math.max(Math.floor(o(e.get(t))+.5),n),r)),c=new Array(a).fill(0),l=new Array(a).fill(0);let u=0;s.forEach((e,t)=>{c[e-n]++,l[i[t]-n]++,u+=(i[t]-e)**2});let m=0;for(let e=0;e<a;e++)for(let t=0;t<a;t++)m+=(e-t)**2*c[e]*l[t];return m/=s.length,0===m?0:1-u/m}(t,s);case"auc":return function(e){const t=u(e.map(([,e])=>e)),s=e.length-t;if(0===t||0===s)return.5;const n=h(e.map(([e])=>e));return(u(n.filter((t,s)=>1===e[s][1]))-t*(t+1)/2)/(t*s)}(c(t,s));case"ap":return function(e){const t=[...e].sort((e,t)=>t[0]-e[0]),s=u(t.map(([,e])=>e));if(0===s)return 0;let n=0,r=0,o=0,a=0,i=0;for(;i<t.length;){let e=i;for(;e<t.length&&t[e][0]===t[i][0];)r+=t[e][1],o++,e++;const c=r/s;n+=r/o*(c-a),a=c,i=e}return n}(c(t,s));case"logloss":return m(c(t,s),(e,t)=>{const s=Math.min(Math.max(e,1e-15),1-1e-15);return 1===t?-Math.log(s):-Math.log(1-s)});case"brier":return m(c(t,s),(e,t)=>(e-t)**2);case"mae":return m(i(t,s),(e,t)=>Math.abs(e-t));case"rmsle":return Math.sqrt(m(i(t,s),(e,t)=>(Math.log1p(Math.max(e,0))-Math.log1p(Math.max(t,0)))**2));case"mape":return m(i(t,s),(e,t)=>Math.abs(e-t)/Math.max(Math.abs(t),2220446049250313e-31));case"smape":return m(i(t,s),(e,t)=>{const s=Math.abs(e)+Math.abs(t);return s?2*Math.abs(e-t)/s:0});case"r2":return function(e){if(0===e.length)return 0;const t=u(e.map(([,e])=>e))/e.length,s=u(e.map(([,e])=>(e-t)**2));return 0===s?0:1-u(e.map(([e,t])=>(e-t)**2))/s}(i(t,s));case"pearson":{const e=i(t,s);return p(e.map(([e])=>e),e.map(([,e])=>e))}case"spearman":{const e=i(t,s);return p(h(e.map(([e])=>e)),h(e.map(([,e])=>e)))}}}export function readSubmission(e,t,s){const n={ok:!1,problems:[],nMissing:0,pred:new Map};if(!t.ok||!t.parsed)return n.problems.push("check the task first"),n;const{table:r,problems:o}=parseCsv(s);if(o.length)return n.problems.push(...o),n;const a=[e.id,...predColumns(e)];if(r.header.join(",")!==a.join(","))return n.problems.push(2===a.length?`the submission needs exactly two columns: ${a.join(",")} — got ${r.header.join(",")}`:`the submission needs exactly these columns, in this order: ${a.join(",")} — got ${r.header.join(",")}`),n;const i=n.pred;for(const e of r.rows){if(i.has(e[0]))return n.problems.push(`duplicate id ${e[0]} in the submission`),n;i.set(e[0],e.slice(1))}const{labels:c}=t.parsed,l=[...i.keys()].filter(e=>!c.has(e));return l.length?(n.problems.push(`${l.length} ids are not in the test set (e.g. ${l.slice(0,3).join(", ")})`),n):(n.nMissing=c.size-i.size,n.nMissing>0?(n.problems.push(`${n.nMissing} test rows have no prediction — the judge refuses an incomplete submission`),n):(n.ok=!0,n))}export function scoreSubmission(e,t,s){const n={ok:!1,problems:[],a:0,b:0,all:0,gap:0,nMissing:0},r=readSubmission(e,t,s);if(n.nMissing=r.nMissing,!r.ok||!t.parsed)return n.problems.push(...r.problems),n;if(e.metric===CUSTOM)return n.problems.push("this task is scored by its scorer.py — setter score runs it"),n;const o=new Map([...r.pred].map(([e,t])=>[e,t[0]])),{labels:a,aIds:i,bIds:c}=t.parsed,l=e=>new Map(e.map(e=>[e,a.get(e)]));return n.a=metricValue(e.metric,o,l(i)),n.b=metricValue(e.metric,o,l(c)),n.all=metricValue(e.metric,o,a),n.gap=LOWER_IS_BETTER.has(e.metric)?n.b-n.a:n.a-n.b,n.ok=!0,n}export function verdictLines(e,t,s,n){const r=[],o=void 0===n?LOWER_IS_BETTER.has(e.metric):!n,a=(e,t)=>o?e<t:e>t,i=Math.abs(t.gap);if(i/Math.max(Math.abs(t.all),1e-9)>.05?r.push(`A and B disagree by ${i.toFixed(4)} on the baseline — check that the two test sets come from the same distribution`):r.push(`A and B agree on the baseline (difference ${i.toFixed(4)})`),s){if(a(s.b,t.b)){const e=Math.abs(s.b-t.b);r.push(`room to climb on B: ${e.toFixed(4)} (baseline ${t.b.toFixed(4)} → reference ${s.b.toFixed(4)})`),e<2*i&&r.push("that room is smaller than twice the A/B difference — the ranking may come down to noise")}else r.push("the reference does NOT beat the baseline on B — contestants have nothing to climb");a(s.a,t.a)!==a(s.b,t.b)&&r.push("baseline and reference rank differently on A and on B — the public board would mislead")}return r}export function tarArchive(e){const t=[],s=new TextEncoder;for(const n of e){const e=new Uint8Array(512),r=(t,n,r)=>{e.set(s.encode(t).subarray(0,r),n)};if(s.encode(n.name).length>99)throw new Error(`file name too long for the archive: ${n.name}`);r(n.name,0,100),r("0000644\0",100,8),r("0000000\0",108,8),r("0000000\0",116,8),r(`${n.data.length.toString(8).padStart(11,"0")}\0`,124,12),r("00000000000\0",136,12),r(" ",148,8),r("0",156,1),r("ustar\0",257,6),r("00",263,2);let o=0;for(const t of e)o+=t;r(`${o.toString(8).padStart(6,"0")}\0 `,148,8),t.push(e,n.data);const a=(512-n.data.length%512)%512;a&&t.push(new Uint8Array(a))}t.push(new Uint8Array(1024));const n=t.reduce((e,t)=>e+t.length,0),r=new Uint8Array(n);let o=0;for(const e of t)r.set(e,o),o+=e.length;return r}export const BASELINE_FILE="baseline.ipynb";export const TASK_FILES=["task.json","train.csv","test_a.csv","labels_a.csv","test_b.csv","labels_b.csv",BASELINE_FILE];export function scorerTemplate(e=["label"]){const t=e[0];return`# scorer.py — how this task is scored: several metrics, each with a weight.\n# 只改标了 1 2 3 的三处,其余不用动。\n# Edit the three numbered places and nothing else, then type: setter check\n# scorer.py may only use Python's maths modules and icoa_metrics.\nfrom icoa_metrics import * # the ready-made metrics; the list is at the top of icoa_metrics.py\n\nCOLUMNS = [${e.map(e=>`"${e}"`).join(", ")}] # 1. what a contestant submits (besides id)\nHIGHER_IS_BETTER = True\n\n\ndef score(pred, truth):\n # pred[column] = the contestant's values, row by row\n # truth[column] = the same rows of your labels file (every column of it, also ones only you have)\n return weighted({\n # 2. one line per metric 3. its weight — the weights add up to 1\n "accuracy": (0.70, accuracy(pred["${t}"], truth["${t}"])),\n "macro_f1": (0.30, macro_f1(pred["${t}"], truth["${t}"])),\n })\n`}export const INLINE_LIMIT_BYTES=5242880;export function baselineCells(e){if(e.metric===CUSTOM)return[{kind:"markdown",source:`# ${e.title}\n\nDescribe the task here: what one row is, what each column to predict means, and how a submission is scored (the parts and their weights).\n\nThis notebook is the baseline every contestant starts from. Keep it short and make sure it runs top to bottom.`},{kind:"code",source:'import pandas as pd\n\ntrain = pd.read_csv("train.csv")\ntest = pd.read_csv("test.csv")\ntargets = [c for c in train.columns if c not in test.columns] # the columns to predict\nprint(train.shape, test.shape, targets)\ntrain.head()'},{kind:"code",source:`# Baseline: predict the most common training value of every column, for every row.\nsub = pd.DataFrame({"${e.id}": test["${e.id}"]})\nfor c in targets:\n sub[c] = train[c].mode()[0]`},{kind:"code",source:'sub.to_csv("submission.csv", index=False)\nprint("wrote submission.csv —", len(sub), "rows,", "columns:", list(sub.columns))'}];const t=METRIC_KIND[e.metric],s="reg"===t||"prob"===t;return[{kind:"markdown",source:`# ${e.title}\n\nDescribe the task here: what one row is, what "${e.label}" means, and how a submission is scored (**${e.metric}**).\n\nThis notebook is the baseline every contestant starts from. Keep it short and make sure it runs top to bottom.`},{kind:"code",source:'import pandas as pd\n\ntrain = pd.read_csv("train.csv")\ntest = pd.read_csv("test.csv")\nprint(train.shape, test.shape)\ntrain.head()'},{kind:"code",source:s?`# Baseline: predict the training mean for every row${"prob"===t?" (the share of 1s — a probability)":""}.\npred = [train["${e.label}"].mean()] * len(test)`:`# Baseline: predict the most common class for every row.\npred = [train["${e.label}"].mode()[0]] * len(test)`},{kind:"code",source:`pd.DataFrame({"${e.id}": test["${e.id}"], "${e.label}": pred}).to_csv("submission.csv", index=False)\nprint("wrote submission.csv —", len(pred), "rows")`}]}export const SQUARE_GIB=3.5;export const CARD_UNITS={l4:6,ada:5,t4:4};export function capsulePlan(e,t,s,n){if(!s)return{tier:"cpu",squares:0,fits:!0,capGib:0,perCard:{l4:0,ada:0,t4:0}};const r=[1,2,3].find(t=>e<3.5*t-.5);if(!r)return{tier:"gpu",squares:3,fits:!1,capGib:10,perCard:{l4:0,ada:0,t4:0}};const o=(e,t)=>3===r&&"l4"!==t?0:Math.floor(e/r),a={l4:o(CARD_UNITS.l4,"l4"),ada:o(CARD_UNITS.ada,"ada"),t4:o(CARD_UNITS.t4,"t4")},i={tier:"gpu",squares:r,fits:!0,capGib:3.5*r-.5,perCard:a},c=e=>({l4:e(a.l4),ada:e(a.ada),t4:e(a.t4)});return n&&n>0&&(i.cards=c(e=>e?Math.ceil(n/e):0)),t>0&&(i.runsPerCardHour=c(e=>Math.floor(60*e/t))),i}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "icoa-cli",
|
|
3
|
-
"version": "2.19.
|
|
3
|
+
"version": "2.19.714",
|
|
4
4
|
"description": "ICOA CLI — AI-native competition runtime of the International Cyber Olympiad in AI: learn, practise, and compete in AI & cybersecurity from the terminal (AI4CTF · CTF4AI · CTF4EAI)",
|
|
5
5
|
"author": "Charlie Zhu",
|
|
6
6
|
"type": "module",
|