nemar-cli 0.10.15-dev.3224 → 0.10.15-dev.3232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -99,7 +99,7 @@ ${t}`,oAe,sAe,aAe=(e,t,r)=>{let n=r===""?"":`with ${r.trim()}() `,i=iAe.bind(nul
|
|
|
99
99
|
`)}var JS=!1;var{spawn:iHe}=globalThis.Bun;async function k(e,t={}){let r={...process.env,GIT_TERMINAL_PROMPT:"0",...t.env};for(let u of t.unsetEnv??[])delete r[u];let n=iHe({cmd:e,cwd:t.cwd,...t.stdin===void 0?{}:{stdin:new TextEncoder().encode(t.stdin)},stdout:"pipe",stderr:"pipe",env:r}),i=!1,o;if(t.timeout)o=setTimeout(()=>{i=!0,n.kill()},t.timeout);if(XS()){let u=t.cwd?` (cwd=${t.cwd})`:"";$p(l.dim(`$ ${e.join(" ")}${u}`))}let s=await new Response(n.stdout).text(),a=await new Response(n.stderr).text(),c=await n.exited;if(o)clearTimeout(o);if(XS()){if(s.trim())$p(l.dim(s.trimEnd()));if(a.trim())$p(l.yellow(a.trimEnd()));$p(l.dim(`(exit ${c})`))}if(i)return{stdout:s,stderr:a||`Command timed out after ${Math.round(t.timeout/1000)}s`,exitCode:c??1,timedOut:!0};return{stdout:s,stderr:a,exitCode:c,timedOut:!1}}var Dt=P(()=>{Xe()});function sHe(){if(process.platform==="darwin")return"macos";if(process.platform==="win32")return"windows";return"linux"}function PX(e,t=sHe()){return e.installInstructions[t]}function aHe(e){let t=e.match(/(\d+\.\d+(?:\.\d+)?)/);return t?t[1]:void 0}function lHe(){let e=Number(process.env.TEST_PROBE_TIMEOUT_MS);return Number.isFinite(e)&&e>0?e:cHe}async function TX(e){try{let{exitCode:t,stdout:r,timedOut:n}=await k(e.cmd,{timeout:lHe()});if(n)return{available:!1,timedOut:!0};if(t!==0)return{available:!1};return{available:!0,version:aHe(r)}}catch(t){let r=t instanceof Error?t.message:String(t);if(!r.includes("ENOENT")&&!r.includes("not found"))console.error(` Warning: ${e.name} check failed unexpectedly: ${r}`);return{available:!1}}}async function kp(){let e=Object.entries(FX);return Promise.all(e.map(async([t,r])=>{let{available:n,version:i,timedOut:o}=await TX(r);return{key:t,name:r.name,purpose:r.purpose,required:r.required,available:n,version:i,timedOut:o,installInstruction:PX(r)}}))}async function IX(){let t=(await kp()).filter((r)=>r.required&&!r.available);if(t.length===0)return[];console.warn(`
|
|
100
100
|
Heads up: some tools NEMAR needs for uploading/validating aren't installed yet:`);for(let r of t)console.warn(` - ${r.name} (${r.purpose})`),console.warn(` Install: ${r.installInstruction}`);return console.warn("Run `nemar doctor` anytime to re-check. You can finish signup without them.\n"),t}async function fc(e){let r=oHe[e].map((s)=>FX[s]).filter(Boolean),n=await Promise.all(r.map(async(s)=>{let a=await TX(s);return{check:s,available:a.available,timedOut:a.timedOut}}));for(let s of n)if(s.timedOut&&s.check.required)console.warn(`Warning: ${s.check.name} did not respond to a version check in time; continuing anyway.`);let i=n.filter((s)=>!s.available&&!s.timedOut&&s.check.required).map((s)=>({tool:s.check.name,installInstruction:PX(s.check)}));if(i.length===0)return;let o=[`
|
|
101
101
|
Missing required tools:`];for(let s of i)o.push(` ${s.tool} not installed`),o.push(` Install: ${s.installInstruction}`);throw Error(o.join(`
|
|
102
|
-
`))}var FX,oHe,cHe=4000;var Cp=P(()=>{Dt();FX={git:{name:"git",purpose:"version control for dataset metadata",cmd:["git","--version"],required:!0,installInstructions:{macos:"brew install git (or install Xcode Command Line Tools)",linux:"sudo apt-get install git OR sudo dnf install git",windows:"winget install Git.Git (or https://git-scm.com/download/win)"}},"git-annex":{name:"git-annex",purpose:"large-file data transfer (upload/download)",cmd:["git-annex","version"],required:!0,installInstructions:{macos:"brew install git-annex",linux:"sudo apt-get install git-annex OR sudo dnf install git-annex",windows:"Download the installer from https://git-annex.branchable.com/install/Windows/"}},gh:{name:"GitHub CLI (gh)",purpose:"GitHub authentication + repository operations",cmd:["gh","--version"],required:!0,installInstructions:{macos:"brew install gh",linux:"See https://cli.github.com/manual/installation",windows:"winget install GitHub.cli (or: scoop install gh)"}},aws:{name:"AWS CLI",purpose:"S3 data upload/download",cmd:["aws","--version"],required:!0,installInstructions:{macos:"brew install awscli",linux:"pip install awscli OR sudo snap install aws-cli --classic",windows:"winget install Amazon.AWSCLI (or the MSI from https://aws.amazon.com/cli/)"}},deno:{name:"Deno",purpose:"BIDS validation",cmd:["deno","--version"],required:!0,installInstructions:{macos:"brew install deno (or: curl -fsSL https://deno.land/install.sh | sh)",linux:"curl -fsSL https://deno.land/install.sh | sh",windows:"winget install DenoLand.Deno (or: irm https://deno.land/install.ps1 | iex)"}}},oHe={upload:["git-annex","gh","aws","deno"],download:["gh"],clone:["git","gh"],push:["git-annex","gh"],publish:["gh","aws"],update:["git","gh"],release:["git","gh"]}});var jX;var NX=P(()=>{jX={name:"nemar-cli",version:"0.10.15-dev.
|
|
102
|
+
`))}var FX,oHe,cHe=4000;var Cp=P(()=>{Dt();FX={git:{name:"git",purpose:"version control for dataset metadata",cmd:["git","--version"],required:!0,installInstructions:{macos:"brew install git (or install Xcode Command Line Tools)",linux:"sudo apt-get install git OR sudo dnf install git",windows:"winget install Git.Git (or https://git-scm.com/download/win)"}},"git-annex":{name:"git-annex",purpose:"large-file data transfer (upload/download)",cmd:["git-annex","version"],required:!0,installInstructions:{macos:"brew install git-annex",linux:"sudo apt-get install git-annex OR sudo dnf install git-annex",windows:"Download the installer from https://git-annex.branchable.com/install/Windows/"}},gh:{name:"GitHub CLI (gh)",purpose:"GitHub authentication + repository operations",cmd:["gh","--version"],required:!0,installInstructions:{macos:"brew install gh",linux:"See https://cli.github.com/manual/installation",windows:"winget install GitHub.cli (or: scoop install gh)"}},aws:{name:"AWS CLI",purpose:"S3 data upload/download",cmd:["aws","--version"],required:!0,installInstructions:{macos:"brew install awscli",linux:"pip install awscli OR sudo snap install aws-cli --classic",windows:"winget install Amazon.AWSCLI (or the MSI from https://aws.amazon.com/cli/)"}},deno:{name:"Deno",purpose:"BIDS validation",cmd:["deno","--version"],required:!0,installInstructions:{macos:"brew install deno (or: curl -fsSL https://deno.land/install.sh | sh)",linux:"curl -fsSL https://deno.land/install.sh | sh",windows:"winget install DenoLand.Deno (or: irm https://deno.land/install.ps1 | iex)"}}},oHe={upload:["git-annex","gh","aws","deno"],download:["gh"],clone:["git","gh"],push:["git-annex","gh"],publish:["gh","aws"],update:["git","gh"],release:["git","gh"]}});var jX;var NX=P(()=>{jX={name:"nemar-cli",version:"0.10.15-dev.3232",description:"CLI for NEMAR (Neuroelectromagnetic Data Archive and Tools Resource) dataset management",type:"module",main:"dist/index.js",bin:{nemar:"dist/index.js"},scripts:{dev:"bun run src/index.ts",build:"bun build src/index.ts --outdir dist --target bun --minify && sed '1s|#!/usr/bin/env node|#!/usr/bin/env bun|' dist/index.js > dist/index.js.tmp && mv dist/index.js.tmp dist/index.js",test:"bun test",lint:"biome check src/","lint:scrub":"biome check scripts/scrub shared/identifier-scrub.ts test/scrub","test:scrub":"bun test --preload ./test/scrub/helpers/temp-dirs-preload.ts --path-ignore-patterns='__run_every_scrub_suite__' test/scrub test/identifier-scan.test.ts test/identifier-scrub.test.ts test/identifier-dates.test.ts test/identifier-fleet.test.ts","migrations:d1-check":"bun run scripts/d1-migration-check.ts","zarr:geometry-check":"bun run backend/scripts/zarr-geometry-conformance.ts","lint:fix":"biome check --fix src/",format:"biome format --write src/",typecheck:"tsc --noEmit && tsc --noEmit -p backend","generate-openapi":"bun run scripts/generate-openapi.ts",prepublishOnly:"bun run build",prepare:"husky",postinstall:"bun scripts/postinstall-check.mjs",deploy:"wrangler deploy",preview:"wrangler dev"},keywords:["nemar","bids","neuroimaging","eeg","emg","datalad","cli"],author:"NEMAR Team",license:"CC-BY-NC-ND-4.0",repository:{type:"git",url:"git+https://github.com/nemarOrg/nemar-cli.git"},bugs:{url:"https://github.com/nemarOrg/nemar-cli/issues"},homepage:"https://docs.nemar.org",engines:{bun:">=1.0.0"},files:["dist","scripts/postinstall-check.mjs","scripts/recover-datasets.json","scripts/withdrawn-datasets.json","README.md","LICENSE"],dependencies:{chalk:"^5.3.0",commander:"^12.1.0",conf:"^13.0.1",inquirer:"^9.2.15",ora:"^8.0.1",semver:"^7.8.5",zod:"^3.23.8"},"lint-staged":{"src/**/*.{ts,tsx}":["biome check --fix"],"backend/src/**/*.{ts,tsx}":["biome check --fix"]},devDependencies:{"@asteasolutions/zod-to-openapi":"^7.3.4","@biomejs/biome":"1.9.4","@types/bun":"latest","@types/inquirer":"^9.0.7","@types/semver":"^7.8.0",ajv:"^8.20.0","ajv-formats":"^3.0.1",bcryptjs:"^3.0.3",husky:"^9.1.7","lint-staged":"^16.3.2",typescript:"^5.5.4",wrangler:"^4.70.0",yaml:"^2.8.4"}}});var Di;var hu=P(()=>{NX();Di=jX.version});import{mkdirSync as dHe,readdirSync as fHe,unlinkSync as pHe,writeFileSync as mHe}from"fs";import{join as ZS}from"path";function zX(){gu=!0}function pc(){return gu}function VX(e,t=process.env){return e.includes("--debug")||t.NEMAR_DEBUG==="1"}function Ap(e){UX=e,Kt(`Step: ${e}`)}function Op(){LX=!0}function HX(){return LX}function so(){BX=!0}function GX(){return BX}function Kt(e){if(gu)process.stderr.write(`[debug] ${e}
|
|
103
103
|
`)}function bHe(e){let t={};for(let[r,n]of Object.entries(e))t[r]=yHe.has(r.toLowerCase())?"[REDACTED]":n;return t}function KX(e){let t=e.indexOf("@");if(t<=0)return"***";return`${e[0]}***${e.slice(t)}`}function QS(e,t,r=0){if(r>_He)return"[REDACTED: depth]";if(Array.isArray(t))return t.map((n)=>QS(e,n,r+1));if(t&&typeof t==="object"){let n={};for(let[i,o]of Object.entries(t))n[i]=QS(i,o,r+1);return n}if(typeof t==="string"){if(xHe.test(e))return"[REDACTED]";return t.replace(WX,KX)}return t}function e$(e){return e.replace(vHe,"Bearer [REDACTED]").replace(wHe,"[REDACTED_AWS_KEY]").replace(DHe,"$1[REDACTED]").replace(EHe,"$1$2=[REDACTED]").replace(WX,KX)}function SHe(e){return e$(e)}function MX(e,t=gHe){if(e===void 0)return;let r=e;try{let n=JSON.parse(e);r=JSON.stringify(QS("",n))}catch{}if(r=e$(r),Buffer.byteLength(r,"utf8")>t)r=`${Buffer.from(r,"utf8").subarray(0,t).toString("utf8")}... [truncated]`;return r}function By(e){if(!gu)return;let t=SHe(e.url);try{let r={...e,url:t,requestHeaders:e.requestHeaders?bHe(e.requestHeaders):void 0,requestBody:MX(e.requestBody),responseBody:MX(e.responseBody)};Ly.push(r),Kt(`${e.method} ${t} -> ${e.status??"ERROR"} (${e.durationMs}ms)`)}catch(r){let n=q(r);Ly.push({method:e.method,url:t,status:e.status,durationMs:e.durationMs,note:`redaction failed: ${n}`}),Kt(`redaction failed for ${e.method} ${t}: ${n}`)}}async function $He(){let e=[];try{let t=process.versions.bun?`Bun ${process.versions.bun}`:`Node ${process.version}`;e.push(`CLI version: ${Di}`),e.push(`OS/Arch: ${process.platform}/${process.arch}`),e.push(`Runtime: ${t}`)}catch(t){e.push(`(could not read CLI/OS/runtime info: ${q(t)})`)}e.push(""),e.push("Active account:");try{let t=mt();if(t.apiKey){if(t.username)e.push(` Username: ${t.username}`);if(e.push(` API URL: ${t.apiUrl}`),t.role)e.push(` Role: ${t.role}`);e.push(` Sandbox completed: ${!!t.sandboxCompleted}`)}else e.push(" (not authenticated)")}catch(t){e.push(` (could not read account config: ${q(t)})`)}e.push(""),e.push("External tools:");try{let t=await kp();for(let r of t){let n=r.available?r.version??"installed":r.timedOut?"(timed out)":"not installed";e.push(` ${r.name}: ${n}`)}}catch(t){e.push(` (could not probe external tools: ${q(t)})`)}return e}async function t$(){return(await $He()).join(`
|
|
104
104
|
`)}async function YX(){if(!gu||qy!==void 0)return;try{qy=await t$()}catch(e){let t=q(e);qy=`(could not build environment report: ${t})`,Kt(`environment snapshot failed: ${t}`)}}function JX(e){if(qX.has(e))return{flag:e};for(let t of qX){if(!e.startsWith(t)||e.length<=t.length)continue;if(t.length===2)return{flag:t,kind:"short-attached"};if(e[t.length]==="=")return{flag:t,kind:"long-eq"}}return null}function kHe(e){let t=[];for(let r=0;r<e.length;r++){let n=e[r],i=JX(n);if(!i){t.push(n);continue}if(i.kind==="long-eq")t.push(`${i.flag}=[REDACTED]`);else if(i.kind==="short-attached")t.push(`${i.flag}[REDACTED]`);else if(t.push(n),r+1<e.length)t.push("[REDACTED]"),r++}return t}function CHe(e){let t=[];for(let r=0;r<e.length;r++){let n=e[r],i=JX(n);if(!i){t.push(n);continue}if(!i.kind&&r+1<e.length)r++}return t}function RHe(e){let t=[];for(let n of CHe(e)){if(n.startsWith("-"))continue;if(t.push(n),t.length===2)break}return(t.length>0?t.join("-"):"nemar").replace(/[^a-zA-Z0-9_-]/g,"_")}function AHe(e,t){let r=[];if(r.push("NEMAR CLI Debug Log"),r.push(`Generated: ${new Date().toISOString()}`),r.push(`Command: nemar ${e$(kHe(e).join(" "))}`),r.push(""),r.push(qy??"(environment snapshot unavailable)"),r.push(""),r.push("HTTP requests:"),Ly.length===0)r.push(" (none)");else Ly.forEach((n,i)=>{if(r.push(` [${i+1}] ${n.method} ${n.url} -> ${n.status??"ERROR"} (${n.durationMs}ms)`),n.error)r.push(` Error: ${n.error}`);if(n.note)r.push(` Note: ${n.note}`);if(n.requestHeaders)r.push(` Request headers: ${JSON.stringify(n.requestHeaders)}`);if(n.requestBody)r.push(` Request body: ${n.requestBody}`);if(n.responseBody)r.push(` Response body: ${n.responseBody}`)});return r.push(""),r.push(`Failing step: ${UX??"(none recorded)"}`),r.push(`Exit code: ${t}`),`${r.join(`
|
|
105
105
|
`)}
|
|
@@ -184,7 +184,7 @@ Privacy correction ${e}: ${n}. The checksums in the provenance file describe the
|
|
|
184
184
|
${m}`:", ";return`${e.slice(0,c)}${h}${p}${e.slice(c)}`}function dm(e){try{return fm(e).isFile()}catch{return!1}}async function EZe(e,t){for(let r of ya(t))if((await k(["git","-c","annex.largefiles=nothing","add","--",...r],{cwd:e})).exitCode!==0)throw Error(`git add failed for ${r.length} scrubbed file(s)`)}function SZe(e){let t=Yn(e,cm);if(!tZe(t))return!1;let r;try{r=Xne(t)}catch{return!0}let n=!1;for(let i of r){if(i.action==="history-rewritten")n=!1;if(i.action==="import-scrubbed"&&((i.counts.json_values_blanked??0)>0||(i.counts.git_held_recordings_scrubbed??0)>0))n=!0}return n}async function $Ze(e){try{return await kZe(e)}catch(t){if(t instanceof St)throw t;throw new St("scrub-failed",pm(t))}}async function kZe(e){let{datasetPath:t}=e,r=_Ze(),n=[...await tm(t)].sort(),i=await go(t),o=n.filter((j)=>kie.test(j));r.recordings=o.length;let s=!e.reimport,a=[],c=[],u=new Set,d=[];for(let j of o){let X=i.get(j);if(X===void 0){a.push(j);continue}if(e.skipData){r.headers_not_read_skip_data++;continue}if(Wn(X)===0){r.headers_not_edf++,u.add(X);continue}let W=e.keyUrlMap.get(X);if(W===void 0){if(e.reimport)r.headers_not_read_nemar_held++;else r.headers_not_read_no_source++;continue}let G=jb(W),Be=G?{kind:"s3",ref:G}:/^https?:\/\//.test(W)?{kind:"url",url:W}:null;if(!Be){d.push("source-unsupported");continue}c.push({path:j,key:X,source:Be})}let f=[];for(let j of a){let X=T2(Yn(t,j));if(X===null){d.push("local-unreadable");continue}r.headers_read++;let W=I2(X,s);if(W===null){r.headers_not_edf++;continue}if(W.change.datesLeft)r.headers_dates_left++;if(W.change.datesUnproven)r.headers_dates_unproven++;if(W.change.scrub||W.change.dates)f.push({path:j,change:W.change})}let m=await vZe(c,e.concurrency??16,(j)=>e.reader.header(j.source)),p=[...d],h=[],g=[];if(m.forEach((j,X)=>{let W=c[X];if(!j.ok){p.push(j.failure);return}let G=Wn(W.key);if(G!==null&&j.total===null){p.push("no-length");return}if(G!==null&&j.total!==G){h.push(W.key);return}r.headers_read++,u.add(W.key);let Be=I2(j.bytes,s);if(Be===null){r.headers_not_edf++;return}if(Be.change.datesLeft)r.headers_dates_left++;if(Be.change.datesUnproven)r.headers_dates_unproven++;if(Be.change.scrub||Be.change.dates)g.push({...W,size:G??0,change:Be.change})}),p.length>0){let j=p.every((X)=>X==="http-403"||X==="http-404");throw new St("header-unreadable",`${p.length} of ${o.length} recording header(s) could not be read: ${wZe(p)}`,j)}if(h.length>0)throw new St("upstream-size-mismatch",`${h.length} upstream object(s) are not the size their annex key declares`);let b=g.filter((j)=>j.change.scrub),x=f.filter((j)=>j.change.scrub),_=b.length+x.length;if(e.reimport&&_>0)throw new St("already-imported-unscrubbed",`${_} recording(s) in the dataset's own tree need a scrub; correct it with ADR 0085's procedure, not by re-importing`);let D=(j)=>wa.test(j.key)&&j.size>0;if(b.some((j)=>!D(j))){let j=b.filter((X)=>!D(X)).length;throw new St("unsupported-key-backend",`${j} recording key(s) to replace are not SHA256E with a declared size`)}let C=e.maxBytes??D2,E=b.reduce((j,X)=>j+X.size,0),F=e.unannexedData.reduce((j,X)=>j+X.size,0);if(E+F>C)throw new St("bound-exceeded",`${b.length} recording(s) to scrub, ${(E/1073741824).toFixed(1)} GiB to download and ${(F/1073741824).toFixed(1)} GiB of git-held data to upload, over the ${(C/1073741824).toFixed(1)} GiB bound; run on a host that can move them with --normalize-max-gb`);let A=g.filter((j)=>!j.change.scrub),U=A.filter(D);r.headers_dates_left+=A.length-U.length;let O=U.reduce((j,X)=>j+X.size,0),T=E+F+O<=C;if(!T)r.headers_dates_over_bound=U.length,r.headers_dates_left+=U.length;let z=new Set(T?U:[]),V=g.filter((j)=>j.change.scrub||z.has(j)),Y=[];for(let j of V){let X=Yn(t,j.path),W=`${X}.nemar-scrub-download`,G=await e.reader.download(j.source,W);if(!j.change.scrub){let En=wa.exec(j.key)?.[2],wo=G.ok&&G.bytes===j.size&&await Die(W)===En;if(wo)try{P2(W,s)}catch{wo=!1}if(!wo){Da(W,{force:!0}),r.headers_dates_failed++,r.headers_dates_left++;continue}r.bytes_downloaded+=j.size,Da(X,{force:!0}),bie(W,X),Y.push(j);continue}if(!G.ok)throw new St("header-unreadable",`a recording to scrub could not be downloaded: ${G.failure}`,G.failure==="http-403"||G.failure==="http-404");let Be=wa.exec(j.key)?.[2];if(G.bytes!==j.size||await Die(W)!==Be)throw Da(W,{force:!0}),new St("content-mismatch","a downloaded recording does not hash to the key it was downloaded for");r.bytes_downloaded+=G.bytes,P2(W,s),Da(X,{force:!0}),bie(W,X),Y.push(j)}for(let j of f)P2(Yn(t,j.path),s);r.headers_scrubbed=_,r.git_held_recordings_scrubbed=x.length;let ue=f.filter((j)=>j.change.dates).length;r.headers_dates_normalized=Y.filter((j)=>j.change.dates).length+ue,r.git_held_recordings_dates_normalized=ue;let N=[],M=new Set,de=new Set,se=[],ir=f.map((j)=>j.path),bt=[...Y.map((j)=>({path:j.path,size:j.size})),...e.skipData?[]:ir.map((j)=>({path:j,size:fm(Yn(t,j)).size}))];if(e.skipData)se.push(...ir);if(bt.length>0){if(!e.upload)throw Error("scrubbed recordings need an upload strategy");let j;try{j=await E2({datasetPath:t,files:bt,remoteName:e.remoteName,bucket:e.bucket,nemarId:e.nemarId,maxBytes:C,upload:e.upload,backend:"SHA256E"})}catch(G){throw new St("upload-failed",`${bt.length} scrubbed recording(s) could not be annexed or uploaded: ${pm(G)}`)}N.push(...j.items);for(let G of ir)de.add(G);let X=new Map(j.files.map((G)=>[G.path,G.key])),W={};for(let G of Y){let Be=X.get(G.path);if(!Be||!wa.test(Be)||Be===G.key)throw new St("scrub-unverified","a scrubbed recording has no new key");W[G.key]=Be,M.add(G.key)}if(r.upstream_keys_replaced=M.size,M.size>0){let G;try{G=await oie({repo:t,keymap:W,remoteUuids:[],execute:!0})}catch(En){throw new St("retire-failed",`the git-annex branch could not be updated: ${pm(En)}`)}let Be=new Set(Object.values(W)).size;if(G.oldStillHeld!==0||G.oldDead!==G.oldKeys||G.deadRefused!==0||G.newPresent!==Be)throw new St("retire-failed",`${G.oldKeys-G.oldDead} of ${G.oldKeys} replaced key(s) not recorded dead, ${Be-G.newPresent} new key(s) not recorded at the remote`)}}for(let j of n){if(!mZe.test(j))continue;if(i.has(j)){r.json_files_annexed++;continue}let X=Yn(t,j);if(!dm(X)||fm(X).size>fZe){r.json_files_unread++;continue}let W;try{W=yie(ex(X))}catch(G){if(G instanceof lm)throw new St("scrub-unverified","a blanked JSON file failed its proof");throw G}if(W.status==="unreadable"){r.json_files_unread++;continue}if(r.json_files_read++,W.status==="blanked"&&W.bytes)um(X,W.bytes),se.push(j),r.json_files_blanked++,r.json_values_blanked+=W.blanked}if(s)for(let j of n){if(!hZe.test(j))continue;if(i.has(j)){r.scans_tables_annexed++;continue}let X=Yn(t,j);if(!dm(X)||fm(X).size>pZe){r.scans_tables_unread++;continue}let W;try{W=gie(ex(X))}catch{W={status:"unreadable",values:0}}if(W.status==="unreadable"){r.scans_tables_unread++;continue}if(r.scans_tables_read++,W.status==="normalized"&&W.bytes)um(X,W.bytes),se.push(j),r.scans_tables_normalized++,r.scans_values_normalized+=W.values}r.images_or_documents_held=kb(n).filter((j)=>j.kind==="image-or-document-file").length;let Or=r.headers_scrubbed>0||r.headers_dates_normalized>0||r.json_values_blanked>0||r.scans_values_normalized>0,Gr=e.now??new Date,Fr=Gr.toISOString().slice(0,10),Wr=r.headers_scrubbed>0?r.headers_dates_normalized>0?"scrubbed-and-dates-set":"scrubbed-in-place":r.headers_dates_normalized>0?"dates-set":null;if(Wr!==null){let j=Yn(t,zu);if(n.includes(zu)&&dm(j)){let W=ex(j,"utf8"),G=W.charCodeAt(0)===65279?String.fromCharCode(65279):"",Be=DZe(W.slice(G.length),k2,Wne(Fr,Wr));if(Be!==null)um(j,`${G}${Be}`),se.push(zu),r.provenance_annotated=1;else r.provenance_not_annotated++}else if(n.includes(zu))r.provenance_not_annotated++;let X=Yn(t,am);if(n.includes(am)&&dm(X)){let W=Kne(Fr,Wr),G=ex(X,"utf8");if(!G.endsWith(W))um(X,`${G}${G===""||G.endsWith(`
|
|
185
185
|
`)?"":`
|
|
186
186
|
`}${W}`),se.push(am);r.provenance_readme_annotated=1}else if(n.includes(am))r.provenance_not_annotated++}let J=!1;if(Or){let j=process.env.GITHUB_ACTOR,X=e.actor??(j&&dZe.test(j)?j:uZe);Jne(Yn(t,cm),{version:1,at:Gr.toISOString(),dataset:e.nemarId,action:"import-scrubbed",versions:[],counts:{...r},scanner:lZe,verification:"scanner-clean+payload-identical",actor:X}),se.push(cm),await EZe(t,se);let W=[`Recording headers scrubbed: ${r.headers_scrubbed} (identification fields only; signal bytes unchanged).`,`Acquisition dates set to 1 January of their year: ${r.headers_dates_normalized} recording header(s), ${r.scans_values_normalized} scans table value(s) in ${r.scans_tables_normalized} file(s).`,`JSON values blanked: ${r.json_values_blanked} in ${r.json_files_blanked} file(s).`,`Images and documents left for review: ${r.images_or_documents_held}.`,`Recorded in ${cm}.`].join(`
|
|
187
|
-
`),G=[...new Set([...se,...bt.map((wo)=>wo.path)])],Be=rZe(Yn(oZe(),"nemar-scrub-pathspec-")),En=Yn(Be,"paths");um(En,`${G.join("\x00")}\x00`,{mode:384});try{if((await k(["git","-c","annex.largefiles=nothing","commit","-m","Privacy correction on import (ADR 0089)","-m",W,`--pathspec-from-file=${En}`,"--pathspec-file-nul"],{cwd:t})).exitCode!==0)throw Error("could not commit the import scrub")}finally{Da(Be,{recursive:!0,force:!0})}J=!0}return{counts:r,items:N,replacedKeys:M,handledPaths:de,historyHoldsOriginals:r.json_values_blanked>0||r.git_held_recordings_scrubbed>0||e.reimport&&SZe(t),headerReadKeys:u,committed:J}}async function CZe(e,t,r){try{return await RZe(e,t,r)}catch(n){if(n instanceof St)throw n;throw new St("scrub-failed",pm(n))}}async function RZe(e,t,r){let n=new Set((await go(e)).values()),i=[...r].filter((d)=>n.has(d)).length;if(i>0)throw new St("old-key-still-named",`${i} replaced key(s) are still named by the tree`);let o=new Map;for(let d of t)if(!o.has(d.key))o.set(d.key,d);let s=[...o.values()].filter((d)=>n.has(d.key)),a=o.size-s.length,c=await Yb(e,s.map((d)=>d.key)),u=s.filter((d,f)=>!Jb(c[f]??""));return{items:u,droppedNotInTree:a,droppedDead:s.length-u.length}}async function Rie(e){let t=await $Ze({datasetPath:e.datasetPath,nemarId:e.nemarId,keyUrlMap:e.keyUrlMap,unannexedData:e.unannexedData,reimport:e.reimport,skipData:e.skipData,maxBytes:e.maxBytes,reader:e.reader,upload:e.upload,remoteName:e.remoteName,bucket:e.bucket,now:e.now,actor:e.actor}),r=await Hb({datasetPath:e.datasetPath,nemarId:e.nemarId,bucket:e.bucket,remoteName:e.remoteName,unannexedData:e.unannexedData.filter((a)=>!t.handledPaths.has(a.path)),upstreamKeys:new Set(e.keyUrlMap.keys()),carryOverUnaccountedKeys:e.reimport&&!e.skipData,upload:e.upload,maxBytes:e.maxBytes}),n=e.skipData?{items:[],droppedNotInTree:0,droppedDead:0}:await CZe(e.datasetPath,[...e.upstreamItems,...t.items,...r.items],t.replacedKeys),i;try{i=await go(e.datasetPath)}catch(a){throw new St("scrub-failed",pm(a))}let o=new Set([...i].filter(([a])=>kie.test(a)).map(([,a])=>a)),s=n.items.filter((a)=>!Tu(a)&&o.has(a.key)&&!t.headerReadKeys.has(a.key)).length;if(s>0)throw new St("header-unreadable",`${s} recording(s) in the copy manifest whose header this run did not read`);return{scrub:t,normalized:r,manifest:n,privacy:{version:1,historyHoldsOriginals:t.historyHoldsOriginals}}}var St,aZe,cZe,lZe,uZe="nemar-importer",dZe,fZe=1048576,pZe=8388608,kie,mZe,hZe,xie="nemar-cli-import-scrub",yZe;var Aie=P(()=>{Wb();sie();R2();bc();F2();ba();Dt();_n();om();$i();rm();St=class St extends Error{code;constructor(e,t,r=!1){super(`${r?`${Gb} `:""}${Mne} refused: ${e} (${t}). Nothing was pushed, and the copy phase did not run.`);this.code=e;this.name="ImportScrubRefused"}};aZe=new Set(["GitScrubError","ContractError","LedgerRefused"]),cZe=/^[A-Z][A-Z0-9_]{1,20}$/;lZe=`identifier-scan@${"
|
|
187
|
+
`),G=[...new Set([...se,...bt.map((wo)=>wo.path)])],Be=rZe(Yn(oZe(),"nemar-scrub-pathspec-")),En=Yn(Be,"paths");um(En,`${G.join("\x00")}\x00`,{mode:384});try{if((await k(["git","-c","annex.largefiles=nothing","commit","-m","Privacy correction on import (ADR 0089)","-m",W,`--pathspec-from-file=${En}`,"--pathspec-file-nul"],{cwd:t})).exitCode!==0)throw Error("could not commit the import scrub")}finally{Da(Be,{recursive:!0,force:!0})}J=!0}return{counts:r,items:N,replacedKeys:M,handledPaths:de,historyHoldsOriginals:r.json_values_blanked>0||r.git_held_recordings_scrubbed>0||e.reimport&&SZe(t),headerReadKeys:u,committed:J}}async function CZe(e,t,r){try{return await RZe(e,t,r)}catch(n){if(n instanceof St)throw n;throw new St("scrub-failed",pm(n))}}async function RZe(e,t,r){let n=new Set((await go(e)).values()),i=[...r].filter((d)=>n.has(d)).length;if(i>0)throw new St("old-key-still-named",`${i} replaced key(s) are still named by the tree`);let o=new Map;for(let d of t)if(!o.has(d.key))o.set(d.key,d);let s=[...o.values()].filter((d)=>n.has(d.key)),a=o.size-s.length,c=await Yb(e,s.map((d)=>d.key)),u=s.filter((d,f)=>!Jb(c[f]??""));return{items:u,droppedNotInTree:a,droppedDead:s.length-u.length}}async function Rie(e){let t=await $Ze({datasetPath:e.datasetPath,nemarId:e.nemarId,keyUrlMap:e.keyUrlMap,unannexedData:e.unannexedData,reimport:e.reimport,skipData:e.skipData,maxBytes:e.maxBytes,reader:e.reader,upload:e.upload,remoteName:e.remoteName,bucket:e.bucket,now:e.now,actor:e.actor}),r=await Hb({datasetPath:e.datasetPath,nemarId:e.nemarId,bucket:e.bucket,remoteName:e.remoteName,unannexedData:e.unannexedData.filter((a)=>!t.handledPaths.has(a.path)),upstreamKeys:new Set(e.keyUrlMap.keys()),carryOverUnaccountedKeys:e.reimport&&!e.skipData,upload:e.upload,maxBytes:e.maxBytes}),n=e.skipData?{items:[],droppedNotInTree:0,droppedDead:0}:await CZe(e.datasetPath,[...e.upstreamItems,...t.items,...r.items],t.replacedKeys),i;try{i=await go(e.datasetPath)}catch(a){throw new St("scrub-failed",pm(a))}let o=new Set([...i].filter(([a])=>kie.test(a)).map(([,a])=>a)),s=n.items.filter((a)=>!Tu(a)&&o.has(a.key)&&!t.headerReadKeys.has(a.key)).length;if(s>0)throw new St("header-unreadable",`${s} recording(s) in the copy manifest whose header this run did not read`);return{scrub:t,normalized:r,manifest:n,privacy:{version:1,historyHoldsOriginals:t.historyHoldsOriginals}}}var St,aZe,cZe,lZe,uZe="nemar-importer",dZe,fZe=1048576,pZe=8388608,kie,mZe,hZe,xie="nemar-cli-import-scrub",yZe;var Aie=P(()=>{Wb();sie();R2();bc();F2();ba();Dt();_n();om();$i();rm();St=class St extends Error{code;constructor(e,t,r=!1){super(`${r?`${Gb} `:""}${Mne} refused: ${e} (${t}). Nothing was pushed, and the copy phase did not run.`);this.code=e;this.name="ImportScrubRefused"}};aZe=new Set(["GitScrubError","ContractError","LedgerRefused"]),cZe=/^[A-Z][A-Z0-9_]{1,20}$/;lZe=`identifier-scan@${"0dc24ef08c95fa1f"}`,dZe=/^[a-z0-9][a-z0-9-]{0,38}$/i,kie=/\.(edf|bdf)$/i,mZe=/\.json$/i,hZe=/_scans\.tsv$/;yZe=new Set(["ENOSPC","EACCES","EPERM","EROFS","EIO","EMFILE","ENFILE"])});import{existsSync as Fie,mkdirSync as AZe,mkdtempSync as Pie,readFileSync as OZe,readdirSync as mm,readlinkSync as FZe,renameSync as PZe,rmSync as TZe,statSync as IZe,writeFileSync as N2}from"fs";import{tmpdir as Tie}from"os";import{join as Hr}from"path";async function j2(e,t,r=120000,n=5000){let i=Date.now()+r,o,s=!1;while(Date.now()<i){try{let c=await Wy(e);if(s=!0,c.bids_validation.present&&c.bids_validation.status!=="no_runs")return{kind:"found"}}catch(c){o=c}let a=Math.max(0,i-Date.now());t.text=`Waiting for BIDS validation run to register... (${Math.ceil(a/1000)}s left)`,await new Promise((c)=>setTimeout(c,Math.min(n,a)))}if(!s&&o!==void 0)return{kind:"error",lastError:o};return{kind:"timeout"}}function MZe(e){if(!e.ciDeployed)return{skipCiCheck:!1,abortReason:"CI workflows did not deploy. Aborting before approval: --trust-upstream bypasses validation lookup, not workflow deployment. Re-run after fixing the deploy failure, or run `nemar admin ci add <id>` manually before approving."};let t=e.poll;if(t===null||t.kind==="found")return{skipCiCheck:!1};if(t.kind==="error")return{skipCiCheck:!1,abortReason:`Every BIDS validation poll attempt failed (last error: ${t.lastError instanceof Error?t.lastError.message:String(t.lastError)}). Refusing to bypass under --trust-upstream because validation state was never actually observed. Investigate API/auth health and re-run.`};if(e.trustUpstream)return{skipCiCheck:!0};return{skipCiCheck:!1,abortReason:"BIDS validation run did not register within the bounded poll window. Re-run with --trust-upstream to bypass (OpenNeuro datasets are pre-validated upstream), or investigate why the deployed CI did not trigger."}}function qZe(e){return e.includes("already exists")||e.includes("409")}function LZe(e){return e==="public"?"skip-already-published":"request"}function BZe(e){return e.originMainSha!==null&&e.originMainSha!==""&&e.currentBranch==="main"}function M2(e){let t=e.match(/^ds(\d{6})$/);if(!t)throw Error(`Invalid OpenNeuro ID "${e}". Expected format: ds###### (e.g., ds007262)`);return`on${t[1]}`}async function tx(e){let[t,r]=await Promise.all([tm(e),Uu(e)]),n=[];for(let i of t){if(r.has(i))continue;let o;try{o=IZe(Hr(e,i)).size}catch(s){let a=s.code;if(a==="ENOENT"||a==="ENOTDIR")continue;throw Error(`Cannot stat tracked file ${i}: ${s instanceof Error?s.message:String(s)}`)}if(Nu(i,o))n.push({path:i,size:o})}return n}function UZe(e){let t;try{t=mm(e,{withFileTypes:!0})}catch(n){let i=n.code;if(i==="ENOENT"||i==="ENOTDIR")return[];throw Error(`Cannot scan ${e} for annexed metadata: ${n instanceof Error?n.message:String(n)}`)}let r=[];for(let n of t){if(!n.isSymbolicLink())continue;if(!tne(n.name))continue;try{if(FZe(Hr(e,n.name)).includes("annex/objects"))r.push(n.name)}catch{}}return r.sort()}function VZe(e,t){let r=t.split("/").map(encodeURIComponent).join("/");return`https://s3.amazonaws.com/openneuro.org/${e}/${r}`}async function HZe(e,t){let r=UZe(e);if(r.length===0)return[];let n=[];for(let o of r){let s=VZe(t,o),a;try{let u=await fetch(s);if(!u.ok){n.push(`${o} (HTTP ${u.status})`);continue}a=Buffer.from(await u.arrayBuffer())}catch(u){n.push(`${o} (fetch failed: ${u instanceof Error?u.message:String(u)})`);continue}if(a.length===0){n.push(`${o} (HTTP 200 but empty body)`);continue}let c=Hr(e,o);try{TZe(c,{force:!0}),N2(c,a)}catch(u){throw Error(`Failed to write fetched metadata ${o} to ${c} (local error, not OpenNeuro): ${u instanceof Error?u.message:String(u)}`)}}if(n.length>0)throw Error(`${Gb} cannot fetch metadata from OpenNeuro: ${n.join(", ")}. These objects are not anonymously readable and NEMAR has no signed OpenNeuro login. This is an OpenNeuro-side access problem, not a NEMAR import bug. (Any files that did fetch are left as regular files on disk, unstaged.)`);let i=await k(["git","-c","annex.largefiles=nothing","add",...r],{cwd:e});if(i.exitCode!==0)throw Error(`Failed to stage un-annexed metadata (${r.join(", ")}): ${i.stderr.trim()}. The files are already regular files on disk; recover with: git -C ${e} -c annex.largefiles=nothing add ${r.join(" ")}`);return r}function GZe(e){let t=Hr(e,"dataset_description.json");if(!Fie(t))throw Error(`dataset_description.json not found at ${t}`);try{return JSON.parse(OZe(t,"utf-8"))}catch(r){throw Error(`Failed to parse dataset_description.json at ${t}: ${r instanceof Error?r.message:String(r)}`)}}function Iie(){let e=process.env.AWS_ACCESS_KEY_ID,t=process.env.AWS_SECRET_ACCESS_KEY;if(!e||!t)throw Error("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set in environment for S3 operations");return{accessKeyId:e,secretAccessKey:t,sessionToken:process.env.AWS_SESSION_TOKEN}}function WZe(e){return typeof e.DatasetDOI==="string"?e.DatasetDOI.replace(/^doi:/,""):null}function KZe(e){let t=new Set,r=[];try{r=mm(e,{withFileTypes:!0}).filter((n)=>n.isDirectory()&&n.name.startsWith("sub-")).map((n)=>n.name)}catch{return[]}for(let n of r){let i=Hr(e,n),o=[];try{o=mm(i,{withFileTypes:!0})}catch{continue}for(let s of o){if(!s.isDirectory())continue;if(s.name.startsWith("ses-")){let a=[];try{a=mm(Hr(i,s.name),{withFileTypes:!0})}catch{continue}for(let c of a)if(c.isDirectory()&&Oie.includes(c.name))t.add(c.name)}else if(Oie.includes(s.name))t.add(s.name)}}return[...t].sort()}function YZe(e){let t=e.Funding;if(!Array.isArray(t))return[];return t.filter((r)=>typeof r==="string"&&r.trim().length>0).map((r)=>({funder_name:r.trim()}))}function JZe(e,t,r,n,i){let o=Hr(e,".nemar");if(!Fie(o))AZe(o,{recursive:!0});let s=i?[{identifier:i,identifier_type:"DOI",relation_type:"IsIdenticalTo"}]:[],a=KZe(e),c=YZe(n),u={version:"2.0",dataset_id:t,source:"openneuro",source_id:r,title:n.Name||t,authors:Array.isArray(n.Authors)?Object.fromEntries(n.Authors.map((d)=>[d,{}])):{},license:n.License||"CC0",dataset_type:n.DatasetType||"raw",related_identifiers:s,pipeline_stage:"seeded"};if(typeof n.BIDSVersion==="string")u.bids_version=n.BIDSVersion;if(a.length>0)u.recording_modality=a;if(a.length>0)u.datatypes=a;if(c.length>0)u.funding_references=c;if(typeof n.Acknowledgements==="string"&&n.Acknowledgements.trim().length>0)u.acknowledgements=n.Acknowledgements.trim();N2(Hr(o,"metadata.json"),`${JSON.stringify(u,null,2)}
|
|
188
188
|
`)}function ZZe(e,t,r){let i=mm(e,{withFileTypes:!0}).filter((f)=>f.isFile()&&XZe.test(f.name)),o=i.find((f)=>f.name.includes("."));if(o)return{kind:"kept",path:o.name};let s=i.find((f)=>!f.name.includes("."));if(s)return PZe(Hr(e,s.name),Hr(e,"README.md")),{kind:"renamed",from:"README",to:"README.md"};let a="<!-- nemar:provenance -->",c=`https://openneuro.org/datasets/${t}`,u=r?`
|
|
189
189
|
- DOI: [${r}](https://doi.org/${r})`:"",d=`${a}
|
|
190
190
|
## Provenance
|
package/package.json
CHANGED