nemar-cli 0.10.10-dev.2983 → 0.10.10-dev.2984
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +3 -3
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -99,7 +99,7 @@ ${t}`,jSe,qSe,MSe=(e,t,r)=>{let n=r===""?"":`with ${r.trim()}() `,o=NSe.bind(nul
|
|
|
99
99
|
`)}var mE=!1;var{spawn:FLe}=globalThis.Bun;async function C(e,t={}){let r={...process.env,GIT_TERMINAL_PROMPT:"0",...t.env};for(let u of t.unsetEnv??[])delete r[u];let n=FLe({cmd:e,cwd:t.cwd,...t.stdin===void 0?{}:{stdin:new TextEncoder().encode(t.stdin)},stdout:"pipe",stderr:"pipe",env:r}),o=!1,i;if(t.timeout)i=setTimeout(()=>{o=!0,n.kill()},t.timeout);if(hE()){let u=t.cwd?` (cwd=${t.cwd})`:"";Tf(l.dim(`$ ${e.join(" ")}${u}`))}let s=await new Response(n.stdout).text(),a=await new Response(n.stderr).text(),c=await n.exited;if(i)clearTimeout(i);if(hE()){if(s.trim())Tf(l.dim(s.trimEnd()));if(a.trim())Tf(l.yellow(a.trimEnd()));Tf(l.dim(`(exit ${c})`))}if(o)return{stdout:s,stderr:a||`Command timed out after ${Math.round(t.timeout/1000)}s`,exitCode:c??1,timedOut:!0};return{stdout:s,stderr:a,exitCode:c,timedOut:!1}}var vt=O(()=>{Xe()});function ILe(){if(process.platform==="darwin")return"macos";if(process.platform==="win32")return"windows";return"linux"}function dY(e,t=ILe()){return e.installInstructions[t]}function TLe(e){let t=e.match(/(\d+\.\d+(?:\.\d+)?)/);return t?t[1]:void 0}function jLe(){let e=Number(process.env.TEST_PROBE_TIMEOUT_MS);return Number.isFinite(e)&&e>0?e:NLe}async function fY(e){try{let{exitCode:t,stdout:r,timedOut:n}=await C(e.cmd,{timeout:jLe()});if(n)return{available:!1,timedOut:!0};if(t!==0)return{available:!1};return{available:!0,version:TLe(r)}}catch(t){let r=t instanceof Error?t.message:String(t);if(!r.includes("ENOENT")&&!r.includes("not found"))console.error(` Warning: ${e.name} check failed unexpectedly: ${r}`);return{available:!1}}}async function Nf(){let e=Object.entries(uY);return Promise.all(e.map(async([t,r])=>{let{available:n,version:o,timedOut:i}=await fY(r);return{key:t,name:r.name,purpose:r.purpose,required:r.required,available:n,version:o,timedOut:i,installInstruction:dY(r)}}))}async function pY(){let t=(await Nf()).filter((r)=>r.required&&!r.available);if(t.length===0)return[];console.warn(`
|
|
100
100
|
Heads up: some tools NEMAR needs for uploading/validating aren't installed yet:`);for(let r of t)console.warn(` - ${r.name} (${r.purpose})`),console.warn(` Install: ${r.installInstruction}`);return console.warn("Run `nemar doctor` anytime to re-check. You can finish signup without them.\n"),t}async function Pa(e){let r=PLe[e].map((s)=>uY[s]).filter(Boolean),n=await Promise.all(r.map(async(s)=>{let a=await fY(s);return{check:s,available:a.available,timedOut:a.timedOut}}));for(let s of n)if(s.timedOut&&s.check.required)console.warn(`Warning: ${s.check.name} did not respond to a version check in time; continuing anyway.`);let o=n.filter((s)=>!s.available&&!s.timedOut&&s.check.required).map((s)=>({tool:s.check.name,installInstruction:dY(s.check)}));if(o.length===0)return;let i=[`
|
|
101
101
|
Missing required tools:`];for(let s of o)i.push(` ${s.tool} not installed`),i.push(` Install: ${s.installInstruction}`);throw Error(i.join(`
|
|
102
|
-
`))}var uY,PLe,NLe=4000;var jf=O(()=>{vt();uY={git:{name:"git",purpose:"version control for dataset metadata",cmd:["git","--version"],required:!0,installInstructions:{macos:"brew install git (or install Xcode Command Line Tools)",linux:"sudo apt-get install git OR sudo dnf install git",windows:"winget install Git.Git (or https://git-scm.com/download/win)"}},"git-annex":{name:"git-annex",purpose:"large-file data transfer (upload/download)",cmd:["git-annex","version"],required:!0,installInstructions:{macos:"brew install git-annex",linux:"sudo apt-get install git-annex OR sudo dnf install git-annex",windows:"Download the installer from https://git-annex.branchable.com/install/Windows/"}},gh:{name:"GitHub CLI (gh)",purpose:"GitHub authentication + repository operations",cmd:["gh","--version"],required:!0,installInstructions:{macos:"brew install gh",linux:"See https://cli.github.com/manual/installation",windows:"winget install GitHub.cli (or: scoop install gh)"}},aws:{name:"AWS CLI",purpose:"S3 data upload/download",cmd:["aws","--version"],required:!0,installInstructions:{macos:"brew install awscli",linux:"pip install awscli OR sudo snap install aws-cli --classic",windows:"winget install Amazon.AWSCLI (or the MSI from https://aws.amazon.com/cli/)"}},deno:{name:"Deno",purpose:"BIDS validation",cmd:["deno","--version"],required:!0,installInstructions:{macos:"brew install deno (or: curl -fsSL https://deno.land/install.sh | sh)",linux:"curl -fsSL https://deno.land/install.sh | sh",windows:"winget install DenoLand.Deno (or: irm https://deno.land/install.ps1 | iex)"}}},PLe={upload:["git-annex","gh","aws","deno"],download:["gh"],clone:["git","gh"],push:["git-annex","gh"],publish:["gh","aws"],update:["git","gh"],release:["git","gh"]}});var hY;var mY=O(()=>{hY={name:"nemar-cli",version:"0.10.10-dev.
|
|
102
|
+
`))}var uY,PLe,NLe=4000;var jf=O(()=>{vt();uY={git:{name:"git",purpose:"version control for dataset metadata",cmd:["git","--version"],required:!0,installInstructions:{macos:"brew install git (or install Xcode Command Line Tools)",linux:"sudo apt-get install git OR sudo dnf install git",windows:"winget install Git.Git (or https://git-scm.com/download/win)"}},"git-annex":{name:"git-annex",purpose:"large-file data transfer (upload/download)",cmd:["git-annex","version"],required:!0,installInstructions:{macos:"brew install git-annex",linux:"sudo apt-get install git-annex OR sudo dnf install git-annex",windows:"Download the installer from https://git-annex.branchable.com/install/Windows/"}},gh:{name:"GitHub CLI (gh)",purpose:"GitHub authentication + repository operations",cmd:["gh","--version"],required:!0,installInstructions:{macos:"brew install gh",linux:"See https://cli.github.com/manual/installation",windows:"winget install GitHub.cli (or: scoop install gh)"}},aws:{name:"AWS CLI",purpose:"S3 data upload/download",cmd:["aws","--version"],required:!0,installInstructions:{macos:"brew install awscli",linux:"pip install awscli OR sudo snap install aws-cli --classic",windows:"winget install Amazon.AWSCLI (or the MSI from https://aws.amazon.com/cli/)"}},deno:{name:"Deno",purpose:"BIDS validation",cmd:["deno","--version"],required:!0,installInstructions:{macos:"brew install deno (or: curl -fsSL https://deno.land/install.sh | sh)",linux:"curl -fsSL https://deno.land/install.sh | sh",windows:"winget install DenoLand.Deno (or: irm https://deno.land/install.ps1 | iex)"}}},PLe={upload:["git-annex","gh","aws","deno"],download:["gh"],clone:["git","gh"],push:["git-annex","gh"],publish:["gh","aws"],update:["git","gh"],release:["git","gh"]}});var hY;var mY=O(()=>{hY={name:"nemar-cli",version:"0.10.10-dev.2984",description:"CLI for NEMAR (Neuroelectromagnetic Data Archive and Tools Resource) dataset management",type:"module",main:"dist/index.js",bin:{nemar:"dist/index.js"},scripts:{dev:"bun run src/index.ts",build:"bun build src/index.ts --outdir dist --target bun --minify && sed '1s|#!/usr/bin/env node|#!/usr/bin/env bun|' dist/index.js > dist/index.js.tmp && mv dist/index.js.tmp dist/index.js",test:"bun test",lint:"biome check src/","migrations:d1-check":"bun run scripts/d1-migration-check.ts","zarr:geometry-check":"bun run backend/scripts/zarr-geometry-conformance.ts","lint:fix":"biome check --fix src/",format:"biome format --write src/",typecheck:"tsc --noEmit && tsc --noEmit -p backend","generate-openapi":"bun run scripts/generate-openapi.ts",prepublishOnly:"bun run build",prepare:"husky",postinstall:"bun scripts/postinstall-check.mjs",deploy:"wrangler deploy",preview:"wrangler dev"},keywords:["nemar","bids","neuroimaging","eeg","emg","datalad","cli"],author:"NEMAR Team",license:"CC-BY-NC-ND-4.0",repository:{type:"git",url:"git+https://github.com/nemarOrg/nemar-cli.git"},bugs:{url:"https://github.com/nemarOrg/nemar-cli/issues"},homepage:"https://docs.nemar.org",engines:{bun:">=1.0.0"},files:["dist","scripts/postinstall-check.mjs","scripts/recover-datasets.json","scripts/withdrawn-datasets.json","README.md","LICENSE"],dependencies:{chalk:"^5.3.0",commander:"^12.1.0",conf:"^13.0.1",inquirer:"^9.2.15",ora:"^8.0.1",semver:"^7.8.5",zod:"^3.23.8"},"lint-staged":{"src/**/*.{ts,tsx}":["biome check --fix"],"backend/src/**/*.{ts,tsx}":["biome check --fix"]},devDependencies:{"@asteasolutions/zod-to-openapi":"^7.3.4","@biomejs/biome":"1.9.4","@types/bun":"latest","@types/inquirer":"^9.0.7","@types/semver":"^7.8.0",ajv:"^8.20.0","ajv-formats":"^3.0.1",bcryptjs:"^3.0.3",husky:"^9.1.7","lint-staged":"^16.3.2",typescript:"^5.5.4",wrangler:"^4.70.0",yaml:"^2.8.4"}}});var Si;var qf=O(()=>{mY();Si=hY.version});import{mkdirSync as MLe,readdirSync as LLe,unlinkSync as BLe,writeFileSync as ULe}from"fs";import{join as gE}from"path";function vY(){Fl=!0}function Ia(){return Fl}function wY(e,t=process.env){return e.includes("--debug")||t.NEMAR_DEBUG==="1"}function Lf(e){_Y=e,$r(`Step: ${e}`)}function Bf(){bY=!0}function DY(){return bY}function bE(){xY=!0}function EY(){return xY}function $r(e){if(Fl)process.stderr.write(`[debug] ${e}
|
|
103
103
|
`)}function HLe(e){let t={};for(let[r,n]of Object.entries(e))t[r]=GLe.has(r.toLowerCase())?"[REDACTED]":n;return t}function $Y(e){let t=e.indexOf("@");if(t<=0)return"***";return`${e[0]}***${e.slice(t)}`}function yE(e,t,r=0){if(r>KLe)return"[REDACTED: depth]";if(Array.isArray(t))return t.map((n)=>yE(e,n,r+1));if(t&&typeof t==="object"){let n={};for(let[o,i]of Object.entries(t))n[o]=yE(o,i,r+1);return n}if(typeof t==="string"){if(WLe.test(e))return"[REDACTED]";return t.replace(SY,$Y)}return t}function xE(e){return e.replace(YLe,"Bearer [REDACTED]").replace(JLe,"[REDACTED_AWS_KEY]").replace(XLe,"$1[REDACTED]").replace(ZLe,"$1$2=[REDACTED]").replace(SY,$Y)}function QLe(e){return xE(e)}function gY(e,t=VLe){if(e===void 0)return;let r=e;try{let n=JSON.parse(e);r=JSON.stringify(yE("",n))}catch{}if(r=xE(r),Buffer.byteLength(r,"utf8")>t)r=`${Buffer.from(r,"utf8").subarray(0,t).toString("utf8")}... [truncated]`;return r}function I0(e){if(!Fl)return;let t=QLe(e.url);try{let r={...e,url:t,requestHeaders:e.requestHeaders?HLe(e.requestHeaders):void 0,requestBody:gY(e.requestBody),responseBody:gY(e.responseBody)};P0.push(r),$r(`${e.method} ${t} -> ${e.status??"ERROR"} (${e.durationMs}ms)`)}catch(r){let n=N(r);P0.push({method:e.method,url:t,status:e.status,durationMs:e.durationMs,note:`redaction failed: ${n}`}),$r(`redaction failed for ${e.method} ${t}: ${n}`)}}async function eBe(){let e=[];try{let t=process.versions.bun?`Bun ${process.versions.bun}`:`Node ${process.version}`;e.push(`CLI version: ${Si}`),e.push(`OS/Arch: ${process.platform}/${process.arch}`),e.push(`Runtime: ${t}`)}catch(t){e.push(`(could not read CLI/OS/runtime info: ${N(t)})`)}e.push(""),e.push("Active account:");try{let t=ot();if(t.apiKey){if(t.username)e.push(` Username: ${t.username}`);if(e.push(` API URL: ${t.apiUrl}`),t.role)e.push(` Role: ${t.role}`);e.push(` Sandbox completed: ${!!t.sandboxCompleted}`)}else e.push(" (not authenticated)")}catch(t){e.push(` (could not read account config: ${N(t)})`)}e.push(""),e.push("External tools:");try{let t=await Nf();for(let r of t){let n=r.available?r.version??"installed":r.timedOut?"(timed out)":"not installed";e.push(` ${r.name}: ${n}`)}}catch(t){e.push(` (could not probe external tools: ${N(t)})`)}return e}async function _E(){return(await eBe()).join(`
|
|
104
104
|
`)}async function CY(){if(!Fl||F0!==void 0)return;try{F0=await _E()}catch(e){let t=N(e);F0=`(could not build environment report: ${t})`,$r(`environment snapshot failed: ${t}`)}}function kY(e){if(yY.has(e))return{flag:e};for(let t of yY){if(!e.startsWith(t)||e.length<=t.length)continue;if(t.length===2)return{flag:t,kind:"short-attached"};if(e[t.length]==="=")return{flag:t,kind:"long-eq"}}return null}function tBe(e){let t=[];for(let r=0;r<e.length;r++){let n=e[r],o=kY(n);if(!o){t.push(n);continue}if(o.kind==="long-eq")t.push(`${o.flag}=[REDACTED]`);else if(o.kind==="short-attached")t.push(`${o.flag}[REDACTED]`);else if(t.push(n),r+1<e.length)t.push("[REDACTED]"),r++}return t}function rBe(e){let t=[];for(let r=0;r<e.length;r++){let n=e[r],o=kY(n);if(!o){t.push(n);continue}if(!o.kind&&r+1<e.length)r++}return t}function nBe(e){let t=[];for(let n of rBe(e)){if(n.startsWith("-"))continue;if(t.push(n),t.length===2)break}return(t.length>0?t.join("-"):"nemar").replace(/[^a-zA-Z0-9_-]/g,"_")}function oBe(e,t){let r=[];if(r.push("NEMAR CLI Debug Log"),r.push(`Generated: ${new Date().toISOString()}`),r.push(`Command: nemar ${xE(tBe(e).join(" "))}`),r.push(""),r.push(F0??"(environment snapshot unavailable)"),r.push(""),r.push("HTTP requests:"),P0.length===0)r.push(" (none)");else P0.forEach((n,o)=>{if(r.push(` [${o+1}] ${n.method} ${n.url} -> ${n.status??"ERROR"} (${n.durationMs}ms)`),n.error)r.push(` Error: ${n.error}`);if(n.note)r.push(` Note: ${n.note}`);if(n.requestHeaders)r.push(` Request headers: ${JSON.stringify(n.requestHeaders)}`);if(n.requestBody)r.push(` Request body: ${n.requestBody}`);if(n.responseBody)r.push(` Response body: ${n.responseBody}`)});return r.push(""),r.push(`Failing step: ${_Y??"(none recorded)"}`),r.push(`Exit code: ${t}`),`${r.join(`
|
|
105
105
|
`)}
|
|
@@ -196,7 +196,7 @@ Exemplar clone + publish complete: ${i} -> ${o}`)),console.log(l.dim(` GitHub:
|
|
|
196
196
|
`)){let n=r.trim().split(/\s+/),o=n[0];if(!o||n.length<2)continue;let i=(a)=>n.find((c)=>c.startsWith(`${a}=`))?.slice(a.length+1),s=i("name");if(!s)continue;t.set(o,{uuid:o,name:s,bucket:i("bucket"),fileprefix:i("fileprefix")})}return t}function AWe(e){if(!e.startsWith("!"))return e;let t=e.slice(1),r=Buffer.from(t,"base64");if(r.toString("base64").replace(/=+$/,"")!==t.replace(/=+$/,""))return e;return r.toString("utf8")}function Dte(e,t){let r=[];for(let o of e.split(`
|
|
197
197
|
`)){let i=/^(\S+?)s?\s+([0-9a-f-]{36}):V\s+([+-])(\S+)$/.exec(o.trim());if(!i)continue;let s=t.get(i[2]);if(!s?.bucket)continue;let a=AWe(i[4]),c=a.indexOf("#");if(c<=0)continue;let u=Number.parseFloat(i[1])||0;r.push({stamp:u,retracted:i[3]==="-",value:a,pin:{bucket:s.bucket,object:a.slice(c+1),version:a.slice(0,c),remoteName:s.name}})}r.sort((o,i)=>o.stamp-i.stamp);let n=new Map;for(let o of r)if(o.retracted)n.delete(o.value);else n.set(o.value,o.pin);return[...n.values()]}async function Ete(e){let t=["aws","s3api","list-object-versions","--bucket",e.bucket,"--prefix",e.prefix,"--output","json","--query","Versions[].[Key,VersionId,Size,ETag]","--page-size","1000"];if(!e.signed)t.push("--no-sign-request");let{stdout:r,stderr:n,exitCode:o,timedOut:i}=await C(t,{unsetEnv:X2,timeout:J2});if(o!==0||i)throw Error(`listing s3://${e.bucket}/${e.prefix} failed: ${i?"timed out":n.trim()||`exit ${o}`}`);return OWe(r)}function OWe(e){let t=new Map,r=e.trim()?JSON.parse(e):[];for(let[n,o,i,s]of r??[]){let a=t.get(n)??[];a.push({object:n,version:o,size:i,etag:(s??"").replace(/"/g,"")}),t.set(n,a)}return t}function FWe(e){if(e.size!==0)return!1;let t=Ste[e.backend];return t!==void 0&&e.hashHex===t}function $te(e){let t=wte(e.key);if(!t)return{key:e.key,size:0,reason:"not a parseable annex key"};let r={key:e.key,size:t.size};if(FWe(t))return{...r,empty:!0};if(t.size===0&&t.hashHex!==null&&Ste[t.backend])return{...r,reason:"declares zero bytes but not the empty file's hash; no content can satisfy it"};let n=e.pins[e.pins.length-1],o=e.upstream?PWe(t.size,e):null;if(n){let i=o&&o.version!==n.version?[o]:void 0;return{...r,source:{...n,origin:"pinned",size:t.size},alternatives:i}}if(!e.upstream)return{...r,reason:"no upstream remote to recover from"};if(o)return{...r,source:o};return{...r,reason:IWe(t.size,e)}}function PWe(e,t){let r=t.upstream;if(!r)return null;let n=t.paths.flatMap((s)=>r.index.get(`${r.prefix}${s}`)?.filter((a)=>a.size===e)??[]),o=new Map(n.map((s)=>[s.etag,s]));if(o.size!==1)return null;let[i]=o.values();return{bucket:r.bucket,object:i.object,version:i.version,etag:i.etag,origin:"version-match",size:e}}function IWe(e,t){let r=t.upstream;if(!r)return"no upstream remote to recover from";let n=t.paths.flatMap((i)=>r.index.get(`${r.prefix}${i}`)?.filter((s)=>s.size===e)??[]),o=new Set(n.map((i)=>i.etag));return o.size>1?`${o.size} distinct upstream objects carry this key's size; refusing to guess`:"no upstream object of this key's size at any of its paths"}function Cte(e){return e==="pinned"}function TWe(e){let t=wte(e.key);if(!t)return{ok:!1,method:"unparseable-key",detail:"unparseable key"};if(e.checksumSha256&&t.backend.startsWith("SHA256")&&t.hashHex){let n=Buffer.from(t.hashHex,"hex").toString("base64");return e.checksumSha256===n?{ok:!0,method:"checksum"}:{ok:!1,method:"checksum",detail:`S3 hashed the object to ${e.checksumSha256}, the key says ${n}`}}let r=e.etag?.replace(/"/g,"");if(r&&!r.includes("-")&&t.backend.startsWith("MD5")&&t.hashHex)return r===t.hashHex?{ok:!0,method:"etag"}:{ok:!1,method:"etag",detail:`the object's MD5 is ${r}, the key says ${t.hashHex}`};if(e.size==null)return{ok:!1,method:"unmeasured",detail:"the bucket did not answer for the copy, so nothing about it was checked"};if(e.size!==t.size)return{ok:!1,method:"size",detail:`the object is ${e.size} bytes, the key says ${t.size}`};if(!Cte(e.origin))return{ok:!1,method:"size-and-pin",detail:"only the size could be checked, and nothing pins this source to this key"};if(e.sourceCrc64&&e.crc64)return e.sourceCrc64===e.crc64?{ok:!0,method:"crc64-of-source"}:{ok:!1,method:"crc64-of-source",detail:`the copy's CRC64 is ${e.crc64}, the source reads ${e.sourceCrc64}`};return{ok:!0,method:"size-and-pin"}}function kte(e){let t=`${e.bucket}/${e.object}`;return e.version?`${t}?versionId=${e.version}`:t}function jWe(e){return NWe.test(e)}async function Wo(e,t){let r=await C(["aws",...e],{env:t,unsetEnv:X2,timeout:J2});for(let n=1;n<=qWe;n++){if(r.exitCode===0||!jWe(r.stderr))return r;let o=MWe*2**(n-1)*(1+Math.random());await new Promise((i)=>setTimeout(i,o)),r=await C(["aws",...e],{env:t,unsetEnv:X2,timeout:J2})}return r}async function LWe(e){if(e.source.size>vte)return{...await UWe(e),multipart:!0};let{stdout:t,stderr:r,exitCode:n,timedOut:o}=await Wo(["s3api","copy-object","--bucket",e.destBucket,"--key",e.destKey,"--copy-source",kte(e.source),"--tagging-directive","REPLACE","--tagging","","--checksum-algorithm","SHA256","--output","json"],e.env);if(n!==0)throw Error(`copy-object failed: ${o?"timed out":r.trim()||`exit ${n}`}`);let i=JSON.parse(t||"{}");return{checksumSha256:i.CopyObjectResult?.ChecksumSHA256??null,etag:i.CopyObjectResult?.ETag?.replace(/"/g,"")??null,multipart:!1}}function BWe(e,t){let r=[];for(let n=0,o=1;n<e;n+=t,o++)r.push({part:o,offset:n,end:Math.min(n+t,e)-1});return r}async function UWe(e){let t=await Wo(["s3api","create-multipart-upload","--bucket",e.destBucket,"--key",e.destKey,"--checksum-algorithm","CRC64NVME","--checksum-type","FULL_OBJECT","--output","json"],e.env);if(t.exitCode!==0)throw Error(`create-multipart-upload failed: ${t.stderr.trim()||`exit ${t.exitCode}`}`);let r=JSON.parse(t.stdout||"{}").UploadId;if(!r)throw Error("create-multipart-upload returned no UploadId");let n=BWe(e.source.size,CWe);try{let o=await gp(n,kWe,async(a)=>{let c=await Wo(["s3api","upload-part-copy","--bucket",e.destBucket,"--key",e.destKey,"--upload-id",r,"--part-number",String(a.part),"--copy-source",kte(e.source),"--copy-source-range",`bytes=${a.offset}-${a.end}`,"--output","json"],e.env);if(c.exitCode!==0)throw Error(`upload-part-copy ${a.part} failed: ${c.stderr.trim()||`exit ${c.exitCode}`}`);let u=JSON.parse(c.stdout||"{}").CopyPartResult?.ETag;if(!u)throw Error(`upload-part-copy ${a.part} returned no ETag`);return{ETag:u,PartNumber:a.part}}),i=await Wo(["s3api","complete-multipart-upload","--bucket",e.destBucket,"--key",e.destKey,"--upload-id",r,"--multipart-upload",JSON.stringify({Parts:o}),"--output","json"],e.env);if(i.exitCode!==0)throw Error(`complete-multipart-upload failed: ${i.stderr.trim()||`exit ${i.exitCode}`}`);return{checksumSha256:null,etag:JSON.parse(i.stdout||"{}").ETag??null}}catch(o){let i=await Wo(["s3api","abort-multipart-upload","--bucket",e.destBucket,"--key",e.destKey,"--upload-id",r],e.env).catch(()=>({exitCode:1,stderr:"abort-multipart-upload could not be invoked"}));if(i.exitCode!==0)console.warn([`[content-recovery] FAILED to abort multipart upload ${r}`,`for ${e.destBucket}/${e.destKey}: ${i.stderr.trim().slice(0,300)}.`,"It is billable storage invisible to every object listing; clear it with",`aws s3api abort-multipart-upload --bucket ${e.destBucket}`,`--key ${e.destKey} --upload-id ${r}`].join(" "));throw o}}async function bte(e){let{stdout:t,stderr:r,exitCode:n,timedOut:o}=await Wo(["s3api","head-object","--bucket",e.bucket,"--key",e.key,"--checksum-mode","ENABLED","--output","json"],e.env);if(n!==0){let s=o?"timed out":r.trim()||`exit ${n}`;return SWe.test(r)?{object:null,failed:null}:{object:null,failed:s}}let i;try{i=JSON.parse(t||"{}")}catch{return{object:null,failed:"the bucket's answer could not be read as JSON"}}return{object:{size:typeof i.ContentLength==="number"?i.ContentLength:null,etag:i.ETag?.replace(/"/g,"")??null,checksumSha256:i.ChecksumSHA256??null,crc64:i.ChecksumType==="FULL_OBJECT"?i.ChecksumCRC64NVME??null:null},failed:null}}async function Y2(e,t,r=!1){let n=["s3api","head-object","--bucket",e.bucket,"--key",e.object];if(!r)n.push("--no-sign-request");if(e.version)n.push("--version-id",e.version);n.push("--output","json");let{stdout:o,stderr:i,exitCode:s,timedOut:a}=await Wo(n,t);if(s===0)return{readable:!0,refusedByUpstream:!1,size:zWe(o),detail:null};let c=a?"timed out":i.trim()||`exit ${s}`;return{readable:!1,refusedByUpstream:$We.test(i),size:null,detail:c}}function zWe(e){try{let t=JSON.parse(e||"{}");return typeof t.ContentLength==="number"?t.ContentLength:null}catch{return null}}function xte(e,t){if(t.size===null||t.size===e)return null;return`the source object is ${t.size} bytes and the key declares ${e}`}async function VWe(e,t){let r=["s3api","head-object","--no-sign-request","--checksum-mode","ENABLED","--bucket",e.bucket,"--key",e.object,"--output","json"];if(e.version)r.push("--version-id",e.version);let{stdout:n,stderr:o,exitCode:i,timedOut:s}=await Wo(r,t);if(i!==0)return{crc64:null,failed:s?"timed out":o.trim()||`exit ${i}`};let a=JSON.parse(n||"{}");return{crc64:a.ChecksumType==="FULL_OBJECT"?a.ChecksumCRC64NVME??null:null,failed:null}}async function GWe(e){let{stderr:t,exitCode:r,timedOut:n}=await Wo(["s3api","put-object","--bucket",e.bucket,"--key",e.key,"--content-length","0"],e.env);if(r===0)return{written:!0,detail:null};return{written:!1,detail:n?"timed out":t.trim()||`exit ${r}`}}async function _te(e){let{stderr:t,exitCode:r,timedOut:n}=await Wo(["s3api","delete-object","--bucket",e.bucket,"--key",e.key],e.env);if(r===0)return{deleted:!0,detail:null};return{deleted:!1,detail:n?"timed out":t.trim()||`exit ${r}`}}async function Rte(e){let{entry:t}=e;if(t.empty){if(!e.apply)return{key:t.key,size:0,action:"would-recover",origin:"empty"};let d=await GWe({bucket:e.destBucket,key:e.destKey,env:e.env});if(!d.written)return{key:t.key,size:0,action:"failed",origin:"empty",detail:d.detail??"the bucket refused the write"};let f=await bte({bucket:e.destBucket,key:e.destKey,env:e.env});if(f.object===null||f.object.size!==0){let m=f.object===null?f.failed?`written, but the bucket then answered ${f.failed}`:"written, but the bucket then reported no such object":f.object.size===null?"written, but the bucket answered without a length, so nothing was checked":`written, but the bucket read back ${f.object.size} bytes`,p=await _te({bucket:e.destBucket,key:e.destKey,env:e.env});return{key:t.key,size:0,action:"failed",origin:"empty",detail:m,leftInBucket:!p.deleted}}return{key:t.key,size:0,action:"recovered",origin:"empty",verification:"empty"}}if(!t.source)return{key:t.key,size:t.size,action:"unrecoverable",detail:t.reason};let r=t.source.origin,n=(d)=>t.size>vte&&!Cte(d.origin),o="above CopyObject's 5 GB limit; a multipart copy cannot carry the key's SHA-256, and matching the source's CRC64 would only prove we faithfully copied a guess";if(n(t.source)&&(t.alternatives??[]).every(n))return{key:t.key,size:t.size,action:"unrecoverable",origin:r,detail:o};if(!e.apply){let d=[];for(let f of[t.source,...t.alternatives??[]]){if(n(f))continue;let m=await Y2(f,e.env);if(!m.readable)continue;let p=xte(t.size,m);if(p){d.push(p);continue}return{key:t.key,size:t.size,action:"would-recover",origin:f.origin}}if(d.length>0)return{key:t.key,size:t.size,action:"unrecoverable",origin:r,detail:d.join("; ")};return{key:t.key,size:t.size,action:"unrecoverable",origin:r,detail:"every recorded source for this key is unreadable"}}let i=[t.source,...t.alternatives??[]].filter((d)=>!n(d)),s=[],a=[];for(let d of i){let f=await Y2(d,e.env,!0);if(f.readable){let v=xte(t.size,f);if(v){s.push(v),a.push(v);continue}}let m;try{m=await LWe({source:d,destBucket:e.destBucket,destKey:e.destKey,env:e.env})}catch(v){let S=v instanceof Error?v.message:String(v);if(s.push(S),EWe.test(S))continue;return{key:t.key,size:t.size,action:"failed",origin:d.origin,detail:S}}let p=await bte({bucket:e.destBucket,key:e.destKey,env:e.env}),h=m.checksumSha256??p.object?.checksumSha256??null,g=!h&&p.object?.crc64?await VWe(d,e.env):null;if(g?.failed)s.push(`could not read the source's CRC64: ${g.failed}`);let b=TWe({key:t.key,origin:d.origin,checksumSha256:h,etag:m.etag??p.object?.etag??null,size:p.object?.size??null,crc64:p.object?.crc64??null,sourceCrc64:g?.crc64??null});if(b.ok)return{key:t.key,size:t.size,action:"recovered",origin:d.origin,verification:b.method};let x=await _te({bucket:e.destBucket,key:e.destKey,env:e.env});return{key:t.key,size:t.size,action:"failed",detail:[`refused by ${b.method}: ${b.detail}`,x.deleted?"the object was deleted":`the object was LEFT IN THE BUCKET (delete failed: ${x.detail})`].join("; "),origin:d.origin,leftInBucket:!x.deleted}}if(a.length===i.length)return{key:t.key,size:t.size,action:"unrecoverable",origin:r,detail:a.join("; ")};let c=await Y2(t.source,e.env),u=s[s.length-1]??"no candidate source could be copied";if(c.refusedByUpstream)return{key:t.key,size:t.size,action:"unrecoverable",origin:t.source.origin,detail:`upstream will not serve any recorded source for this key (${c.detail}): ${u.slice(0,160)}`};return{key:t.key,size:t.size,action:"failed",origin:t.source.origin,detail:c.readable?u:`the source could not be probed (${c.detail}), so whether upstream serves it is unknown: ${u.slice(0,160)}`}}var EWe,SWe,$We,vte=5368709120,CWe=1073741824,kWe=8,J2=1800000,X2,RWe,Ste,NWe,qWe=4,MWe=750;var Ate=O(()=>{yp();vt();Vo();EWe=/\((?:AccessDenied|NoSuchVersion|NoSuchKey|NotFound|Forbidden|InvalidArgument|403|404)\)/i,SWe=/\((?:404|NoSuchKey|NotFound)\)/i,$We=/\((?:403|404|AccessDenied|Forbidden|NoSuchKey|NotFound|NoSuchVersion|InvalidArgument)\)/i,X2=["AWS_ENDPOINT_URL","AWS_ENDPOINT_URL_S3"],RWe={SHA256:64,SHA256E:64,MD5:32,MD5E:32};Ste={SHA256:"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",SHA256E:"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",MD5:"d41d8cd98f00b204e9800998ecf8427e",MD5E:"d41d8cd98f00b204e9800998ecf8427e"};NWe=/\((?:429|503|SlowDown|Throttling\w*|ThrottledException|RequestTimeout|RequestTimeoutException|InternalError|ServiceUnavailable)\)|Rate exceeded/i});import{rmSync as Q2}from"fs";import{join as HWe}from"path";async function KWe(e){let t=await Jl(e),r=new Map;for(let[n,o]of t){let i=r.get(o)??[];i.push(n),r.set(o,i)}return r}function Fte(e){return/does not exist in/.test(e)}async function YWe(e,t,r){let n=new Map;if(r.length===0)return n;let o=Bun.spawn({cmd:["git","cat-file","--batch"],cwd:e,stdin:new TextEncoder().encode(r.map((u)=>`${t}:${u}
|
|
198
198
|
`).join("")),stdout:"pipe",stderr:"pipe"}),i=Buffer.from(await new Response(o.stdout).arrayBuffer()),s=await new Response(o.stderr).text(),a=await o.exited;if(a!==0)throw Error(`git cat-file --batch failed over ${r.length} record(s): ${s.trim()||`exit ${a}`}`);let c=0;for(let u of r){let d=i.indexOf(10,c);if(d<0)throw Error(`git cat-file --batch returned ${n.size} of ${r.length} record(s); the stream ended early`);let f=i.subarray(c,d).toString("utf8").split(" ");if(f[1]==="missing"){c=d+1;continue}let m=Number(f[2]);if(!Number.isFinite(m)||m<0)throw Error(`git cat-file --batch returned an unparseable header for ${u}`);let p=d+1;n.set(u,i.subarray(p,p+m).toString("utf8")),c=p+m+1}return n}async function JWe(e,t,r="git-annex"){let n=await C(["git","show",`${r}:remote.log`],{cwd:e});if(n.exitCode!==0&&!Fte(n.stderr))throw Error(`could not read ${r}:remote.log: ${n.stderr.trim()||`exit ${n.exitCode}`}`);let o=Z2(n.stdout),i=await C(["git","ls-tree","-r","--name-only",r],{cwd:e});if(i.exitCode!==0)throw Error(`could not list ${r}: ${i.stderr.trim()||`exit ${i.exitCode}`}`);let s=new Set(t),a=[];for(let d of i.stdout.split(`
|
|
199
|
-
`)){if(!d.endsWith(".log.rmet"))continue;let f=d.slice(d.lastIndexOf("/")+1,-9);if(s.has(f))a.push([f,d])}let c=await YWe(e,r,a.map(([,d])=>d)),u=new Map;for(let[d,f]of a){let m=c.get(f);if(!m)continue;let p=Dte(m,o);if(p.length>0)u.set(d,p)}return u}async function XWe(e,t,r=hp,n="git-annex"){let{stdout:o,stderr:i,exitCode:s}=await C(["git","show",`${n}:remote.log`],{cwd:e});if(s!==0&&!Fte(i))throw Error(`could not read ${n}:remote.log to find ${r}'s object prefix: ${i.trim()||`exit ${s}`}`);for(let a of Z2(o).values())if(a.name===r&&a.fileprefix)return a.fileprefix;return`${t}/objects/`}function qy(e,t,r){let n=(d)=>r.filter(d).reduce((f,m)=>f+m.size,0),o=r.filter((d)=>d.action==="recovered").length,i=r.filter((d)=>d.action==="unrecoverable").length,s=r.filter((d)=>d.action==="failed").length,a=r.filter((d)=>d.action==="would-recover").length,c=t.length-r.length,u;if(t.length===0)u="nothing-missing";else if(s>0)u="failed";else if(a>0)u="would-recover";else if(o>0)u=i>0||c>0?"partial":"recovered";else u="unrecoverable";return{datasetId:e,action:u,measured:!0,missing:t.length,missingBytes:t.reduce((d,f)=>d+f.size,0),recovered:o,recoveredBytes:n((d)=>d.action==="recovered"),unrecoverable:i,unrecoverableBytes:n((d)=>d.action==="unrecoverable"),failed:s,keys:r}}async function ZWe(e,t,r){let n={...qy(e,[],[]),measured:!1,action:"failed"};if(!WWe.test(e))return{...n,error:`"${e}" is not a dataset id, and it would become a directory this deletes`};let o=HWe(r.workRoot,e);Q2(o,{recursive:!0,force:!0});let i=r.originUrl??`https://github.com/nemarDatasets/${e}.git`,s=await K2(i,o,{partial:!1,description:"fleet-content-recovery"});if(s)return Q2(o,{recursive:!0,force:!0}),{...n,error:s};try{let a=await KWe(o),c=await t(e),u=[...a.keys()].filter((x)=>!rn(x,c));if(u.length===0)return qy(e,[],[]);let d=await JWe(o,u),f;if(e.startsWith("on")){let x=`ds${e.slice(2)}/`;f={bucket:Ote,prefix:x,index:await Ete({bucket:Ote,prefix:x})}}let m=u.map((x)=>$te({key:x,paths:a.get(x)??[],pins:d.get(x)??[],upstream:f})).sort((x,v)=>x.size-v.size),p=r.limit?m.slice(0,r.limit):m,h=await XWe(o,e,r.remoteName),g=0,b=await gp(p,r.concurrency??8,async(x)=>{let v=await Rte({entry:x,destBucket:"nemar",destKey:`${h}${x.key}`,apply:r.apply===!0,env:r.env});return r.onKey?.(v,++g,p.length),v});return qy(e,m,b)}catch(a){return{...n,error:a instanceof Error?a.message:String(a)}}finally{Q2(o,{recursive:!0,force:!0})}}async function Pte(e,t,r){let n=[...new Set(e)],o=[],i=0;for(let u of n){let d;try{d=await ZWe(u,t,r)}catch(f){d={...qy(u,[],[]),measured:!1,action:"failed",error:f instanceof Error?f.message:String(f)}}o.push(d),r.onDataset?.(d,++i,n.length)}let s={"nothing-missing":0,recovered:0,"would-recover":0,partial:0,unrecoverable:0,failed:0},a=0,c=0;for(let u of o)s[u.action]++,a+=u.recovered,c+=u.recoveredBytes;return{outcomes:o,tally:s,recoveredKeys:a,recoveredBytes:c}}var WWe,Ote="openneuro.org";var Ite=O(()=>{Ate();yp();vt();kn();Vo();WWe=/^[a-z]{2}[0-9]{6}$/});function SYe(e){return e.replace(/\/+$/,"")}function qre(e,t=EYe){return`${SYe(t)}/dataset/${e}`}var jre,F_t,EYe="https://nemar.org";var hS=O(()=>{jre=["IsCitedBy","Cites","IsSupplementTo","IsSupplementedBy","IsContinuedBy","Continues","IsDescribedBy","Describes","HasMetadata","IsMetadataFor","HasVersion","IsVersionOf","IsNewVersionOf","IsPreviousVersionOf","IsPartOf","HasPart","IsReferencedBy","References","IsDocumentedBy","Documents","IsCompiledBy","Compiles","IsVariantFormOf","IsOriginalFormOf","IsIdenticalTo","IsCollectedBy","Collects","IsRequiredBy","Requires","IsObsoletedBy","Obsoletes","IsDerivedFrom","IsSourceOf","IsReviewedBy","Reviews","IsPublishedIn","HasTranslation","IsTranslationOf"],F_t=new Set(jre)});var Lne=()=>{};function Bne(e){return e.replace(/^(doi:|https?:\/\/(dx\.)?doi\.org\/)/i,"").trim()}function Une(e){let t=e.split(",").map((r)=>r.trim());if(t.length>=2)return{name:e,familyName:t[0],givenName:t.slice(1).join(", ")};return{name:e}}var zne=O(()=>{hS();Lne()});function bJe(e){let t=e,r=(t.match(/\(/g)??[]).length,n=(t.match(/\)/g)??[]).length;while(n>r&&t.endsWith(")"))t=t.slice(0,-1),n-=1;return t}function Vne(e){let t=e.trim().replace(/^(?:doi:|https?:\/\/(?:dx\.)?doi\.org\/)/i,""),r;do r=t,t=bJe(t).replace(/[.,;:]+$/,"").trim();while(t!==r);return t.toLowerCase()}var Gne=()=>{};function DJe(e){if(vJe.has(e))return"absent";return e>=200&&e<300?"found":"failed"}function EJe(e){let t=e instanceof Error?`${e.name} ${e.message}`:String(e);return/AbortError|TimeoutError|timeout|fetch|network/i.test(t)}async function SJe(e,t,r){let{base:n,accept:o}=wJe[e],i=AbortSignal.timeout(_Je);try{let s=await fetch(`${n}/${encodeURIComponent(t)}`,{headers:{Accept:o,"User-Agent":xJe},signal:r?AbortSignal.any([i,r]):i}),a=DJe(s.status);if(a==="absent")return{outcome:"absent"};if(a==="failed"){let c=`HTTP ${s.status}`;return console.warn(`[doi-registry] ${e} returned ${c} for ${t}`),{outcome:"failed",detail:c}}return{outcome:"found",body:await s.json()}}catch(s){if(r?.aborted)return{outcome:"failed",detail:"deadline reached"};let a=s instanceof Error?s.message:String(s);if(EJe(s))console.warn(`[doi-registry] ${e} lookup failed for ${t}: ${a}`);else console.error(`[doi-registry] Unexpected error querying ${e} for ${t}:`,s);return{outcome:"failed",detail:a}}}function PS(e,t,r,n){let o=Vne(t),i=`${e}:${o}`,s=r?.get(i);if(s)return s;let a=SJe(e,o,n);if(r){if(r.set(i,a),n)a.then((c)=>{if(c.outcome==="failed"&&n.aborted&&r.get(i)===a)r.delete(i)})}return a}var xJe="NEMAR/1.0 (https://nemar.org; mailto:nemar@ucsd.edu)",_Je=1e4,vJe,wJe;var Hne=O(()=>{Gne();vJe=new Set([400,404,410]),wJe={DataCite:{base:"https://api.datacite.org/application/vnd.datacite.datacite+json",accept:"application/vnd.datacite.datacite+json"},Crossref:{base:"https://api.crossref.org/works",accept:"application/json"}}});function kJe(e){let t=[],r=new Set,n=(o,i)=>{let s=Bne(o).trim();if($Je.test(s)&&!r.has(s))r.add(s),t.push({doi:s,source:i})};if(Array.isArray(e.SourceDatasets))for(let o of e.SourceDatasets){if(!o||typeof o!=="object")continue;let i=o;if(typeof i.DOI==="string")n(i.DOI,"SourceDatasets");else if(typeof i.URL==="string"){let s=i.URL.match(/(?:doi\.org\/|doi:)(10\.\d{4,}\/.+)/);if(s)n(s[1],"SourceDatasets")}}if(Array.isArray(e.ReferencesAndLinks))for(let o of e.ReferencesAndLinks){if(typeof o!=="string"||!o)continue;let i=o.match(/(?:doi\.org\/|^)(10\.\d{4,}\/[^\s]+)/);if(i)n(i[1],"ReferencesAndLinks")}if(typeof e.HowToAcknowledge==="string"){let i=e.HowToAcknowledge.matchAll(/(?:doi\.org\/|(?<=\s|^))(10\.\d{4,}\/[^\s,)]+)/g);for(let s of i)n(s[1].replace(/[.;:]+$/,""),"HowToAcknowledge")}return t}async function RJe(e,t){let r=await PS("DataCite",e,t);if(r.outcome!=="found")return null;let n=r.body;return{doi:e,creators:Array.isArray(n?.creators)?n.creators.map((o)=>({...o})):[]}}async function AJe(e,t){let r=await PS("Crossref",e,t);if(r.outcome!=="found")return null;let o=r.body?.message?.author;if(!Array.isArray(o)||o.length===0)return null;let i=o.map((s)=>{let a=[];if(s.ORCID)a.push({nameIdentifier:s.ORCID,nameIdentifierScheme:"ORCID"});return{name:s.family&&s.given?`${s.family}, ${s.given}`:s.name||s.family||"",givenName:s.given,familyName:s.family,nameIdentifiers:a,affiliation:(s.affiliation||[]).filter((c)=>c.name)}}).filter((s)=>s.name.trim()!=="");return{doi:e,creators:i}}async function OJe(e,t){let r=await RJe(e,t),n=await AJe(e,t);if(!r)return n;if(!n)return r;for(let o of r.creators){if(o.nameIdentifiers?.some((c)=>c.nameIdentifierScheme==="ORCID")||!o.familyName)continue;let a=n.creators.find((c)=>c.familyName&&An(c.familyName)===An(o.familyName))?.nameIdentifiers?.find((c)=>c.nameIdentifierScheme==="ORCID");if(a)o.nameIdentifiers=[...o.nameIdentifiers??[],a]}return r}function An(e){return e.normalize("NFD").replace(/\p{Diacritic}/gu,"").replace(/\./g,"").toLowerCase().trim()}function Wne(e){if(!e)return"";return An(e).charAt(0)}function FJe(e,t){let r=[],n=new Set,o=new Set,i=t.map((s)=>({original:s,parsed:Une(s)}));for(let s=0;s<e.length;s++){if(o.has(s))continue;let a=e[s],c=An(a.name);for(let u of i){if(n.has(u.original))continue;if(An(u.original)===c){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"exact"}),n.add(u.original),o.add(s);break}if(u.parsed.familyName&&u.parsed.givenName){if(An(`${u.parsed.givenName} ${u.parsed.familyName}`)===c){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"exact"}),n.add(u.original),o.add(s);break}}if(a.givenName&&a.familyName){let d=An(`${a.givenName} ${a.familyName}`);if(An(u.original)===d){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"exact"}),n.add(u.original),o.add(s);break}}}}for(let s=0;s<e.length;s++){if(o.has(s))continue;let a=e[s];if(!a.familyName)continue;let c=An(a.familyName),u=Wne(a.givenName);for(let d of i){if(n.has(d.original))continue;if(!d.parsed.familyName)continue;if(An(d.parsed.familyName)!==c)continue;let m=Wne(d.parsed.givenName);if(m&&u&&m===u){r.push({bidsAuthor:d.original,matchedCreator:a,confidence:"high"}),n.add(d.original),o.add(s);break}}}for(let s=0;s<e.length;s++){if(o.has(s))continue;let a=e[s];if(!a.familyName)continue;let c=An(a.familyName);for(let u of i){if(n.has(u.original))continue;if(!u.parsed.familyName)continue;if(An(u.parsed.familyName)===c){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"medium"}),n.add(u.original),o.add(s);break}}}return r}async function Yne(e,t,r,n){let o=kJe(e);if(r?.length)o.push(...r);if(o.length===0)return{discoveries:{},unresolvedDois:[],totalDoisQueried:0};let i=Array.isArray(e.Authors)?e.Authors.filter((f)=>typeof f==="string"):[];if(i.length===0)return{discoveries:{},unresolvedDois:[],totalDoisQueried:o.length};let s=new Set;if(t){for(let[f,m]of Object.entries(t))if(m.orcid)s.add(f)}let a=i.filter((f)=>!s.has(f));if(a.length===0)return{discoveries:{},unresolvedDois:[],totalDoisQueried:o.length};let c=[],u=new Map;for(let f=0;f<o.length;f+=Kne){let m=o.slice(f,f+Kne),p=await Promise.all(m.map((h)=>OJe(h.doi,n)));for(let h=0;h<m.length;h++)if(p[h])u.set(m[h].doi,p[h]?.creators??[]);else c.push(m[h].doi)}let d={};for(let[f,m]of u){let p=m.filter((g)=>g.nameIdentifiers?.some((b)=>b.nameIdentifierScheme==="ORCID"));if(p.length===0)continue;let h=FJe(p,a);for(let g of h){if(d[g.bidsAuthor])continue;let b=g.matchedCreator.nameIdentifiers.find((S)=>S.nameIdentifierScheme==="ORCID");if(!b)continue;let x=b.nameIdentifier.replace(/^https?:\/\/orcid\.org\//i,"").trim();if(!CJe.test(x))continue;let v=g.matchedCreator.affiliation?.filter((S)=>S.name).map((S)=>({name:S.name,...S.affiliationIdentifier&&{identifier:S.affiliationIdentifier},...S.affiliationIdentifierScheme&&{scheme:S.affiliationIdentifierScheme}}));d[g.bidsAuthor]={orcid:x,affiliations:v?.length?v:void 0,sourceDoi:f,confidence:g.confidence}}}return{discoveries:d,unresolvedDois:c,totalDoisQueried:o.length}}var $Je,CJe,Kne=5;var Jne=O(()=>{zne();Hne();$Je=/^10\.\d{4,}\/[^\s]+$/,CJe=/^\d{4}-\d{4}-\d{4}-[\dX]{4}$/});Xe();var Ob={};HS(Ob,{program:()=>x$,createCommand:()=>_$,createOption:()=>v$,createArgument:()=>w$,Command:()=>D$,Option:()=>E$,Argument:()=>S$,Help:()=>$$,CommanderError:()=>C$,InvalidArgumentError:()=>k$,InvalidOptionArgumentError:()=>R$},{program:(ZXe)=>x$=ZXe,createCommand:(ZXe)=>_$=ZXe,createOption:(ZXe)=>v$=ZXe,createArgument:(ZXe)=>w$=ZXe,Command:(ZXe)=>D$=ZXe,Option:(ZXe)=>E$=ZXe,Argument:(ZXe)=>S$=ZXe,Help:(ZXe)=>$$=ZXe,CommanderError:(ZXe)=>C$=ZXe,InvalidArgumentError:(ZXe)=>k$=ZXe,InvalidOptionArgumentError:(ZXe)=>R$=ZXe});var{Argument:g$}=Ip(),{Command:Ab}=h$(),{CommanderError:Kie,InvalidArgumentError:y$}=cu(),{Help:Yie}=Eb(),{Option:b$}=Sb();var x$=new Ab,_$=(e)=>new Ab(e),v$=(e,t)=>new b$(e,t),w$=(e,t)=>new g$(e,t),D$=Ab,E$=b$,S$=g$,$$=Yie,C$=Kie,k$=y$,R$=y$;var{program:eZe,createCommand:tZe,createArgument:rZe,createOption:nZe,CommanderError:oZe,InvalidArgumentError:Fb,InvalidOptionArgumentError:iZe,Command:ue,Argument:sZe,Option:lu,Help:aZe}=Ob;var Pb={dataset:["search","download","upload","validate","status","list"],admin:["users","approve","revoke","publish","doi","make-public","delete-dataset","notify","doctor"]};var Ib=process.argv.includes("--help-all");if(process.argv.includes("--no-color"))l.level=0;var A$=process.argv.indexOf("--help-all");if(A$!==-1)process.argv[A$]="--help";var uu=ue.prototype.addHelpText,O$=`
|
|
199
|
+
`)){if(!d.endsWith(".log.rmet"))continue;let f=d.slice(d.lastIndexOf("/")+1,-9);if(s.has(f))a.push([f,d])}let c=await YWe(e,r,a.map(([,d])=>d)),u=new Map;for(let[d,f]of a){let m=c.get(f);if(!m)continue;let p=Dte(m,o);if(p.length>0)u.set(d,p)}return u}async function XWe(e,t,r=hp,n="git-annex"){let{stdout:o,stderr:i,exitCode:s}=await C(["git","show",`${n}:remote.log`],{cwd:e});if(s!==0&&!Fte(i))throw Error(`could not read ${n}:remote.log to find ${r}'s object prefix: ${i.trim()||`exit ${s}`}`);for(let a of Z2(o).values())if(a.name===r&&a.fileprefix)return a.fileprefix;return`${t}/objects/`}function qy(e,t,r){let n=(d)=>r.filter(d).reduce((f,m)=>f+m.size,0),o=r.filter((d)=>d.action==="recovered").length,i=r.filter((d)=>d.action==="unrecoverable").length,s=r.filter((d)=>d.action==="failed").length,a=r.filter((d)=>d.action==="would-recover").length,c=t.length-r.length,u;if(t.length===0)u="nothing-missing";else if(s>0)u="failed";else if(a>0)u="would-recover";else if(o>0)u=i>0||c>0?"partial":"recovered";else u="unrecoverable";return{datasetId:e,action:u,measured:!0,missing:t.length,missingBytes:t.reduce((d,f)=>d+f.size,0),recovered:o,recoveredBytes:n((d)=>d.action==="recovered"),unrecoverable:i,unrecoverableBytes:n((d)=>d.action==="unrecoverable"),failed:s,keys:r}}async function ZWe(e,t,r){let n={...qy(e,[],[]),measured:!1,action:"failed"};if(!WWe.test(e))return{...n,error:`"${e}" is not a dataset id, and it would become a directory this deletes`};let o=HWe(r.workRoot,e);Q2(o,{recursive:!0,force:!0});let i=r.originUrl??`https://github.com/nemarDatasets/${e}.git`,s=await K2(i,o,{partial:!1,description:"fleet-content-recovery"});if(s)return Q2(o,{recursive:!0,force:!0}),{...n,error:s};try{let a=await KWe(o),c=await t(e),u=[...a.keys()].filter((x)=>!rn(x,c));if(u.length===0)return qy(e,[],[]);let d=await JWe(o,u),f;if(e.startsWith("on")){let x=`ds${e.slice(2)}/`;f={bucket:Ote,prefix:x,index:await Ete({bucket:Ote,prefix:x})}}let m=u.map((x)=>$te({key:x,paths:a.get(x)??[],pins:d.get(x)??[],upstream:f})).sort((x,v)=>x.size-v.size),p=r.limit?m.slice(0,r.limit):m,h=await XWe(o,e,r.remoteName),g=0,b=await gp(p,r.concurrency??8,async(x)=>{let v=await Rte({entry:x,destBucket:"nemar",destKey:`${h}${x.key}`,apply:r.apply===!0,env:r.env});return r.onKey?.(v,++g,p.length),v});return qy(e,m,b)}catch(a){return{...n,error:a instanceof Error?a.message:String(a)}}finally{Q2(o,{recursive:!0,force:!0})}}async function Pte(e,t,r){let n=[...new Set(e)],o=[],i=0;for(let u of n){let d;try{d=await ZWe(u,t,r)}catch(f){d={...qy(u,[],[]),measured:!1,action:"failed",error:f instanceof Error?f.message:String(f)}}o.push(d),r.onDataset?.(d,++i,n.length)}let s={"nothing-missing":0,recovered:0,"would-recover":0,partial:0,unrecoverable:0,failed:0},a=0,c=0;for(let u of o)s[u.action]++,a+=u.recovered,c+=u.recoveredBytes;return{outcomes:o,tally:s,recoveredKeys:a,recoveredBytes:c}}var WWe,Ote="openneuro.org";var Ite=O(()=>{Ate();yp();vt();kn();Vo();WWe=/^[a-z]{2}[0-9]{6}$/});function SYe(e){return e.replace(/\/+$/,"")}function qre(e,t=EYe){return`${SYe(t)}/dataset/${e}`}var jre,F_t,EYe="https://nemar.org";var hS=O(()=>{jre=["IsCitedBy","Cites","IsSupplementTo","IsSupplementedBy","IsContinuedBy","Continues","IsDescribedBy","Describes","HasMetadata","IsMetadataFor","HasVersion","IsVersionOf","IsNewVersionOf","IsPreviousVersionOf","IsPartOf","HasPart","IsReferencedBy","References","IsDocumentedBy","Documents","IsCompiledBy","Compiles","IsVariantFormOf","IsOriginalFormOf","IsIdenticalTo","IsCollectedBy","Collects","IsRequiredBy","Requires","IsObsoletedBy","Obsoletes","IsDerivedFrom","IsSourceOf","IsReviewedBy","Reviews","IsPublishedIn","HasTranslation","IsTranslationOf"],F_t=new Set(jre)});var Lne=()=>{};function Bne(e){return e.replace(/^(doi:|https?:\/\/(dx\.)?doi\.org\/)/i,"").trim()}function Une(e){let t=e.split(",").map((r)=>r.trim());if(t.length>=2)return{name:e,familyName:t[0],givenName:t.slice(1).join(", ")};return{name:e}}var zne=O(()=>{hS();Lne()});function bJe(e){let t=e,r=(t.match(/\(/g)??[]).length,n=(t.match(/\)/g)??[]).length;while(n>r&&t.endsWith(")"))t=t.slice(0,-1),n-=1;return t}function Vne(e){let t=e.trim().replace(/^(?:doi:|https?:\/\/(?:dx\.)?doi\.org\/)/i,""),r;do r=t,t=bJe(t).replace(/[.,;:]+$/,"").trim();while(t!==r);return t.toLowerCase()}var Gne=()=>{};function DJe(e){if(vJe.has(e))return"absent";return e>=200&&e<300?"found":"failed"}function EJe(e){let t=e instanceof Error?`${e.name} ${e.message}`:String(e);return/AbortError|TimeoutError|timeout|fetch|network/i.test(t)}async function SJe(e,t,r){let{base:n,accept:o}=wJe[e],i=AbortSignal.timeout(_Je);try{let s=await fetch(`${n}/${encodeURIComponent(t)}`,{headers:{Accept:o,"User-Agent":xJe},signal:r?AbortSignal.any([i,r]):i}),a=DJe(s.status);if(a==="absent")return{outcome:"absent"};if(a==="failed"){let c=`HTTP ${s.status}`;return console.warn(`[doi-registry] ${e} returned ${c} for ${t}`),{outcome:"failed",detail:c}}return{outcome:"found",body:await s.json()}}catch(s){if(r?.aborted)return{outcome:"failed",detail:"deadline reached"};let a=s instanceof Error?s.message:String(s);if(EJe(s))console.warn(`[doi-registry] ${e} lookup failed for ${t}: ${a}`);else console.error(`[doi-registry] Unexpected error querying ${e} for ${t}:`,s);return{outcome:"failed",detail:a}}}function PS(e,t,r,n){let o=Vne(t),i=`${e}:${o}`,s=r?.get(i);if(s)return s;if(n?.aborted)return Promise.resolve({outcome:"failed",detail:"deadline reached"});let a=SJe(e,o,n);if(r){if(r.set(i,a),n)a.then((c)=>{if(c.outcome==="failed"&&n.aborted&&r.get(i)===a)r.delete(i)})}return a}var xJe="NEMAR/1.0 (https://nemar.org; mailto:nemar@ucsd.edu)",_Je=1e4,vJe,wJe;var Hne=O(()=>{Gne();vJe=new Set([400,404,410]),wJe={DataCite:{base:"https://api.datacite.org/application/vnd.datacite.datacite+json",accept:"application/vnd.datacite.datacite+json"},Crossref:{base:"https://api.crossref.org/works",accept:"application/json"}}});function kJe(e){let t=[],r=new Set,n=(o,i)=>{let s=Bne(o).trim();if($Je.test(s)&&!r.has(s))r.add(s),t.push({doi:s,source:i})};if(Array.isArray(e.SourceDatasets))for(let o of e.SourceDatasets){if(!o||typeof o!=="object")continue;let i=o;if(typeof i.DOI==="string")n(i.DOI,"SourceDatasets");else if(typeof i.URL==="string"){let s=i.URL.match(/(?:doi\.org\/|doi:)(10\.\d{4,}\/.+)/);if(s)n(s[1],"SourceDatasets")}}if(Array.isArray(e.ReferencesAndLinks))for(let o of e.ReferencesAndLinks){if(typeof o!=="string"||!o)continue;let i=o.match(/(?:doi\.org\/|^)(10\.\d{4,}\/[^\s]+)/);if(i)n(i[1],"ReferencesAndLinks")}if(typeof e.HowToAcknowledge==="string"){let i=e.HowToAcknowledge.matchAll(/(?:doi\.org\/|(?<=\s|^))(10\.\d{4,}\/[^\s,)]+)/g);for(let s of i)n(s[1].replace(/[.;:]+$/,""),"HowToAcknowledge")}return t}async function RJe(e,t){let r=await PS("DataCite",e,t);if(r.outcome!=="found")return null;let n=r.body;return{doi:e,creators:Array.isArray(n?.creators)?n.creators.map((o)=>({...o})):[]}}async function AJe(e,t){let r=await PS("Crossref",e,t);if(r.outcome!=="found")return null;let o=r.body?.message?.author;if(!Array.isArray(o)||o.length===0)return null;let i=o.map((s)=>{let a=[];if(s.ORCID)a.push({nameIdentifier:s.ORCID,nameIdentifierScheme:"ORCID"});return{name:s.family&&s.given?`${s.family}, ${s.given}`:s.name||s.family||"",givenName:s.given,familyName:s.family,nameIdentifiers:a,affiliation:(s.affiliation||[]).filter((c)=>c.name)}}).filter((s)=>s.name.trim()!=="");return{doi:e,creators:i}}async function OJe(e,t){let r=await RJe(e,t),n=await AJe(e,t);if(!r)return n;if(!n)return r;for(let o of r.creators){if(o.nameIdentifiers?.some((c)=>c.nameIdentifierScheme==="ORCID")||!o.familyName)continue;let a=n.creators.find((c)=>c.familyName&&An(c.familyName)===An(o.familyName))?.nameIdentifiers?.find((c)=>c.nameIdentifierScheme==="ORCID");if(a)o.nameIdentifiers=[...o.nameIdentifiers??[],a]}return r}function An(e){return e.normalize("NFD").replace(/\p{Diacritic}/gu,"").replace(/\./g,"").toLowerCase().trim()}function Wne(e){if(!e)return"";return An(e).charAt(0)}function FJe(e,t){let r=[],n=new Set,o=new Set,i=t.map((s)=>({original:s,parsed:Une(s)}));for(let s=0;s<e.length;s++){if(o.has(s))continue;let a=e[s],c=An(a.name);for(let u of i){if(n.has(u.original))continue;if(An(u.original)===c){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"exact"}),n.add(u.original),o.add(s);break}if(u.parsed.familyName&&u.parsed.givenName){if(An(`${u.parsed.givenName} ${u.parsed.familyName}`)===c){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"exact"}),n.add(u.original),o.add(s);break}}if(a.givenName&&a.familyName){let d=An(`${a.givenName} ${a.familyName}`);if(An(u.original)===d){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"exact"}),n.add(u.original),o.add(s);break}}}}for(let s=0;s<e.length;s++){if(o.has(s))continue;let a=e[s];if(!a.familyName)continue;let c=An(a.familyName),u=Wne(a.givenName);for(let d of i){if(n.has(d.original))continue;if(!d.parsed.familyName)continue;if(An(d.parsed.familyName)!==c)continue;let m=Wne(d.parsed.givenName);if(m&&u&&m===u){r.push({bidsAuthor:d.original,matchedCreator:a,confidence:"high"}),n.add(d.original),o.add(s);break}}}for(let s=0;s<e.length;s++){if(o.has(s))continue;let a=e[s];if(!a.familyName)continue;let c=An(a.familyName);for(let u of i){if(n.has(u.original))continue;if(!u.parsed.familyName)continue;if(An(u.parsed.familyName)===c){r.push({bidsAuthor:u.original,matchedCreator:a,confidence:"medium"}),n.add(u.original),o.add(s);break}}}return r}async function Yne(e,t,r,n){let o=kJe(e);if(r?.length)o.push(...r);if(o.length===0)return{discoveries:{},unresolvedDois:[],totalDoisQueried:0};let i=Array.isArray(e.Authors)?e.Authors.filter((f)=>typeof f==="string"):[];if(i.length===0)return{discoveries:{},unresolvedDois:[],totalDoisQueried:o.length};let s=new Set;if(t){for(let[f,m]of Object.entries(t))if(m.orcid)s.add(f)}let a=i.filter((f)=>!s.has(f));if(a.length===0)return{discoveries:{},unresolvedDois:[],totalDoisQueried:o.length};let c=[],u=new Map;for(let f=0;f<o.length;f+=Kne){let m=o.slice(f,f+Kne),p=await Promise.all(m.map((h)=>OJe(h.doi,n)));for(let h=0;h<m.length;h++)if(p[h])u.set(m[h].doi,p[h]?.creators??[]);else c.push(m[h].doi)}let d={};for(let[f,m]of u){let p=m.filter((g)=>g.nameIdentifiers?.some((b)=>b.nameIdentifierScheme==="ORCID"));if(p.length===0)continue;let h=FJe(p,a);for(let g of h){if(d[g.bidsAuthor])continue;let b=g.matchedCreator.nameIdentifiers.find((S)=>S.nameIdentifierScheme==="ORCID");if(!b)continue;let x=b.nameIdentifier.replace(/^https?:\/\/orcid\.org\//i,"").trim();if(!CJe.test(x))continue;let v=g.matchedCreator.affiliation?.filter((S)=>S.name).map((S)=>({name:S.name,...S.affiliationIdentifier&&{identifier:S.affiliationIdentifier},...S.affiliationIdentifierScheme&&{scheme:S.affiliationIdentifierScheme}}));d[g.bidsAuthor]={orcid:x,affiliations:v?.length?v:void 0,sourceDoi:f,confidence:g.confidence}}}return{discoveries:d,unresolvedDois:c,totalDoisQueried:o.length}}var $Je,CJe,Kne=5;var Jne=O(()=>{zne();Hne();$Je=/^10\.\d{4,}\/[^\s]+$/,CJe=/^\d{4}-\d{4}-\d{4}-[\dX]{4}$/});Xe();var Ob={};HS(Ob,{program:()=>x$,createCommand:()=>_$,createOption:()=>v$,createArgument:()=>w$,Command:()=>D$,Option:()=>E$,Argument:()=>S$,Help:()=>$$,CommanderError:()=>C$,InvalidArgumentError:()=>k$,InvalidOptionArgumentError:()=>R$},{program:(ZXe)=>x$=ZXe,createCommand:(ZXe)=>_$=ZXe,createOption:(ZXe)=>v$=ZXe,createArgument:(ZXe)=>w$=ZXe,Command:(ZXe)=>D$=ZXe,Option:(ZXe)=>E$=ZXe,Argument:(ZXe)=>S$=ZXe,Help:(ZXe)=>$$=ZXe,CommanderError:(ZXe)=>C$=ZXe,InvalidArgumentError:(ZXe)=>k$=ZXe,InvalidOptionArgumentError:(ZXe)=>R$=ZXe});var{Argument:g$}=Ip(),{Command:Ab}=h$(),{CommanderError:Kie,InvalidArgumentError:y$}=cu(),{Help:Yie}=Eb(),{Option:b$}=Sb();var x$=new Ab,_$=(e)=>new Ab(e),v$=(e,t)=>new b$(e,t),w$=(e,t)=>new g$(e,t),D$=Ab,E$=b$,S$=g$,$$=Yie,C$=Kie,k$=y$,R$=y$;var{program:eZe,createCommand:tZe,createArgument:rZe,createOption:nZe,CommanderError:oZe,InvalidArgumentError:Fb,InvalidOptionArgumentError:iZe,Command:ue,Argument:sZe,Option:lu,Help:aZe}=Ob;var Pb={dataset:["search","download","upload","validate","status","list"],admin:["users","approve","revoke","publish","doi","make-public","delete-dataset","notify","doctor"]};var Ib=process.argv.includes("--help-all");if(process.argv.includes("--no-color"))l.level=0;var A$=process.argv.indexOf("--help-all");if(A$!==-1)process.argv[A$]="--help";var uu=ue.prototype.addHelpText,O$=`
|
|
200
200
|
Run with --help-all for examples and detailed descriptions.
|
|
201
201
|
`,Tp=new WeakSet;ue.prototype.addHelpText=function(e,t){if(e==="after"||e==="afterAll"){if(Ib)return uu.call(this,e,t);if(!Tp.has(this))return Tp.add(this),uu.call(this,e,O$);return this}return uu.call(this,e,t)};function Jie(e){let t=[];for(let r=e;r?.parent;r=r.parent)t.unshift(r.name());return t.join(" ")}function Ft(e){let t=[];for(let r=e;r;r=r.parent)t.unshift(r.name());return t.join(" ")}var Xie={formatHelp(e,t){let r=t.helpWidth??80,n=2,o=2,i=40,s=Math.max(8,r-2-2-40),a=(A)=>Math.min(s,A.reduce((q,T)=>Math.max(q,T.length),0));function c(A,q,T,B){if(!T)return q;let Q=B+2;if(A.length>B){let I=t.wrap(T,r-2-4,0);return`${q}
|
|
202
202
|
${I.replace(/^/gm," ")}`}let z=" ".repeat(Math.max(0,Q-A.length)),ee=`${A}${z}${T}`,ie=t.wrap(ee,r-2,Q);return q+ie.slice(A.length)}function u(A){return A.join(`
|
|
@@ -568,7 +568,7 @@ ${s}/${i.length} data_complete`))});pe.addCommand(nS);var oKe=/^xx0999\d{2}$/,iK
|
|
|
568
568
|
${u} of ${c.length} fleet entries failed`)),process.exit(1);return}if(!e)console.error(l.red("An xx-id argument is required unless --all is passed")),process.exit(1);if(!oKe.test(e))console.error(l.red(`Invalid exemplar id "${e}". Expected xx099900-xx099999.`)),process.exit(1);let s;try{s=pp(o)}catch(c){console.error(l.red(`Failed to load fleet file: ${N(c)}`)),process.exit(1)}let a=t.source??s.find((c)=>c.xx_id===e)?.source_id;if(!a)console.error(l.red(`No source dataset for ${e}. Pass --source <nm/on id> or add it to the fleet file.`)),process.exit(1);if(!iKe.test(a))console.error(l.red(`Invalid source id "${a}". Expected an nm/on dataset id.`)),process.exit(1);try{await z2({xxId:e,sourceId:a,...i})}catch(c){console.error(l.red(`
|
|
569
569
|
Exemplar clone failed: ${N(c)}`)),process.exit(1)}});Gy.command("status").description("Show which fleet entries have been cloned and their current state").option("--fleet-file <path>","Path to the fleet JSON (default: scripts/exemplar-fleet.json)").action(async(e)=>{if(!Z())return;await Promise.resolve().then(() => V2());let r=e.fleetFile||Wte(),n;try{n=pp(r)}catch(o){console.error(l.red(`Failed to load fleet file: ${N(o)}`)),process.exit(1)}console.log(l.bold(`
|
|
570
570
|
Exemplar Fleet (${n.length})
|
|
571
|
-
`));for(let o of n)try{let i=await Tt(o.xx_id);console.log(` ${o.xx_id.padEnd(10)} ${o.modality.padEnd(6)} <- ${o.source_id.padEnd(10)} ${l.green(i.visibility)} doi=${i.concept_doi??"none"}`)}catch(i){if(i instanceof L&&i.statusCode===404)console.log(` ${o.xx_id.padEnd(10)} ${o.modality.padEnd(6)} <- ${o.source_id.padEnd(10)} ${l.dim("not created")}`);else console.log(` ${o.xx_id.padEnd(10)} ${o.modality.padEnd(6)} <- ${o.source_id.padEnd(10)} ${l.red(`status check failed (${N(i)})`)}`)}});Gy.command("remint-dois").description("Re-mint an exemplar's sandbox concept DOI (idempotent)").argument("<xx-id>","Exemplar dataset id (e.g., xx099900)").action(async(e)=>{if(!Z())return;let t=D(`Re-minting DOIs for ${e}...`).start();try{let r=await sJ(e);t.succeed(`Re-minted ${r.concept_doi} (status: ${r.status})`);for(let n of r.warnings??[])console.log(l.yellow(` ${n}`))}catch(r){Qe(r,t,"Failed to re-mint exemplar DOIs",{400:"Not an exemplar dataset",403:"Exemplar DOI re-mint is disabled in production",404:"Dataset not found"}),process.exit(1)}});pe.addCommand(Gy);function sKe(e,t){let r=e.enrichment.status,n=r==="ok"?l.green("enrich:ok"):r==="failed"?l.red("enrich:failed"):l.dim("enrich:skipped"),o=e.sync.status,i=o==="ok"?l.green("sync:ok"):o==="failed"?l.red("sync:failed"):l.dim("sync:skipped"),s=e.sync.metadata_columns_written===!0?l.green("cols:written"):e.sync.metadata_columns_error?l.red("cols:failed"):"",a=t?.showRef&&e.enrichment.ref?l.dim(`@${e.enrichment.ref}`):"",c=[` ${e.dataset_id.padEnd(12)} ${n}${a} ${i} ${s}`];if(e.enrichment.error)c.push(l.red(` enrichment: ${e.enrichment.error}`));for(let d of e.enrichment.warnings??[])c.push(l.yellow(` warning: ${d}`));let u=e.enrichment.doi_resolution;if(u){if(c.push(l.dim(` DOI lookups: ${u.resolved} resolved, ${u.unresolved} unresolved, ${u.failed} failed`)),u.skipped>0)c.push(l.yellow(` ${u.skipped} DOI(s) past the per-run lookup cap were not looked up; their relation labels were chosen without registry metadata`))}if(e.sync.metadata_columns_error)c.push(l.red(` metadata columns: ${e.sync.metadata_columns_error}`));return c}function jte(e,t){for(let r of sKe(e,t))console.log(r)}var Ba=new ue("fleet").description("Governance drift reporting and enforcement (epic #713)");Ba.command("drift").description("Report dataset repos that are off the governance spec").option("--prefix <prefix>","Filter datasets by id prefix (e.g. nm, on)").option("--visibility <vis>","Filter by visibility: public or private").option("--limit <n>","Max repos to scan (default 25, max 50)","25").option("--json","Output raw JSON").action(async(e)=>{if(!je())console.log(l.red("Error: Not authenticated")),process.exit(1);if(e.visibility&&!["public","private"].includes(e.visibility))console.log(l.red("Error: --visibility must be public or private")),process.exit(1);let t=D("Scanning fleet for drift...").start();try{let r=await wJ({prefix:e.prefix,visibility:e.visibility,limit:Number.parseInt(e.limit,10)||25});if(t.stop(),e.json){console.log(JSON.stringify(r,null,2));return}console.log(),console.log(l.bold(`Fleet drift (scanned ${r.scanned}):`));let n=["PUBLIC_UNPROTECTED","RED_REQUIRED_CHECK","CONTEXT_NAME_MISMATCH","MISSING_REQUIRED_WORKFLOW","PRIVATE_WITH_STRAY_READ","DEFAULT_BRANCH_OUTLIER","DEPRECATED_WORKFLOW_PRESENT","COMPLIANT"];for(let o of n){let i=r.buckets[o];if(!i||i.length===0)continue;let s=o==="COMPLIANT"?l.green:l.yellow;if(console.log(` ${s(o.padEnd(28))} ${i.length}`),o!=="COMPLIANT"){let a=i.slice(0,20).join(", ");console.log(l.dim(` ${a}${i.length>20?` ... +${i.length-20}`:""}`))}}console.log()}catch(r){if(r instanceof L)t.fail(r.message);else t.fail("Failed to scan fleet"),console.log(l.red(` ${r.message}`));process.exit(1)}});Ba.command("enforce").description("Bring dataset repos to the governance spec (single or --all). Dry-run by default.").argument("[dataset-id]","Dataset to enforce (omit when using --all)").option("--all","Bulk enforce across a filtered set (owner-only)").option("--apply","Actually apply changes (default is a dry run)").option("--prefix <prefix>","Bulk: filter by id prefix").option("--visibility <vis>","Bulk: filter by visibility (public|private)").option("--limit <n>","Bulk: max repos (default 25, max 50)","25").option("--json","Output raw JSON").action(async(e,t)=>{if(!je())console.log(l.red("Error: Not authenticated")),process.exit(1);if(!e&&!t.all)console.log(l.red("Error: provide a dataset-id or use --all")),process.exit(1);if(e&&t.all)console.log(l.red("Error: provide a dataset-id OR --all, not both")),process.exit(1);let r=!t.apply,n=r?"[dry-run] ":"",o=D(`${n}Enforcing spec...`).start();try{if(e){let i=await Nl(e,r);if(o.succeed(`${n}${e} (${i.result.visibility})`),t.json){console.log(JSON.stringify(i,null,2));return}for(let[s,a]of Object.entries(i.result.steps)){let c=a.status==="ok"?l.green:a.status==="skipped"?l.yellow:l.red;console.log(` ${c(a.status.padEnd(8))} ${s}${a.detail?l.dim(` (${a.detail})`):""}`)}}else{let i=await DJ({prefix:t.prefix,visibility:t.visibility,limit:Number.parseInt(t.limit,10)||25,dryRun:r});if(o.succeed(`${n}processed ${i.count} repo(s)`),t.json){console.log(JSON.stringify(i,null,2));return}for(let s of i.results){if(s.error){console.log(` ${l.red("error")} ${s.dataset_id}: ${s.error}`);continue}let a=Object.entries(s.steps??{}).filter(([,u])=>u.status!=="ok").map(([u,d])=>`${u}=${d.status}`),c=a.length?l.yellow("drift"):l.green("ok");console.log(` ${c} ${s.dataset_id}${a.length?l.dim(` (${a.join(", ")})`):""}`)}}}catch(i){if(i instanceof L)o.fail(i.message);else o.fail("Enforcement failed"),console.log(l.red(` ${i.message}`));process.exit(1)}});var eu=(e)=>new Promise((t)=>setTimeout(t,e));async function aKe(e,t,r){let n=Date.now()+t,o="no probe completed";while(Date.now()<n){try{let i=(await Nl(e,!0)).result?.steps?.branch_ruleset;if(i?.status==="ok")return{status:"green"};if(i?.detail)o=i.detail}catch(i){o=i instanceof L?i.message:String(i)}await eu(r)}return{status:"timeout",detail:o}}Ba.command("revalidate").description("Re-run BIDS validation on main HEAD for unprotected datasets, then optionally enforce (epic #713)").argument("[dataset-id]","Dataset to revalidate (omit when using --all/--prefix)").option("--all","Revalidate across unprotected public datasets").option("--prefix <prefix>","Filter by id prefix (with --all), e.g. nm000 / on").option("--enforce","After a dataset goes green, run enforce (dry-run unless --apply)").option("--apply","With --enforce, actually apply the ruleset (default is a dry run)").option("--force","Override the live-dataset guard (nm000103-107)").option("--limit <n>","Max repos for --all/--prefix (default 25)","25").option("--json","Output raw JSON").action(async(e,t)=>{if(!je())console.log(l.red("Error: Not authenticated")),process.exit(1);let r=Boolean(t.all||!e&&t.prefix);if(!e&&!r)console.log(l.red("Error: provide a dataset-id, or --all/--prefix")),process.exit(1);let n;if(e)n=[e];else{let d=D("Fetching dataset list...").start();try{let f=[];for(let p=0;;p+=200){let h=await Ns({limit:200,offset:p});if(f.push(...h.datasets),h.datasets.length<200||f.length>=h.total_count)break}let m=Number.parseInt(t.limit,10)||25;n=RX(f,{prefix:t.prefix}).slice(0,m),d.succeed(`${n.length} target(s) (from ${f.length} datasets)`)}catch(f){Qe(f,d,"Failed to fetch datasets");return}}let o=Boolean(t.enforce),i=Boolean(t.apply),s={revalidated:0,green:0,locked:0,stillRed:0,skipped:0,errors:0},a=[],c=[],u=0;for(let d of n){if(u++,Ri.has(d)&&!t.force){console.log(` [${u}/${n.length}] ${l.dim(`${d}: skip (live; use --force)`)}`),s.skipped++;continue}try{let f=await EJ(d,t.force);if(a.push(f),f.skipped)console.log(` [${u}/${n.length}] ${d}: ${l.dim(`skipped (${f.skipped})`)}`),s.skipped++;else s.revalidated++,c.push(d),console.log(` [${u}/${n.length}] ${d}: ${l.cyan(`revalidating (${f.triggered_by})`)}`)}catch(f){s.errors++,console.log(` [${u}/${n.length}] ${d}: ${l.red(f instanceof L?f.message:String(f))}`)}if(u<n.length)await eu(4000)}if(o&&c.length>0)if(e){let d=c[0],f=D(`Waiting for ${d} validation to land...`).start(),m=await aKe(d,300000,15000);if(m.status!=="green")f.warn(`${d}: not green within 5m \u2014 last: ${m.detail} (real errors or still running)`),s.stillRed++;else if(s.green++,!i)f.succeed(`${d}: green (would lock; pass --apply)`);else{let p=await Nl(d,!1);if(p.result?.steps?.branch_ruleset?.status==="ok")f.succeed(`${d}: LOCKED`),s.locked++;else f.warn(`${d}: enforce ${p.result?.steps?.branch_ruleset?.status??"?"}`)}}else{let d=D("Waiting ~90s for validations to drain...").start();await eu(90000),d.text="Enforcing green datasets...";for(let f of c)try{if((await Nl(f,!0)).result?.steps?.branch_ruleset?.status!=="ok"){s.stillRed++;continue}if(s.green++,i){if((await Nl(f,!1)).result?.steps?.branch_ruleset?.status==="ok")s.locked++}}catch{s.errors++}d.stop()}if(t.json){console.log(JSON.stringify({tally:s,raw:a},null,2));return}console.log(),console.log(l.cyan(`Done: revalidated=${s.revalidated} green=${s.green} locked=${s.locked} still-red=${s.stillRed} skipped=${s.skipped} errors=${s.errors}`))});Ba.command("annex-policy").description("Report, and optionally fix, dataset repos that NEMAR's annex policy does not actually govern (#1374, ADR 0060)").argument("[dataset-id...]","Specific datasets to act on (omit to sweep by prefix)").option("--prefix <prefix>","Dataset id prefix to sweep (default on: the imported fleet)","on").option("--limit <n>","Act on at most this many datasets, in id order").option("--apply","Make the change (default is a read-only report)").option("--include-data","Also move data a repo keeps in git, which uploads to S3").option("--via-aws-cli","With --include-data, move the bytes with this machine's aws CLI").option("--no-push","With --apply, commit locally and push nothing (a rehearsal)").option("--dir <path>","Where clones go while they are being fixed (default: a temp dir)").option("--keep-clones","Do not delete each clone after its dataset is done").option("--concurrency <n>","Parallel GitHub reads during the scan (default 4)","4").option("--json <path>","Write the full report as JSON").option("--force","Include the live datasets (nm000103-107)").option(Pe,Ie).option(He,qe).addHelpText("after",`
|
|
571
|
+
`));for(let o of n)try{let i=await Tt(o.xx_id);console.log(` ${o.xx_id.padEnd(10)} ${o.modality.padEnd(6)} <- ${o.source_id.padEnd(10)} ${l.green(i.visibility)} doi=${i.concept_doi??"none"}`)}catch(i){if(i instanceof L&&i.statusCode===404)console.log(` ${o.xx_id.padEnd(10)} ${o.modality.padEnd(6)} <- ${o.source_id.padEnd(10)} ${l.dim("not created")}`);else console.log(` ${o.xx_id.padEnd(10)} ${o.modality.padEnd(6)} <- ${o.source_id.padEnd(10)} ${l.red(`status check failed (${N(i)})`)}`)}});Gy.command("remint-dois").description("Re-mint an exemplar's sandbox concept DOI (idempotent)").argument("<xx-id>","Exemplar dataset id (e.g., xx099900)").action(async(e)=>{if(!Z())return;let t=D(`Re-minting DOIs for ${e}...`).start();try{let r=await sJ(e);t.succeed(`Re-minted ${r.concept_doi} (status: ${r.status})`);for(let n of r.warnings??[])console.log(l.yellow(` ${n}`))}catch(r){Qe(r,t,"Failed to re-mint exemplar DOIs",{400:"Not an exemplar dataset",403:"Exemplar DOI re-mint is disabled in production",404:"Dataset not found"}),process.exit(1)}});pe.addCommand(Gy);function sKe(e,t){let r=e.enrichment.status,n=r==="ok"?l.green("enrich:ok"):r==="failed"?l.red("enrich:failed"):l.dim("enrich:skipped"),o=e.sync.status,i=o==="ok"?l.green("sync:ok"):o==="failed"?l.red("sync:failed"):l.dim("sync:skipped"),s=e.sync.metadata_columns_written===!0?l.green("cols:written"):e.sync.metadata_columns_error?l.red("cols:failed"):"",a=t?.showRef&&e.enrichment.ref?l.dim(`@${e.enrichment.ref}`):"",c=[` ${e.dataset_id.padEnd(12)} ${n}${a} ${i} ${s}`];if(e.enrichment.error)c.push(l.red(` enrichment: ${e.enrichment.error}`));for(let d of e.enrichment.warnings??[])c.push(l.yellow(` warning: ${d}`));let u=e.enrichment.doi_resolution;if(u){let d=` DOI lookups: ${u.resolved} resolved, ${u.unresolved} unresolved, ${u.failed} failed`;if(u.failed>0){let f=(e.enrichment.warnings??[]).some((m)=>m.startsWith("doi_resolution:"));c.push(l.yellow(f?d:`${d} (reindex this dataset again)`))}else c.push(l.dim(d));if(u.skipped>0)c.push(l.yellow(` ${u.skipped} DOI(s) past the per-run lookup cap were not looked up; their relation labels were chosen without registry metadata`))}if(e.sync.metadata_columns_error)c.push(l.red(` metadata columns: ${e.sync.metadata_columns_error}`));return c}function jte(e,t){for(let r of sKe(e,t))console.log(r)}var Ba=new ue("fleet").description("Governance drift reporting and enforcement (epic #713)");Ba.command("drift").description("Report dataset repos that are off the governance spec").option("--prefix <prefix>","Filter datasets by id prefix (e.g. nm, on)").option("--visibility <vis>","Filter by visibility: public or private").option("--limit <n>","Max repos to scan (default 25, max 50)","25").option("--json","Output raw JSON").action(async(e)=>{if(!je())console.log(l.red("Error: Not authenticated")),process.exit(1);if(e.visibility&&!["public","private"].includes(e.visibility))console.log(l.red("Error: --visibility must be public or private")),process.exit(1);let t=D("Scanning fleet for drift...").start();try{let r=await wJ({prefix:e.prefix,visibility:e.visibility,limit:Number.parseInt(e.limit,10)||25});if(t.stop(),e.json){console.log(JSON.stringify(r,null,2));return}console.log(),console.log(l.bold(`Fleet drift (scanned ${r.scanned}):`));let n=["PUBLIC_UNPROTECTED","RED_REQUIRED_CHECK","CONTEXT_NAME_MISMATCH","MISSING_REQUIRED_WORKFLOW","PRIVATE_WITH_STRAY_READ","DEFAULT_BRANCH_OUTLIER","DEPRECATED_WORKFLOW_PRESENT","COMPLIANT"];for(let o of n){let i=r.buckets[o];if(!i||i.length===0)continue;let s=o==="COMPLIANT"?l.green:l.yellow;if(console.log(` ${s(o.padEnd(28))} ${i.length}`),o!=="COMPLIANT"){let a=i.slice(0,20).join(", ");console.log(l.dim(` ${a}${i.length>20?` ... +${i.length-20}`:""}`))}}console.log()}catch(r){if(r instanceof L)t.fail(r.message);else t.fail("Failed to scan fleet"),console.log(l.red(` ${r.message}`));process.exit(1)}});Ba.command("enforce").description("Bring dataset repos to the governance spec (single or --all). Dry-run by default.").argument("[dataset-id]","Dataset to enforce (omit when using --all)").option("--all","Bulk enforce across a filtered set (owner-only)").option("--apply","Actually apply changes (default is a dry run)").option("--prefix <prefix>","Bulk: filter by id prefix").option("--visibility <vis>","Bulk: filter by visibility (public|private)").option("--limit <n>","Bulk: max repos (default 25, max 50)","25").option("--json","Output raw JSON").action(async(e,t)=>{if(!je())console.log(l.red("Error: Not authenticated")),process.exit(1);if(!e&&!t.all)console.log(l.red("Error: provide a dataset-id or use --all")),process.exit(1);if(e&&t.all)console.log(l.red("Error: provide a dataset-id OR --all, not both")),process.exit(1);let r=!t.apply,n=r?"[dry-run] ":"",o=D(`${n}Enforcing spec...`).start();try{if(e){let i=await Nl(e,r);if(o.succeed(`${n}${e} (${i.result.visibility})`),t.json){console.log(JSON.stringify(i,null,2));return}for(let[s,a]of Object.entries(i.result.steps)){let c=a.status==="ok"?l.green:a.status==="skipped"?l.yellow:l.red;console.log(` ${c(a.status.padEnd(8))} ${s}${a.detail?l.dim(` (${a.detail})`):""}`)}}else{let i=await DJ({prefix:t.prefix,visibility:t.visibility,limit:Number.parseInt(t.limit,10)||25,dryRun:r});if(o.succeed(`${n}processed ${i.count} repo(s)`),t.json){console.log(JSON.stringify(i,null,2));return}for(let s of i.results){if(s.error){console.log(` ${l.red("error")} ${s.dataset_id}: ${s.error}`);continue}let a=Object.entries(s.steps??{}).filter(([,u])=>u.status!=="ok").map(([u,d])=>`${u}=${d.status}`),c=a.length?l.yellow("drift"):l.green("ok");console.log(` ${c} ${s.dataset_id}${a.length?l.dim(` (${a.join(", ")})`):""}`)}}}catch(i){if(i instanceof L)o.fail(i.message);else o.fail("Enforcement failed"),console.log(l.red(` ${i.message}`));process.exit(1)}});var eu=(e)=>new Promise((t)=>setTimeout(t,e));async function aKe(e,t,r){let n=Date.now()+t,o="no probe completed";while(Date.now()<n){try{let i=(await Nl(e,!0)).result?.steps?.branch_ruleset;if(i?.status==="ok")return{status:"green"};if(i?.detail)o=i.detail}catch(i){o=i instanceof L?i.message:String(i)}await eu(r)}return{status:"timeout",detail:o}}Ba.command("revalidate").description("Re-run BIDS validation on main HEAD for unprotected datasets, then optionally enforce (epic #713)").argument("[dataset-id]","Dataset to revalidate (omit when using --all/--prefix)").option("--all","Revalidate across unprotected public datasets").option("--prefix <prefix>","Filter by id prefix (with --all), e.g. nm000 / on").option("--enforce","After a dataset goes green, run enforce (dry-run unless --apply)").option("--apply","With --enforce, actually apply the ruleset (default is a dry run)").option("--force","Override the live-dataset guard (nm000103-107)").option("--limit <n>","Max repos for --all/--prefix (default 25)","25").option("--json","Output raw JSON").action(async(e,t)=>{if(!je())console.log(l.red("Error: Not authenticated")),process.exit(1);let r=Boolean(t.all||!e&&t.prefix);if(!e&&!r)console.log(l.red("Error: provide a dataset-id, or --all/--prefix")),process.exit(1);let n;if(e)n=[e];else{let d=D("Fetching dataset list...").start();try{let f=[];for(let p=0;;p+=200){let h=await Ns({limit:200,offset:p});if(f.push(...h.datasets),h.datasets.length<200||f.length>=h.total_count)break}let m=Number.parseInt(t.limit,10)||25;n=RX(f,{prefix:t.prefix}).slice(0,m),d.succeed(`${n.length} target(s) (from ${f.length} datasets)`)}catch(f){Qe(f,d,"Failed to fetch datasets");return}}let o=Boolean(t.enforce),i=Boolean(t.apply),s={revalidated:0,green:0,locked:0,stillRed:0,skipped:0,errors:0},a=[],c=[],u=0;for(let d of n){if(u++,Ri.has(d)&&!t.force){console.log(` [${u}/${n.length}] ${l.dim(`${d}: skip (live; use --force)`)}`),s.skipped++;continue}try{let f=await EJ(d,t.force);if(a.push(f),f.skipped)console.log(` [${u}/${n.length}] ${d}: ${l.dim(`skipped (${f.skipped})`)}`),s.skipped++;else s.revalidated++,c.push(d),console.log(` [${u}/${n.length}] ${d}: ${l.cyan(`revalidating (${f.triggered_by})`)}`)}catch(f){s.errors++,console.log(` [${u}/${n.length}] ${d}: ${l.red(f instanceof L?f.message:String(f))}`)}if(u<n.length)await eu(4000)}if(o&&c.length>0)if(e){let d=c[0],f=D(`Waiting for ${d} validation to land...`).start(),m=await aKe(d,300000,15000);if(m.status!=="green")f.warn(`${d}: not green within 5m \u2014 last: ${m.detail} (real errors or still running)`),s.stillRed++;else if(s.green++,!i)f.succeed(`${d}: green (would lock; pass --apply)`);else{let p=await Nl(d,!1);if(p.result?.steps?.branch_ruleset?.status==="ok")f.succeed(`${d}: LOCKED`),s.locked++;else f.warn(`${d}: enforce ${p.result?.steps?.branch_ruleset?.status??"?"}`)}}else{let d=D("Waiting ~90s for validations to drain...").start();await eu(90000),d.text="Enforcing green datasets...";for(let f of c)try{if((await Nl(f,!0)).result?.steps?.branch_ruleset?.status!=="ok"){s.stillRed++;continue}if(s.green++,i){if((await Nl(f,!1)).result?.steps?.branch_ruleset?.status==="ok")s.locked++}}catch{s.errors++}d.stop()}if(t.json){console.log(JSON.stringify({tally:s,raw:a},null,2));return}console.log(),console.log(l.cyan(`Done: revalidated=${s.revalidated} green=${s.green} locked=${s.locked} still-red=${s.stillRed} skipped=${s.skipped} errors=${s.errors}`))});Ba.command("annex-policy").description("Report, and optionally fix, dataset repos that NEMAR's annex policy does not actually govern (#1374, ADR 0060)").argument("[dataset-id...]","Specific datasets to act on (omit to sweep by prefix)").option("--prefix <prefix>","Dataset id prefix to sweep (default on: the imported fleet)","on").option("--limit <n>","Act on at most this many datasets, in id order").option("--apply","Make the change (default is a read-only report)").option("--include-data","Also move data a repo keeps in git, which uploads to S3").option("--via-aws-cli","With --include-data, move the bytes with this machine's aws CLI").option("--no-push","With --apply, commit locally and push nothing (a rehearsal)").option("--dir <path>","Where clones go while they are being fixed (default: a temp dir)").option("--keep-clones","Do not delete each clone after its dataset is done").option("--concurrency <n>","Parallel GitHub reads during the scan (default 4)","4").option("--json <path>","Write the full report as JSON").option("--force","Include the live datasets (nm000103-107)").option(Pe,Ie).option(He,qe).addHelpText("after",`
|
|
572
572
|
Two things govern what git-annex takes, and an imported dataset has neither (#1374):
|
|
573
573
|
upstream's .gitattributes outranks NEMAR's configured expression, and the expression
|
|
574
574
|
was never written to the git-annex branch at all. The scan reads both from GitHub
|
package/package.json
CHANGED