@rizom/ops 0.2.0-alpha.713 → 0.2.0-alpha.715
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/brains-ops.js
CHANGED
|
@@ -737,7 +737,7 @@ printf 'REMOTE_ARTIFACT_DIR=%s\\n' "$remote_dir"
|
|
|
737
737
|
`).replace(/\\n/g,`
|
|
738
738
|
`).trimEnd()}
|
|
739
739
|
`}var C8,tge;var V$=v(async()=>{pe();T8();hn();Yf();await A8();C8=c.object({HCLOUD_TOKEN:c.string().min(1),KAMAL_SSH_PRIVATE_KEY:c.string().min(1)}),tge=c.object({servers:c.array(c.object({status:c.string(),public_net:c.object({ipv4:c.object({ip:c.string().min(1)})})}))})});import{parseArgs as vK}from"util";function nt(e){return e}class np extends Error{constructor(e){super(e);this.name="CliUsageError"}}function wK(e,t){let r={...t};for(let o of e)for(let[n,i]of Object.entries(o.flags??{})){let s=r[n];if(s===void 0){r[n]=i;continue}if(s.type!==i.type||s.short!==i.short||s.default!==i.default)throw Error(`Conflicting definitions for flag --${n}: commands must agree on type, short, and default`)}return r}function xK(e){let t={};for(let[r,o]of Object.entries(e))t[r]={type:o.type,...o.short!==void 0?{short:o.short}:{},...o.default!==void 0?{default:o.default}:{}};return t}function _K(e){if(!(e instanceof Error)||!("code"in e))return!1;return e.code==="ERR_PARSE_ARGS_UNKNOWN_OPTION"||e.code==="ERR_PARSE_ARGS_INVALID_OPTION_VALUE"}function Q$(e,t){let r=wK(t.commands,t.globalFlags??{}),o,n;try{({values:o,positionals:n}=vK({args:e,options:xK(r),allowPositionals:!0,strict:t.strict??!0}))}catch(s){if(_K(s))throw new np(s.message.split(". ")[0]??s.message);throw s}if("help"in r&&o.help===!0)return{command:"help",args:[],flags:{help:!0}};if("version"in r&&o.version===!0)return{command:"version",args:[],flags:{version:!0}};let i={};for(let[s,a]of Object.entries(r)){let l=o[s];if(typeof l==="string"&&a.type==="string")i[s]=l;else if(typeof l==="boolean"&&a.type==="boolean")i[s]=l}return{command:n[0]??"help",args:n.slice(1),flags:i}}function Y$(e){let t=e.reduce((r,o)=>Math.max(r,o.label.length),0);return e.map((r)=>` ${r.label.padEnd(t)} ${r.description}`)}function EK(e){return Y$(e.map((t)=>({label:t.usage?`${t.name} ${t.usage}`:t.name,description:t.description})))}function kK(e,t){let r=t.type==="string"?` ${t.placeholder??"<value>"}`:"",o=t.short!==void 0?`, -${t.short}`:"";return`--${e}${r}${o}`}function X$(e){return Y$(Object.entries(e).map(([t,r])=>({label:kK(t,r),description:r.description})))}function eO(e,t){return`Usage: ${e} ${t.name}${t.usage?` ${t.usage}`:""}`}function IK(e){return JSON.stringify(Object.entries(e).map(([t,r])=>[t,r.type,r.short??"",r.description,r.placeholder??"",r.default??""]).sort((t,r)=>String(t[0]).localeCompare(String(r[0]))))}function AK(e){let t=[],r=new Map;for(let n of e){let i=n.flags??{};if(Object.keys(i).length===0)continue;let s=IK(i),a=r.get(s);if(a){a.names.push(n.name);continue}let l={names:[n.name],flags:i};r.set(s,l),t.push(l)}let o=[];for(let n of t)o.push("",`${n.names.join(" / ")} options:`),o.push(...X$(n.flags));return o}function tO(e){let t=[e.intro,"",`Usage: ${e.cliName} ${e.usage??"<command> [args]"}`,"","Commands:",...EK(e.commands)];if(e.globalFlags&&Object.keys(e.globalFlags).length>0)t.push("","Options:",...X$(e.globalFlags));return t.push(...AK(e.commands)),t.join(`
|
|
740
|
-
`)}function rO(e){let t=new Map;for(let r of e){if(t.has(r.name))throw Error(`Duplicate command name: ${r.name}`);t.set(r.name,r)}return t}function ot(e,t){let r=e[t];return typeof r==="string"?r:void 0}function ds(e,t){let r=e[t];return typeof r==="boolean"?r:void 0}var op={name:"@rizom/ops",version:"0.2.0-alpha.
|
|
740
|
+
`)}function rO(e){let t=new Map;for(let r of e){if(t.has(r.name))throw Error(`Duplicate command name: ${r.name}`);t.set(r.name,r)}return t}function ot(e,t){let r=e[t];return typeof r==="string"?r:void 0}function ds(e,t){let r=e[t];return typeof r==="boolean"?r:void 0}var op={name:"@rizom/ops",version:"0.2.0-alpha.715",description:"Operator CLI for managing private brain fleet registry repos",keywords:["brains","ops","cli","deploy","fleet","operator"],license:"AGPL-3.0-only",author:"Yeehaa <yeehaa@rizom.ai> (https://rizom.ai)",bin:{"brains-ops":"./dist/brains-ops.js"},bugs:"https://github.com/rizom-ai/brains/issues",dependencies:{"@libsql/client":"^0.17.4","age-encryption":"^0.3.0",yaml:"^2.9.0"},devDependencies:{"@brains/contracts":"workspace:*","@brains/deploy-support":"workspace:*","@brains/eslint-config":"workspace:*","@brains/job-queue":"workspace:*","@brains/profile":"workspace:*","@brains/site-composition":"workspace:*","@brains/test-utils":"workspace:*","@brains/typescript-config":"workspace:*","@brains/utils":"workspace:*","@types/bun":"^1.4.0",typescript:"^7.0.2"},engines:{bun:">=1.3.3"},exports:{".":{types:"./dist/index.d.ts",import:"./dist/index.js"},"./deploy":{types:"./dist/deploy.d.ts",import:"./dist/deploy.js",bun:"./dist/deploy.js"}},files:["dist","templates"],homepage:"https://github.com/rizom-ai/brains/tree/main/packages/brains-ops#readme",main:"./dist/index.js",publishConfig:{access:"public"},repository:{type:"git",url:"https://github.com/rizom-ai/brains.git",directory:"packages/brains-ops"},scripts:{build:"bun scripts/build.ts",lint:"eslint . --max-warnings 0","lint:fix":"eslint . --max-warnings 0 --fix",prepublishOnly:"bun scripts/build.ts",test:"bun test --timeout 20000","test:smoke":"bun run build && RUN_SMOKE_TESTS=1 bun test --timeout 60000",typecheck:"tsc --noEmit"},type:"module",types:"./dist/index.d.ts"};kv();bi();import{writeFile as LY}from"fs/promises";import{join as ij}from"path";import{createSign as hW,generateKeyPairSync as gW}from"crypto";import{mkdir as yW,writeFile as rD}from"fs/promises";import{join as nD}from"path";async function kV(e){let t=await e.text();if(!t.trim())return;try{return JSON.parse(t)}catch{return t}}async function Wa(e,t,r){let o=await kV(e),n=t.safeParse(o);if(!e.ok||!n.success){let i=r.formatError??IV;throw Error(`${r.label}${i(e,o)}`)}return n.data}function IV(e,t){let r=t===void 0?"":`: ${typeof t==="string"?t:JSON.stringify(t)}`;return` (${e.status} ${e.statusText}${r})`}pe();var bW="2.5.4.3",SW="1.2.840.113549.1.1.11",sD="https://api.cloudflare.com/client/v4",vW="origin-rsa",wW=5475,aD=15000;function cD(e){return{Authorization:`Bearer ${e}`,"Content-Type":"application/json"}}var oD=c.looseObject({message:c.string().optional()}),xW=c.looseObject({errors:c.array(oD).optional(),messages:c.array(oD).optional()}),_W=c.looseObject({certificate:c.string().min(1),expires_on:c.string().optional()}),EW=c.looseObject({success:c.literal(!0),result:_W}),kW=c.looseObject({success:c.literal(!0)});function IW(){let{privateKey:e,publicKey:t}=gW("rsa",{modulusLength:2048,privateKeyEncoding:{type:"pkcs8",format:"pem"},publicKeyEncoding:{type:"spki",format:"der"}});return{privateKeyPem:e,publicKeyDer:Buffer.from(t)}}function AW(e,t){let r=bd(zW(bd(iD(bW),OW(e)))),o=bd($W(0),r,Buffer.from(t.publicKeyDer),NW(0,Buffer.alloc(0))),n=hW("sha256").update(o).sign(t.privateKeyPem),i=bd(o,bd(iD(SW),PW()),CW(n));return{csrPem:RW("CERTIFICATE REQUEST",i),certificationRequestInfoDer:o,signature:n}}async function TW(e,t,r,o){let n=await e(`${sD}/certificates`,{method:"POST",headers:cD(t),body:JSON.stringify({hostnames:[o,`*.${o}`],requested_validity:wW,request_type:vW,csr:r}),signal:AbortSignal.timeout(aD)}),i=await Wa(n,EW,{label:"Cloudflare Origin CA request failed",formatError:uD}),a={certificatePem:`${i.result.certificate.trim()}
|
|
741
741
|
`};if(i.result.expires_on)a.expiresOn=i.result.expires_on;return a}async function lD(e,t,r){let o=await e(`${sD}/zones/${r}/settings/ssl`,{method:"PATCH",headers:cD(t),body:JSON.stringify({value:"strict"}),signal:AbortSignal.timeout(aD)});await Wa(o,kW,{label:"Cloudflare zone SSL update failed",formatError:uD})}function RW(e,t){let o=t.toString("base64").match(/.{1,64}/g)??[];return[`-----BEGIN ${e}-----`,...o,`-----END ${e}-----`,""].join(`
|
|
742
742
|
`)}function $W(e){if(e!==0)throw Error("Only zero-valued INTEGER encoding is currently supported");return Buffer.from([2,1,0])}function iD(e){let t=e.split(".").map((s)=>Number(s));if(t.length<2||t.some((s)=>Number.isNaN(s)))throw Error(`Invalid OID: ${e}`);let r=t[0],o=t[1];if(r===void 0||o===void 0)throw Error(`Invalid OID: ${e}`);let i=[r*40+o];for(let s of t.slice(2)){let a=LW(s);i.push(...a)}return Ac(6,Buffer.from(i))}function OW(e){return Ac(12,Buffer.from(e,"utf8"))}function CW(e){return Ac(3,Buffer.concat([Buffer.from([0]),e]))}function PW(){return Buffer.from([5,0])}function bd(...e){return Ac(48,Buffer.concat(e))}function zW(...e){return Ac(49,Buffer.concat(e))}function NW(e,t){return Ac(160+e,t)}function Ac(e,t){return Buffer.concat([Buffer.from([e]),DW(t.length),t])}function DW(e){if(e<128)return Buffer.from([e]);let t=[],r=e;while(r>0)t.unshift(r&255),r>>=8;return Buffer.from([128|t.length,...t])}function LW(e){if(!Number.isInteger(e)||e<0)throw Error(`Invalid OID component: ${e}`);if(e===0)return[0];let t=[],r=e;while(r>0)t.unshift(r&127),r>>=7;for(let o=0;o<t.length-1;o+=1)t[o]=(t[o]??0)|128;return t}function uD(e,t){let r=BW(t),o=r.length>0?`: ${r.join("; ")}`:"";return` (${e.status} ${e.statusText}${o})`}function BW(e){let t=xW.safeParse(e);if(!t.success)return[];return[...t.data.errors??[],...t.data.messages??[]].map((r)=>r.message).filter((r)=>typeof r==="string"&&r.length>0)}async function dD(e,t,r){let o=IW(),{csrPem:n}=AW(r,o),i=await TW(e,t,n,r);return{certificatePem:i.certificatePem,privateKeyPem:o.privateKeyPem,...i.expiresOn!==void 0&&{expiresOn:i.expiresOn}}}async function fD(e,t){let r=nD(e,"origin.pem"),o=nD(e,"origin.key");return await yW(e,{recursive:!0}),await Promise.all([rD(r,t.certificatePem,"utf-8"),rD(o,t.privateKeyPem,{encoding:"utf-8",mode:384})]),{certificatePath:r,privateKeyPath:o}}hn();Ru();Un();async function aj(e,t={}){try{return await BY(e,t),{success:!0}}catch(r){return{success:!1,message:Ve(r,"Certificate bootstrap failed")}}}async function BY(e,t={}){let r=await wt(e),o=ZY(r,t.handle),n=o.domain,i=t.env??process.env,s=Dr(e),a=t.cfApiToken??jt("CF_API_TOKEN",i,s),l=t.cfZoneId??o.cloudflareZoneId??jt("CF_ZONE_ID",i,s);if(!a)throw Error("Missing CF_API_TOKEN");if(!l)throw Error("Missing CF_ZONE_ID");let u=t.fetchImpl??fetch,d=t.logger??console.log,f=await dD(u,a,n),m=ij(e,".brains-ops","certs",o.id),{certificatePath:h,privateKeyPath:b}=await fD(m,f),y=ij(m,"secrets.yaml");await LY(y,jY(f.certificatePem,f.privateKeyPem),"utf-8"),await lD(u,a,l);let S=Ha(t.pushTo);if(S)await gi([["CERTIFICATE_PEM",f.certificatePem],["PRIVATE_KEY_PEM",f.privateKeyPem]],{logger:d,runCommand:t.runCommand??tr});if(d(`Issued ${o.id} Origin CA cert for ${n} and *.${n}`),d(`Wrote ${h}`),d(`Wrote ${b}`),d(`Wrote ${y}`),f.expiresOn)d(`Expires on ${f.expiresOn}`);if(d("Cloudflare zone SSL mode set to Full (strict)"),S)d(`Pushed CERTIFICATE_PEM and PRIVATE_KEY_PEM to ${S}`);return{domain:n,certificatePath:h,privateKeyPath:b,certificatePem:f.certificatePem,secretsSnippetPath:y}}function jY(e,t){return[`certificatePem: ${sj(e)}`,`privateKeyPem: ${sj(t)}`,""].join(`
|
|
743
743
|
`)}function sj(e){return`"${e.replace(/"/g,"\\\"").replace(/\n/g,"\\n")}"`}function ZY(e,t){if(!t)return{id:"shared",domain:UY(e.pilot.domainSuffix)};let r=e.users.find((o)=>o.handle===t);if(!r)throw Error(`Unknown user handle: ${t}`);return{id:t,domain:r.domain,...r.cloudflareZoneId?{cloudflareZoneId:r.cloudflareZoneId}:{}}}function UY(e){let t=e.trim().replace(/^\./,"").replace(/\.$/,"");if(!t||t.includes("*"))throw Error(`Invalid pilot domainSuffix: ${e}`);return t}import{access as lj,chmod as NI,mkdir as rg,readFile as dj,writeFile as uj}from"fs/promises";import{dirname as JY,join as Dc}from"path";import{fileURLToPath as HY}from"url";bi();Ad();var VY=["pilot.yaml","package.json",".env.schema",".gitignore","cohorts/cohort-1.yaml","users/alice.yaml",".github/actions/varlock-env/action.yml",".github/workflows/build.yml",".github/workflows/deploy.yml",".github/workflows/directory-sync-stress.yml",".github/workflows/health-watchdog-smoke.yml",".github/workflows/offboard.yml",".github/workflows/reconcile.yml",".github/workflows/upgrade.yml","deploy/Dockerfile","deploy/kamal/deploy.yml","deploy/scripts/helpers.ts","deploy/scripts/create-predeploy-backup.ts","deploy/scripts/install-health-watchdog.ts","deploy/scripts/provision-server.ts","deploy/scripts/pull-local-models.ts","deploy/scripts/update-dns.ts","deploy/scripts/write-ssh-key.ts","deploy/scripts/decrypt-user-secrets.ts","deploy/scripts/validate-secrets.ts","deploy/scripts/write-kamal-secrets.ts","deploy/scripts/resolve-user-config.ts","deploy/scripts/resolve-missing-images.ts","deploy/scripts/verify-runtime-image.ts","deploy/scripts/resolve-deploy-handles.ts","deploy/scripts/sync-content-repo.ts",".kamal/hooks/pre-deploy","docs/onboarding-checklist.md","docs/canonical-crossover-record.md","docs/operator-playbook.md","docs/user-onboarding.md","README.md"],DI=new Set([".kamal/hooks/pre-deploy"]),WY=new Set(["cohorts/cohort-1.yaml","users/alice.yaml","docs/canonical-crossover-record.md"]),GY=HY(new URL("../templates/rover-pilot/",import.meta.url));function Io(e){return(t)=>t.includes(e)}function QY(e){return FP.find((t)=>e===`deploy/scripts/${t}`)}function LI(e){return e.replace(/\n {2}secret:\n(?: {4}- .*\n)+\nvolumes:\n/,`
|
package/dist/index.js
CHANGED
|
@@ -747,7 +747,7 @@ printf 'REMOTE_ARTIFACT_DIR=%s\\n' "$remote_dir"
|
|
|
747
747
|
`),i}async function K8(e,t){let r=await ut(e),o=r.users.find((n)=>n.handle===t);if(!o)throw Error(`Unknown pilot user: ${t}`);return{pilot:r.pilot,user:o}}async function H8(e){let t=await rge(io(nge(),"brains-ops-watchdog-smoke-")),r=io(t,"id_ed25519"),o=io(t,"known_hosts");try{let n=encodeURIComponent(`brain=${e.target.pilot.contentRepoPrefix}${e.target.user.handle}`),i=await e.fetchImpl(`https://api.hetzner.cloud/v1/servers?label_selector=${n}`,{headers:{Authorization:`Bearer ${e.environment.HCLOUD_TOKEN}`}});if(!i.ok)throw Error(`Hetzner server lookup failed with ${i.status}`);let s=oge.parse(await i.json());if(s.servers.length!==1)throw Error(`Expected one smoke server, found ${s.servers.length}`);let a=s.servers[0];if(a?.status!=="running")throw Error("Smoke server is not running");await Ul(r,age(e.environment.KAMAL_SSH_PRIVATE_KEY),{mode:384}),await tge(r,384);let l=await e.commandRunner("ssh-keyscan",["-T","10","-t","ed25519",a.public_net.ipv4.ip]);if(l.exitCode!==0||!l.stdout.trim())throw Error("Unable to resolve watchdog smoke SSH host key");return await Ul(o,l.stdout,{mode:384}),{temporaryRoot:t,sshKeyPath:r,knownHostsPath:o,serverIp:a.public_net.ipv4.ip,commandRunner:e.commandRunner}}catch(n){throw await uO(t,{recursive:!0,force:!0}),n}}async function lO(e,t,r){return e.commandRunner("ssh",[...J8(e),`root@${e.serverIp}`,"bash","-s","--",t,r],{stdin:dO()})}async function ige(e,t,r){return e.commandRunner("scp",[...J8(e),"-r",`root@${e.serverIp}:${t}/.`,r])}function J8(e){return["-i",e.sshKeyPath,"-o","BatchMode=yes","-o","IdentitiesOnly=yes","-o","ConnectTimeout=10","-o","StrictHostKeyChecking=yes","-o",`UserKnownHostsFile=${e.knownHostsPath}`]}async function sge(e){try{return await ege(e),!0}catch{return!1}}async function Nb(e,t,r){await Promise.all([Ul(io(e,`${t}.stdout.log`),r.stdout),Ul(io(e,`${t}.stderr.log`),r.stderr)])}function age(e){return`${e.replace(/\r\n/g,`
|
|
748
748
|
`).replace(/\\n/g,`
|
|
749
749
|
`).trimEnd()}
|
|
750
|
-
`}var M8,oge;var Lb=v(async()=>{fe();B8();Hr();rf();await L8();M8=c.object({HCLOUD_TOKEN:c.string().min(1),KAMAL_SSH_PRIVATE_KEY:c.string().min(1)}),oge=c.object({servers:c.array(c.object({status:c.string(),public_net:c.object({ipv4:c.object({ip:c.string().min(1)})})}))})});import{access as k2,chmod as WE,mkdir as ah,readFile as A2,writeFile as I2}from"fs/promises";import{dirname as LW,join as dc}from"path";import{fileURLToPath as BW}from"url";var cf={name:"@rizom/ops",version:"0.2.0-alpha.
|
|
750
|
+
`}var M8,oge;var Lb=v(async()=>{fe();B8();Hr();rf();await L8();M8=c.object({HCLOUD_TOKEN:c.string().min(1),KAMAL_SSH_PRIVATE_KEY:c.string().min(1)}),oge=c.object({servers:c.array(c.object({status:c.string(),public_net:c.object({ipv4:c.object({ip:c.string().min(1)})})}))})});import{access as k2,chmod as WE,mkdir as ah,readFile as A2,writeFile as I2}from"fs/promises";import{dirname as LW,join as dc}from"path";import{fileURLToPath as BW}from"url";var cf={name:"@rizom/ops",version:"0.2.0-alpha.715",description:"Operator CLI for managing private brain fleet registry repos",keywords:["brains","ops","cli","deploy","fleet","operator"],license:"AGPL-3.0-only",author:"Yeehaa <yeehaa@rizom.ai> (https://rizom.ai)",bin:{"brains-ops":"./dist/brains-ops.js"},bugs:"https://github.com/rizom-ai/brains/issues",dependencies:{"@libsql/client":"^0.17.4","age-encryption":"^0.3.0",yaml:"^2.9.0"},devDependencies:{"@brains/contracts":"workspace:*","@brains/deploy-support":"workspace:*","@brains/eslint-config":"workspace:*","@brains/job-queue":"workspace:*","@brains/profile":"workspace:*","@brains/site-composition":"workspace:*","@brains/test-utils":"workspace:*","@brains/typescript-config":"workspace:*","@brains/utils":"workspace:*","@types/bun":"^1.4.0",typescript:"^7.0.2"},engines:{bun:">=1.3.3"},exports:{".":{types:"./dist/index.d.ts",import:"./dist/index.js"},"./deploy":{types:"./dist/deploy.d.ts",import:"./dist/deploy.js",bun:"./dist/deploy.js"}},files:["dist","templates"],homepage:"https://github.com/rizom-ai/brains/tree/main/packages/brains-ops#readme",main:"./dist/index.js",publishConfig:{access:"public"},repository:{type:"git",url:"https://github.com/rizom-ai/brains.git",directory:"packages/brains-ops"},scripts:{build:"bun scripts/build.ts",lint:"eslint . --max-warnings 0","lint:fix":"eslint . --max-warnings 0 --fix",prepublishOnly:"bun scripts/build.ts",test:"bun test --timeout 20000","test:smoke":"bun run build && RUN_SMOKE_TESTS=1 bun test --timeout 60000",typecheck:"tsc --noEmit"},type:"module",types:"./dist/index.d.ts"};Qo();uc();var jW=["pilot.yaml","package.json",".env.schema",".gitignore","cohorts/cohort-1.yaml","users/alice.yaml",".github/actions/varlock-env/action.yml",".github/workflows/build.yml",".github/workflows/deploy.yml",".github/workflows/directory-sync-stress.yml",".github/workflows/health-watchdog-smoke.yml",".github/workflows/offboard.yml",".github/workflows/reconcile.yml",".github/workflows/upgrade.yml","deploy/Dockerfile","deploy/kamal/deploy.yml","deploy/scripts/helpers.ts","deploy/scripts/create-predeploy-backup.ts","deploy/scripts/install-health-watchdog.ts","deploy/scripts/provision-server.ts","deploy/scripts/pull-local-models.ts","deploy/scripts/update-dns.ts","deploy/scripts/write-ssh-key.ts","deploy/scripts/decrypt-user-secrets.ts","deploy/scripts/validate-secrets.ts","deploy/scripts/write-kamal-secrets.ts","deploy/scripts/resolve-user-config.ts","deploy/scripts/resolve-missing-images.ts","deploy/scripts/verify-runtime-image.ts","deploy/scripts/resolve-deploy-handles.ts","deploy/scripts/sync-content-repo.ts",".kamal/hooks/pre-deploy","docs/onboarding-checklist.md","docs/canonical-crossover-record.md","docs/operator-playbook.md","docs/user-onboarding.md","README.md"],GE=new Set([".kamal/hooks/pre-deploy"]),ZW=new Set(["cohorts/cohort-1.yaml","users/alice.yaml","docs/canonical-crossover-record.md"]),UW=BW(new URL("../templates/rover-pilot/",import.meta.url));function fo(e){return(t)=>t.includes(e)}function MW(e){return vO.find((t)=>e===`deploy/scripts/${t}`)}function QE(e){return e.replace(/\n {2}secret:\n(?: {4}- .*\n)+\nvolumes:\n/,`
|
|
751
751
|
secret:
|
|
752
752
|
- __DYNAMIC_SECRETS__
|
|
753
753
|
|
package/package.json
CHANGED
|
@@ -40,6 +40,20 @@ export interface PredeployDatabaseSource {
|
|
|
40
40
|
method: DatabaseCaptureMethod;
|
|
41
41
|
quickCheck: QuickCheckDriver;
|
|
42
42
|
logicalVector: boolean;
|
|
43
|
+
/** Verify the image assets the snapshot's entities reference. */
|
|
44
|
+
assets?: boolean | undefined;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The asset inventory a verified snapshot holds. */
|
|
48
|
+
interface AssetSnapshotReport {
|
|
49
|
+
/** Entity rows referencing an asset. */
|
|
50
|
+
references: number;
|
|
51
|
+
/** Distinct referenced assets, each re-hashed from its chunks. */
|
|
52
|
+
assets: number;
|
|
53
|
+
totalBytes: number;
|
|
54
|
+
/** Staged uploads no asset publishes; harmless, reported only. */
|
|
55
|
+
orphanUploads: number;
|
|
56
|
+
digests: string[];
|
|
43
57
|
}
|
|
44
58
|
|
|
45
59
|
export interface PredeployBackupMetadata {
|
|
@@ -92,6 +106,7 @@ interface DatabaseCaptureRecord extends Omit<
|
|
|
92
106
|
sha256: string;
|
|
93
107
|
quickCheckDriver: QuickCheckDriver;
|
|
94
108
|
quickCheck: string;
|
|
109
|
+
assetVerification?: Omit<AssetSnapshotReport, "digests">;
|
|
95
110
|
}
|
|
96
111
|
|
|
97
112
|
interface GitCaptureRecord {
|
|
@@ -262,6 +277,98 @@ function vectorDigestDatabase(database: Database): {
|
|
|
262
277
|
return { counts, sha256: hasher.digest("hex") };
|
|
263
278
|
}
|
|
264
279
|
|
|
280
|
+
const ASSET_REF_PREFIX = "asset://sha256/";
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Check every asset the snapshot's entities reference: a published header
|
|
284
|
+
* whose chunks run contiguously, add up to its size and hash to its digest.
|
|
285
|
+
* Any failure blocks the backup. A database from before staged asset
|
|
286
|
+
* storage references none.
|
|
287
|
+
*/
|
|
288
|
+
function verifySnapshotAssets(path: string, name: string): AssetSnapshotReport {
|
|
289
|
+
const database = new Database(path, { readonly: true });
|
|
290
|
+
try {
|
|
291
|
+
const tables = new Set(
|
|
292
|
+
database
|
|
293
|
+
.query("SELECT name FROM sqlite_master WHERE type = 'table'")
|
|
294
|
+
.all()
|
|
295
|
+
.filter(isPlainRecord)
|
|
296
|
+
.map((row) => String(row["name"])),
|
|
297
|
+
);
|
|
298
|
+
if (!tables.has("asset_chunks")) {
|
|
299
|
+
return {
|
|
300
|
+
references: 0,
|
|
301
|
+
assets: 0,
|
|
302
|
+
totalBytes: 0,
|
|
303
|
+
orphanUploads: 0,
|
|
304
|
+
digests: [],
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
const fail = (detail: string): never => {
|
|
308
|
+
throw new Error(`${name}: asset verification failed: ${detail}`);
|
|
309
|
+
};
|
|
310
|
+
const referenced = database
|
|
311
|
+
.query(
|
|
312
|
+
"SELECT content, COUNT(*) AS count FROM entities WHERE content LIKE ? GROUP BY content ORDER BY content",
|
|
313
|
+
)
|
|
314
|
+
.all(`${ASSET_REF_PREFIX}%`)
|
|
315
|
+
.filter(isPlainRecord)
|
|
316
|
+
.map((row) => ({
|
|
317
|
+
digest: String(row["content"]).slice(ASSET_REF_PREFIX.length),
|
|
318
|
+
count: Number(row["count"]),
|
|
319
|
+
}));
|
|
320
|
+
const header = database.query(
|
|
321
|
+
"SELECT upload_id, size_bytes, chunk_count FROM assets WHERE digest = ?",
|
|
322
|
+
);
|
|
323
|
+
const chunks = database.query(
|
|
324
|
+
"SELECT ordinal, bytes FROM asset_chunks WHERE upload_id = ? ORDER BY ordinal",
|
|
325
|
+
);
|
|
326
|
+
const sizes = referenced.map(({ digest }) => {
|
|
327
|
+
const row = header.get(digest);
|
|
328
|
+
if (!isPlainRecord(row)) return fail(`${digest} has no published asset`);
|
|
329
|
+
const sizeBytes = Number(row["size_bytes"]);
|
|
330
|
+
const chunkCount = Number(row["chunk_count"]);
|
|
331
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
332
|
+
const read = { chunks: 0, bytes: 0 };
|
|
333
|
+
for (const chunk of chunks.iterate(String(row["upload_id"]))) {
|
|
334
|
+
if (!isPlainRecord(chunk) || Number(chunk["ordinal"]) !== read.chunks) {
|
|
335
|
+
return fail(`${digest} is missing chunk ${read.chunks}`);
|
|
336
|
+
}
|
|
337
|
+
const bytes = chunk["bytes"];
|
|
338
|
+
if (!(bytes instanceof Uint8Array)) {
|
|
339
|
+
return fail(`${digest} chunk ${read.chunks} is not binary`);
|
|
340
|
+
}
|
|
341
|
+
hasher.update(bytes);
|
|
342
|
+
read.chunks += 1;
|
|
343
|
+
read.bytes += bytes.byteLength;
|
|
344
|
+
}
|
|
345
|
+
if (read.chunks !== chunkCount || read.bytes !== sizeBytes) {
|
|
346
|
+
return fail(
|
|
347
|
+
`${digest} holds ${read.chunks} chunks and ${read.bytes} bytes; expected ${chunkCount} and ${sizeBytes}`,
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
if (hasher.digest("hex") !== digest) {
|
|
351
|
+
return fail(`${digest} chunks hash to a different digest`);
|
|
352
|
+
}
|
|
353
|
+
return sizeBytes;
|
|
354
|
+
});
|
|
355
|
+
const orphans = database
|
|
356
|
+
.query(
|
|
357
|
+
"SELECT COUNT(*) AS count FROM asset_uploads WHERE upload_id NOT IN (SELECT upload_id FROM assets)",
|
|
358
|
+
)
|
|
359
|
+
.get();
|
|
360
|
+
return {
|
|
361
|
+
references: referenced.reduce((sum, { count }) => sum + count, 0),
|
|
362
|
+
assets: referenced.length,
|
|
363
|
+
totalBytes: sizes.reduce((sum, size) => sum + size, 0),
|
|
364
|
+
orphanUploads: isPlainRecord(orphans) ? Number(orphans["count"]) : 0,
|
|
365
|
+
digests: referenced.map(({ digest }) => digest),
|
|
366
|
+
};
|
|
367
|
+
} finally {
|
|
368
|
+
database.close(false);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
265
372
|
function vectorDigest(path: string): {
|
|
266
373
|
counts: Record<string, number>;
|
|
267
374
|
sha256: string;
|
|
@@ -329,6 +436,16 @@ async function captureDatabases(
|
|
|
329
436
|
);
|
|
330
437
|
}
|
|
331
438
|
|
|
439
|
+
const assetReport = source.assets
|
|
440
|
+
? verifySnapshotAssets(destination, source.name)
|
|
441
|
+
: undefined;
|
|
442
|
+
if (assetReport) {
|
|
443
|
+
await Bun.write(
|
|
444
|
+
`${config.backupDir}/${source.name}.asset-verify.json`,
|
|
445
|
+
`${JSON.stringify(assetReport, null, 2)}\n`,
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
|
|
332
449
|
const file = Bun.file(destination);
|
|
333
450
|
const { quickCheck: quickCheckDriver, ...sourceMetadata } = source;
|
|
334
451
|
captures.push({
|
|
@@ -341,6 +458,14 @@ async function captureDatabases(
|
|
|
341
458
|
sha256: await sha256(destination),
|
|
342
459
|
quickCheckDriver,
|
|
343
460
|
quickCheck: check,
|
|
461
|
+
...(assetReport && {
|
|
462
|
+
assetVerification: {
|
|
463
|
+
references: assetReport.references,
|
|
464
|
+
assets: assetReport.assets,
|
|
465
|
+
totalBytes: assetReport.totalBytes,
|
|
466
|
+
orphanUploads: assetReport.orphanUploads,
|
|
467
|
+
},
|
|
468
|
+
}),
|
|
344
469
|
});
|
|
345
470
|
}
|
|
346
471
|
|
|
@@ -537,6 +662,7 @@ function captureConfigFromEnvironment(): PredeployCaptureConfig {
|
|
|
537
662
|
method: "vacuum",
|
|
538
663
|
quickCheck: "bun",
|
|
539
664
|
logicalVector: false,
|
|
665
|
+
assets: true,
|
|
540
666
|
},
|
|
541
667
|
{
|
|
542
668
|
source: "/data/brain-jobs.db",
|