@north-light/cli 0.3.552 → 0.3.553

package/dist/core/canvas/db.js

@@ -1,4 +1,4 @@
-var L=Object.defineProperty;var a=(e,t)=>L(e,"name",{value:t,configurable:!0});import{AsyncLocalStorage as x}from"node:async_hooks";import{createHash as N,randomBytes as P}from"node:crypto";import{appendFileSync as D,mkdirSync as M,readFileSync as b}from"node:fs";import{dirname as q,join as T}from"node:path";import{fileURLToPath as I}from"node:url";import p from"pg";import{resolveDatabaseUrl as B}from"../../daemon/postgres/supervisor.js";import{publishFileExclusive as F}from"../fs-utils.js";import{initializeEmptyLayout as U,runtimeRoot as d}from"../layout.js";import{translatePlaceholders as H,translateOnce as K}from"../statements.js";import{ensureHome as V}from"./paths.js";import{CANVAS_SCHEMA_VERSION as w}from"./schema-version.js";const G="crtr_test_template";class W extends Error{static{a(this,"CanvasConstraintError")}constraint;code="23505";constructor(t,n,r){super(t,r),this.name="CanvasConstraintError",this.constraint=n}}function v(e){const t=process.env.CRTR_DB_OPEN_LEDGER;if(t)try{const n=`${new Date().toISOString()} ${process.pid} ${process.argv[1]??"<unknown>"} ${e.replace(/\/\/([^:@/]*):[^@/]*@/,"//$1@")}
-`;D(t,n)}catch{}}a(v,"recordOpenLedger");const Y=4,z=/\b(?:FOR\s+(?:NO\s+KEY\s+)?(?:UPDATE|SHARE)|FOR\s+KEY\s+SHARE|INTO|nextval|setval|set_config|pg_notify|pg_advisory\w*|pg_try_advisory\w*|lo_\w+)\b/i;function J(e){return/^\s*SELECT\b/i.test(e)&&!z.test(e)}a(J,"isReadStatement");const X=1e3,h=new x;class j{static{a(this,"FifoLock")}tail=Promise.resolve();pending=0;get busy(){return this.pending>0}drained(){return this.tail}async acquire(){this.pending++;const t=this.tail;let n;return this.tail=new Promise(r=>{n=r}),await t,()=>{this.pending--,n()}}}const Z=new Set([114,3802]),C={getTypeParser:a(((e,t)=>Z.has(e)?n=>n:e===20||e===1700?n=>{const r=Number(n);if(e===20&&!Number.isSafeInteger(r))throw new Error(`canvas db: bigint ${n} does not fit a JS number`);return r}:p.types.getTypeParser(e,t)),"getTypeParser")};class Q{static{a(this,"Handle")}url;lock=new j;db;client=null;pool=null;connecting=null;schemaChecked=!1;active=0;inFlight=new Set;constructor(t){this.url=t;const n=this;this.db={prepare:a(r=>this.statement(r,!0),"prepare"),prepareUncached:a(r=>this.statement(r,!1),"prepareUncached"),exec:a(r=>this.gated(async()=>{await this.send({text:r})}),"exec"),get inTransaction(){return E(n)}}}async connected(){return this.client!==null?this.client:(this.connecting??=this.connect().finally(()=>{this.connecting=null}),this.connecting)}async connect(){v(this.url);const t=new p.Client({connectionString:this.url,types:C,application_name:`crtr-${process.pid}`,connectionTimeoutMillis:1e4});t.on("error",n=>{console.warn(`[canvas db] connection error: ${n.message}`),this.drop(t)}),t.on("end",()=>this.drop(t)),await t.connect();try{this.schemaChecked||(await rt(t),this.schemaChecked=!0)}catch(n){throw await t.end().catch(()=>{}),n}return t.connection.stream.unref(),this.client=t,t}readPool(){if(this.pool!==null)return this.pool;v(this.url);const t=new p.Pool({connectionString:this.url,types:C,application_name:`crtr-${process.pid}`,connectionTimeoutMillis:1e4,max:Y,idleTimeoutMillis:1e4,allowExitOnIdle:!0});return t.on("error",n=>console.warn(`[canvas db] read connection error: ${n.message}`)),this.pool=t}async read(t){await this.lock.drained(),this.inFlight.size>0&&await Promise.all([...this.inFlight]);try{return await this.readPool().query(t)}catch(n){throw this.mapError(n,null)}}drop(t){this.client===t&&(this.client=null)}async send(t){const n=await this.connected(),r=h.getStore();if(r!==void 0&&r.open&&r.handle===this&&r.client!==n)throw new Error("canvas db: the connection was lost during this transaction; it was rolled back");return this.sendOn(n,t)}async sendOn(t,n){const r=t.connection.stream;this.active++===0&&r.ref();const o=t.query(n),s=o.then(()=>{},()=>{});this.inFlight.add(s),s.then(()=>this.inFlight.delete(s));try{return await o}catch(i){throw this.mapError(i,t)}finally{--this.active===0&&r.unref()}}mapError(t,n){const r=t;return r.code==="23505"?new W(r.message??"unique constraint violation",r.constraint,{cause:t}):((r.code==="57P01"||r.code==="ECONNRESET"||r.code==="EPIPE"||/Connection terminated|connection error|Client was closed/i.test(r.message??""))&&n!==null&&(this.drop(n),n.end().catch(()=>{})),t)}async gated(t){if(E(this)){const r=h.getStore(),o=performance.now();try{return await t()}finally{r.statementMs+=performance.now()-o}}if(!this.lock.busy)return t();const n=await this.lock.acquire();try{return await t()}finally{n()}}statement(t,n){const r=n?(()=>{const i=H(t);return{text:i.text,name:i.name}})():{text:K(t).text},o=J(r.text),s=a(i=>{const m={...r,values:i.map(tt)};return o&&!E(this)?this.read(m):this.gated(()=>this.send(m))},"query");return{get:a(async(...i)=>(await s(i)).rows[0],"get"),all:a(async(...i)=>(await s(i)).rows,"all"),run:a(async(...i)=>({changes:(await s(i)).rowCount??0}),"run")}}async close(){const t=this.pool;this.pool=null,await t?.end().catch(()=>{}),(await this.lock.acquire())();const n=this.client;this.client=null,n!==null&&(n.connection.stream.ref(),await n.end().catch(()=>{}))}}function tt(e){return ArrayBuffer.isView(e)&&!Buffer.isBuffer(e)?Buffer.from(e.buffer,e.byteOffset,e.byteLength):e}a(tt,"bindable");function E(e){const t=h.getStore();return t!==void 0&&t.open&&t.handle===e}a(E,"ownsTransaction");const et=q(I(import.meta.url)),nt=T(et,"schema-pg.sql"),A=7427001;async function O(e,t){await e.query("INSERT INTO canvas_meta (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",[t.key,t.hash])}a(O,"recordBlock");async function R(e){if((await e.query("SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'canvas_meta'")).rowCount===0)return;const n=await e.query("SELECT value FROM canvas_meta WHERE key = 'schema_version'");return n.rows[0]===void 0?void 0:Number(n.rows[0].value)}a(R,"storedSchemaVersion");async function rt(e){await e.query("SELECT pg_advisory_lock($1)",[A]);try{const t=b(nt,"utf8"),n=[...t.matchAll(/^-- (\w+):begin\n([\s\S]*?)^-- \1:end$/gm)].map(([,o,s])=>({key:`schema_block.${o}`,sql:s,hash:N("sha256").update(s).digest("hex")}));let r=await R(e);if(r===void 0){await e.query(t),r=await R(e);for(const o of n)await O(e,o)}else if(r===w){const o=new Map((await e.query("SELECT key, value FROM canvas_meta WHERE key LIKE 'schema\\_block.%'")).rows.map(s=>[s.key,s.value]));for(const s of n)o.get(s.key)!==s.hash&&(await e.query(s.sql),await O(e,s))}if(r!==w)throw new Error(r!==void 0&&r>w?`unsupported canvas schema version ${r} (this build supports ${w}); this build is older than the database. Select a newer runtime generation.`:`canvas schema version ${String(r)} does not match this build's ${w}`)}finally{await e.query("SELECT pg_advisory_unlock($1)",[A]).catch(()=>{})}}a(rt,"ensureSchema");function at(){return process.env.NODE_TEST_CONTEXT!==void 0||process.execArgv.includes("--test")}a(at,"runningUnderNodeTest");function y(e,t){const n=/^(postgres(?:ql)?:\/\/[^/?#]*)(\/[^?#]*)?(.*)$/s.exec(e);if(n===null)throw new Error(`not a postgres URL: ${e.replace(/\/\/[^@/]*@/,"//")}`);return`${n[1]}/${t}${n[3]}`}a(y,"withDatabase");async function st(e){const t=T(d(),"test-db"),n=a(()=>{try{return b(t,"utf8").trim()||void 0}catch(i){if(i.code==="ENOENT")return;throw i}},"recorded"),r=n();if(r!==void 0)return y(e,r);M(d(),{recursive:!0});const o=`t_${P(8).toString("hex")}`,s=new p.Client({connectionString:y(e,"postgres")});await s.connect();try{if(await s.query(`CREATE DATABASE ${o} TEMPLATE ${G} STRATEGY file_copy`),!F(t,`${o}
-`,384))return await s.query(`DROP DATABASE ${o}`),y(e,n())}finally{await s.end()}return y(e,o)}a(st,"testDatabaseUrl");async function ot(){const e=process.env.CRTR_TEST_PG_URL;if(e)return st(e);if(at())throw new Error("CRTR_TEST_PG_URL is not set: a test must reach its own Postgres (run the suite through the lane runner); refusing to open a real canvas database");return B(d())}a(ot,"canvasDatabaseUrl");const f=new Map,g=new Map;async function k(){U(),V();const e=await ot();g.set(d(),e);let t=f.get(e);return t===void 0&&f.set(e,t=new Q(e)),await t.connected(),t}a(k,"openHandle");async function _t(){return(await k()).db}a(_t,"openDb");async function bt(){const e=g.get(d());if(e===void 0)return;g.delete(d());const t=f.get(e);t!==void 0&&(f.delete(e),await t.close())}a(bt,"closeDb");async function Tt(){const e=[...f.values()];f.clear(),g.clear(),await Promise.all(e.map(t=>t.close()))}a(Tt,"closeAllDbs");function vt(e){const t=h.getStore();if(t===void 0||!t.open){queueMicrotask(e);return}t.afterCommit.push(e)}a(vt,"afterCommit");let it=0;async function Ct(e,t={}){const n=await k(),r=n.db;if(E(n)){if(!t.savepoint&&!t.rollback)return e(r);const c=`canvas_write_${++it}`;await n.send({text:`SAVEPOINT ${c}`});const l=h.getStore()?.afterCommit,S=l?.length??0;let _;try{_=await e(r)}catch($){throw await n.send({text:`ROLLBACK TO ${c}; RELEASE ${c}`}),l?.splice(S),$}return await n.send({text:t.rollback?`ROLLBACK TO ${c}; RELEASE ${c}`:`RELEASE ${c}`}),t.rollback&&l?.splice(S),_}const o=await n.lock.acquire(),s=performance.now(),i=new Error("canvas write transaction entered here");let m=!1;const u={handle:n,client:void 0,open:!0,afterCommit:[],statementMs:0};try{const c=await n.connected();await n.sendOn(c,{text:"BEGIN"}),u.client=c;try{const l=await h.run(u,()=>e(r));return await n.sendOn(c,{text:t.rollback?"ROLLBACK":"COMMIT"}),m=!t.rollback,l}catch(l){try{await n.sendOn(c,{text:"ROLLBACK"})}catch{}throw l}}finally{u.open=!1;const c=performance.now()-s;if(c-u.statementMs>X&&console.warn(`[canvas db] write transaction held ${Math.round(c)}ms (${Math.round(u.statementMs)}ms in its statements); its body did non-db work while every other db caller waited
-${i.stack}`),o(),m)for(const l of u.afterCommit)queueMicrotask(l)}}a(Ct,"withCanvasWrite");export{W as CanvasConstraintError,vt as afterCommit,ot as canvasDatabaseUrl,Tt as closeAllDbs,bt as closeDb,J as isReadStatement,_t as openDb,Ct as withCanvasWrite};
+var C=Object.defineProperty;var a=(n,t)=>C(n,"name",{value:t,configurable:!0});import{AsyncLocalStorage as R}from"node:async_hooks";import{randomBytes as x}from"node:crypto";import{appendFileSync as $,mkdirSync as P,readFileSync as k}from"node:fs";import{join as L}from"node:path";import m from"pg";import{resolveDatabaseUrl as D}from"../../daemon/postgres/supervisor.js";import{publishFileExclusive as M}from"../fs-utils.js";import{initializeEmptyLayout as N,runtimeRoot as h}from"../layout.js";import{translatePlaceholders as B,translateOnce as q}from"../statements.js";import{ensureHome as U}from"./paths.js";import{emitEvent as F}from"../events/emit.js";import{ensureSchema as T}from"./schema-chain.js";import{canvasDatabaseSchema as I,canvasSchemaDatabaseUrl as K,canvasSearchPathOptions as b}from"./schema-version.js";const G="crtr_test_template";class H extends Error{static{a(this,"CanvasConstraintError")}constraint;code="23505";constructor(t,e,r){super(t,r),this.name="CanvasConstraintError",this.constraint=e}}function _(n){const t=process.env.CRTR_DB_OPEN_LEDGER;if(t)try{const e=`${new Date().toISOString()} ${process.pid} ${process.argv[1]??"<unknown>"} ${n.replace(/\/\/([^:@/]*):[^@/]*@/,"//$1@")}
+`;$(t,e)}catch{}}a(_,"recordOpenLedger");const Y=4,z=/\b(?:FOR\s+(?:NO\s+KEY\s+)?(?:UPDATE|SHARE)|FOR\s+KEY\s+SHARE|INTO|nextval|setval|set_config|pg_notify|pg_advisory\w*|pg_try_advisory\w*|lo_\w+)\b/i;function J(n){return/^\s*SELECT\b/i.test(n)&&!z.test(n)}a(J,"isReadStatement");const V=1e3,f=new R;class W{static{a(this,"FifoLock")}tail=Promise.resolve();pending=0;get busy(){return this.pending>0}drained(){return this.tail}async acquire(){this.pending++;const t=this.tail;let e;return this.tail=new Promise(r=>{e=r}),await t,()=>{this.pending--,e()}}}const j=new Set([114,3802]),O={getTypeParser:a(((n,t)=>j.has(n)?e=>e:n===20||n===1700?e=>{const r=Number(e);if(n===20&&!Number.isSafeInteger(r))throw new Error(`canvas db: bigint ${e} does not fit a JS number`);return r}:m.types.getTypeParser(n,t)),"getTypeParser")};class X{static{a(this,"Handle")}url;lock=new W;db;client=null;pool=null;connecting=null;schemaChecked=!1;active=0;inFlight=new Set;constructor(t){this.url=t;const e=this;this.db={prepare:a(r=>this.statement(r,!0),"prepare"),prepareUncached:a(r=>this.statement(r,!1),"prepareUncached"),exec:a(r=>this.gated(async()=>{await this.send({text:r})}),"exec"),get inTransaction(){return w(e)}}}async connected(){return this.client!==null?this.client:(this.connecting??=this.connect().finally(()=>{this.connecting=null}),this.connecting)}async connect(){_(this.url);const t=new m.Client({connectionString:this.url,types:O,application_name:`crtr-${process.pid}`,connectionTimeoutMillis:1e4,options:b()});t.on("error",e=>{console.warn(`[canvas db] connection error: ${e.message}`),this.drop(t)}),t.on("end",()=>this.drop(t)),await t.connect();try{this.schemaChecked||(await Q(t),this.schemaChecked=!0)}catch(e){throw await t.end().catch(()=>{}),e}return t.connection.stream.unref(),this.client=t,t}readPool(){if(this.pool!==null)return this.pool;_(this.url);const t=new m.Pool({connectionString:this.url,types:O,application_name:`crtr-${process.pid}`,connectionTimeoutMillis:1e4,options:b(),max:Y,idleTimeoutMillis:1e4,allowExitOnIdle:!0});return t.on("error",e=>console.warn(`[canvas db] read connection error: ${e.message}`)),this.pool=t}async read(t){await this.lock.drained(),this.inFlight.size>0&&await Promise.all([...this.inFlight]);try{return await this.readPool().query(t)}catch(e){throw this.mapError(e,null)}}drop(t){this.client===t&&(this.client=null)}async send(t){const e=await this.connected(),r=f.getStore();if(r!==void 0&&r.open&&r.handle===this&&r.client!==e)throw new Error("canvas db: the connection was lost during this transaction; it was rolled back");return this.sendOn(e,t)}async sendOn(t,e){const r=t.connection.stream;this.active++===0&&r.ref();const o=t.query(e),i=o.then(()=>{},()=>{});this.inFlight.add(i),i.then(()=>this.inFlight.delete(i));try{return await o}catch(s){throw this.mapError(s,t)}finally{--this.active===0&&r.unref()}}mapError(t,e){const r=t;return r.code==="23505"?new H(r.message??"unique constraint violation",r.constraint,{cause:t}):((r.code==="57P01"||r.code==="ECONNRESET"||r.code==="EPIPE"||/Connection terminated|connection error|Client was closed/i.test(r.message??""))&&e!==null&&(this.drop(e),e.end().catch(()=>{})),t)}async gated(t){if(w(this)){const r=f.getStore(),o=performance.now();try{return await t()}finally{r.statementMs+=performance.now()-o}}if(!this.lock.busy)return t();const e=await this.lock.acquire();try{return await t()}finally{e()}}statement(t,e){const r=e?(()=>{const s=B(t);return{text:s.text,name:s.name}})():{text:q(t).text},o=J(r.text),i=a(s=>{const u={...r,values:s.map(Z)};return o&&!w(this)?this.read(u):this.gated(()=>this.send(u))},"query");return{get:a(async(...s)=>(await i(s)).rows[0],"get"),all:a(async(...s)=>(await i(s)).rows,"all"),run:a(async(...s)=>({changes:(await i(s)).rowCount??0}),"run")}}async close(){const t=this.pool;this.pool=null,await t?.end().catch(()=>{}),(await this.lock.acquire())();const e=this.client;this.client=null,e!==null&&(e.connection.stream.ref(),await e.end().catch(()=>{}))}}function Z(n){return ArrayBuffer.isView(n)&&!Buffer.isBuffer(n)?Buffer.from(n.buffer,n.byteOffset,n.byteLength):n}a(Z,"bindable");function w(n){const t=f.getStore();return t!==void 0&&t.open&&t.handle===n}a(w,"ownsTransaction");async function Q(n){const t=a((i,s,u)=>{F({level:i,event:s,fields:u})},"emit"),e=I(),r=K();if(r===void 0){await T(n,{schema:e,emit:t});return}const o=new m.Client({connectionString:r,application_name:`crtr-schema-${process.pid}`,connectionTimeoutMillis:1e4});await o.connect();try{await T(o,{schema:e,emit:t})}finally{await o.end().catch(()=>{})}}a(Q,"applySchema");function tt(){return process.env.NODE_TEST_CONTEXT!==void 0||process.execArgv.includes("--test")}a(tt,"runningUnderNodeTest");function y(n,t){const e=/^(postgres(?:ql)?:\/\/[^/?#]*)(\/[^?#]*)?(.*)$/s.exec(n);if(e===null)throw new Error(`not a postgres URL: ${n.replace(/\/\/[^@/]*@/,"//")}`);return`${e[1]}/${t}${e[3]}`}a(y,"withDatabase");async function et(n){const t=L(h(),"test-db"),e=a(()=>{try{return k(t,"utf8").trim()||void 0}catch(s){if(s.code==="ENOENT")return;throw s}},"recorded"),r=e();if(r!==void 0)return y(n,r);P(h(),{recursive:!0});const o=`t_${x(8).toString("hex")}`,i=new m.Client({connectionString:y(n,"postgres")});await i.connect();try{if(await i.query(`CREATE DATABASE ${o} TEMPLATE ${G} STRATEGY file_copy`),!M(t,`${o}
+`,384))return await i.query(`DROP DATABASE ${o}`),y(n,e())}finally{await i.end()}return y(n,o)}a(et,"testDatabaseUrl");async function nt(){const n=process.env.CRTR_TEST_PG_URL;if(n)return et(n);if(tt())throw new Error("CRTR_TEST_PG_URL is not set: a test must reach its own Postgres (run the suite through the lane runner); refusing to open a real canvas database");return D(h())}a(nt,"canvasDatabaseUrl");const p=new Map,g=new Map;async function v(){N(),U();const n=await nt();g.set(h(),n);let t=p.get(n);return t===void 0&&p.set(n,t=new X(n)),await t.connected(),t}a(v,"openHandle");async function gt(){return(await v()).db}a(gt,"openDb");async function Et(){const n=g.get(h());if(n===void 0)return;g.delete(h());const t=p.get(n);t!==void 0&&(p.delete(n),await t.close())}a(Et,"closeDb");async function St(){const n=[...p.values()];p.clear(),g.clear(),await Promise.all(n.map(t=>t.close()))}a(St,"closeAllDbs");function Tt(n){const t=f.getStore();if(t===void 0||!t.open){queueMicrotask(n);return}t.afterCommit.push(n)}a(Tt,"afterCommit");let rt=0;async function bt(n,t={}){const e=await v(),r=e.db;if(w(e)){if(!t.savepoint&&!t.rollback)return n(r);const c=`canvas_write_${++rt}`;await e.send({text:`SAVEPOINT ${c}`});const l=f.getStore()?.afterCommit,E=l?.length??0;let S;try{S=await n(r)}catch(A){throw await e.send({text:`ROLLBACK TO ${c}; RELEASE ${c}`}),l?.splice(E),A}return await e.send({text:t.rollback?`ROLLBACK TO ${c}; RELEASE ${c}`:`RELEASE ${c}`}),t.rollback&&l?.splice(E),S}const o=await e.lock.acquire(),i=performance.now(),s=new Error("canvas write transaction entered here");let u=!1;const d={handle:e,client:void 0,open:!0,afterCommit:[],statementMs:0};try{const c=await e.connected();await e.sendOn(c,{text:"BEGIN"}),d.client=c;try{const l=await f.run(d,()=>n(r));return await e.sendOn(c,{text:t.rollback?"ROLLBACK":"COMMIT"}),u=!t.rollback,l}catch(l){try{await e.sendOn(c,{text:"ROLLBACK"})}catch{}throw l}}finally{d.open=!1;const c=performance.now()-i;if(c-d.statementMs>V&&console.warn(`[canvas db] write transaction held ${Math.round(c)}ms (${Math.round(d.statementMs)}ms in its statements); its body did non-db work while every other db caller waited
+${s.stack}`),o(),u)for(const l of d.afterCommit)queueMicrotask(l)}}a(bt,"withCanvasWrite");export{H as CanvasConstraintError,Tt as afterCommit,nt as canvasDatabaseUrl,St as closeAllDbs,Et as closeDb,J as isReadStatement,gt as openDb,bt as withCanvasWrite};

package/dist/core/canvas/export-sqlite.js

@@ -1,3 +1,3 @@
-var V=Object.defineProperty;var l=(i,a)=>V(i,"name",{value:a,configurable:!0});import{DatabaseSync as W}from"node:sqlite";import{existsSync as Y,mkdirSync as k,mkdtempSync as J,renameSync as X,rmSync as P}from"node:fs";import{tmpdir as Q}from"node:os";import{join as A}from"node:path";import j from"pg";import{migrate as z,MIGRATIONS as I}from"./migrations.js";import{canvasDbPath as Z}from"./paths.js";import{EXPORT_MARK_KEY as F,IMPORT_MARK_KEY as ee,OBJECT_COLUMNS as te,pgColumns as ne,quote as c,TABLE_ORDER as C}from"./import-sqlite.js";const re=5e3,se=new Set(["environments","relations","schemas","charges","charge_totals","limits"]),oe={exposures:"node, object, channel",rules:"object, position",edges:"type, src, dst",unresolved_links:"src, name",revision_parents:"child, parent",canvas_meta:"key",runtime_state:"key",message_causes:"message_id",peer_ingested:"key",peer_ingested_destinations:"key",message_sequence:"id",messages:"seq",review_comment_events:"seq",audit_rows:"id",peer_outbox:"id",app_users:"uid"},ae={objects:"id",revisions:"rev_id",watches:"watch_id"},H=new Set(["daemon_owner","schema_version","ledger_migrated",ee,F]);function ie(i){const a=J(A(Q(),"crtr-export-")),r=process.env.CRTR_HOME;process.env.CRTR_HOME=a,k(A(a,"runtime","nodes"),{recursive:!0});try{const s=new W(i);return s.exec("PRAGMA journal_mode = DELETE;"),s.exec("PRAGMA foreign_keys = ON;"),z(s),s.exec("DELETE FROM canvas_meta; DELETE FROM message_sequence;"),s}finally{r===void 0?delete process.env.CRTR_HOME:process.env.CRTR_HOME=r,P(a,{recursive:!0,force:!0})}}l(ie,"createEmptyDatabase");async function _e(i,a=()=>{}){const r=Z();if(Y(r))throw new Error(`${r} already exists; refusing to overwrite it`);k(A(r,".."),{recursive:!0,mode:448});const s=`${r}.exporting-${process.pid}`,m=Date.now(),t=new j.Client({connectionString:i,application_name:`crtr-export-${process.pid}`,types:{getTypeParser:l(((f,o)=>f===20?d=>Number(d):j.types.getTypeParser(f,o)),"getTypeParser")}});await t.connect();let u;try{if((await t.query("SELECT value FROM canvas_meta WHERE key = 'schema_version'").catch(()=>({rows:[]}))).rows.length===0)throw new Error("the Postgres database has no canvas schema; nothing to export");u=ie(s);const o=u;o.exec("PRAGMA foreign_keys = OFF;");const d=await ne(t),L=[...d.keys()].filter(e=>!se.has(e)&&!C.includes(e));if(L.length>0)throw new Error(`export: Postgres tables with no export rule: ${L.join(", ")}`);await t.query("BEGIN READ ONLY ISOLATION LEVEL REPEATABLE READ");const h={};for(const[e,O]of Object.entries(ae)){const _=new Map;for(const E of(await t.query(`SELECT ${O} AS id, seq FROM ${e}`)).rows)_.set(E.id,Number(E.seq));h[e]=_}const B=h.objects;let M=0;o.exec("BEGIN");for(const e of C){const O=Date.now(),_=new Set(o.prepare(`PRAGMA table_info(${c(e)})`).all().map(n=>n.name)),E=d.get(e);if(E===void 0)throw new Error(`export: Postgres has no table ${e}`);const G=E.some(n=>n.name==="seq"),$=E.filter(n=>_.has(n.name)),K=new Set(te[e]??[]),R=e==="objects"||e==="revisions"||e==="watches"||e==="watch_subscriptions",v=$.map(n=>n.type==="jsonb"?`${c(n.name)}::text AS ${c(n.name)}`:c(n.name));e==="watches"&&v.push("watch_id AS _watch_id"),R&&v.push("seq AS _seq");const g=$.map(n=>c(n.name)),U=o.prepare(`INSERT INTO ${c(e)} (${R?["rowid",...g]:g}) VALUES (${(R?["?",...g]:g).map(()=>"?").join(", ")})`),q=G?"seq":oe[e],T=`export_${e}`;await t.query(`DECLARE ${T} NO SCROLL CURSOR FOR SELECT ${v.join(", ")} FROM ${c(e)}${e==="canvas_meta"?` WHERE key NOT IN (${[...H].map(n=>`'${n}'`).join(", ")})`:""}${q===void 0?"":` ORDER BY ${q}`}`);let b=0;for(;;){const n=await t.query(`FETCH ${re} FROM ${T}`);if(n.rows.length===0)break;for(const N of n.rows){const D=R?[Number(N._seq)]:[];for(const p of $){let w=N[p.name];e==="revision_parents"&&(p.name==="child"||p.name==="parent")?w=x(h.revisions,"revision",e,p.name,w):e==="watch_subscriptions"&&p.name==="watch"?w=x(h.watches,"watch",e,p.name,w):K.has(p.name)&&(w=x(B,"object",e,p.name,w)),D.push(w??null)}U.run(...D)}b+=n.rows.length}await t.query(`CLOSE ${T}`),M+=b,a(`export ${e}: ${b} rows in ${Date.now()-O} ms`)}o.exec("COMMIT"),await t.query("COMMIT");const y=o.prepare("PRAGMA foreign_key_check").all();if(y.length>0)throw new Error(`export: ${y.length} foreign key violation(s) in the exported canvas.db, first: ${JSON.stringify(y[0])}`);await ce(t,o);const S=o.prepare("PRAGMA user_version").get().user_version;if(S!==I.length)throw new Error(`export: exported canvas.db is at user_version ${S}, expected ${I.length}`);return o.close(),u=void 0,await t.query("INSERT INTO canvas_meta (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",[F,JSON.stringify({at:new Date().toISOString(),path:r})]),X(s,r),{path:r,rows:M,ms:Date.now()-m,userVersion:S}}catch(f){await t.query("ROLLBACK").catch(()=>{});try{u?.close()}catch{}for(const o of["","-wal","-shm","-journal"])P(`${s}${o}`,{force:!0});throw f}finally{await t.end().catch(()=>{})}}l(_e,"exportPostgresToSqlite");function x(i,a,r,s,m){if(m==null)return null;const t=i.get(m);if(t===void 0)throw new Error(`export: ${r}.${s} references ${a} ${String(m)}, which was not exported (`);return t}l(x,"mapId");async function ce(i,a){const r=[];for(const s of C){const m=Number((await i.query(`SELECT count(*) AS n FROM ${c(s)}`)).rows[0].n);let t=Number(a.prepare(`SELECT count(*) AS n FROM ${c(s)}`).get().n),u=m;s==="canvas_meta"&&(u=Number((await i.query("SELECT count(*) AS n FROM canvas_meta WHERE key <> ALL($1)",[[...H]])).rows[0].n),t=Number(a.prepare("SELECT count(*) AS n FROM canvas_meta").get().n)),u!==t&&r.push(`${s}: Postgres ${u} rows, SQLite ${t}`)}if(r.length>0)throw new Error(`Postgres \u2192 SQLite export verification failed:
+var V=Object.defineProperty;var l=(i,a)=>V(i,"name",{value:a,configurable:!0});import{DatabaseSync as W}from"node:sqlite";import{existsSync as Y,mkdirSync as P,mkdtempSync as J,renameSync as X,rmSync as k}from"node:fs";import{tmpdir as Q}from"node:os";import{join as A}from"node:path";import j from"pg";import{migrate as z,MIGRATIONS as I}from"./migrations.js";import{canvasDbPath as Z}from"./paths.js";import{canvasSearchPathOptions as ee}from"./schema-version.js";import{EXPORT_MARK_KEY as F,IMPORT_MARK_KEY as te,OBJECT_COLUMNS as ne,pgColumns as re,quote as c,TABLE_ORDER as C}from"./import-sqlite.js";const se=5e3,oe=new Set(["environments","relations","schemas","charges","charge_totals","limits"]),ae={exposures:"node, object, channel",rules:"object, position",edges:"type, src, dst",unresolved_links:"src, name",revision_parents:"child, parent",canvas_meta:"key",runtime_state:"key",message_causes:"message_id",peer_ingested:"key",peer_ingested_destinations:"key",message_sequence:"id",messages:"seq",review_comment_events:"seq",audit_rows:"id",peer_outbox:"id",app_users:"uid"},ie={objects:"id",revisions:"rev_id",watches:"watch_id"},H=new Set(["daemon_owner","schema_version","schema_min_reader","ledger_migrated",te,F]);function ce(i){const a=J(A(Q(),"crtr-export-")),r=process.env.CRTR_HOME;process.env.CRTR_HOME=a,P(A(a,"runtime","nodes"),{recursive:!0});try{const s=new W(i);return s.exec("PRAGMA journal_mode = DELETE;"),s.exec("PRAGMA foreign_keys = ON;"),z(s),s.exec("DELETE FROM canvas_meta; DELETE FROM message_sequence;"),s}finally{r===void 0?delete process.env.CRTR_HOME:process.env.CRTR_HOME=r,k(a,{recursive:!0,force:!0})}}l(ce,"createEmptyDatabase");async function ge(i,a=()=>{}){const r=Z();if(Y(r))throw new Error(`${r} already exists; refusing to overwrite it`);P(A(r,".."),{recursive:!0,mode:448});const s=`${r}.exporting-${process.pid}`,m=Date.now(),t=new j.Client({connectionString:i,application_name:`crtr-export-${process.pid}`,options:ee(),types:{getTypeParser:l(((f,o)=>f===20?d=>Number(d):j.types.getTypeParser(f,o)),"getTypeParser")}});await t.connect();let p;try{if((await t.query("SELECT value FROM canvas_meta WHERE key = 'schema_version'").catch(()=>({rows:[]}))).rows.length===0)throw new Error("the Postgres database has no canvas schema; nothing to export");p=ce(s);const o=p;o.exec("PRAGMA foreign_keys = OFF;");const d=await re(t),L=[...d.keys()].filter(e=>!oe.has(e)&&!C.includes(e));if(L.length>0)throw new Error(`export: Postgres tables with no export rule: ${L.join(", ")}`);await t.query("BEGIN READ ONLY ISOLATION LEVEL REPEATABLE READ");const h={};for(const[e,O]of Object.entries(ie)){const _=new Map;for(const E of(await t.query(`SELECT ${O} AS id, seq FROM ${e}`)).rows)_.set(E.id,Number(E.seq));h[e]=_}const B=h.objects;let M=0;o.exec("BEGIN");for(const e of C){const O=Date.now(),_=new Set(o.prepare(`PRAGMA table_info(${c(e)})`).all().map(n=>n.name)),E=d.get(e);if(E===void 0)throw new Error(`export: Postgres has no table ${e}`);const G=E.some(n=>n.name==="seq"),$=E.filter(n=>_.has(n.name)),K=new Set(ne[e]??[]),R=e==="objects"||e==="revisions"||e==="watches"||e==="watch_subscriptions",v=$.map(n=>n.type==="jsonb"?`${c(n.name)}::text AS ${c(n.name)}`:c(n.name));e==="watches"&&v.push("watch_id AS _watch_id"),R&&v.push("seq AS _seq");const g=$.map(n=>c(n.name)),U=o.prepare(`INSERT INTO ${c(e)} (${R?["rowid",...g]:g}) VALUES (${(R?["?",...g]:g).map(()=>"?").join(", ")})`),q=G?"seq":ae[e],T=`export_${e}`;await t.query(`DECLARE ${T} NO SCROLL CURSOR FOR SELECT ${v.join(", ")} FROM ${c(e)}${e==="canvas_meta"?` WHERE key NOT IN (${[...H].map(n=>`'${n}'`).join(", ")})`:""}${q===void 0?"":` ORDER BY ${q}`}`);let b=0;for(;;){const n=await t.query(`FETCH ${se} FROM ${T}`);if(n.rows.length===0)break;for(const N of n.rows){const D=R?[Number(N._seq)]:[];for(const u of $){let w=N[u.name];e==="revision_parents"&&(u.name==="child"||u.name==="parent")?w=x(h.revisions,"revision",e,u.name,w):e==="watch_subscriptions"&&u.name==="watch"?w=x(h.watches,"watch",e,u.name,w):K.has(u.name)&&(w=x(B,"object",e,u.name,w)),D.push(w??null)}U.run(...D)}b+=n.rows.length}await t.query(`CLOSE ${T}`),M+=b,a(`export ${e}: ${b} rows in ${Date.now()-O} ms`)}o.exec("COMMIT"),await t.query("COMMIT");const y=o.prepare("PRAGMA foreign_key_check").all();if(y.length>0)throw new Error(`export: ${y.length} foreign key violation(s) in the exported canvas.db, first: ${JSON.stringify(y[0])}`);await me(t,o);const S=o.prepare("PRAGMA user_version").get().user_version;if(S!==I.length)throw new Error(`export: exported canvas.db is at user_version ${S}, expected ${I.length}`);return o.close(),p=void 0,await t.query("INSERT INTO canvas_meta (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",[F,JSON.stringify({at:new Date().toISOString(),path:r})]),X(s,r),{path:r,rows:M,ms:Date.now()-m,userVersion:S}}catch(f){await t.query("ROLLBACK").catch(()=>{});try{p?.close()}catch{}for(const o of["","-wal","-shm","-journal"])k(`${s}${o}`,{force:!0});throw f}finally{await t.end().catch(()=>{})}}l(ge,"exportPostgresToSqlite");function x(i,a,r,s,m){if(m==null)return null;const t=i.get(m);if(t===void 0)throw new Error(`export: ${r}.${s} references ${a} ${String(m)}, which was not exported (`);return t}l(x,"mapId");async function me(i,a){const r=[];for(const s of C){const m=Number((await i.query(`SELECT count(*) AS n FROM ${c(s)}`)).rows[0].n);let t=Number(a.prepare(`SELECT count(*) AS n FROM ${c(s)}`).get().n),p=m;s==="canvas_meta"&&(p=Number((await i.query("SELECT count(*) AS n FROM canvas_meta WHERE key <> ALL($1)",[[...H]])).rows[0].n),t=Number(a.prepare("SELECT count(*) AS n FROM canvas_meta").get().n)),p!==t&&r.push(`${s}: Postgres ${p} rows, SQLite ${t}`)}if(r.length>0)throw new Error(`Postgres \u2192 SQLite export verification failed:
   ${r.join(`
-  `)}`)}l(ce,"verify");export{_e as exportPostgresToSqlite};
+  `)}`)}l(me,"verify");export{ge as exportPostgresToSqlite};

package/dist/core/canvas/import-sqlite.d.ts

@@ -8,10 +8,12 @@ export declare const EXPORT_MARK_KEY = "exported_to_sqlite";
 export declare const TABLE_ORDER: readonly ["objects", "documents", "revisions", "revision_parents", "jobs", "identities", "custom_objects", "context_sessions", "peer_stubs", "object_kinds", "watches", "watch_subscriptions", "exposures", "rules", "edges", "unresolved_links", "nodes", "focuses", "canvas_meta", "runtime_state", "warm_pool", "sessions", "executions", "worktrees", "node_faults", "node_tokens", "node_outcome_deliveries", "deleted_isolated_nodes", "create_requests", "waits", "messages", "message_sequence", "message_causes", "deliveries", "structured_requests", "daemon_actions", "human_requests", "human_action_deliveries", "review_comments", "review_comment_events", "run_events", "idempotency_keys", "grants", "memory_reads", "app_users", "uploads", "upload_uses", "shares", "audit_rows", "grantee_usage", "provider_receipts", "provider_approvals", "usage", "crons", "cron_runs", "peer_outbox", "peer_locations", "peer_ingested", "peer_ingested_destinations"];
 /** Columns holding `objects.rowid` in SQLite and `objects.id` in Postgres. */
 export declare const OBJECT_COLUMNS: Record<string, readonly string[]>;
+/** `optional`: the column may be left out of an INSERT (nullable, or it has a default or is an identity). */
 export interface PgColumn {
     name: string;
     type: string;
     identity: boolean;
+    optional: boolean;
 }
 export interface ImportTableStat {
     table: string;

package/dist/core/canvas/import-sqlite.js

@@ -1,7 +1,7 @@
-var Q=Object.defineProperty;var m=(t,o)=>Q(t,"name",{value:o,configurable:!0});import{DatabaseSync as X}from"node:sqlite";import{createHash as z,randomBytes as Z}from"node:crypto";import{existsSync as O,mkdirSync as ee,renameSync as F}from"node:fs";import{dirname as te}from"node:path";import oe from"pg";import{withExclusiveDirectoryLockAsync as se}from"../exclusive-lock.js";import{DAEMON_STARTUP_WINDOW_MS as re}from"../../daemon/startup-policy.js";import{migrate as ne}from"./migrations.js";import{readLegacyPrivacy as ae}from"../graph/types.js";import{canvasDbPath as B,onDiskMigrationLockPath as ie}from"./paths.js";const q="imported_from_sqlite",U="exported_to_sqlite",M=1e4,L=["objects","documents","revisions","revision_parents","jobs","identities","custom_objects","context_sessions","peer_stubs","object_kinds","watches","watch_subscriptions","exposures","rules","edges","unresolved_links","nodes","focuses","canvas_meta","runtime_state","warm_pool","sessions","executions","worktrees","node_faults","node_tokens","node_outcome_deliveries","deleted_isolated_nodes","create_requests","waits","messages","message_sequence","message_causes","deliveries","structured_requests","daemon_actions","human_requests","human_action_deliveries","review_comments","review_comment_events","run_events","idempotency_keys","grants","memory_reads","app_users","uploads","upload_uses","shares","audit_rows","grantee_usage","provider_receipts","provider_approvals","usage","crons","cron_runs","peer_outbox","peer_locations","peer_ingested","peer_ingested_destinations"],ce=new Set(["sqlite_sequence","sqlite_stat1","sqlite_stat4","reviews"]),ue={objects:["owner"],documents:["object"],revisions:["document","author"],jobs:["object"],identities:["object"],custom_objects:["object"],context_sessions:["node"],peer_stubs:["object"],object_kinds:["owner"],watches:["watcher","object","link_app"],exposures:["node","object"],rules:["object","owner"],edges:["src","dst"],unresolved_links:["src"]},pe={objects:["schema_type"],nodes:["app","caused_by"],revisions:["acting_app","acting_for"]},W=new Set(["daemon_owner","schema_version","ledger_migrated"]),de={canvas_meta:"key",message_sequence:"id"};function H(){return Z(16).toString("hex")}m(H,"newId");function f(t){return`"${t.replace(/"/g,'""')}"`}m(f,"quote");function me(t){switch(t){case"bigint":return"bigint[]";case"double precision":return"float8[]";case"bytea":return"bytea[]";case"text":case"jsonb":return"text[]";default:throw new Error(`import: no array type for Postgres column type ${t}`)}}m(me,"arrayType");async function fe(t){const o=await t.query(`SELECT c.relname AS tbl, a.attname AS col, format_type(a.atttypid, a.atttypmod) AS type, a.attidentity <> '' AS identity
+var X=Object.defineProperty;var p=(t,o)=>X(t,"name",{value:o,configurable:!0});import{DatabaseSync as z}from"node:sqlite";import{createHash as Z,randomBytes as ee}from"node:crypto";import{existsSync as O,mkdirSync as te,renameSync as F}from"node:fs";import{dirname as oe}from"node:path";import se from"pg";import{withExclusiveDirectoryLockAsync as re}from"../exclusive-lock.js";import{DAEMON_STARTUP_WINDOW_MS as ne}from"../../daemon/startup-policy.js";import{migrate as ae}from"./migrations.js";import{readLegacyPrivacy as ie}from"../graph/types.js";import{canvasDbPath as B,onDiskMigrationLockPath as ce}from"./paths.js";import{canvasSearchPathOptions as ue}from"./schema-version.js";const C="imported_from_sqlite",U="exported_to_sqlite",M=1e4,L=["objects","documents","revisions","revision_parents","jobs","identities","custom_objects","context_sessions","peer_stubs","object_kinds","watches","watch_subscriptions","exposures","rules","edges","unresolved_links","nodes","focuses","canvas_meta","runtime_state","warm_pool","sessions","executions","worktrees","node_faults","node_tokens","node_outcome_deliveries","deleted_isolated_nodes","create_requests","waits","messages","message_sequence","message_causes","deliveries","structured_requests","daemon_actions","human_requests","human_action_deliveries","review_comments","review_comment_events","run_events","idempotency_keys","grants","memory_reads","app_users","uploads","upload_uses","shares","audit_rows","grantee_usage","provider_receipts","provider_approvals","usage","crons","cron_runs","peer_outbox","peer_locations","peer_ingested","peer_ingested_destinations"],pe=new Set(["sqlite_sequence","sqlite_stat1","sqlite_stat4","reviews"]),de={objects:["owner"],documents:["object"],revisions:["document","author"],jobs:["object"],identities:["object"],custom_objects:["object"],context_sessions:["node"],peer_stubs:["object"],object_kinds:["owner"],watches:["watcher","object","link_app"],exposures:["node","object"],rules:["object","owner"],edges:["src","dst"],unresolved_links:["src"]},W=new Set(["daemon_owner","schema_version","schema_min_reader","ledger_migrated"]),me={canvas_meta:"key",message_sequence:"id"};function H(){return ee(16).toString("hex")}p(H,"newId");function f(t){return`"${t.replace(/"/g,'""')}"`}p(f,"quote");function fe(t){switch(t){case"bigint":return"bigint[]";case"double precision":return"float8[]";case"bytea":return"bytea[]";case"text":case"jsonb":return"text[]";default:throw new Error(`import: no array type for Postgres column type ${t}`)}}p(fe,"arrayType");async function we(t){const o=await t.query(`SELECT c.relname AS tbl, a.attname AS col, format_type(a.atttypid, a.atttypmod) AS type, a.attidentity <> '' AS identity, (NOT a.attnotnull OR a.atthasdef OR a.attidentity <> '') AS optional
        FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid
       WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped
-      ORDER BY c.relname, a.attnum`),e=new Map;for(const a of o.rows){const r=e.get(a.tbl)??[];r.push({name:a.col,type:a.type,identity:a.identity}),e.set(a.tbl,r)}return e}m(fe,"pgColumns");const we="\uFFFD";function Ee(t){if(!t.includes("\\u0000"))return{text:t,replaced:0};let o;try{o=JSON.parse(t)}catch{return{text:t,replaced:0}}let e=0;const a=m(i=>{const u=i.split("\0");return e+=u.length-1,u.join(we)},"clean"),r=m(i=>typeof i=="string"?a(i):Array.isArray(i)?i.map(r):i!==null&&typeof i=="object"?Object.fromEntries(Object.entries(i).map(([u,l])=>[a(u),r(l)])):i,"walk"),E=r(o);return e===0?{text:t,replaced:e}:{text:JSON.stringify(E),replaced:e}}m(Ee,"scrubNul");async function le(t,o,e,a,r,E,i){for(let u=0;u<a.length;u++)for(const l of e){const n=r[E.indexOf(l)][u];if(n!=null){await t.query("SAVEPOINT import_probe");try{await t.query("SELECT $1::jsonb",[n]),await t.query("RELEASE SAVEPOINT import_probe")}catch(c){return await t.query("ROLLBACK TO SAVEPOINT import_probe"),new Error(`import: ${o}.${l.name} at ${i(a[u])} is not a value jsonb accepts: ${c.message}`,{cause:c})}}}}m(le,"describeJsonbFailure");function j(t,o,e,a,r){if(r==null)return null;const E=t.get(Number(r));if(E===void 0)throw new Error(`import: ${e}.${a} references ${o} rowid ${String(r)}, which does not exist`);return E}m(j,"lookup");async function ye(t,o,e,a,r,E){const i=a.filter(s=>!(pe[e]??[]).includes(s.name)),u=t.prepare(`PRAGMA table_info(${f(e)})`).all(),l=new Set(u.map(s=>s.name)),c=!t.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(e).sql.toUpperCase().includes("WITHOUT ROWID"),_=new Set(ue[e]??[]),h=[];for(const s of i)if(s.name==="seq"&&!l.has("seq")){if(!c)throw new Error(`import: ${e} needs seq from rowid but has none`);h.push("rowid AS seq")}else if(s.name==="watch_id"||s.name==="subscription_id")h.push("NULL AS "+f(s.name));else if(l.has(s.name))h.push(f(s.name));else throw new Error(`import: SQLite table ${e} has no column ${s.name}`);c&&h.push("rowid AS _rowid");const S=e==="canvas_meta"?` WHERE key NOT IN (${[...W].map(s=>`'${s}'`).join(",")})`:"",y=t.prepare(`SELECT ${h.join(", ")} FROM ${f(e)}${S}${c?" ORDER BY rowid":""}`),$=i.map(s=>f(s.name)),g=i.map((s,p)=>`$${p+1}::${me(s.type)}`),T=i.map((s,p)=>`c${p}`),A=i.map((s,p)=>s.type==="jsonb"?`c${p}::jsonb`:`c${p}`);let v=`INSERT INTO ${f(e)} (${$.join(", ")}) SELECT ${A.join(", ")} FROM unnest(${g.join(", ")}) AS u(${T.join(", ")})`;const N=de[e];if(N!==void 0){const s=i.filter(p=>p.name!==N).map(p=>`${f(p.name)} = EXCLUDED.${f(p.name)}`);v+=` ON CONFLICT (${f(N)}) DO UPDATE SET ${s.join(", ")}`}const I=e==="objects",D=i.filter(s=>s.type==="jsonb"),k=c?"_rowid":i[0].name,V=m(s=>`${c?"rowid":k} ${String(s[k])}`,"key");let x=0,b=[];const P=m(async()=>{if(b.length===0)return;const s=i.map(()=>new Array(b.length));if(b.forEach((p,R)=>{i.forEach((d,J)=>{let w=p[d.name];if(e==="objects"&&d.name==="privacy"?w=ae(w):d.name==="watch_id"||d.name==="subscription_id"?w=H():e==="revision_parents"&&(d.name==="child"||d.name==="parent")?w=j(r.revisions,"revision",e,d.name,w):e==="watch_subscriptions"&&d.name==="watch"?w=j(r.watches,"watch",e,d.name,w):_.has(d.name)&&(w=I?null:j(r.objects,"object",e,d.name,w)),d.type==="bytea"&&w instanceof Uint8Array&&!Buffer.isBuffer(w)&&(w=Buffer.from(w.buffer,w.byteOffset,w.byteLength)),d.type==="jsonb"&&typeof w=="string"){const C=Ee(w);C.replaced>0&&(w=C.text,E.set(`${e}.${d.name}`,(E.get(`${e}.${d.name}`)??0)+C.replaced))}s[J][R]=w??null}),e==="watches"&&r.watches.set(Number(p._rowid),s[i.findIndex(d=>d.name==="watch_id")][R])}),D.length===0)await o.query(v,s);else{await o.query("SAVEPOINT import_batch");try{await o.query(v,s)}catch(p){throw await o.query("ROLLBACK TO SAVEPOINT import_batch"),await le(o,e,D,b,s,i,V)??p}await o.query("RELEASE SAVEPOINT import_batch")}x+=b.length,b=[]},"flush");for(const s of y.iterate())e==="objects"&&r.objects.set(Number(s._rowid),s.id),e==="revisions"&&r.revisions.set(Number(s._rowid),s.rev_id),b.push(s),b.length>=M&&await P();if(await P(),I){const s=t.prepare("SELECT id, owner FROM objects WHERE owner IS NOT NULL").all();for(let p=0;p<s.length;p+=M){const R=s.slice(p,p+M);await o.query("UPDATE objects o SET owner = v.owner FROM unnest($1::text[], $2::text[]) AS v(id, owner) WHERE o.id = v.id",[R.map(d=>d.id),R.map(d=>j(r.objects,"object","objects","owner",d.owner))])}}return x}m(ye,"copyTable");async function _e(t,o){for(const[e,a]of o)for(const r of a)r.identity&&await t.query(`SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(${f(r.name)}) FROM ${f(e)}), 1), (SELECT max(${f(r.name)}) IS NOT NULL FROM ${f(e)}))`,[f(e),r.name])}m(_e,"advanceIdentities");function G(t,o,e=""){return Number(t.prepare(`SELECT count(*) AS n FROM ${f(o)}${e}`).get().n)}m(G,"sqliteCount");async function he(t,o,e){const a=[];for(const n of L){if(n==="canvas_meta")continue;const c=G(t,n),_=Number((await o.query(`SELECT count(*) AS n FROM ${f(n)}`)).rows[0].n);c!==_&&a.push(`${n}: SQLite ${c} rows, Postgres ${_}`)}for(const n of t.prepare(`SELECT key, value FROM canvas_meta WHERE key NOT IN (${[...W].map(c=>`'${c}'`).join(",")})`).all())(await o.query("SELECT value FROM canvas_meta WHERE key = $1",[n.key])).rows[0]?.value!==n.value&&a.push(`canvas_meta ${n.key}: value differs`);const r=m(n=>{const c=z("sha256");for(const _ of n)c.update(_).update(`
-`);return c.digest("hex")},"hash"),E=t.prepare("SELECT id FROM objects ORDER BY rowid").all().map(n=>n.id),i=(await o.query("SELECT id FROM objects ORDER BY seq")).rows.map(n=>n.id);r(E)!==r(i)&&a.push("objects: ids in insertion order differ");const u=t.prepare("SELECT o.id AS id, p.id AS owner FROM objects o JOIN objects p ON p.rowid = o.owner ORDER BY o.rowid").all(),l=(await o.query("SELECT id, owner FROM objects WHERE owner IS NOT NULL ORDER BY seq")).rows;r(u.map(n=>`${n.id}>${n.owner}`))!==r(l.map(n=>`${n.id}>${n.owner}`))&&a.push("objects: owner links differ");for(const[n,c]of[["run_events","body_json"],["messages","body_json"],["deliveries","last_failure_json"]]){const _=Number(t.prepare(`SELECT COALESCE(sum(length(CAST(${f(c)} AS TEXT))), 0) AS n FROM ${f(n)}`).get().n),h=e.get(n).find($=>$.name===c).type,S=h==="jsonb"?`${f(c)}::text`:f(c);if(h==="jsonb")continue;const y=Number((await o.query(`SELECT COALESCE(sum(length(${S})), 0)::bigint AS n FROM ${f(n)}`)).rows[0].n);_!==y&&a.push(`${n}.${c}: total length SQLite ${_}, Postgres ${y}`)}if(a.length>0)throw new Error(`SQLite \u2192 Postgres import verification failed (rolled back; canvas.db untouched):
+      ORDER BY c.relname, a.attnum`),e=new Map;for(const a of o.rows){const n=e.get(a.tbl)??[];n.push({name:a.col,type:a.type,identity:a.identity,optional:a.optional}),e.set(a.tbl,n)}return e}p(we,"pgColumns");const le="\uFFFD";function Ee(t){if(!t.includes("\\u0000"))return{text:t,replaced:0};let o;try{o=JSON.parse(t)}catch{return{text:t,replaced:0}}let e=0;const a=p(c=>{const i=c.split("\0");return e+=i.length-1,i.join(le)},"clean"),n=p(c=>typeof c=="string"?a(c):Array.isArray(c)?c.map(n):c!==null&&typeof c=="object"?Object.fromEntries(Object.entries(c).map(([i,_])=>[a(i),n(_)])):c,"walk"),l=n(o);return e===0?{text:t,replaced:e}:{text:JSON.stringify(l),replaced:e}}p(Ee,"scrubNul");async function ye(t,o,e,a,n,l,c){for(let i=0;i<a.length;i++)for(const _ of e){const s=n[l.indexOf(_)][i];if(s!=null){await t.query("SAVEPOINT import_probe");try{await t.query("SELECT $1::jsonb",[s]),await t.query("RELEASE SAVEPOINT import_probe")}catch(u){return await t.query("ROLLBACK TO SAVEPOINT import_probe"),new Error(`import: ${o}.${_.name} at ${c(a[i])} is not a value jsonb accepts: ${u.message}`,{cause:u})}}}}p(ye,"describeJsonbFailure");function j(t,o,e,a,n){if(n==null)return null;const l=t.get(Number(n));if(l===void 0)throw new Error(`import: ${e}.${a} references ${o} rowid ${String(n)}, which does not exist`);return l}p(j,"lookup");async function _e(t,o,e,a,n,l){const c=t.prepare(`PRAGMA table_info(${f(e)})`).all(),i=new Set(c.map(r=>r.name)),_=p(r=>r==="seq"||r==="watch_id"||r==="subscription_id","minted"),s=a.filter(r=>i.has(r.name)||_(r.name)||!r.optional),E=!t.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(e).sql.toUpperCase().includes("WITHOUT ROWID"),b=new Set(de[e]??[]),$=[];for(const r of s)if(r.name==="seq"&&!i.has("seq")){if(!E)throw new Error(`import: ${e} needs seq from rowid but has none`);$.push("rowid AS seq")}else if(r.name==="watch_id"||r.name==="subscription_id")$.push("NULL AS "+f(r.name));else if(i.has(r.name))$.push(f(r.name));else throw new Error(`import: SQLite table ${e} has no column ${r.name}`);E&&$.push("rowid AS _rowid");const y=e==="canvas_meta"?` WHERE key NOT IN (${[...W].map(r=>`'${r}'`).join(",")})`:"",h=t.prepare(`SELECT ${$.join(", ")} FROM ${f(e)}${y}${E?" ORDER BY rowid":""}`),R=s.map(r=>f(r.name)),T=s.map((r,d)=>`$${d+1}::${fe(r.type)}`),A=s.map((r,d)=>`c${d}`),V=s.map((r,d)=>r.type==="jsonb"?`c${d}::jsonb`:`c${d}`);let v=`INSERT INTO ${f(e)} (${R.join(", ")}) SELECT ${V.join(", ")} FROM unnest(${T.join(", ")}) AS u(${A.join(", ")})`;const N=me[e];if(N!==void 0){const r=s.filter(d=>d.name!==N).map(d=>`${f(d.name)} = EXCLUDED.${f(d.name)}`);v+=` ON CONFLICT (${f(N)}) DO UPDATE SET ${r.join(", ")}`}const I=e==="objects",D=s.filter(r=>r.type==="jsonb"),k=E?"_rowid":s[0].name,J=p(r=>`${E?"rowid":k} ${String(r[k])}`,"key");let x=0,S=[];const P=p(async()=>{if(S.length===0)return;const r=s.map(()=>new Array(S.length));if(S.forEach((d,g)=>{s.forEach((m,Q)=>{let w=d[m.name];if(e==="objects"&&m.name==="privacy"?w=ie(w):m.name==="watch_id"||m.name==="subscription_id"?w=H():e==="revision_parents"&&(m.name==="child"||m.name==="parent")?w=j(n.revisions,"revision",e,m.name,w):e==="watch_subscriptions"&&m.name==="watch"?w=j(n.watches,"watch",e,m.name,w):b.has(m.name)&&(w=I?null:j(n.objects,"object",e,m.name,w)),m.type==="bytea"&&w instanceof Uint8Array&&!Buffer.isBuffer(w)&&(w=Buffer.from(w.buffer,w.byteOffset,w.byteLength)),m.type==="jsonb"&&typeof w=="string"){const q=Ee(w);q.replaced>0&&(w=q.text,l.set(`${e}.${m.name}`,(l.get(`${e}.${m.name}`)??0)+q.replaced))}r[Q][g]=w??null}),e==="watches"&&n.watches.set(Number(d._rowid),r[s.findIndex(m=>m.name==="watch_id")][g])}),D.length===0)await o.query(v,r);else{await o.query("SAVEPOINT import_batch");try{await o.query(v,r)}catch(d){throw await o.query("ROLLBACK TO SAVEPOINT import_batch"),await ye(o,e,D,S,r,s,J)??d}await o.query("RELEASE SAVEPOINT import_batch")}x+=S.length,S=[]},"flush");for(const r of h.iterate())e==="objects"&&n.objects.set(Number(r._rowid),r.id),e==="revisions"&&n.revisions.set(Number(r._rowid),r.rev_id),S.push(r),S.length>=M&&await P();if(await P(),I){const r=t.prepare("SELECT id, owner FROM objects WHERE owner IS NOT NULL").all();for(let d=0;d<r.length;d+=M){const g=r.slice(d,d+M);await o.query("UPDATE objects o SET owner = v.owner FROM unnest($1::text[], $2::text[]) AS v(id, owner) WHERE o.id = v.id",[g.map(m=>m.id),g.map(m=>j(n.objects,"object","objects","owner",m.owner))])}}return x}p(_e,"copyTable");async function he(t,o){for(const[e,a]of o)for(const n of a)n.identity&&await t.query(`SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(${f(n.name)}) FROM ${f(e)}), 1), (SELECT max(${f(n.name)}) IS NOT NULL FROM ${f(e)}))`,[f(e),n.name])}p(he,"advanceIdentities");function G(t,o,e=""){return Number(t.prepare(`SELECT count(*) AS n FROM ${f(o)}${e}`).get().n)}p(G,"sqliteCount");async function $e(t,o,e){const a=[];for(const s of L){if(s==="canvas_meta")continue;const u=G(t,s),E=Number((await o.query(`SELECT count(*) AS n FROM ${f(s)}`)).rows[0].n);u!==E&&a.push(`${s}: SQLite ${u} rows, Postgres ${E}`)}for(const s of t.prepare(`SELECT key, value FROM canvas_meta WHERE key NOT IN (${[...W].map(u=>`'${u}'`).join(",")})`).all())(await o.query("SELECT value FROM canvas_meta WHERE key = $1",[s.key])).rows[0]?.value!==s.value&&a.push(`canvas_meta ${s.key}: value differs`);const n=p(s=>{const u=Z("sha256");for(const E of s)u.update(E).update(`
+`);return u.digest("hex")},"hash"),l=t.prepare("SELECT id FROM objects ORDER BY rowid").all().map(s=>s.id),c=(await o.query("SELECT id FROM objects ORDER BY seq")).rows.map(s=>s.id);n(l)!==n(c)&&a.push("objects: ids in insertion order differ");const i=t.prepare("SELECT o.id AS id, p.id AS owner FROM objects o JOIN objects p ON p.rowid = o.owner ORDER BY o.rowid").all(),_=(await o.query("SELECT id, owner FROM objects WHERE owner IS NOT NULL ORDER BY seq")).rows;n(i.map(s=>`${s.id}>${s.owner}`))!==n(_.map(s=>`${s.id}>${s.owner}`))&&a.push("objects: owner links differ");for(const[s,u]of[["run_events","body_json"],["messages","body_json"],["deliveries","last_failure_json"]]){const E=Number(t.prepare(`SELECT COALESCE(sum(length(CAST(${f(u)} AS TEXT))), 0) AS n FROM ${f(s)}`).get().n),b=e.get(s).find(h=>h.name===u).type,$=b==="jsonb"?`${f(u)}::text`:f(u);if(b==="jsonb")continue;const y=Number((await o.query(`SELECT COALESCE(sum(length(${$})), 0)::bigint AS n FROM ${f(s)}`)).rows[0].n);E!==y&&a.push(`${s}.${u}: total length SQLite ${E}, Postgres ${y}`)}if(a.length>0)throw new Error(`SQLite \u2192 Postgres import verification failed (rolled back; canvas.db untouched):
   ${a.join(`
-  `)}`)}m(he,"verify");async function $e(t,o,e={}){const a=e.log??(()=>{}),r=Date.now(),E=await fe(o),i=t.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map(y=>y.name),u=i.filter(y=>!ce.has(y)&&!L.includes(y));if(u.length>0)throw new Error(`import: SQLite tables with no copy rule: ${u.join(", ")}`);const l=L.filter(y=>!E.has(y));if(l.length>0)throw new Error(`import: Postgres tables missing from the schema: ${l.join(", ")}`);const n=i.includes("reviews")?G(t,"reviews"):0;if(n>0)throw new Error(`import: the legacy reviews table still holds ${n} rows after the SQLite migration chain`);const c={objects:new Map,revisions:new Map,watches:new Map},_=[],h=new Map;let S=0;await o.query("BEGIN");try{await o.query("ALTER TABLE nodes DISABLE TRIGGER USER"),await o.query("ALTER TABLE deliveries DISABLE TRIGGER USER");for(const $ of L){const g=Date.now(),T=await ye(t,o,$,E.get($),c,h),A=Date.now()-g;_.push({table:$,rows:T,ms:A}),a(`import ${$}: ${T} rows in ${A} ms`)}await o.query("ALTER TABLE nodes ENABLE TRIGGER USER"),await o.query("ALTER TABLE deliveries ENABLE TRIGGER USER"),await _e(o,E);const y=Date.now();await he(t,o,E),S=Date.now()-y,a(`import verify: ${S} ms`),await o.query("INSERT INTO canvas_meta (key, value) VALUES ($1, $2)",[q,JSON.stringify({at:new Date().toISOString(),rows:_.reduce(($,g)=>$+g.rows,0)})]),await o.query("COMMIT")}catch(y){throw await o.query("ROLLBACK").catch(()=>{}),y}return h.size>0&&a(`import: replaced NUL (\\u0000) with U+FFFD in jsonb values: ${[...h].map(([y,$])=>`${y} ${$}`).join(", ")}`),{tables:_,verifyMs:S,totalMs:Date.now()-r,nulReplaced:Object.fromEntries(h)}}m($e,"copySqliteToPostgres");const Y=["","-wal","-shm"];function K(t=B()){if(O(`${t}.imported`)){const o=`${t}.imported.${Date.now()}`;for(const e of Y)O(`${t}.imported${e}`)&&F(`${t}.imported${e}`,`${o}${e}`)}for(const o of Y)O(`${t}${o}`)&&F(`${t}${o}`,`${t}.imported${o}`)}m(K,"renameImportedDatabase");async function Ce(t,o=()=>{}){const e=B();if(!O(e))return null;const a=ie();return ee(te(a),{recursive:!0,mode:448}),se(a,async()=>{if(!O(e))return null;const r=new oe.Client({connectionString:t,application_name:`crtr-import-${process.pid}`});await r.connect();try{const E=await r.query("SELECT key FROM canvas_meta WHERE key = ANY($1)",[[q,U]]),i=new Set(E.rows.map(n=>n.key));if(i.has(q)){if(i.has(U))throw new Error(`${e} was written by \`crtr sys db export-sqlite\` and the Postgres database predates it; importing would drop the exported writes. Stop the daemon and delete the Postgres cluster (runtime/pg/data) to import ${e} afresh, or move ${e} aside to keep using Postgres.`);return K(e),o(`import: finished the rename of ${e} (already imported)`),null}const u=new X(e);let l;try{u.exec("PRAGMA busy_timeout = 5000;"),u.exec("PRAGMA foreign_keys = ON;"),ne(u),l=await $e(u,r,{log:o})}finally{u.close()}return K(e),o(`import: ${l.tables.reduce((n,c)=>n+c.rows,0)} rows in ${l.totalMs} ms; canvas.db renamed to canvas.db.imported`),l}finally{await r.end().catch(()=>{})}},{timeoutMs:re,timeoutError:m(()=>new Error(`timed out waiting for the canvas import lock: ${a}`),"timeoutError")})}m(Ce,"importSqliteCanvasIfPresent");export{U as EXPORT_MARK_KEY,q as IMPORT_MARK_KEY,ue as OBJECT_COLUMNS,L as TABLE_ORDER,$e as copySqliteToPostgres,Ce as importSqliteCanvasIfPresent,fe as pgColumns,f as quote,K as renameImportedDatabase};
+  `)}`)}p($e,"verify");async function be(t,o,e={}){const a=e.log??(()=>{}),n=Date.now(),l=await we(o),c=t.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map(y=>y.name),i=c.filter(y=>!pe.has(y)&&!L.includes(y));if(i.length>0)throw new Error(`import: SQLite tables with no copy rule: ${i.join(", ")}`);const _=L.filter(y=>!l.has(y));if(_.length>0)throw new Error(`import: Postgres tables missing from the schema: ${_.join(", ")}`);const s=c.includes("reviews")?G(t,"reviews"):0;if(s>0)throw new Error(`import: the legacy reviews table still holds ${s} rows after the SQLite migration chain`);const u={objects:new Map,revisions:new Map,watches:new Map},E=[],b=new Map;let $=0;await o.query("BEGIN");try{await o.query("ALTER TABLE nodes DISABLE TRIGGER USER"),await o.query("ALTER TABLE deliveries DISABLE TRIGGER USER");for(const h of L){const R=Date.now(),T=await _e(t,o,h,l.get(h),u,b),A=Date.now()-R;E.push({table:h,rows:T,ms:A}),a(`import ${h}: ${T} rows in ${A} ms`)}await o.query("ALTER TABLE nodes ENABLE TRIGGER USER"),await o.query("ALTER TABLE deliveries ENABLE TRIGGER USER"),await he(o,l);const y=Date.now();await $e(t,o,l),$=Date.now()-y,a(`import verify: ${$} ms`),await o.query("INSERT INTO canvas_meta (key, value) VALUES ($1, $2)",[C,JSON.stringify({at:new Date().toISOString(),rows:E.reduce((h,R)=>h+R.rows,0)})]),await o.query("COMMIT")}catch(y){throw await o.query("ROLLBACK").catch(()=>{}),y}return b.size>0&&a(`import: replaced NUL (\\u0000) with U+FFFD in jsonb values: ${[...b].map(([y,h])=>`${y} ${h}`).join(", ")}`),{tables:E,verifyMs:$,totalMs:Date.now()-n,nulReplaced:Object.fromEntries(b)}}p(be,"copySqliteToPostgres");const Y=["","-wal","-shm"];function K(t=B()){if(O(`${t}.imported`)){const o=`${t}.imported.${Date.now()}`;for(const e of Y)O(`${t}.imported${e}`)&&F(`${t}.imported${e}`,`${o}${e}`)}for(const o of Y)O(`${t}${o}`)&&F(`${t}${o}`,`${t}.imported${o}`)}p(K,"renameImportedDatabase");async function Me(t,o=()=>{}){const e=B();if(!O(e))return null;const a=ce();return te(oe(a),{recursive:!0,mode:448}),re(a,async()=>{if(!O(e))return null;const n=new se.Client({connectionString:t,application_name:`crtr-import-${process.pid}`,options:ue()});await n.connect();try{const l=await n.query("SELECT key FROM canvas_meta WHERE key = ANY($1)",[[C,U]]),c=new Set(l.rows.map(s=>s.key));if(c.has(C)){if(c.has(U))throw new Error(`${e} was written by \`crtr sys db export-sqlite\` and the Postgres database predates it; importing would drop the exported writes. Stop the daemon and delete the Postgres cluster (runtime/pg/data) to import ${e} afresh, or move ${e} aside to keep using Postgres.`);return K(e),o(`import: finished the rename of ${e} (already imported)`),null}const i=new z(e);let _;try{i.exec("PRAGMA busy_timeout = 5000;"),i.exec("PRAGMA foreign_keys = ON;"),ae(i),_=await be(i,n,{log:o})}finally{i.close()}return K(e),o(`import: ${_.tables.reduce((s,u)=>s+u.rows,0)} rows in ${_.totalMs} ms; canvas.db renamed to canvas.db.imported`),_}finally{await n.end().catch(()=>{})}},{timeoutMs:ne,timeoutError:p(()=>new Error(`timed out waiting for the canvas import lock: ${a}`),"timeoutError")})}p(Me,"importSqliteCanvasIfPresent");export{U as EXPORT_MARK_KEY,C as IMPORT_MARK_KEY,de as OBJECT_COLUMNS,L as TABLE_ORDER,be as copySqliteToPostgres,Me as importSqliteCanvasIfPresent,we as pgColumns,f as quote,K as renameImportedDatabase};

package/dist/core/canvas/schema-chain.d.ts

@@ -0,0 +1,34 @@
+import type pg from 'pg';
+/** The version `schema-pg.sql` writes; every delta number is above it. */
+export declare const BASELINE_VERSION = 1;
+export declare const DEFAULT_STEP_LOCK_TIMEOUT_MS = 5000;
+export declare const DEFAULT_STEP_RETRY_BUDGET_MS = 120000;
+export interface SchemaDelta {
+    version: number;
+    name: string;
+    file: string;
+}
+/** The delta files in `dir`, in number order. They must be numbered contiguously from {@link BASELINE_VERSION} + 1; any other
+ *  entry except `.gitkeep` (which keeps the directory in git while it is empty) is an error. */
+export declare function listDeltas(dir?: string): SchemaDelta[];
+export type SchemaEmit = (level: 'info' | 'warn', event: string, fields: Record<string, string | number>) => void;
+export interface SchemaChainOptions {
+    /** The Postgres schema holding the canvas tables (`CRTR_DATABASE_SCHEMA`, see `canvasDatabaseSchema`). Created when absent and set as the session's `search_path`. Default: the role's own (`public`). */
+    schema?: string;
+    /** The build's schema version. Default {@link CANVAS_SCHEMA_VERSION}; a test passes another to drive fixture deltas. */
+    version?: number;
+    /** The directory of delta files. Default `schema-pg/` beside this module; a test passes a fixture directory. */
+    deltaDir?: string;
+    /** `lock_timeout` of each step. */
+    lockTimeoutMs?: number;
+    /** How long one step keeps retrying after lock timeouts before the open fails. */
+    retryBudgetMs?: number;
+    /** Receives `schema.step.applied` after each committed step and `schema.step.lock_timeout` before each retry. */
+    emit?: SchemaEmit;
+}
+/** Bring the connected database to this build's schema. Takes `pg_advisory_lock` for the session and reads the version under
+ *  it, so processes opening together queue and a late one applies nothing. Refuses a database whose `schema_min_reader` is above
+ *  this build's version; opens one whose `schema_version` is above it when `schema_min_reader` allows. A database with no
+ *  `schema_version` gets `schema-pg.sql`, which runs as one multi-statement simple query, one transaction. A database at the
+ *  baseline version first runs the file's `-- <name>:begin … -- <name>:end` blocks, in file order, each idempotent (S1 deletes this). */
+export declare function ensureSchema(client: pg.Client, options?: SchemaChainOptions): Promise<void>;

package/dist/core/canvas/schema-chain.js

@@ -0,0 +1 @@
+var _=Object.defineProperty;var u=(a,e)=>_(a,"name",{value:e,configurable:!0});import{createHash as v}from"node:crypto";import{readdirSync as S,readFileSync as E}from"node:fs";import{dirname as T,join as d}from"node:path";import{fileURLToPath as p}from"node:url";import{CANVAS_SCHEMA_VERSION as $,readStoredVersions as f}from"./schema-version.js";const w=T(p(import.meta.url)),g=d(w,"schema-pg.sql"),k=d(w,"schema-pg"),y=7427001,h=1,A=5e3,L=12e4,q="55P03",C=/^(\d{3})-([a-z0-9][a-z0-9-]*)\.sql$/;function M(a=k){const e=[];for(const t of S(a).sort()){if(t===".gitkeep")continue;const o=C.exec(t);if(o===null)throw new Error(`schema delta directory ${a}: ${t} is not named NNN-<name>.sql`);e.push({version:Number(o[1]),name:o[2],file:d(a,t)})}return e.forEach((t,o)=>{if(t.version!==h+1+o)throw new Error(`schema delta directory ${a}: expected delta ${h+1+o} but found ${t.version}-${t.name}.sql; deltas are numbered contiguously from ${h+1}`)}),e}u(M,"listDeltas");async function l(a,e){await a.query("INSERT INTO canvas_meta (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",[e.key,e.hash])}u(l,"recordBlock");const b=u(a=>new Promise(e=>setTimeout(e,a)),"sleep");async function D(a,e,t){const o=E(e.file,"utf8"),c=Date.now();let m=250;for(let r=1;;r++)try{if(await a.query("BEGIN"),await a.query(`SET LOCAL lock_timeout = '${t.lockTimeoutMs}ms'`),await a.query(o),(await a.query("UPDATE canvas_meta SET value = $1 WHERE key = 'schema_version' AND value = $2",[String(e.version),String(e.version-1)])).rowCount!==1)throw new Error(`schema step ${e.version} (${e.name}): schema_version was not ${e.version-1}; another writer changed the schema outside the chain`);await a.query("COMMIT"),t.emit?.("info","schema.step.applied",{version:e.version,name:e.name,ms:Date.now()-c,attempts:r});return}catch(s){if(await a.query("ROLLBACK").catch(()=>{}),s.code!==q)throw s;if(Date.now()+m-c>t.retryBudgetMs)throw new Error(`schema step ${e.version} (${e.name}) could not get its locks within ${t.retryBudgetMs} ms (${r} attempts); a long transaction holds a table it needs`,{cause:s});t.emit?.("warn","schema.step.lock_timeout",{version:e.version,name:e.name,attempt:r,retry_in_ms:m}),await b(m),m=Math.min(m*2,1e4)}}u(D,"applyStep");async function F(a,e={}){const t=e.version??$,o=M(e.deltaDir);if(e.schema!==void 0){const c=`"${e.schema.replace(/"/g,'""')}"`;await a.query(`CREATE SCHEMA IF NOT EXISTS ${c}`),await a.query(`SET search_path TO ${c}`)}await a.query("SELECT pg_advisory_lock($1)",[y]);try{const c=E(g,"utf8"),m=[...c.matchAll(/^-- (\w+):begin\n([\s\S]*?)^-- \1:end$/gm)].map(([,i,n])=>({key:`schema_block.${i}`,sql:n,hash:v("sha256").update(n).digest("hex")}));let r=await f(a);if(r!==void 0&&t<r.minReader)throw new Error(`unsupported canvas schema: this build is at version ${t} but the database requires readers at version ${r.minReader} or above (it is at ${r.version}); this build is older than the database. Select a newer runtime generation.`);let s;if(r===void 0){if(await a.query(c),r=await f(a),r===void 0)throw new Error("schema-pg.sql did not write canvas_meta.schema_version");s=r.version;for(const i of m)await l(a,i)}else if(s=r.version,s===h){const i=new Map((await a.query("SELECT key, value FROM canvas_meta WHERE key LIKE 'schema\\_block.%'")).rows.map(n=>[n.key,n.value]));for(const n of m)i.get(n.key)!==n.hash&&(await a.query(n.sql),await l(a,n))}for(const i of o)i.version<=s||(s=i.version,await D(a,i,{lockTimeoutMs:e.lockTimeoutMs??A,retryBudgetMs:e.retryBudgetMs??L,emit:e.emit}));if(s<t)throw new Error(`canvas schema version ${s} does not match this build's ${t}: no delta reaches it`)}finally{await a.query("SELECT pg_advisory_unlock($1)",[y]).catch(()=>{})}}u(F,"ensureSchema");export{h as BASELINE_VERSION,A as DEFAULT_STEP_LOCK_TIMEOUT_MS,L as DEFAULT_STEP_RETRY_BUDGET_MS,F as ensureSchema,M as listDeltas};

package/dist/core/canvas/schema-version.d.ts

@@ -1,4 +1,30 @@
-/** The canvas schema version `schema-pg.sql` writes to `canvas_meta.schema_version`. `db.ts` refuses a database at
- *  any other version, and `scripts/install-runtime.mjs` reads this module from a candidate generation to refuse one older
- *  than the live cluster. Kept free of imports so the installer can load it alone. */
+/** The canvas schema version this build supports: the number of the highest delta in `schema-pg/` (the baseline
+ *  `schema-pg.sql` is version 1). `scripts/install-runtime.mjs` reads this module from a candidate generation to refuse
+ *  one below the live database's `schema_min_reader`. */
 export declare const CANVAS_SCHEMA_VERSION = 1;
+/** The Postgres schema holding the canvas tables: `CRTR_DATABASE_SCHEMA` (`crtr` in the cloud deploy). Unset or empty
+ *  means the role's own search_path (`public` on a Mac and on a VM). A name that is not a plain identifier is an error:
+ *  it is put in a `search_path` connection option. */
+export declare function canvasDatabaseSchema(env?: Record<string, string | undefined>): string | undefined;
+/** The direct, non-pooled connection the schema chain runs on when `CRTR_SCHEMA_DATABASE_URL` is set (its advisory lock
+ *  needs one session). Unset: the chain runs on the daemon's own connection. */
+export declare function canvasSchemaDatabaseUrl(env?: Record<string, string | undefined>): string | undefined;
+/** The pg `options` connection parameter that makes every connection resolve unqualified names in the canvas schema. */
+export declare function canvasSearchPathOptions(env?: Record<string, string | undefined>): string | undefined;
+interface Queryable {
+    query(text: string): Promise<{
+        rows: Array<Record<string, unknown>>;
+    }>;
+}
+export interface StoredVersions {
+    /** `canvas_meta.schema_version`: the number of the last committed step. */
+    version: number;
+    /** `canvas_meta.schema_min_reader`: the lowest build version that may open the database; absent means `version`. */
+    minReader: number;
+}
+/** The versions stored in `schema` (default: the session's search_path), or undefined when the database has no canvas schema yet. */
+export declare function readStoredVersions(client: Queryable, schema?: string): Promise<StoredVersions | undefined>;
+/** The installer's rollback gate: refuse a candidate whose version is below the live database's `schema_min_reader`. A
+ *  database with no canvas schema yet accepts any candidate. */
+export declare function assertCandidateMayOpen(client: Queryable, supported: number, schema?: string): Promise<void>;
+export {};

package/dist/core/canvas/schema-version.js

@@ -1 +1 @@
-const A=1;export{A as CANVAS_SCHEMA_VERSION};
+var m=Object.defineProperty;var r=(n,e)=>m(n,"name",{value:e,configurable:!0});const A=1;function _(n=process.env){const e=n.CRTR_DATABASE_SCHEMA?.trim();if(e){if(!/^[A-Za-z_][A-Za-z0-9_]*$/.test(e))throw new Error(`CRTR_DATABASE_SCHEMA must be a plain identifier (letters, digits, underscore); got ${JSON.stringify(e)}`);return e}}r(_,"canvasDatabaseSchema");function h(n=process.env){return n.CRTR_SCHEMA_DATABASE_URL?.trim()||void 0}r(h,"canvasSchemaDatabaseUrl");function E(n=process.env){const e=_(n);return e===void 0?void 0:`-c search_path=${e}`}r(E,"canvasSearchPathOptions");async function v(n,e){const t=e===void 0?"canvas_meta":`"${e.replace(/"/g,'""')}".canvas_meta`;if((await n.query(`SELECT to_regclass('${t.replace(/'/g,"''")}') IS NOT NULL AS found`)).rows[0].found!==!0)return;const u=(await n.query(`SELECT key, value FROM ${t} WHERE key IN ('schema_version', 'schema_min_reader')`)).rows,o=r(c=>{const i=u.find(f=>f.key===c);if(i===void 0)return;const d=Number(i.value);if(!Number.isSafeInteger(d))throw new Error(`canvas_meta.${c} is not an integer: ${String(i.value)}`);return d},"value"),s=o("schema_version");return s===void 0?void 0:{version:s,minReader:o("schema_min_reader")??s}}r(v,"readStoredVersions");async function l(n,e,t){const a=await v(n,t);if(a!==void 0&&a.minReader>e)throw new Error(`refusing to select runtime with canvas schema ${e}: the live database requires readers at version ${a.minReader} or above (it is at ${a.version}); install a descendant build instead`)}r(l,"assertCandidateMayOpen");export{A as CANVAS_SCHEMA_VERSION,l as assertCandidateMayOpen,_ as canvasDatabaseSchema,h as canvasSchemaDatabaseUrl,E as canvasSearchPathOptions,v as readStoredVersions};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.552",
+  "version": "0.3.553",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -71,6 +71,7 @@
     "scripts/patch-pi-tree-root-ordering.mjs",
     "scripts/install-runtime.mjs",
     "scripts/lib/isolation-env.mjs",
+    "scripts/lib/canvas-schema-gate.mjs",
     "scripts/lib/pack-args.mjs",
     "scripts/lib/update-target.mjs",
     "scripts/lib/runtime-deps-key.mjs",
@@ -140,7 +141,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.552",
+    "@north-light/identity": "0.3.553",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.552",
+  "version": "0.3.553",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.552",
+  "version": "0.3.553",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.552",
+      "version": "0.3.553",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -24,7 +24,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.552",
+        "@north-light/identity": "0.3.553",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12455,17 +12455,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.552",
+      "version": "0.3.553",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.552",
+      "version": "0.3.553",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.552",
+      "version": "0.3.553",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12473,21 +12473,21 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.552",
+      "version": "0.3.553",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.552",
-        "@north-light/identity": "0.3.552"
+        "@north-light/api": "0.3.553",
+        "@north-light/identity": "0.3.553"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.552",
+      "version": "0.3.553",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.552",
-        "@north-light/identity": "0.3.552",
-        "@north-light/plugin": "0.3.552",
+        "@north-light/api": "0.3.553",
+        "@north-light/identity": "0.3.553",
+        "@north-light/plugin": "0.3.553",
         "jose": "^6.2.1"
       },
       "bin": {

package/scripts/install-runtime.mjs

@@ -13,6 +13,7 @@ import { smokeValidate } from './runtime-smoke.mjs';
 import { scrubInheritedBrokerEnv, unsafeIsolationReason } from './lib/isolation-env.mjs';
 import { packArgs } from './lib/pack-args.mjs';
 import { runtimeDepsKeyFromFiles } from './lib/runtime-deps-key.mjs';
+import { assertCandidateMayOpenLiveDatabase, candidateSchemaVersion } from './lib/canvas-schema-gate.mjs';
 
 scrubInheritedBrokerEnv();
 // CI has no account store to protect, and its checkout sits inside the runner's
@@ -198,18 +199,11 @@ async function assertCanvasSchemaCompatible(stage) {
   }
 }
 
-/** The candidate's Postgres canvas schema version, read from its own `schema-version.js`. */
-async function candidatePostgresSchemaVersion(stage) {
-  const { CANVAS_SCHEMA_VERSION: version } = await import(pathToFileURL(join(stage, 'dist', 'core', 'canvas', 'schema-version.js')).href);
-  if (!Number.isSafeInteger(version) || version < 1) throw new Error('candidate runtime does not expose a valid canvas schema version');
-  return version;
-}
-
-/** Read `canvas_meta.schema_version` from the live Postgres cluster and refuse a candidate older than it. A cluster that
- *  is not running cannot be read, so the candidate is refused until the cluster is started. */
+/** Refuse a candidate whose version is below `canvas_meta.schema_min_reader` (absent: `schema_version`) in the live Postgres
+ *  cluster, read from `CRTR_DATABASE_SCHEMA` as the daemon reads it (`scripts/lib/canvas-schema-gate.mjs`). A cluster that is
+ *  not running cannot be read, so the candidate is refused until the cluster is started. */
 async function assertPostgresSchemaCompatible(stage, runtime) {
   const { default: pg } = await import('pg');
-  const supported = await candidatePostgresSchemaVersion(stage);
   let socketDir = join(runtime, 'pg');
   try { socketDir = (await readFile(join(runtime, 'pg', 'socket-dir'), 'utf8')).trim(); } catch (error) { if (error?.code !== 'ENOENT') throw error; }
   const connectionString = process.env.CRTR_DATABASE_URL
@@ -220,22 +214,14 @@ async function assertPostgresSchemaCompatible(stage, runtime) {
   } catch (error) {
     if (!process.env.CRTR_DATABASE_URL && (error?.code === 'ENOENT' || error?.code === 'ECONNREFUSED')) {
       throw new Error(
-        `refusing to select runtime: the Postgres cluster at ${socketDir} is not running, so its canvas schema version cannot be checked against this build (${supported}). ` +
+        `refusing to select runtime: the Postgres cluster at ${socketDir} is not running, so its canvas schema version cannot be checked against this build (${await candidateSchemaVersion(stage)}). ` +
         'Start it with the currently selected generation (`crtr sys daemon start`) and install again.',
       );
     }
     throw error;
   }
   try {
-    const exists = await client.query(`SELECT to_regclass('canvas_meta') AS t`);
-    const row = exists.rows[0].t === null ? { rows: [] } : await client.query(`SELECT value FROM canvas_meta WHERE key = 'schema_version'`);
-    const current = row.rows[0] === undefined ? undefined : Number(row.rows[0].value);
-    if (current !== undefined && current > supported) {
-      throw new Error(
-        `refusing to select runtime with canvas schema ${supported}: ` +
-        `the live database is already at version ${current}; install a descendant build instead`,
-      );
-    }
+    await assertCandidateMayOpenLiveDatabase(stage, client);
   } finally {
     await client.end();
   }

package/scripts/lib/canvas-schema-gate.mjs

@@ -0,0 +1,20 @@
+// The installer's rollback gate on the live Postgres canvas: loads the candidate generation's own schema module (its
+// supported version, which Postgres schema holds the canvas — `CRTR_DATABASE_SCHEMA`, read by the same resolver the daemon
+// uses — and the check itself) and refuses a candidate below the database's `schema_min_reader`. Takes a connected client so
+// a test can drive it without the installer's side effects.
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+/** `client` is a connected `pg` client for the live database; `env` is the installer's environment. */
+export async function assertCandidateMayOpenLiveDatabase(stage, client, env = process.env) {
+  const candidate = await import(pathToFileURL(join(stage, 'dist', 'core', 'canvas', 'schema-version.js')).href);
+  const supported = candidate.CANVAS_SCHEMA_VERSION;
+  if (!Number.isSafeInteger(supported) || supported < 1) throw new Error('candidate runtime does not expose a valid canvas schema version');
+  await candidate.assertCandidateMayOpen(client, supported, candidate.canvasDatabaseSchema(env));
+}
+
+/** The candidate's supported canvas schema version, for messages. */
+export async function candidateSchemaVersion(stage) {
+  const candidate = await import(pathToFileURL(join(stage, 'dist', 'core', 'canvas', 'schema-version.js')).href);
+  return candidate.CANVAS_SCHEMA_VERSION;
+}