@lunora/shard-engine 0.0.0 → 1.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +122 -0
- package/README.md +10 -38
- package/dist/conformance/index.d.mts +24 -0
- package/dist/conformance/index.d.ts +24 -0
- package/dist/conformance/index.mjs +1 -0
- package/dist/index.d.mts +1906 -0
- package/dist/index.d.ts +1906 -0
- package/dist/index.mjs +1 -0
- package/dist/packem_shared/ADMIN_FUNCTIONS-B0xlQJ4w.mjs +1 -0
- package/dist/packem_shared/AGGREGATE_SQL_FUNCTION-QYaiuty4.mjs +1 -0
- package/dist/packem_shared/AGG_COUNT-BWXe3gtQ.mjs +1 -0
- package/dist/packem_shared/AUDIT_LOG_TABLE-CaA0jL0L.mjs +8 -0
- package/dist/packem_shared/CDC_LOG_TABLE-BnqlH2eZ.mjs +8 -0
- package/dist/packem_shared/CLIENT_WATERMARK_TABLE-CXsSLU8D.mjs +7 -0
- package/dist/packem_shared/ConflictError-C8GtJmjS.mjs +1 -0
- package/dist/packem_shared/CountRlsUnsupportedError-B2WKJD9v.mjs +1 -0
- package/dist/packem_shared/DATA_MIGRATION_STATE_TABLE-CaO6L0Ee.mjs +31 -0
- package/dist/packem_shared/DEFAULT_MAX_RELATION_KEYS-bq00btqV.mjs +1 -0
- package/dist/packem_shared/DEFAULT_MAX_RELAYS-CnCyh6oX.mjs +1 -0
- package/dist/packem_shared/DEFAULT_PROMOTION_THRESHOLDS-Dteg0sZF.mjs +1 -0
- package/dist/packem_shared/FTS_COUNT_COLUMN-D1gY3wwZ-CZHdaRrd.mjs +1 -0
- package/dist/packem_shared/GEO_DEFAULT_PRECISION-okRCkZ6r.mjs +1 -0
- package/dist/packem_shared/GLOBAL_SHAPE_SNAPSHOT_TABLE-DX-6-gBP.mjs +7 -0
- package/dist/packem_shared/IDEMPOTENCY_TABLE-DjkEMp6w.mjs +7 -0
- package/dist/packem_shared/MAIL_RETENTION-KmozO2NQ.mjs +18 -0
- package/dist/packem_shared/MAX_SQL_ROWS-nMwkxv5f.mjs +2 -0
- package/dist/packem_shared/NotFoundError-BhF7FeFr.mjs +1 -0
- package/dist/packem_shared/NotUniqueError-Bw1Oh2FL.mjs +1 -0
- package/dist/packem_shared/QUEUE_TABLE-DYCDvzTG.mjs +17 -0
- package/dist/packem_shared/RANK_TIEBREAK-9NU5s_mi.mjs +1 -0
- package/dist/packem_shared/RELATION_EXISTS_KEY-BaqSIFU1.mjs +1 -0
- package/dist/packem_shared/RLS_UNWRAP_SYMBOL-C6_WX3dG.mjs +1 -0
- package/dist/packem_shared/ReactiveCache-1_9Rs7J_.mjs +1 -0
- package/dist/packem_shared/SCAN_DEP-D_yR9EeV.mjs +1 -0
- package/dist/packem_shared/SCHEMA_HISTORY_MAX_VERSIONS-CHXz6p_z.mjs +6 -0
- package/dist/packem_shared/SEARCH_STATE_TABLE-eLN8U5tz.mjs +1 -0
- package/dist/packem_shared/ShardRunner-DIqVdWtk.mjs +1 -0
- package/dist/packem_shared/aggregateTableName-G-eXyjcz.mjs +1 -0
- package/dist/packem_shared/applyOnDelete-uFRC5p1d.mjs +1 -0
- package/dist/packem_shared/applySelect-B0CF8T7y.mjs +1 -0
- package/dist/packem_shared/armRestore-BNzdvQ_o.mjs +1 -0
- package/dist/packem_shared/awaitWsDrain-Dk50ISgE.mjs +1 -0
- package/dist/packem_shared/backfillAggregateIndexes-BHiewuB-.mjs +1 -0
- package/dist/packem_shared/buildPokeFrames-DaPpm5Ss.mjs +1 -0
- package/dist/packem_shared/buildSettings-DT18_DkX.mjs +1 -0
- package/dist/packem_shared/compileWhereSql-BLcfs4QW.mjs +1 -0
- package/dist/packem_shared/computeRankPage-IUSS-zIB.mjs +1 -0
- package/dist/packem_shared/createCompanionSync-BCBUOnMk.mjs +1 -0
- package/dist/packem_shared/createSystemReader-DcDcrtM3.mjs +1 -0
- package/dist/packem_shared/defineEngineContractSuite-ii3_DS_I.mjs +1 -0
- package/dist/packem_shared/diffExternalSource-DMpkJta1.mjs +1 -0
- package/dist/packem_shared/exportShardRows-kt42wijd.mjs +1 -0
- package/dist/packem_shared/hasTrigger-CbkOHExZ.mjs +1 -0
- package/dist/packem_shared/isSoftDeleted-B-SXkGUo.mjs +9 -0
- package/dist/packem_shared/liftSourceId-DApAbu7Q.mjs +1 -0
- package/dist/packem_shared/materializeExternalRows-DOQJV9p2.mjs +1 -0
- package/dist/packem_shared/param-B5lF5Jd9.mjs +1 -0
- package/dist/packem_shared/quote-identifier-CGiYFBvY.mjs +1 -0
- package/dist/packem_shared/runDrizzle-GKR3y97k.mjs +1 -0
- package/dist/packem_shared/runShardMigrations-CPxqCh3O.mjs +5 -0
- package/dist/packem_shared/runSocketPool-DPQLVyWF.mjs +1 -0
- package/dist/packem_shared/selectExpiredIds-FzhIEeG1.mjs +1 -0
- package/dist/packem_shared/selectShapeMemberIds-DvE7K6zG.mjs +1 -0
- package/dist/packem_shared/serializeSqlValue-DnpyaLcw.mjs +1 -0
- package/dist/packem_shared/stableStringify-BjLh4gvA.mjs +1 -0
- package/dist/packem_shared/stableWireKey-YEHLaX6X.mjs +1 -0
- package/dist/packem_shared/wire-codec-Ctnni0h6.mjs +1 -0
- package/package.json +55 -7
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{createShardCtxDb as b}from"./NotUniqueError-Bw1Oh2FL.mjs";import{createRelayLink as I}from"./DEFAULT_MAX_RELAYS-CnCyh6oX.mjs";import{ConflictError as N}from"./ConflictError-C8GtJmjS.mjs";import{runShardMigrations as v}from"./runShardMigrations-CPxqCh3O.mjs";import{relayName as q}from"./DEFAULT_PROMOTION_THRESHOLDS-Dteg0sZF.mjs";const C=(_,m={})=>({_meta:{column:{notNull:!0,...m}},kind:_}),D=(_,m,E)=>{const{describe:k,expect:i,it:f}=E;k(`engine contract: ${_}`,()=>{k("optimistic concurrency",()=>{const g=p=>({tables:{items:{indexes:[],shape:{title:C("string"),version:C("number",{notNull:!1})},triggerMap:{clobber:{handler:()=>{p.exec(`UPDATE "items" SET "__doc__" = json_set("__doc__", '$.version', 99) WHERE "id" = 'i1'`)},op:"update",timing:"before"}}}}});f("raises a CONFLICT of kind `occ` when a write's snapshot is clobbered",async()=>{const{close:p,host:d}=m();try{const c=d.sql,l=g(c);v(c,l);const s=b({schema:l,sql:c});await s.insert("items",{_id:"i1",title:"first",version:1},{allowExplicitId:!0}),await i(s.patch("i1",{title:"second"})).rejects.toBeInstanceOf(N)}finally{p?.()}}),f("reports the conflict as CONFLICT/409 rather than retrying it away",async()=>{const{close:p,host:d}=m();try{const c=d.sql,l=g(c);v(c,l);const s=b({schema:l,sql:c});await s.insert("items",{_id:"i1",title:"first",version:1},{allowExplicitId:!0});let r;try{await s.patch("i1",{title:"second"})}catch(t){r=t}const e=r;i(e.code).toBe("CONFLICT"),i(e.kind).toBe("occ")}finally{p?.()}}),f("leaves the row readable and unchanged after a conflict",async()=>{const{close:p,host:d}=m();try{const c=d.sql,l=g(c);v(c,l);const s=b({schema:l,sql:c});await s.insert("items",{_id:"i1",title:"first",version:1},{allowExplicitId:!0});try{await s.patch("i1",{title:"second"})}catch{}const r=await s.get("i1");i(r?.title).toBe("first"),i(r?.version).toBe(99)}finally{p?.()}}),f("raises a CONFLICT of kind `trigger` when a trigger writes its own row through the db",async()=>{const{close:p,host:d}=m();try{const c=d.sql,l={tables:{items:{indexes:[],shape:{title:C("string"),version:C("number",{notNull:!1})},triggerMap:{recurse:{handler:async(t,o)=>{await t.db.patch(o.doc._id,{version:99})},op:"update",timing:"before"}}}}};v(c,l);const s=b({schema:l,sql:c});await s.insert("items",{_id:"i1",title:"first",version:1},{allowExplicitId:!0});let r;try{await s.patch("i1",{title:"second"})}catch(t){r=t}const e=r;i(e).toBeInstanceOf(N),i(e.code).toBe("CONFLICT"),i(e.kind).toBe("trigger")}finally{p?.()}})}),k("shape-poke ordering",()=>{const g="shard-a",p={args:{},name:"messages"},d=(e,t,o,a)=>e.accept(t?.()??{},{connectionId:o,shapes:{[a]:p}}),c=(e,t)=>{let o=0,a=0;const n=u=>()=>{throw new Error(`the relay poke path must not reach RelayHost.${u}`)},h={fetch:(u,O)=>{if(JSON.parse(O?.body??"{}").type!=="relay_shape_subscribe")return Promise.resolve(new Response(void 0,{status:204}));const B=t[a];if(a+=1,B===void 0)throw new Error("the relay asked for more seeds than this fixture supplies");return Promise.resolve(Response.json(B))}},y={get:()=>h,getByName:()=>h,idFromName:u=>u},w={buildShapeDiff:n("buildShapeDiff"),computeOpLogShapeSeed:n("computeOpLogShapeSeed"),currentCdcEpoch:n("currentCdcEpoch"),deliverWhisperLocal:n("deliverWhisperLocal"),doName:()=>q(g,0),env:()=>({SHARD:y}),getWebSockets:()=>e.getSockets(),maskMetadata:n("maskMetadata"),nextPokeId:()=>(o+=1,`poke-${String(o)}`),readAttachment:u=>u.deserializeAttachment?.(),recordShapePokeFanout:()=>{},resolveShape:n("resolveShape"),rlsMetadata:n("rlsMetadata"),shardBinding:()=>"SHARD",sql:n("sql")},S=I(w);if(S===void 0)throw new Error("expected a relay link for a `…::relay::N` name");return S},l=e=>new Request("https://relay.internal/_lunora/relay",{body:JSON.stringify(e),headers:{"content-type":"application/json"},method:"POST"}),s=async(e,t,o)=>{const a=await e.seedRelayShape(t,o,p,{identity:void 0,userId:void 0});if(a!=="ok")throw new Error(`seed failed: ${JSON.stringify(a)}`)},r=(e={})=>l({...p,checkpoint:20,epoch:"e1",fromCursor:10,rowsPatch:[{id:"r1",op:"upsert",value:{title:"hello"}}],type:"relay_shape_poke",...e});f("frames a poke as pokeStart → pokePart per shape → pokeEnd, under one poke id",async()=>{const{close:e,createSocket:t,readFrames:o,sockets:a}=m();try{const n=c(a,[{cursor:10,epoch:"e1",frames:[]}]),h=d(a,t,"c-alice","s1");await s(n,h,"s1"),await n.handleControl(r());const y=(await o(h)).map(w=>JSON.parse(w));i(y.map(w=>w.type)).toStrictEqual(["pokeStart","pokePart","pokeEnd"]),i(new Set(y.map(w=>w.pokeId)).size).toBe(1),i(y[1]?.shapeId).toBe("s1"),i(y[2]?.checkpoint).toBe(20)}finally{e?.()}}),f("delivers a poke only to sockets whose cursor matches, and advances them past it",async()=>{const{close:e,createSocket:t,readFrames:o,sockets:a}=m();try{const n=c(a,[{cursor:10,epoch:"e1",frames:[]},{cursor:7,epoch:"e1",frames:[]}]),h=d(a,t,"c-alice","s1"),y=d(a,t,"c-bob","s2");await s(n,h,"s1"),await s(n,y,"s2"),await n.handleControl(r());const w=await o(h);i(w.length).toBe(3);const S=await o(y);i(S.length).toBe(0),await n.handleControl(r());const u=await o(h);i(u.length).toBe(3)}finally{e?.()}}),f("skips a socket whose cursor matches under a different CDC epoch",async()=>{const{close:e,createSocket:t,readFrames:o,sockets:a}=m();try{const n=c(a,[{cursor:10,epoch:"e1",frames:[]}]),h=d(a,t,"c-alice","s1");await s(n,h,"s1"),await n.handleControl(r({epoch:"e2"}));const y=await o(h);i(y.length).toBe(0),await n.handleControl(r());const w=await o(h);i(w.length).toBe(3)}finally{e?.()}})}),k("RLS identity under live subscription",()=>{const g="shard-a",p={args:{},name:"lobby-messages"},d={args:{},name:"my-orders"},c=s=>{const r=[],e=[],t={fetch:(a,n)=>(r.push(JSON.parse(n?.body??"{}")),Promise.resolve(new Response(void 0,{status:204})))},o=I({buildShapeDiff:()=>[{id:"r1",op:"upsert",value:{}}],computeOpLogShapeSeed:()=>({baseCheckpoint:void 0,cursor:10,epoch:"e1",rowsPatch:[]}),currentCdcEpoch:()=>"e1",deliverWhisperLocal:()=>0,doName:()=>g,env:()=>({SHARD:{get:()=>t,getByName:()=>t,idFromName:a=>a}}),getWebSockets:()=>[],maskMetadata:()=>({columns:[]}),nextPokeId:()=>"poke-1",readAttachment:()=>({}),recordShapePokeFanout:()=>{},resolveShape:(a,n,h)=>(e.push(h),a===d.name?{columns:["id"],effectiveWhere:{org:h?.identity?.org??"anonymous"},global:!1,table:"orders"}:{columns:["id"],effectiveWhere:{room:"lobby"},global:!1,table:"messages"}),rlsMetadata:()=>({policies:[]}),shardBinding:()=>"SHARD",sql:()=>s});if(o===void 0)throw new Error("expected an owner link for an un-suffixed DO name");return{owner:o,posts:r,resolvedUnder:e}},l=async(s,r)=>{await s.handleControl(new Request("https://owner.internal/_lunora/relay",{body:JSON.stringify({...r,connectionId:"c-alice",identity:{org:"acme"},relayIndex:0,subId:"s1",type:"relay_shape_subscribe",userId:"u1"}),headers:{"content-type":"application/json"},method:"POST"}))};f("seeds a subscriber under its own forwarded identity, never the anonymous one",async()=>{const{close:s,host:r}=m();try{const{owner:e,resolvedUnder:t}=c(r.sql);await l(e,d),i(t.some(o=>o?.userId==="u1"&&o.identity?.org==="acme")).toBe(!0)}finally{s?.()}}),f("routes an identity-scoped shape to a per-socket poke, not the cohort multicast",async()=>{const{close:s,host:r}=m();try{const{owner:e,posts:t}=c(r.sql);await l(e,d),t.length=0,await e.onFlush(new Set(["orders"]),20);const o=t.filter(a=>a.type==="relay_shape_poke");i(o.length).toBe(1),i(o[0]?.targetConnectionId).toBe("c-alice")}finally{s?.()}}),f("still multicasts an identity-blind shape to the whole cohort",async()=>{const{close:s,host:r}=m();try{const{owner:e,posts:t}=c(r.sql);await l(e,p),t.length=0,await e.onFlush(new Set(["messages"]),20);const o=t.filter(a=>a.type==="relay_shape_poke");i(o.length).toBe(1),i(o[0]?.targetConnectionId).toBeUndefined()}finally{s?.()}})})})};export{D as defineEngineContractSuite};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{stableStringify as u}from"./stableStringify-BjLh4gvA.mjs";const l=(t,s)=>{const o={_id:t._id};if(s){for(const e of s)Object.hasOwn(t,e)&&(o[e]=t[e]);return o}for(const[e,n]of Object.entries(t))e!=="_id"&&e!=="_creationTime"&&(o[e]=n);return o},h=(t,s,o)=>{const{columns:e,table:n}=o,i=[],d=new Map;for(const r of t){const c=l(r,e),a=String(c._id),f=u(c);d.set(a,f);const p=s.get(a);p===void 0?i.push({doc:c,id:a,op:"insert",seq:0,table:n,ts:0}):p!==f&&i.push({doc:c,id:a,op:"update",seq:0,table:n,ts:0})}for(const r of s.keys())d.has(r)||i.push({id:r,op:"delete",seq:0,table:n,ts:0});return{changes:i,nextBaseline:d}};export{h as diffExternalSource,l as projectExternalSourceRow};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{toErrorBody as f}from"@lunora/errors";const d=200,b=(r,s)=>{const n=t=>{const e=r.tables[t];return e?e.shardMode?.kind!=="global":!1};if(s&&s.length>0){const t=[];for(const e of s)n(e)&&t.push(e);return t}const o=[];for(const t of Object.keys(r.tables))n(t)&&o.push(t);return o},u=async function*(r,s,n=d){let o=null,t=!1;for(;!t;){const e=await r.findMany(s,{cursor:o,limit:n});for(const a of e.page)yield{doc:a,table:s};o=e.continueCursor,t=e.isDone||o===null}},k=async function*(r,s,n){const o=b(s,n.tables),t=n.batchSize??d;for(const e of o)yield*u(r,e,t)},p=new Set(["_creationTime","_id"]),y=(r,s)=>{for(const[n,o]of Object.entries(r.shape)){const t=s[n];if(t===void 0&&o.kind==="optional")continue;const e=o.parse;if(typeof e=="function")try{e(t)}catch(a){const c=a instanceof Error?a.message:String(a);return`field "${n}": ${c}`}}},h=(r,s,n)=>{const o=r.tables[s];if(!o)return`unknown table: ${s}`;if(o.shardMode?.kind==="global")return`table "${s}" is a global (.global()) table and is not importable through the shard import path`;const t=Object.fromEntries(Object.entries(n).filter(([e])=>!p.has(e)));for(const e of Object.keys(t))if(!(e in o.shape))return`unexpected field "${e}": not declared in table "${s}"`;return y(o,t)},g=async(r,s)=>{try{return await r.get(s)!==null}catch{return!1}},m=async(r,s,n,o)=>{const{doc:t,table:e}=n;if(typeof e!="string"||e.length===0)return{error:{code:"BAD_ROW",line:o,message:"row is missing `table`",table:e},kind:"error"};if(!t||typeof t!="object"||Array.isArray(t))return{error:{code:"BAD_ROW",line:o,message:"row is missing or malformed `doc`",table:e},kind:"error"};const a=h(s,e,t);if(a!==void 0)return{error:{code:"VALIDATION_ERROR",line:o,message:a,table:e},kind:"error"};const c=typeof t._id=="string"?t._id:void 0;if(c!==void 0&&await g(r,c))return{kind:"conflict"};try{return await r.insert(e,t,{allowExplicitId:!0}),{kind:"inserted",table:e}}catch(i){const{body:l}=f(i,{fallbackCode:"INSERT_FAILED"});return{error:{code:l.code,line:o,message:l.message,table:e},kind:"error"}}},A=async(r,s,n)=>{const o=[],t={};let e=0,a=(n.startLine??1)-1;for(const c of n.rows){a+=1;const i=await m(r,s,c,a);i.kind==="error"?o.push(i.error):i.kind==="conflict"?e+=1:t[i.table]=(t[i.table]??0)+1}return{conflicts:e,errors:o,inserted:t}},S=r=>{const s=Array.isArray(r.tables)?r.tables.filter(n=>typeof n=="string"):void 0;return{batchSize:typeof r.batchSize=="number"?r.batchSize:void 0,tables:s}},E=r=>{const s=Array.isArray(r.rows)?r.rows:[],n=[];for(const t of s){if(!t||typeof t!="object")continue;const e=t;if(typeof e.table!="string"||!e.doc||typeof e.doc!="object"||Array.isArray(e.doc)){n.push({doc:e.doc??{},table:typeof e.table=="string"?e.table:""});continue}n.push({doc:e.doc,table:e.table})}const o=typeof r.startLine=="number"?r.startLine:void 0;return{rows:n,startLine:o}};export{k as exportShardRows,u as exportShardTable,A as importShardRows,S as parseExportShardArgs,E as parseImportShardArgs,b as selectExportTables,h as validateImportRow};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const g=async t=>{const e=t.schema.tables[t.tableName]?.triggerMap;if(e)for(const a of Object.values(e))a.timing===t.timing&&a.op===t.op&&await a.handler(t.ctx,t.event)},i=(t,e,a)=>{const r=t.tables[e]?.triggerMap;return r?Object.values(r).some(s=>s.op===a):!1};export{i as hasTrigger,g as runTriggers};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import{LunoraError as E}from"@lunora/errors";import{sql as m}from"drizzle-orm";import{runDrizzle as h}from"./runDrizzle-GKR3y97k.mjs";import{liftSourceId as x}from"./liftSourceId-DApAbu7Q.mjs";import{normalizeSourceValue as K}from"./liftSourceId-DApAbu7Q.mjs";import{runExternalSourceTick as v,materializeExternalRowsIncremental as M}from"./materializeExternalRows-DOQJV9p2.mjs";const _="__lunora_source_cursor",D=e=>e instanceof Date?`d:${e.toISOString()}`:typeof e=="bigint"?`b:${e.toString()}`:typeof e=="number"?`n:${e.toString()}`:`s:${e}`,R=e=>{const t=e.slice(2);switch(e[0]){case"b":return BigInt(t);case"d":return new Date(t);case"n":return Number(t);default:return t}},k=/^-?\d+$/,I=/^-?\d+(?:\.\d+)?$/,A=(e,t)=>e instanceof Date&&t instanceof Date?e.getTime()>t.getTime():typeof e=="bigint"&&typeof t=="bigint"||typeof e=="number"&&typeof t=="number"?e>t:typeof e=="string"&&typeof t=="string"&&I.test(e)&&I.test(t)?k.test(e)&&k.test(t)?BigInt(e)>BigInt(t):Number(e)>Number(t):String(e)>String(t),q=(e,t,a)=>{let n=a===null?void 0:R(a);for(const r of e){const s=r[t];if(s==null)continue;const i=s;(n===void 0||A(i,n))&&(n=i)}return n===void 0?null:D(n)},O=e=>{h(e,m`CREATE TABLE IF NOT EXISTS ${m.identifier(_)} (
|
|
2
|
+
table_name TEXT NOT NULL,
|
|
3
|
+
shard_key TEXT NOT NULL,
|
|
4
|
+
watermark TEXT,
|
|
5
|
+
last_reconcile_ms INTEGER,
|
|
6
|
+
PRIMARY KEY (table_name, shard_key)
|
|
7
|
+
)`)},B=(e,t,a)=>{const n=h(e,m`SELECT watermark, last_reconcile_ms FROM ${m.identifier(_)} WHERE table_name = ${t} AND shard_key = ${a} LIMIT 1`).toArray()[0];return{lastReconcileMs:n?.last_reconcile_ms??null,watermark:n?.watermark??null}},C=(e,t,a,n)=>{h(e,m`INSERT INTO ${m.identifier(_)} (table_name, shard_key, watermark, last_reconcile_ms)
|
|
8
|
+
VALUES (${t}, ${a}, ${n.watermark}, ${n.lastReconcileMs})
|
|
9
|
+
ON CONFLICT(table_name, shard_key) DO UPDATE SET watermark = excluded.watermark, last_reconcile_ms = excluded.last_reconcile_ms`)},P=(e,t,a)=>e==="manual"?!1:e===void 0||t===void 0?!0:a-t>=e.everyMs,T=async(e,t,a,n)=>{const r=await e.query(t,a);return{documents:r.map(s=>x(s,{idColumn:n.idColumn,map:n.map})),rows:r}},V=async(e,t,a,n,r,s)=>{const i=r.tenantBy?r.tenantBy(s):[],{documents:l}=await T(a,r.query,i,r);return v(e,t,l,{columns:r.columns,table:n})},N=(e,t)=>{const a=e[t];return a!=null&&a!==!1&&a!==0},Y=async(e,t,a,n,r,s,i)=>{const{cursor:l}=r;if(!l)throw new E("INTERNAL",`external-source: table "${n}" is mode "incremental" but has no \`cursor\` — this should have been rejected at defineSchema`);O(e);const u=B(e,n,s),b=r.tenantBy?r.tenantBy(s):[],g=r.reconcileEveryMs!==void 0&&(u.lastReconcileMs===null||i-u.lastReconcileMs>=r.reconcileEveryMs),w=u.watermark===null||g;let o,f;if(u.watermark===null||g){o=await T(a,r.query,b,r);const{softDeleteColumn:c}=r,y=c?o.documents.filter(($,p)=>{const d=o.rows[p];return!(d&&N(d,c))}):o.documents;({applied:f}=await v(e,t,y,{columns:r.columns,table:n}))}else{o=await T(a,l.query,[...b,R(u.watermark)],r);const{softDeleteColumn:c}=r,y=c?new Set(o.documents.flatMap(($,p)=>{const d=o.rows[p];return d&&N(d,c)?[String($._id)]:[]})):void 0;({applied:f}=await M(t,o.documents,{columns:r.columns,deletedIds:y,table:n}))}const S=q(o.rows,l.column,u.watermark);if(w&&S===null&&o.rows.length>0)throw new E("INTERNAL",`external-source: table "${n}" (mode "incremental") pulled ${String(o.rows.length)} rows but none carry the cursor column "${l.column}" — the seed \`query\` must project it (matching \`cursor.query\`'s alias), or the watermark can never advance.`);const L=o.rows.length>0&&o.rows.every(c=>c[l.column]===null||c[l.column]===void 0);if(!w&&L)throw new E("INTERNAL",`external-source: table "${n}" (mode "incremental") pulled ${String(o.rows.length)} rows but none carry the cursor column "${l.column}" — \`cursor.query\` must project it (matching the seed \`query\`'s alias), or the watermark can never advance and every tick re-pulls the same stranded slice.`);return C(e,n,s,{lastReconcileMs:w?i:u.lastReconcileMs,watermark:S}),{applied:f}};export{N as isSoftDeleted,P as isSourceDue,x as liftSourceId,K as normalizeSourceValue,Y as pullExternalSourceIncrementalTick,V as pullExternalSourceTick};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{LunoraError as m}from"@lunora/errors";const f=r=>r instanceof Date?r.toISOString():typeof r=="bigint"?String(r):Array.isArray(r)?r.map(t=>f(t)):r!==null&&typeof r=="object"&&Object.getPrototypeOf(r)===Object.prototype?n(r):r,n=r=>{const t={};for(const[o,i]of Object.entries(r))t[o]=f(i);return t},p=(r,t={})=>{const{idColumn:o="id",map:i}=t,e=r[o];if(e==null)throw new m("INTERNAL",`external-source: row is missing id column "${o}"`);if(typeof e!="string"&&typeof e!="number"&&typeof e!="bigint")throw new TypeError(`external-source: id column "${o}" must be a string or number`);const s=String(e);if(i)return n({...i(r),_id:s});const c={};for(const[u,a]of Object.entries(r))u!==o&&(c[u]=a);return n({...c,_id:s})};export{p as liftSourceId,n as normalizeSourceDocument,f as normalizeSourceValue};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{applyCdcChanges as m}from"./CDC_LOG_TABLE-BnqlH2eZ.mjs";import{selectShapeRows as f}from"./selectShapeMemberIds-DvE7K6zG.mjs";import{diffExternalSource as g,projectExternalSourceRow as c}from"./diffExternalSource-DMpkJta1.mjs";import{stableStringify as p}from"./stableStringify-BjLh4gvA.mjs";const x=async(t,s,r,e)=>{const{changes:n,nextBaseline:a}=g(s,r,e);return await m(t,n),{applied:n.length,nextBaseline:a}},E=async(t,s,r)=>{const{columns:e,deletedIds:n,table:a}=r,i=[];for(const u of s){const l=c(u,e),o=String(l._id);if(n?.has(o)){await t.get(o,a)&&i.push({id:o,op:"delete",seq:0,table:a,ts:0});continue}const d=await t.get(o,a);d&&p(c({...d,_id:o},e))===p(l)||i.push({doc:l,id:o,op:"insert",seq:0,table:a,ts:0})}return await m(t,i),{applied:i.length}},w=(t,s,r)=>{const e=new Map;for(const{doc:n,id:a}of f(t,s,void 0))e.set(a,p(c({...n,_id:a},r)));return e},R=async(t,s,r,e)=>{const n=w(t,e.table,e.columns);return x(s,r,n,e)};export{x as materializeExternalRows,E as materializeExternalRowsIncremental,w as readExternalSourceBaseline,R as runExternalSourceTick};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{sql as m}from"drizzle-orm";import{MySqlDialect as s}from"drizzle-orm/mysql-core";import{PgDialect as l}from"drizzle-orm/pg-core";import{SQLiteSyncDialect as n}from"drizzle-orm/sqlite-core";const p=new l,a=new s,i=new n,q={mysql:a,postgres:p,sqlite:i},w=(r,t)=>{const{params:e,sql:o}=q[r].sqlToQuery(t);return{params:e,sql:o}},D=r=>m`${r}`;export{D as param,w as renderSql};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const t=e=>`"${e.replaceAll('"','""')}"`;export{t as q};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{renderSql as o}from"./param-B5lF5Jd9.mjs";const t=(r,e,...l)=>r.exec.call(r,e,...l),c=(r,e)=>{const{params:l,sql:n}=o("sqlite",e);return t(r,n,...l)};export{c as runDrizzle,t as runSql};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import{l as I,T as N,c as l}from"./FTS_COUNT_COLUMN-D1gY3wwZ-CZHdaRrd.mjs";import"@lunora/errors";import{sql as e}from"drizzle-orm";import{aggregateTableName as S}from"./aggregateTableName-G-eXyjcz.mjs";import{backfillSearchIndexesForTable as p}from"./backfillAggregateIndexes-BHiewuB-.mjs";import{migrateCdcLog as g,migrateCdcMeta as h}from"./CDC_LOG_TABLE-BnqlH2eZ.mjs";import{migrateClientWatermark as R}from"./CLIENT_WATERMARK_TABLE-CXsSLU8D.mjs";import{migrateGlobalShapeSnapshot as C}from"./GLOBAL_SHAPE_SNAPSHOT_TABLE-DX-6-gBP.mjs";import{migrateIdempotency as O}from"./IDEMPOTENCY_TABLE-DjkEMp6w.mjs";import{migrateSearchState as U}from"./SEARCH_STATE_TABLE-eLN8U5tz.mjs";import{runDrizzle as t}from"./runDrizzle-GKR3y97k.mjs";import{DOC_COLUMN as b,jsonPathSql as E,createIndexSql as T,tableColumns as u,isFtsAvailable as X,AGG_KEY as x,AGG_VALUE as B,AGG_COUNT as $,geoTableName as G}from"./AGG_COUNT-BWXe3gtQ.mjs";import{sortColumnName as Y,rankTableName as F}from"./RANK_TIEBREAK-9NU5s_mi.mjs";import{recordSchemaVersion as M}from"./SCHEMA_HISTORY_MAX_VERSIONS-CHXz6p_z.mjs";const D=(r,o,a)=>{for(const i of a.indexes){const n=`${o}_${i.name}`,s=e.join(i.fields.map(f=>E(f)),e`, `);t(r,T(n,o,s,i.unique??!1))}for(const[i,n]of u(a)){if(!n.unique)continue;const s=`${o}_unique_${i}`;t(r,T(s,o,E(i),!0))}},j=(r,o,a)=>{if(!(!a.searchIndexes||a.searchIndexes.length===0||!X(r))){for(const i of a.searchIndexes){const n=l(o,i.name);t(r,e`CREATE VIRTUAL TABLE IF NOT EXISTS ${e.identifier(n)} USING fts5(${e.identifier(I)}, ${e.identifier(N)} UNINDEXED)`),t(r,e`CREATE VIRTUAL TABLE IF NOT EXISTS ${e.identifier(`${n}__vocab`)} USING fts5vocab(${e.identifier(n)}, ${e.raw("instance")})`)}p(r,o,a)}},k=(r,o,a)=>{if(a.geoIndexes)for(const i of a.geoIndexes){const n=G(o,i.name);t(r,e`CREATE TABLE IF NOT EXISTS ${e.identifier(n)} (${e.identifier("__id__")} TEXT PRIMARY KEY, ${e.identifier("__geohash__")} TEXT NOT NULL, ${e.identifier("__lat__")} REAL NOT NULL, ${e.identifier("__lng__")} REAL NOT NULL)`);const s=`${o}__geo_${i.name}__btree`;t(r,T(s,n,e`${e.identifier("__geohash__")} ASC, ${e.identifier("__id__")} ASC`,!1))}},q=(r,o,a)=>{if(a.aggregateIndexes)for(const i of a.aggregateIndexes){const n=S(o,i.name);t(r,e`CREATE TABLE IF NOT EXISTS ${e.identifier(n)} (${x} TEXT PRIMARY KEY, ${B} REAL, ${$} INTEGER NOT NULL DEFAULT 0)`),t(r,e`PRAGMA table_info(${e.identifier(n)})`).toArray().some(s=>s.name==="__count__")||t(r,e`ALTER TABLE ${e.identifier(n)} ADD COLUMN ${$} INTEGER NOT NULL DEFAULT 0`)}},P=(r,o,a)=>{if(a.rankIndexes)for(const i of a.rankIndexes){const n=F(o,i.name),s=i.sortBy.map((_,d)=>Y(d)),f=s.map(_=>e`${e.identifier(_)} BLOB`),c=f.length>0?e`, ${e.join(f,e`, `)}`:e``;t(r,e`CREATE TABLE IF NOT EXISTS ${e.identifier(n)} (${e.identifier("__id__")} TEXT PRIMARY KEY, ${e.identifier("__partition__")} TEXT NOT NULL${c})`);const m=[e`${e.identifier("__partition__")} ASC`];for(const[_,d]of s.entries()){const L=i.sortBy[_]?.direction;m.push(e`${e.identifier(d)} ${e.raw(L==="desc"?"DESC":"ASC")}`)}m.push(e`${e.identifier("__id__")} ASC`);const A=`${o}__rank_${i.name}__btree`;t(r,T(A,n,e.join(m,e`, `),!1))}},ne=(r,o,a={})=>{a.schemaSnapshot!==void 0&&M(r,a.schemaSnapshot.hash,a.schemaSnapshot.json),U(r);for(const[i,n]of Object.entries(o.tables))n.shardMode?.kind!=="global"&&(t(r,e`CREATE TABLE IF NOT EXISTS ${e.identifier(i)} (
|
|
2
|
+
id TEXT PRIMARY KEY,
|
|
3
|
+
_creationTime REAL NOT NULL,
|
|
4
|
+
${e.identifier(b)} TEXT NOT NULL
|
|
5
|
+
)`),D(r,i,n),j(r,i,n),k(r,i,n),q(r,i,n),P(r,i,n));a.cdc&&(g(r),h(r),R(r)),O(r),C(r)};export{ne as runShardMigrations};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const l=async(o,e,n=8)=>{let t=0;const c=async()=>{let a=o[t];for(t+=1;a!==void 0;){try{await e(a)}catch{}a=o[t],t+=1}};await Promise.all(Array.from({length:Math.min(n,o.length)},()=>c()))};export{l as runSocketPool};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{sql as e}from"drizzle-orm";import{runDrizzle as p}from"./runDrizzle-GKR3y97k.mjs";import{jsonPathSql as r}from"./AGG_COUNT-BWXe3gtQ.mjs";const E=(s,i,a,o)=>{const d=a-(i.after??0),t=[e`${r(i.field)} IS NOT NULL`,e`${r(i.field)} < ${d}`];i.softDeleteField!==void 0&&t.push(e`${r(i.softDeleteField)} IS NULL`);const f=e`SELECT id FROM ${e.identifier(i.table)} WHERE ${e.join(t,e` AND `)} LIMIT ${e.raw(String(Math.max(0,Math.floor(o))+1))}`,l=p(s,f).toArray(),n=l.length>o,m=l.slice(0,o).map($=>$.id);return{hasMore:n,ids:m}};export{E as selectExpiredIds};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{sql as e}from"drizzle-orm";import{runDrizzle as m}from"./runDrizzle-GKR3y97k.mjs";import{DOC_COLUMN as a,rowToDocument as c,jsonPathSql as u}from"./AGG_COUNT-BWXe3gtQ.mjs";import{compileWhereSql as h}from"./compileWhereSql-BLcfs4QW.mjs";import{serializeSqlValue as $}from"./serializeSqlValue-DnpyaLcw.mjs";const S={fieldRef:u,serialize:$},g=i=>{if(i.length!==0)return e`id IN (${e.join(i.map(o=>e`${o}`),e`, `)})`},p=(i,o)=>{const r=[];o&&r.push(o);const t=h(i,S);return t&&r.push(t),r.length===0?e``:e` WHERE ${e.join(r,e` AND `)}`},C=(i,o,r)=>{const t=p(r,void 0),d=m(i,e`SELECT id, _creationTime, ${e.identifier(a)} FROM ${e.identifier(o)}${t}`).toArray(),n=[];for(const s of d){const f=c(s),{id:l}=s;f!==void 0&&typeof l=="string"&&n.push({doc:f,id:l})}return n},D=(i,o,r,t)=>{if(t.length===0)return new Set;const d=p(r,g(t)),n=m(i,e`SELECT id FROM ${e.identifier(o)}${d}`).toArray();return new Set(n.map(s=>s.id))};export{D as selectShapeMemberIds,C as selectShapeRows};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const e=t=>typeof t=="boolean"?t?1:0:t===null||typeof t=="string"||typeof t=="number"?t:typeof t=="bigint"?t.toString():JSON.stringify(t);export{e as serializeSqlValue};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const y=(e,t)=>e<t?-1:e>t?1:0,s=e=>{if(e===void 0)return"null";if(typeof e=="bigint")throw new TypeError("stableStringify: cannot use a bigint in a stable JSON cache key — pass it as a string, or use stableWireKey");if(e===null||typeof e!="object")return JSON.stringify(e);if(Array.isArray(e))return`[${e.map(r=>s(r)).join(",")}]`;const t=Object.getPrototypeOf(e);if(t!==null&&t!==Object.prototype){const r=e.constructor?.name??"value";throw new TypeError(`stableStringify: cannot use a ${r} in a stable JSON cache key — only plain objects, arrays, and JSON primitives are supported (wire-typed values key via stableWireKey)`)}const n=e,a=Object.keys(n).toSorted(y),i=[];for(const r of a){const o=n[r];o!==void 0&&i.push(`${JSON.stringify(r)}:${s(o)}`)}return`{${i.join(",")}}`};export{s as stableStringify};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{stableStringify as r}from"./stableStringify-BjLh4gvA.mjs";import{f as e}from"./wire-codec-Ctnni0h6.mjs";const s=t=>r(e(t));export{s as stableWireKey};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const b=e=>{let t="";for(let i=0;i<e.length;i+=32768)t+=String.fromCharCode(...e.subarray(i,i+32768));return btoa(t)},w=e=>{const t=atob(e),i=new Uint8Array(t.length);for(let c=0;c<t.length;c+=1)i[c]=t.codePointAt(c)??0;return i},o="$lunora.wire$";const g="__proto__",d={BigInt64Array,BigUint64Array,Float32Array,Float64Array,Int8Array,Int16Array,Int32Array,Uint8Array,Uint8ClampedArray,Uint16Array,Uint32Array},p={Error,EvalError,RangeError,ReferenceError,SyntaxError,TypeError,URIError},l=(e,t=0)=>{if(t>64)throw new RangeError("wire-codec: value nesting exceeds the 64-level limit");if(e===void 0)return[o,"undefined"];if(e===null)return null;const i=typeof e;if(i==="bigint")return[o,"bigint",e.toString()];if(i==="number"){const r=e;return Number.isNaN(r)?[o,"nan"]:r===1/0?[o,"inf"]:r===-1/0?[o,"-inf"]:r}if(i!=="object")return e;if(e instanceof Date)return[o,"date",l(e.getTime(),t+1)];if(e instanceof Error){const r=e,a={};for(const y of Object.keys(r))r[y]!==void 0&&(a[y]=l(r[y],t+1));const f=[o,"error",r.name,r.message,a];return r.cause!==void 0&&f.push(l(r.cause,t+1)),f}if(e instanceof URL)return[o,"url",e.href];if(e instanceof Map)return[o,"map",[...e.entries()].map(([r,a])=>[l(r,t+1),l(a,t+1)])];if(e instanceof Set)return[o,"set",[...e].map(r=>l(r,t+1))];if(e instanceof ArrayBuffer)return[o,"bytes",b(new Uint8Array(e)),"ArrayBuffer"];if(ArrayBuffer.isView(e)){const r=e,a=r.constructor.name,f=new Uint8Array(r.buffer,r.byteOffset,r.byteLength);return a==="Uint8Array"?[o,"bytes",b(f)]:[o,"bytes",b(f),a]}if(Array.isArray(e)){const r=e.map(a=>l(a,t+1));return r.length>0&&r[0]===o?[o,"arr",r]:r}const c=Object.getPrototypeOf(e);if(c!==null&&c!==Object.prototype){const r=e.constructor?.name??"value";throw new TypeError(`wire-codec: cannot encode a ${r} over the Lunora wire — only plain objects, arrays, and the supported built-ins (Date, Error, URL, Map, Set, ArrayBuffer/typed arrays, bigint) round-trip`)}const n=e,u={};for(const r of Object.keys(n)){const a=n[r];a!==void 0&&(u[r]=l(a,t+1))}return u},s=(e,t=0)=>{if(t>64)throw new RangeError("wire-codec: value nesting exceeds the 64-level limit");if(e===null||typeof e!="object")return e;if(Array.isArray(e)){if(e[0]===o)switch(e[1]){case"-inf":return-1/0;case"arr":return e[2].map(n=>s(n,t+1));case"bigint":{const n=e[2];if(typeof n!="string"||n.length>1024||!/^-?\d+$/.test(n))throw new RangeError("wire-codec: invalid or over-long bigint (max 1024 digits)");return BigInt(n)}case"date":return new Date(s(e[2],t+1));case"map":return new Map(e[2].map(([n,u])=>[s(n,t+1),s(u,t+1)]));case"set":return new Set(e[2].map(n=>s(n,t+1)));case"url":return new URL(e[2]);case"error":{const n=e[2],u=e[3],r=(Object.hasOwn(p,n)?p[n]:void 0)??Error,a=new r(u);a.name!==n&&Object.defineProperty(a,"name",{configurable:!0,value:n,writable:!0});const f=s(e[4],t+1);for(const y of Object.keys(f))y===g?Object.defineProperty(a,y,{configurable:!0,enumerable:!0,value:f[y],writable:!0}):a[y]=f[y];return e.length>5&&Object.defineProperty(a,"cause",{configurable:!0,value:s(e[5],t+1),writable:!0}),a}case"bytes":{const n=w(e[2]),u=e[3]??"Uint8Array";if(u==="ArrayBuffer")return n.buffer.byteLength===n.byteLength?n.buffer:n.slice().buffer;const r=Object.hasOwn(d,u)?d[u]:void 0;return r?new r(n.slice().buffer):n}case"inf":return 1/0;case"nan":return Number.NaN;case"undefined":return;default:return e.map(n=>s(n,t+1))}return e.map(n=>s(n,t+1))}const i=e,c={};for(const n of Object.keys(i)){const u=s(i[n],t+1);n===g?Object.defineProperty(c,n,{configurable:!0,enumerable:!0,value:u,writable:!0}):c[n]=u}return c};export{s as c,l as f};
|
package/package.json
CHANGED
|
@@ -1,10 +1,58 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lunora/shard-engine",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.0.0-alpha.2",
|
|
4
|
+
"description": "Host-neutral reactive engine for Lunora: per-shard state, OCC, CDC, reactive subscriptions, and the poke protocol. Consumes @lunora/platform host contracts and can be mounted on any platform host.",
|
|
5
5
|
"keywords": [
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
"cloudflare",
|
|
7
|
+
"durable-objects",
|
|
8
|
+
"lunora",
|
|
9
|
+
"platform",
|
|
10
|
+
"reactivity",
|
|
11
|
+
"shard-engine",
|
|
12
|
+
"workers"
|
|
13
|
+
],
|
|
14
|
+
"homepage": "https://lunora.sh",
|
|
15
|
+
"bugs": "https://github.com/anolilab/lunora/issues",
|
|
16
|
+
"license": "FSL-1.1-Apache-2.0",
|
|
17
|
+
"author": {
|
|
18
|
+
"name": "Daniel Bannert",
|
|
19
|
+
"email": "d.bannert@anolilab.de"
|
|
20
|
+
},
|
|
21
|
+
"repository": {
|
|
22
|
+
"type": "git",
|
|
23
|
+
"url": "git+https://github.com/anolilab/lunora.git",
|
|
24
|
+
"directory": "packages/shard-engine"
|
|
25
|
+
},
|
|
26
|
+
"files": [
|
|
27
|
+
"./dist",
|
|
28
|
+
"README.md",
|
|
29
|
+
"LICENSE.md"
|
|
30
|
+
],
|
|
31
|
+
"type": "module",
|
|
32
|
+
"sideEffects": false,
|
|
33
|
+
"main": "./dist/index.mjs",
|
|
34
|
+
"module": "./dist/index.mjs",
|
|
35
|
+
"types": "./dist/index.d.ts",
|
|
36
|
+
"exports": {
|
|
37
|
+
".": {
|
|
38
|
+
"types": "./dist/index.d.ts",
|
|
39
|
+
"import": "./dist/index.mjs"
|
|
40
|
+
},
|
|
41
|
+
"./conformance": {
|
|
42
|
+
"types": "./dist/conformance/index.d.ts",
|
|
43
|
+
"import": "./dist/conformance/index.mjs"
|
|
44
|
+
},
|
|
45
|
+
"./package.json": "./package.json"
|
|
46
|
+
},
|
|
47
|
+
"publishConfig": {
|
|
48
|
+
"access": "public"
|
|
49
|
+
},
|
|
50
|
+
"dependencies": {
|
|
51
|
+
"@lunora/errors": "1.0.0-alpha.9",
|
|
52
|
+
"@lunora/platform": "1.0.0-alpha.1",
|
|
53
|
+
"drizzle-orm": "^0.45.2"
|
|
54
|
+
},
|
|
55
|
+
"engines": {
|
|
56
|
+
"node": "^22.15.0 || >=24.11.0"
|
|
57
|
+
}
|
|
58
|
+
}
|