@sixb/core 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/dist/actions/index.js +8 -8
  2. package/dist/agents/index.js +16 -15
  3. package/dist/agents/index.js.map +1 -1
  4. package/dist/bootstrap/index.js +20 -19
  5. package/dist/bootstrap/index.js.map +1 -1
  6. package/dist/broker/in-memory.d.ts +8 -2
  7. package/dist/broker/in-memory.d.ts.map +1 -1
  8. package/dist/broker/index.d.ts +4 -2
  9. package/dist/broker/index.d.ts.map +1 -1
  10. package/dist/broker/index.js +7 -3
  11. package/dist/broker/index.js.map +1 -1
  12. package/dist/broker/retention.d.ts +4 -0
  13. package/dist/broker/retention.d.ts.map +1 -0
  14. package/dist/broker/subscriber.d.ts +10 -0
  15. package/dist/broker/subscriber.d.ts.map +1 -0
  16. package/dist/broker/types.d.ts +14 -1
  17. package/dist/broker/types.d.ts.map +1 -1
  18. package/dist/{chunk-k3cqkkn2.js → chunk-1824gssz.js} +30 -65
  19. package/dist/chunk-1824gssz.js.map +12 -0
  20. package/dist/{chunk-h1gfhhbh.js → chunk-28ja2cjs.js} +1 -1
  21. package/dist/chunk-2c4j899j.js.map +1 -1
  22. package/dist/{chunk-ppytyr6w.js → chunk-358caqf5.js} +4 -4
  23. package/dist/chunk-358caqf5.js.map +12 -0
  24. package/dist/{chunk-kxwa347f.js → chunk-3qms4nax.js} +2 -2
  25. package/dist/{chunk-kxwa347f.js.map → chunk-3qms4nax.js.map} +3 -3
  26. package/dist/{chunk-emyyk7dx.js → chunk-5c7ga8zz.js} +22 -145
  27. package/dist/chunk-5c7ga8zz.js.map +12 -0
  28. package/dist/{chunk-hpr1r6av.js → chunk-5p1v802y.js} +46 -39
  29. package/dist/{chunk-hpr1r6av.js.map → chunk-5p1v802y.js.map} +4 -4
  30. package/dist/{chunk-zcnq6105.js → chunk-85cd7e8c.js} +2 -1
  31. package/dist/chunk-85cd7e8c.js.map +10 -0
  32. package/dist/{chunk-1enhpacz.js → chunk-976rxdpx.js} +5 -5
  33. package/dist/{chunk-3f7veca0.js → chunk-9x8dybsb.js} +3 -3
  34. package/dist/chunk-9x8dybsb.js.map +10 -0
  35. package/dist/{chunk-r29fxdhn.js → chunk-aa6z4a19.js} +160 -44
  36. package/dist/chunk-aa6z4a19.js.map +12 -0
  37. package/dist/{chunk-r4tfx1sf.js → chunk-agbbygvh.js} +4 -4
  38. package/dist/{chunk-jjy0fhrn.js → chunk-b0ec55a1.js} +20 -3
  39. package/dist/chunk-b0ec55a1.js.map +12 -0
  40. package/dist/{chunk-xfbydgzy.js → chunk-bn138h9w.js} +7 -8
  41. package/dist/{chunk-xfbydgzy.js.map → chunk-bn138h9w.js.map} +3 -3
  42. package/dist/{chunk-an5tt9y0.js → chunk-ftv470j8.js} +42 -4
  43. package/dist/{chunk-an5tt9y0.js.map → chunk-ftv470j8.js.map} +7 -7
  44. package/dist/{chunk-0w35wavj.js → chunk-g7knyczr.js} +1 -1
  45. package/dist/{chunk-x57ycxq0.js → chunk-gqh45d2n.js} +42 -3
  46. package/dist/{chunk-x57ycxq0.js.map → chunk-gqh45d2n.js.map} +3 -3
  47. package/dist/{chunk-0rdrecsb.js → chunk-kfee1h52.js} +116 -32
  48. package/dist/chunk-kfee1h52.js.map +13 -0
  49. package/dist/{chunk-ypbtykv6.js → chunk-nmbxcaqx.js} +1 -1
  50. package/dist/{chunk-3tgwrcyh.js → chunk-nt99sxd6.js} +2 -2
  51. package/dist/{chunk-3tgwrcyh.js.map → chunk-nt99sxd6.js.map} +3 -3
  52. package/dist/{chunk-vf691g1r.js → chunk-p5vcsw9c.js} +2 -2
  53. package/dist/{chunk-mjbk2djy.js → chunk-p6a4npdx.js} +211 -106
  54. package/dist/{chunk-mjbk2djy.js.map → chunk-p6a4npdx.js.map} +9 -8
  55. package/dist/{chunk-3frsnm7x.js → chunk-qma80ych.js} +4 -6
  56. package/dist/chunk-qma80ych.js.map +11 -0
  57. package/dist/{chunk-qhcpj9wc.js → chunk-qyeptdyv.js} +96 -2
  58. package/dist/chunk-qyeptdyv.js.map +15 -0
  59. package/dist/{chunk-52hbv0ak.js → chunk-r7fj830t.js} +101 -48
  60. package/dist/chunk-r7fj830t.js.map +36 -0
  61. package/dist/{chunk-xyr8kppc.js → chunk-rtk71jzp.js} +1 -1
  62. package/dist/{chunk-m3z3x9v0.js → chunk-t1f4f1ve.js} +1 -1
  63. package/dist/chunk-v2vcg0w6.js +151 -0
  64. package/dist/chunk-v2vcg0w6.js.map +11 -0
  65. package/dist/connectors/connections/authorizations.d.ts.map +1 -1
  66. package/dist/connectors/connections/contracts.d.ts +3 -0
  67. package/dist/connectors/connections/contracts.d.ts.map +1 -1
  68. package/dist/connectors/connections/internal.js +1 -1
  69. package/dist/connectors/connections/request.d.ts +3 -1
  70. package/dist/connectors/connections/request.d.ts.map +1 -1
  71. package/dist/connectors/connections/runs.d.ts.map +1 -1
  72. package/dist/connectors/connections/tokens.d.ts.map +1 -1
  73. package/dist/connectors/connections/validation.d.ts +5 -1
  74. package/dist/connectors/connections/validation.d.ts.map +1 -1
  75. package/dist/connectors/types.d.ts +19 -4
  76. package/dist/connectors/types.d.ts.map +1 -1
  77. package/dist/events/execution.d.ts +1 -1
  78. package/dist/events/execution.d.ts.map +1 -1
  79. package/dist/events/index.js +1 -1
  80. package/dist/events/service.d.ts +2 -2
  81. package/dist/events/service.d.ts.map +1 -1
  82. package/dist/execution/agent.js +16 -15
  83. package/dist/execution/agent.js.map +1 -1
  84. package/dist/execution/primitive.js +16 -15
  85. package/dist/execution/primitive.js.map +2 -2
  86. package/dist/execution/request.js +16 -15
  87. package/dist/execution/request.js.map +2 -2
  88. package/dist/index.d.ts +3 -3
  89. package/dist/index.d.ts.map +1 -1
  90. package/dist/index.js +30 -28
  91. package/dist/index.js.map +1 -1
  92. package/dist/lake-storage/changes.d.ts +5 -0
  93. package/dist/lake-storage/changes.d.ts.map +1 -0
  94. package/dist/lake-storage/in-memory.d.ts +2 -1
  95. package/dist/lake-storage/in-memory.d.ts.map +1 -1
  96. package/dist/lake-storage/index.d.ts +2 -1
  97. package/dist/lake-storage/index.d.ts.map +1 -1
  98. package/dist/lake-storage/index.js +4 -2
  99. package/dist/lake-storage/index.js.map +1 -1
  100. package/dist/lake-storage/types.d.ts +36 -0
  101. package/dist/lake-storage/types.d.ts.map +1 -1
  102. package/dist/logging/internal.js +1 -1
  103. package/dist/materialization/index.d.ts +1 -1
  104. package/dist/materialization/index.d.ts.map +1 -1
  105. package/dist/materialization/model.d.ts +12 -1
  106. package/dist/materialization/model.d.ts.map +1 -1
  107. package/dist/materializer/index.d.ts +1 -1
  108. package/dist/materializer/index.d.ts.map +1 -1
  109. package/dist/materializer/index.js +5 -5
  110. package/dist/materializer/materializer.d.ts +5 -1
  111. package/dist/materializer/materializer.d.ts.map +1 -1
  112. package/dist/materializer/projections/entry-validator.d.ts +2 -1
  113. package/dist/materializer/projections/entry-validator.d.ts.map +1 -1
  114. package/dist/materializer/projections/replace.d.ts.map +1 -1
  115. package/dist/materializer/projections/source-materialization.d.ts +4 -3
  116. package/dist/materializer/projections/source-materialization.d.ts.map +1 -1
  117. package/dist/materializer/shared/normalize.d.ts +2 -1
  118. package/dist/materializer/shared/normalize.d.ts.map +1 -1
  119. package/dist/models/execution/index.js +5 -5
  120. package/dist/objects/index.js +9 -9
  121. package/dist/objects/query/index.js +5 -5
  122. package/dist/pipelines/index.js +6 -6
  123. package/dist/pipelines/run-dispatch.d.ts.map +1 -1
  124. package/dist/projections/internal.js +6 -8
  125. package/dist/projections/internal.js.map +1 -1
  126. package/dist/projections/run-dispatch.d.ts.map +1 -1
  127. package/dist/queues/in-memory.d.ts +7 -3
  128. package/dist/queues/in-memory.d.ts.map +1 -1
  129. package/dist/queues/index.js +1 -1
  130. package/dist/queues/indexed-heap.d.ts +15 -0
  131. package/dist/queues/indexed-heap.d.ts.map +1 -0
  132. package/dist/runtime/host.d.ts.map +1 -1
  133. package/dist/runtime/internal.js +1 -1
  134. package/dist/runtime/ontology-mutations.d.ts +3 -1
  135. package/dist/runtime/ontology-mutations.d.ts.map +1 -1
  136. package/dist/shares/internal.js +4 -4
  137. package/dist/storage/connector-connections/in-memory.d.ts.map +1 -1
  138. package/dist/storage/connector-connections/provider/runs.d.ts.map +1 -1
  139. package/dist/storage/connector-connections/provider.js +2 -2
  140. package/dist/storage/connector-connections/provider.js.map +3 -3
  141. package/dist/storage/connector-connections/types.d.ts +3 -2
  142. package/dist/storage/connector-connections/types.d.ts.map +1 -1
  143. package/dist/storage/index.d.ts +1 -1
  144. package/dist/storage/index.d.ts.map +1 -1
  145. package/dist/storage/index.js +4 -4
  146. package/dist/storage/internal.js +1 -1
  147. package/dist/storage/ontology/in-memory/materializations-state.d.ts.map +1 -1
  148. package/dist/storage/ontology/in-memory/materializations.d.ts +1 -1
  149. package/dist/storage/ontology/in-memory/materializations.d.ts.map +1 -1
  150. package/dist/storage/ontology/in-memory/shared-state.d.ts +15 -2
  151. package/dist/storage/ontology/in-memory/shared-state.d.ts.map +1 -1
  152. package/dist/storage/ontology/in-memory/source-roots.d.ts +6 -0
  153. package/dist/storage/ontology/in-memory/source-roots.d.ts.map +1 -0
  154. package/dist/storage/ontology/in-memory/sources.d.ts.map +1 -1
  155. package/dist/storage/ontology/index.d.ts +1 -1
  156. package/dist/storage/ontology/index.d.ts.map +1 -1
  157. package/dist/storage/ontology/provider.d.ts +16 -0
  158. package/dist/storage/ontology/provider.d.ts.map +1 -1
  159. package/dist/storage/ontology/provider.js +4 -2
  160. package/dist/storage/ontology/provider.js.map +1 -1
  161. package/dist/storage/ontology/sources.d.ts +13 -4
  162. package/dist/storage/ontology/sources.d.ts.map +1 -1
  163. package/dist/storage/projection-runs/lifecycle.d.ts.map +1 -1
  164. package/dist/storage/projection-runs/provider.js +2 -2
  165. package/dist/storage/projection-runs/types.d.ts +2 -0
  166. package/dist/storage/projection-runs/types.d.ts.map +1 -1
  167. package/dist/syncs/index.js +8 -7
  168. package/dist/syncs/index.js.map +1 -1
  169. package/dist/syncs/run-dispatch.d.ts.map +1 -1
  170. package/dist/testing/broker-contract.d.ts.map +1 -1
  171. package/dist/testing/connector-connection-storage-contract.d.ts.map +1 -1
  172. package/dist/testing/index.d.ts +1 -0
  173. package/dist/testing/index.d.ts.map +1 -1
  174. package/dist/testing/index.js +1459 -951
  175. package/dist/testing/index.js.map +9 -8
  176. package/dist/testing/lake-changes-contract.d.ts +6 -0
  177. package/dist/testing/lake-changes-contract.d.ts.map +1 -0
  178. package/dist/testing/materializer-storage-contract.d.ts.map +1 -1
  179. package/dist/testing/ontology-storage-contract.d.ts.map +1 -1
  180. package/dist/testing/projection-run-storage-contract.d.ts.map +1 -1
  181. package/dist/webhooks/connections.d.ts +6 -0
  182. package/dist/webhooks/connections.d.ts.map +1 -0
  183. package/dist/webhooks/index.d.ts +1 -1
  184. package/dist/webhooks/index.d.ts.map +1 -1
  185. package/dist/webhooks/internal.d.ts +2 -0
  186. package/dist/webhooks/internal.d.ts.map +1 -1
  187. package/dist/webhooks/internal.js +54 -9
  188. package/dist/webhooks/internal.js.map +4 -3
  189. package/dist/webhooks/registry.d.ts +1 -1
  190. package/dist/webhooks/registry.d.ts.map +1 -1
  191. package/dist/webhooks/types.d.ts +15 -4
  192. package/dist/webhooks/types.d.ts.map +1 -1
  193. package/dist/workflows/index.js +17 -16
  194. package/dist/workflows/index.js.map +1 -1
  195. package/dist/workflows/run-dispatch.d.ts +3 -1
  196. package/dist/workflows/run-dispatch.d.ts.map +1 -1
  197. package/package.json +1 -1
  198. package/src/broker/in-memory.ts +104 -39
  199. package/src/broker/index.ts +4 -1
  200. package/src/broker/retention.ts +28 -0
  201. package/src/broker/subscriber.ts +21 -0
  202. package/src/broker/types.ts +15 -1
  203. package/src/connectors/builders.ts +0 -6
  204. package/src/connectors/connections/authorizations.ts +11 -5
  205. package/src/connectors/connections/contracts.ts +3 -0
  206. package/src/connectors/connections/request.ts +56 -22
  207. package/src/connectors/connections/runs.ts +1 -0
  208. package/src/connectors/connections/tokens.ts +4 -0
  209. package/src/connectors/connections/validation.ts +91 -5
  210. package/src/connectors/types.ts +19 -4
  211. package/src/events/execution.ts +2 -2
  212. package/src/events/service.ts +3 -3
  213. package/src/index.ts +10 -2
  214. package/src/lake-storage/changes.ts +43 -0
  215. package/src/lake-storage/in-memory.ts +73 -0
  216. package/src/lake-storage/index.ts +4 -0
  217. package/src/lake-storage/types.ts +40 -0
  218. package/src/logging/service.ts +1 -1
  219. package/src/materialization/index.ts +2 -0
  220. package/src/materialization/model.ts +14 -1
  221. package/src/materializer/index.ts +2 -0
  222. package/src/materializer/materializer.ts +10 -0
  223. package/src/materializer/projections/entry-validator.ts +19 -1
  224. package/src/materializer/projections/replace.ts +36 -1
  225. package/src/materializer/projections/source-materialization.ts +26 -6
  226. package/src/materializer/shared/normalize.ts +1 -1
  227. package/src/pipelines/run-dispatch.ts +21 -1
  228. package/src/projections/run-dispatch.ts +30 -68
  229. package/src/queues/in-memory.ts +105 -62
  230. package/src/queues/indexed-heap.ts +74 -0
  231. package/src/runtime/host.ts +0 -14
  232. package/src/runtime/ontology-mutations.ts +4 -0
  233. package/src/storage/connector-connections/in-memory.ts +3 -1
  234. package/src/storage/connector-connections/provider/runs.ts +3 -1
  235. package/src/storage/connector-connections/types.ts +3 -2
  236. package/src/storage/index.ts +1 -0
  237. package/src/storage/ontology/in-memory/materializations-state.ts +5 -18
  238. package/src/storage/ontology/in-memory/materializations.ts +73 -92
  239. package/src/storage/ontology/in-memory/shared-state.ts +26 -1
  240. package/src/storage/ontology/in-memory/source-roots.ts +123 -0
  241. package/src/storage/ontology/in-memory/sources.ts +78 -23
  242. package/src/storage/ontology/index.ts +1 -0
  243. package/src/storage/ontology/provider.ts +70 -0
  244. package/src/storage/ontology/sources.ts +14 -3
  245. package/src/storage/projection-runs/lifecycle.ts +5 -2
  246. package/src/storage/projection-runs/types.ts +4 -3
  247. package/src/syncs/run-dispatch.ts +19 -1
  248. package/src/testing/broker-contract.ts +95 -0
  249. package/src/testing/connector-connection-storage-contract.ts +18 -1
  250. package/src/testing/index.ts +1 -0
  251. package/src/testing/lake-changes-contract.ts +240 -0
  252. package/src/testing/materializer-storage-contract.ts +172 -1
  253. package/src/testing/ontology-storage-contract.ts +7 -0
  254. package/src/testing/projection-run-storage-contract.ts +34 -0
  255. package/src/webhooks/connections.ts +56 -0
  256. package/src/webhooks/index.ts +2 -0
  257. package/src/webhooks/internal.ts +2 -0
  258. package/src/webhooks/registry.ts +3 -4
  259. package/src/webhooks/types.ts +17 -4
  260. package/src/workflows/run-dispatch.ts +42 -29
  261. package/dist/chunk-0rdrecsb.js.map +0 -11
  262. package/dist/chunk-3f7veca0.js.map +0 -10
  263. package/dist/chunk-3frsnm7x.js.map +0 -11
  264. package/dist/chunk-52hbv0ak.js.map +0 -36
  265. package/dist/chunk-emyyk7dx.js.map +0 -14
  266. package/dist/chunk-jjy0fhrn.js.map +0 -12
  267. package/dist/chunk-k3cqkkn2.js.map +0 -12
  268. package/dist/chunk-ppytyr6w.js.map +0 -12
  269. package/dist/chunk-qhcpj9wc.js.map +0 -14
  270. package/dist/chunk-r29fxdhn.js.map +0 -11
  271. package/dist/chunk-zcnq6105.js.map +0 -10
  272. /package/dist/{chunk-h1gfhhbh.js.map → chunk-28ja2cjs.js.map} +0 -0
  273. /package/dist/{chunk-1enhpacz.js.map → chunk-976rxdpx.js.map} +0 -0
  274. /package/dist/{chunk-r4tfx1sf.js.map → chunk-agbbygvh.js.map} +0 -0
  275. /package/dist/{chunk-0w35wavj.js.map → chunk-g7knyczr.js.map} +0 -0
  276. /package/dist/{chunk-ypbtykv6.js.map → chunk-nmbxcaqx.js.map} +0 -0
  277. /package/dist/{chunk-vf691g1r.js.map → chunk-p5vcsw9c.js.map} +0 -0
  278. /package/dist/{chunk-xyr8kppc.js.map → chunk-rtk71jzp.js.map} +0 -0
  279. /package/dist/{chunk-m3z3x9v0.js.map → chunk-t1f4f1ve.js.map} +0 -0
@@ -13,6 +13,7 @@ import {
13
13
  isExactStagingManifest,
14
14
  sourceConflict,
15
15
  sourceMaterializationIdentity,
16
+ sourceStageRoots,
16
17
  } from "../provider"
17
18
  import type {
18
19
  AbandonSourceMaterializationCandidateInput,
@@ -32,6 +33,7 @@ import type {
32
33
  SummarizeTerminalSourceMaterializationsInput,
33
34
  TerminalSourceMaterializationSummary,
34
35
  } from "../sources"
36
+ import { findActiveSourceMaterialization } from "./materializations-state"
35
37
  import {
36
38
  assertNonblank,
37
39
  type InMemoryOntologyState,
@@ -93,6 +95,7 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
93
95
  projectionRevision: input.projectionRevision,
94
96
  ownershipHash: input.ownershipHash,
95
97
  ontologyRevision: input.ontologyRevision,
98
+ ...(input.base ? { base: structuredClone(input.base) } : {}),
96
99
  rootCount: null,
97
100
  assertionCount: null,
98
101
  createdAt: input.createdAt,
@@ -102,6 +105,7 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
102
105
  lastCommitId: null,
103
106
  updatedAt: input.createdAt,
104
107
  rowsByEntity: new Map(),
108
+ roots: new Map(),
105
109
  rootOrdinals: new Map(),
106
110
  ordinalRoots: new Map(),
107
111
  }
@@ -121,7 +125,7 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
121
125
  `Source materialization '${input.materializationId}' is '${materialization.status}' and cannot accept rows.`
122
126
  )
123
127
  }
124
- return stageRowsAtomically(materialization, input.rows)
128
+ return stageRowsAtomically(materialization, input)
125
129
  })
126
130
  }
127
131
 
@@ -178,18 +182,12 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
178
182
  async getActive(input: GetActiveOntologySourceInput): Promise<OntologySourceRecord | null> {
179
183
  return this.runRootOperation(() => {
180
184
  assertProjectAndSource(input)
181
- const active = [...this.state.sourceMaterializations.values()].filter(
182
- (materialization) =>
183
- materialization.projectId === input.projectId &&
184
- materialization.source.projectionId === input.source.projectionId &&
185
- materialization.status === "active"
185
+ const active = findActiveSourceMaterialization(
186
+ this.state,
187
+ input.projectId,
188
+ input.source.projectionId
186
189
  )
187
- if (active.length > 1) {
188
- throw sourceConflict(
189
- `Source '${input.source.projectionId}' has more than one active materialization.`
190
- )
191
- }
192
- return active[0] ? sourceMaterializationRecord(active[0]) : null
190
+ return active ? sourceMaterializationRecord(active) : null
193
191
  })
194
192
  }
195
193
 
@@ -230,6 +228,14 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
230
228
  ) {
231
229
  continue
232
230
  }
231
+ if (
232
+ materialization.roots.size > 0 &&
233
+ ![...materialization.roots.values()].some(
234
+ (root) =>
235
+ !root.active && root.retiredAt !== null && root.retiredAt < input.terminalBefore
236
+ )
237
+ )
238
+ continue
233
239
  insertBounded(candidates, entry, input.limit, compareTerminalMaterializations)
234
240
  }
235
241
 
@@ -240,18 +246,28 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
240
246
  if (remaining === 0) break
241
247
  const current = this.state.sourceMaterializations.get(key)
242
248
  if (current !== candidate) continue
243
- // Terminal manifests never participate in reads or activation again. Release their
244
- // ingestion-only indexes immediately instead of retaining O(root count) memory until the
245
- // last child row is removed by a later bounded cleanup pass.
246
249
  current.rootOrdinals.clear()
247
250
  current.ordinalRoots.clear()
248
- for (const entityKey of current.rowsByEntity.keys()) {
251
+ for (const [rootKey, root] of current.roots) {
249
252
  if (remaining === 0) break
250
- current.rowsByEntity.delete(entityKey)
251
- rowsDeleted += 1
252
- remaining -= 1
253
+ if (root.active || root.retiredAt === null || root.retiredAt >= input.terminalBefore)
254
+ continue
255
+ for (const entityKey of root.entityKeys) {
256
+ if (remaining === 0) break
257
+ current.rowsByEntity.delete(entityKey)
258
+ root.entityKeys.delete(entityKey)
259
+ rowsDeleted += 1
260
+ remaining -= 1
261
+ }
262
+ if (root.entityKeys.size === 0 && remaining > 0) {
263
+ current.roots.delete(rootKey)
264
+ current.rootOrdinals.delete(rootKey)
265
+ current.ordinalRoots.delete(root.stagingOrdinal)
266
+ rowsDeleted += 1
267
+ remaining -= 1
268
+ }
253
269
  }
254
- if (current.rowsByEntity.size === 0 && remaining > 0) {
270
+ if (current.roots.size === 0 && current.rowsByEntity.size === 0 && remaining > 0) {
255
271
  this.state.sourceMaterializations.delete(key)
256
272
  materializationsDeleted += 1
257
273
  remaining -= 1
@@ -372,6 +388,7 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
372
388
  if (materialization.readyAt !== null) {
373
389
  assertNotBefore(abandonedAt, materialization.readyAt, "Source abandonedAt", "readyAt")
374
390
  }
391
+ for (const root of materialization.roots.values()) root.retiredAt = abandonedAt
375
392
  const abandoned: InMemorySourceMaterialization = {
376
393
  ...materialization,
377
394
  status: "abandoned",
@@ -407,14 +424,33 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
407
424
 
408
425
  function stageRowsAtomically(
409
426
  materialization: InMemorySourceMaterialization,
410
- rows: StageSourceRowsInput["rows"]
427
+ input: StageSourceRowsInput
411
428
  ): StageSourceRowsResult {
412
429
  let inserted = 0
413
430
  let unchanged = 0
414
431
  const newRows = new Map<string, StageSourceRowsInput["rows"][number]>()
415
432
  const newRootOrdinals = new Map<string, number>()
416
433
  const newOrdinalRoots = new Map<number, string>()
417
- for (const row of rows) {
434
+ const roots = sourceStageRoots(materialization.projectionKind, input)
435
+ if (!materialization.base && roots.some((root) => root.deleted)) {
436
+ throw new MaterializationValidationError("Root deletions require a source delta base.")
437
+ }
438
+ for (const root of roots) {
439
+ const previous = materialization.roots.get(root.rootKey)
440
+ const otherRoot = materialization.ordinalRoots.get(root.stagingOrdinal)
441
+ if (
442
+ (previous &&
443
+ (previous.deleted !== root.deleted || previous.stagingOrdinal !== root.stagingOrdinal)) ||
444
+ (otherRoot !== undefined && otherRoot !== root.rootKey)
445
+ ) {
446
+ throw new MaterializationValidationError(
447
+ "Source materialization repeats root or stream ordinal with different content."
448
+ )
449
+ }
450
+ newRootOrdinals.set(root.rootKey, root.stagingOrdinal)
451
+ newOrdinalRoots.set(root.stagingOrdinal, root.rootKey)
452
+ }
453
+ for (const row of input.rows) {
418
454
  assertSourceStagedRow(materialization.projectionKind, row)
419
455
 
420
456
  const rootKey = projectionEntityKey(row.root)
@@ -458,7 +494,21 @@ function stageRowsAtomically(
458
494
  for (const [ordinal, rootKey] of newOrdinalRoots) {
459
495
  materialization.ordinalRoots.set(ordinal, rootKey)
460
496
  }
461
- for (const [entityKey, row] of newRows) materialization.rowsByEntity.set(entityKey, row)
497
+ for (const root of roots) {
498
+ if (!materialization.roots.has(root.rootKey))
499
+ materialization.roots.set(root.rootKey, {
500
+ root: structuredClone(root.root),
501
+ stagingOrdinal: root.stagingOrdinal,
502
+ deleted: root.deleted,
503
+ entityKeys: new Set(),
504
+ active: false,
505
+ retiredAt: null,
506
+ })
507
+ }
508
+ for (const [entityKey, row] of newRows) {
509
+ materialization.rowsByEntity.set(entityKey, row)
510
+ materialization.roots.get(projectionEntityKey(row.root))!.entityKeys.add(entityKey)
511
+ }
462
512
  return { inserted, unchanged }
463
513
  }
464
514
 
@@ -497,6 +547,11 @@ function assertReadyTopology(materialization: InMemorySourceMaterialization): vo
497
547
 
498
548
  for (const rootKey of materialization.rootOrdinals.keys()) {
499
549
  const rows = rowsByRoot.get(rootKey) ?? []
550
+ if (materialization.roots.get(rootKey)?.deleted) {
551
+ if (rows.length > 0)
552
+ throw new MaterializationValidationError("Deleted source roots cannot contain assertions.")
553
+ continue
554
+ }
500
555
  const first = rows[0]
501
556
  if (!first) {
502
557
  throw new MaterializationValidationError(`Source root ${rootKey} has no staged assertions.`)
@@ -109,6 +109,7 @@ export type {
109
109
  OntologySourceStorage,
110
110
  ReclaimSourceMaterializationInput,
111
111
  StageSourceAssertion,
112
+ StageSourceRoot,
112
113
  StageSourceRowsInput,
113
114
  StageSourceRowsResult,
114
115
  StoredSourceAssertion,
@@ -164,6 +164,7 @@ function isJsonValue(value: unknown): value is import("../../json").JsonValue {
164
164
  }
165
165
 
166
166
  export interface ProviderReplacementSessionState {
167
+ readonly incremental?: boolean
167
168
  readonly sourceId: string
168
169
  readonly candidateMaterializationId: string
169
170
  readonly previousMaterializationId: string | null
@@ -654,6 +655,66 @@ export function sourceStageRow(row: StageSourceAssertion): SourceStageRow {
654
655
  }
655
656
  }
656
657
 
658
+ export interface SourceStageRoot {
659
+ readonly root: StageSourceAssertion["root"]
660
+ readonly rootKey: string
661
+ readonly sortKey: string
662
+ readonly stagingOrdinal: number
663
+ readonly deleted: boolean
664
+ }
665
+
666
+ /** Shared root identity rules, including deletions that deliberately have no assertion rows. */
667
+ export function sourceStageRoots(
668
+ projectionKind: OntologySourceRecord["projectionKind"],
669
+ input: {
670
+ readonly rows: readonly StageSourceAssertion[]
671
+ readonly deletions?: readonly {
672
+ readonly root: StageSourceAssertion["root"]
673
+ readonly stagingOrdinal: number
674
+ }[]
675
+ }
676
+ ): readonly SourceStageRoot[] {
677
+ for (const row of input.rows) assertSourceStagedRow(projectionKind, row)
678
+ const roots = new Map<string, SourceStageRoot>()
679
+ const ordinals = new Map<number, string>()
680
+ for (const [items, deleted] of [
681
+ [input.rows, false],
682
+ [input.deletions ?? [], true],
683
+ ] as const) {
684
+ for (const row of items) {
685
+ assertSourceEntity(row.root, "Source root")
686
+ if (
687
+ row.root.kind !== projectionKind ||
688
+ !Number.isSafeInteger(row.stagingOrdinal) ||
689
+ row.stagingOrdinal < 0
690
+ ) {
691
+ throw new MaterializationValidationError("Source root kind or staging ordinal is invalid.")
692
+ }
693
+ const key = projectionEntityKey(row.root)
694
+ const previous = roots.get(key)
695
+ const otherRoot = ordinals.get(row.stagingOrdinal)
696
+ if (
697
+ (previous &&
698
+ (previous.deleted !== deleted || previous.stagingOrdinal !== row.stagingOrdinal)) ||
699
+ (otherRoot !== undefined && otherRoot !== key)
700
+ ) {
701
+ throw new MaterializationValidationError(
702
+ `Source materialization repeats root or stream ordinal ${row.stagingOrdinal} with different content.`
703
+ )
704
+ }
705
+ roots.set(key, {
706
+ root: row.root,
707
+ rootKey: key,
708
+ sortKey: utf8SortKey(key),
709
+ stagingOrdinal: row.stagingOrdinal,
710
+ deleted,
711
+ })
712
+ ordinals.set(row.stagingOrdinal, key)
713
+ }
714
+ }
715
+ return [...roots.values()]
716
+ }
717
+
657
718
  export function reconcileSourceStageRows(
658
719
  rows: readonly SourceStageRow[],
659
720
  existingRows: readonly SourceStageRow[]
@@ -732,6 +793,13 @@ export function assertSourceBeginInput(input: BeginSourceMaterializationInput):
732
793
  assertNonblank(input.datasetVersion.versionId, "Source dataset version id")
733
794
  assertTimestamp(input.datasetVersion.createdAt, "Source dataset version createdAt", true)
734
795
  assertTimestamp(input.createdAt, "Source createdAt", true)
796
+ if (input.base) {
797
+ assertNonblank(input.base.materializationId, "Source base materialization id")
798
+ assertNonblank(input.base.lastCommitId, "Source base commit id")
799
+ if (input.base.materializationId === input.materializationId) {
800
+ throw new MaterializationValidationError("A source candidate cannot be its own base.")
801
+ }
802
+ }
735
803
  }
736
804
 
737
805
  export function assertSourceWriteIdentity(input: {
@@ -863,6 +931,8 @@ export function isExactStagingManifest(
863
931
  row.datasetVersion.createdAt === input.datasetVersion.createdAt &&
864
932
  row.projectionRevision === input.projectionRevision &&
865
933
  row.ownershipHash === input.ownershipHash &&
934
+ row.base?.materializationId === input.base?.materializationId &&
935
+ row.base?.lastCommitId === input.base?.lastCommitId &&
866
936
  row.ontologyRevision === input.ontologyRevision
867
937
  )
868
938
  }
@@ -3,6 +3,7 @@ import type {
3
3
  ProjectionEntityRef,
4
4
  ProjectionExecution,
5
5
  ProjectionSourceAssertion,
6
+ ProjectionSourceBase,
6
7
  ProjectionSourceRef,
7
8
  } from "../../materialization/model"
8
9
 
@@ -32,7 +33,8 @@ export interface OntologySourceRecord {
32
33
  readonly projectionRevision: string
33
34
  readonly ownershipHash: string
34
35
  readonly ontologyRevision: string
35
- /** Null until staging is sealed by markReady; zero explicitly represents an empty output. */
36
+ readonly base?: ProjectionSourceBase
37
+ /** Staged roots, including explicit deletions for a delta; null until sealed by markReady. */
36
38
  readonly rootCount: number | null
37
39
  /** Null until staging is sealed by markReady; zero explicitly represents an empty output. */
38
40
  readonly assertionCount: number | null
@@ -78,6 +80,7 @@ export interface BeginSourceMaterializationInput {
78
80
  readonly projectionRevision: string
79
81
  readonly ownershipHash: string
80
82
  readonly ontologyRevision: string
83
+ readonly base?: ProjectionSourceBase
81
84
  readonly createdAt: string
82
85
  }
83
86
 
@@ -94,6 +97,13 @@ export interface StageSourceRowsInput {
94
97
  readonly materializationId: string
95
98
  readonly execution: ProjectionExecution
96
99
  readonly rows: readonly StageSourceAssertion[]
100
+ /** Explicit root removals, allowed only for a candidate with a pinned base. */
101
+ readonly deletions?: readonly StageSourceRoot[]
102
+ }
103
+
104
+ export interface StageSourceRoot {
105
+ readonly root: ProjectionEntityRef
106
+ readonly stagingOrdinal: number
97
107
  }
98
108
 
99
109
  export interface StageSourceRowsResult {
@@ -139,13 +149,14 @@ export type AbandonSourceMaterializationInput =
139
149
 
140
150
  export interface CleanupTerminalSourceMaterializationsInput {
141
151
  readonly projectId: string
142
- /** Exclusive cutoff applied only to superseded/abandoned terminalAt. */
152
+ /** Exclusive cutoff for retired roots in terminal manifests, and for empty terminal manifests. */
143
153
  readonly terminalBefore: string
144
- /** Maximum total child-row plus manifest deletions performed by one call. */
154
+ /** Maximum total assertion, root-reference, and manifest deletions performed by one call. */
145
155
  readonly limit: number
146
156
  }
147
157
 
148
158
  export interface CleanupTerminalSourceMaterializationsResult {
159
+ /** Assertion rows and root references deleted. */
149
160
  readonly rowsDeleted: number
150
161
  readonly materializationsDeleted: number
151
162
  }
@@ -287,7 +287,7 @@ export function mergeProjectionRunProgress(
287
287
  const value = patch[key]
288
288
  if (value === undefined) continue
289
289
  assertProjectionRunCounter(value, key)
290
- if (value < current[key]) {
290
+ if (value < (current[key] ?? 0)) {
291
291
  throw new ProjectionRunError(`[Sixb] Projection run ${key} must not decrease.`)
292
292
  }
293
293
  progress[key] = value
@@ -706,7 +706,10 @@ function projectionTargetsEqual(left: ProjectionTarget, right: ProjectionTarget)
706
706
 
707
707
  function assertProjectionRunProgress(progress: ProjectionRunProgress): void {
708
708
  for (const key of PROJECTION_RUN_PROGRESS_KEYS) {
709
- assertProjectionRunCounter(progress[key], key)
709
+ assertProjectionRunCounter(
710
+ key === "sourceChangesRead" ? (progress[key] ?? 0) : progress[key],
711
+ key
712
+ )
710
713
  }
711
714
  if (progress.sourceRowsSkipped > progress.sourceRowsRead) {
712
715
  throw new ProjectionRunError(
@@ -23,11 +23,14 @@ export interface ProjectionRunProgress {
23
23
  readonly sourceRowsRead: number
24
24
  /** Physical rows intentionally skipped because a required mapped value was blank. */
25
25
  readonly sourceRowsSkipped: number
26
+ /** Maximum complete change records consumed by one incremental attempt; not a recovery cursor. */
27
+ readonly sourceChangesRead?: number
26
28
  }
27
29
 
28
30
  const progressKeyFlags: Record<keyof ProjectionRunProgress, true> = {
29
31
  sourceRowsRead: true,
30
32
  sourceRowsSkipped: true,
33
+ sourceChangesRead: true,
31
34
  }
32
35
 
33
36
  export const PROJECTION_RUN_PROGRESS_KEYS = Object.keys(
@@ -35,9 +38,7 @@ export const PROJECTION_RUN_PROGRESS_KEYS = Object.keys(
35
38
  ) as readonly (keyof ProjectionRunProgress)[]
36
39
 
37
40
  export function zeroProjectionRunProgress(): ProjectionRunProgress {
38
- const progress = {} as Record<keyof ProjectionRunProgress, number>
39
- for (const key of PROJECTION_RUN_PROGRESS_KEYS) progress[key] = 0
40
- return progress
41
+ return { sourceRowsRead: 0, sourceRowsSkipped: 0 }
41
42
  }
42
43
 
43
44
  export interface ProjectionTelemetryCheckpoint {
@@ -93,7 +93,10 @@ export class SyncRunDispatcher implements SyncRunDispatchPort {
93
93
  /** Persist a Sync execution and queued run before publishing its queue job. */
94
94
  export async function dispatchSyncRun(input: DispatchSyncRunInput): Promise<SyncRunRequestResult> {
95
95
  const runId = createSyncRunId(input.runId)
96
- const persisted = await persistSyncRun(input, runId)
96
+ let persisted = await persistSyncRun(input, runId)
97
+ if (!persisted.publish && input.validateExistingRun && persisted.run.status === "queued") {
98
+ persisted = await reuseSyncRun(input, persisted.run)
99
+ }
97
100
  if (!persisted.publish) return existingSyncRunResult(persisted.run)
98
101
  return publishSyncRun(input, persisted)
99
102
  }
@@ -151,6 +154,15 @@ async function reuseSyncRun(
151
154
  ): Promise<PersistedSyncRun> {
152
155
  assertExistingRunMatchesRequest(existing, input)
153
156
  await input.validateExistingRun?.(existing, input.storage)
157
+ if (input.validateExistingRun && existing.status === "queued") {
158
+ const execution = await input.storage.executions.getById({
159
+ projectId: input.projectId,
160
+ id: existing.executionId,
161
+ })
162
+ if (!execution)
163
+ throw new SyncRunError(`[Sixb] Execution '${existing.executionId}' was not found.`)
164
+ return { publish: true, run: existing, execution, queuedAt: existing.queuedAt, created: false }
165
+ }
154
166
  if (
155
167
  existing.status !== "failed" ||
156
168
  existing.error?.code !== "queue.enqueue_failed" ||
@@ -200,6 +212,9 @@ async function publishSyncRun(
200
212
  },
201
213
  ],
202
214
  })
215
+ if (!job || job.id !== persisted.run.id) {
216
+ throw new SyncRunError(`[Sixb] Queue did not acknowledge Sync run '${persisted.run.id}'.`)
217
+ }
203
218
  return {
204
219
  syncId: persisted.run.syncId,
205
220
  runId: persisted.run.id,
@@ -208,6 +223,9 @@ async function publishSyncRun(
208
223
  created: persisted.created,
209
224
  }
210
225
  } catch (error) {
226
+ // Automatic consumers retain the batch and retry. Keep uncertain delivery queued:
227
+ // a lost reply does not prove that the queue rejected the job.
228
+ if (input.validateExistingRun) throw error
211
229
  const failedAt = new Date()
212
230
  const failure = toEnqueueFailure(error, persisted.run, failedAt)
213
231
  const failed = await requireSyncRunStorage(input.storage).finish({
@@ -446,6 +446,101 @@ export function runBrokerContractSuite<TBroker extends Broker>(
446
446
  })
447
447
 
448
448
  describe("subscribe", () => {
449
+ // Removing awaited provider delivery makes the second callback enter before release.
450
+ test("awaits async handlers without blocking publishers or other subscribers", async () => {
451
+ await withBroker(async (broker) => {
452
+ await broker.ensureStream({ projectId: "project-a", stream: eventsStream })
453
+ let release!: () => void
454
+ const blocked = new Promise<void>((resolve) => {
455
+ release = resolve
456
+ })
457
+ const received: string[] = []
458
+ const observer: string[] = []
459
+ const unsubscribe = await broker.subscribe(
460
+ { projectId: "project-a", streamId: eventsStream.id },
461
+ async (records) => {
462
+ received.push(...records.map((record) => String(record.payload)))
463
+ await blocked
464
+ }
465
+ )
466
+ const observe = await broker.subscribe(
467
+ { projectId: "project-a", streamId: eventsStream.id },
468
+ (records) => {
469
+ observer.push(...records.map((record) => String(record.payload)))
470
+ }
471
+ )
472
+ const wait = (predicate: () => boolean) =>
473
+ waitUntil(predicate, {
474
+ timeoutMs: subscriptionDeliveryTimeoutMs,
475
+ intervalMs: subscriptionPollIntervalMs,
476
+ message: "Async broker subscription did not progress",
477
+ })
478
+ try {
479
+ if (subscriptionSetupMs > 0) await Bun.sleep(subscriptionSetupMs)
480
+ await broker.append({
481
+ projectId: "project-a",
482
+ streamId: eventsStream.id,
483
+ records: [{ payload: "one" }],
484
+ })
485
+ await wait(() => received.length === 1)
486
+ await broker.append({
487
+ projectId: "project-a",
488
+ streamId: eventsStream.id,
489
+ records: [{ payload: "two" }],
490
+ })
491
+ await wait(() => observer.length === 2)
492
+ await Bun.sleep(30)
493
+ expect(received).toEqual(["one"])
494
+ release()
495
+ await wait(() => received.length === 2)
496
+ expect(received).toEqual(["one", "two"])
497
+ } finally {
498
+ release()
499
+ unsubscribe()
500
+ observe()
501
+ }
502
+ })
503
+ })
504
+
505
+ test("isolates rejected async handlers and permits appending from a handler", async () => {
506
+ await withBroker(async (broker) => {
507
+ await broker.ensureStream({ projectId: "project-a", stream: eventsStream })
508
+ const received: string[] = []
509
+ const unsubscribe = await broker.subscribe(
510
+ { projectId: "project-a", streamId: eventsStream.id },
511
+ async (records) => {
512
+ for (const record of records) {
513
+ received.push(String(record.payload))
514
+ if (record.payload === "one") {
515
+ await broker.append({
516
+ projectId: "project-a",
517
+ streamId: eventsStream.id,
518
+ records: [{ payload: "two" }],
519
+ })
520
+ throw new Error("observer failure")
521
+ }
522
+ }
523
+ }
524
+ )
525
+ try {
526
+ if (subscriptionSetupMs > 0) await Bun.sleep(subscriptionSetupMs)
527
+ await broker.append({
528
+ projectId: "project-a",
529
+ streamId: eventsStream.id,
530
+ records: [{ payload: "one" }],
531
+ })
532
+ await waitUntil(() => received.length === 2, {
533
+ timeoutMs: subscriptionDeliveryTimeoutMs,
534
+ intervalMs: subscriptionPollIntervalMs,
535
+ message: "Rejected async observer blocked later delivery",
536
+ })
537
+ expect(received).toEqual(["one", "two"])
538
+ } finally {
539
+ unsubscribe()
540
+ }
541
+ })
542
+ })
543
+
449
544
  test("requires streams to be ensured before subscribing", async () => {
450
545
  await withBroker(async (broker) => {
451
546
  await expect(
@@ -90,6 +90,22 @@ export function runConnectorConnectionStorageContractSuite<
90
90
  })
91
91
  })
92
92
 
93
+ test("persists explicitly non-PKCE attempts without a verifier", async () => {
94
+ // Removal proof: restore required code_verifier storage or omit migration 038; SQL providers fail.
95
+ await withStorage(async (storage) => {
96
+ const { codeVerifier: _verifier, ...input } = authorizationAttempt()
97
+ const created = await storage.createAuthorizationAttempt(input)
98
+ expect(created).not.toHaveProperty("codeVerifier")
99
+ const consumed = await storage.consumeAuthorizationAttempt(
100
+ authorizationAttemptConsumption()
101
+ )
102
+ expect(consumed).not.toHaveProperty("codeVerifier")
103
+ await expect(
104
+ storage.consumeAuthorizationAttempt(authorizationAttemptConsumption())
105
+ ).rejects.toThrow("invalid, expired, or already used")
106
+ })
107
+ })
108
+
93
109
  test("expires authorization attempts from the authoritative storage clock", async () => {
94
110
  await withStorage(async (storage, root) => {
95
111
  const ttlMs = 60_000
@@ -110,12 +126,13 @@ export function runConnectorConnectionStorageContractSuite<
110
126
  test("returns detached authorization attempt records", async () => {
111
127
  await withStorage(async (storage) => {
112
128
  const created = await storage.createAuthorizationAttempt(authorizationAttempt())
129
+ if (!created.codeVerifier) throw new Error("Expected a stored PKCE verifier.")
113
130
  Object.assign(created.codeVerifier, { ciphertext: "mutated" })
114
131
 
115
132
  const consumed = await storage.consumeAuthorizationAttempt(
116
133
  authorizationAttemptConsumption()
117
134
  )
118
- expect(consumed.codeVerifier.ciphertext).toBe(credentials.ciphertext)
135
+ expect(consumed.codeVerifier?.ciphertext).toBe(credentials.ciphertext)
119
136
  })
120
137
  })
121
138
 
@@ -59,6 +59,7 @@ export {
59
59
  type ExecutionStorageContractSuiteOptions,
60
60
  runExecutionStorageContractSuite,
61
61
  } from "./execution-storage-contract"
62
+ export { runLakeChangesContractSuite } from "./lake-changes-contract"
62
63
  export {
63
64
  type LakeMergeStorageContractSuiteOptions,
64
65
  runLakeMergeStorageContractSuite,