@sixb/core 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/dist/actions/index.js +8 -8
  2. package/dist/agents/index.js +16 -15
  3. package/dist/agents/index.js.map +1 -1
  4. package/dist/bootstrap/index.js +20 -19
  5. package/dist/bootstrap/index.js.map +1 -1
  6. package/dist/broker/in-memory.d.ts +8 -2
  7. package/dist/broker/in-memory.d.ts.map +1 -1
  8. package/dist/broker/index.d.ts +4 -2
  9. package/dist/broker/index.d.ts.map +1 -1
  10. package/dist/broker/index.js +7 -3
  11. package/dist/broker/index.js.map +1 -1
  12. package/dist/broker/retention.d.ts +4 -0
  13. package/dist/broker/retention.d.ts.map +1 -0
  14. package/dist/broker/subscriber.d.ts +10 -0
  15. package/dist/broker/subscriber.d.ts.map +1 -0
  16. package/dist/broker/types.d.ts +14 -1
  17. package/dist/broker/types.d.ts.map +1 -1
  18. package/dist/{chunk-k3cqkkn2.js → chunk-1824gssz.js} +30 -65
  19. package/dist/chunk-1824gssz.js.map +12 -0
  20. package/dist/{chunk-h1gfhhbh.js → chunk-28ja2cjs.js} +1 -1
  21. package/dist/chunk-2c4j899j.js.map +1 -1
  22. package/dist/{chunk-ppytyr6w.js → chunk-358caqf5.js} +4 -4
  23. package/dist/chunk-358caqf5.js.map +12 -0
  24. package/dist/{chunk-kxwa347f.js → chunk-3qms4nax.js} +2 -2
  25. package/dist/{chunk-kxwa347f.js.map → chunk-3qms4nax.js.map} +3 -3
  26. package/dist/{chunk-emyyk7dx.js → chunk-5c7ga8zz.js} +22 -145
  27. package/dist/chunk-5c7ga8zz.js.map +12 -0
  28. package/dist/{chunk-hpr1r6av.js → chunk-5p1v802y.js} +46 -39
  29. package/dist/{chunk-hpr1r6av.js.map → chunk-5p1v802y.js.map} +4 -4
  30. package/dist/{chunk-zcnq6105.js → chunk-85cd7e8c.js} +2 -1
  31. package/dist/chunk-85cd7e8c.js.map +10 -0
  32. package/dist/{chunk-1enhpacz.js → chunk-976rxdpx.js} +5 -5
  33. package/dist/{chunk-3f7veca0.js → chunk-9x8dybsb.js} +3 -3
  34. package/dist/chunk-9x8dybsb.js.map +10 -0
  35. package/dist/{chunk-r29fxdhn.js → chunk-aa6z4a19.js} +160 -44
  36. package/dist/chunk-aa6z4a19.js.map +12 -0
  37. package/dist/{chunk-r4tfx1sf.js → chunk-agbbygvh.js} +4 -4
  38. package/dist/{chunk-jjy0fhrn.js → chunk-b0ec55a1.js} +20 -3
  39. package/dist/chunk-b0ec55a1.js.map +12 -0
  40. package/dist/{chunk-xfbydgzy.js → chunk-bn138h9w.js} +7 -8
  41. package/dist/{chunk-xfbydgzy.js.map → chunk-bn138h9w.js.map} +3 -3
  42. package/dist/{chunk-an5tt9y0.js → chunk-ftv470j8.js} +42 -4
  43. package/dist/{chunk-an5tt9y0.js.map → chunk-ftv470j8.js.map} +7 -7
  44. package/dist/{chunk-0w35wavj.js → chunk-g7knyczr.js} +1 -1
  45. package/dist/{chunk-x57ycxq0.js → chunk-gqh45d2n.js} +42 -3
  46. package/dist/{chunk-x57ycxq0.js.map → chunk-gqh45d2n.js.map} +3 -3
  47. package/dist/{chunk-0rdrecsb.js → chunk-kfee1h52.js} +116 -32
  48. package/dist/chunk-kfee1h52.js.map +13 -0
  49. package/dist/{chunk-ypbtykv6.js → chunk-nmbxcaqx.js} +1 -1
  50. package/dist/{chunk-3tgwrcyh.js → chunk-nt99sxd6.js} +2 -2
  51. package/dist/{chunk-3tgwrcyh.js.map → chunk-nt99sxd6.js.map} +3 -3
  52. package/dist/{chunk-vf691g1r.js → chunk-p5vcsw9c.js} +2 -2
  53. package/dist/{chunk-mjbk2djy.js → chunk-p6a4npdx.js} +211 -106
  54. package/dist/{chunk-mjbk2djy.js.map → chunk-p6a4npdx.js.map} +9 -8
  55. package/dist/{chunk-3frsnm7x.js → chunk-qma80ych.js} +4 -6
  56. package/dist/chunk-qma80ych.js.map +11 -0
  57. package/dist/{chunk-qhcpj9wc.js → chunk-qyeptdyv.js} +96 -2
  58. package/dist/chunk-qyeptdyv.js.map +15 -0
  59. package/dist/{chunk-52hbv0ak.js → chunk-r7fj830t.js} +101 -48
  60. package/dist/chunk-r7fj830t.js.map +36 -0
  61. package/dist/{chunk-xyr8kppc.js → chunk-rtk71jzp.js} +1 -1
  62. package/dist/{chunk-m3z3x9v0.js → chunk-t1f4f1ve.js} +1 -1
  63. package/dist/chunk-v2vcg0w6.js +151 -0
  64. package/dist/chunk-v2vcg0w6.js.map +11 -0
  65. package/dist/connectors/connections/authorizations.d.ts.map +1 -1
  66. package/dist/connectors/connections/contracts.d.ts +3 -0
  67. package/dist/connectors/connections/contracts.d.ts.map +1 -1
  68. package/dist/connectors/connections/internal.js +1 -1
  69. package/dist/connectors/connections/request.d.ts +3 -1
  70. package/dist/connectors/connections/request.d.ts.map +1 -1
  71. package/dist/connectors/connections/runs.d.ts.map +1 -1
  72. package/dist/connectors/connections/tokens.d.ts.map +1 -1
  73. package/dist/connectors/connections/validation.d.ts +5 -1
  74. package/dist/connectors/connections/validation.d.ts.map +1 -1
  75. package/dist/connectors/types.d.ts +19 -4
  76. package/dist/connectors/types.d.ts.map +1 -1
  77. package/dist/events/execution.d.ts +1 -1
  78. package/dist/events/execution.d.ts.map +1 -1
  79. package/dist/events/index.js +1 -1
  80. package/dist/events/service.d.ts +2 -2
  81. package/dist/events/service.d.ts.map +1 -1
  82. package/dist/execution/agent.js +16 -15
  83. package/dist/execution/agent.js.map +1 -1
  84. package/dist/execution/primitive.js +16 -15
  85. package/dist/execution/primitive.js.map +2 -2
  86. package/dist/execution/request.js +16 -15
  87. package/dist/execution/request.js.map +2 -2
  88. package/dist/index.d.ts +3 -3
  89. package/dist/index.d.ts.map +1 -1
  90. package/dist/index.js +30 -28
  91. package/dist/index.js.map +1 -1
  92. package/dist/lake-storage/changes.d.ts +5 -0
  93. package/dist/lake-storage/changes.d.ts.map +1 -0
  94. package/dist/lake-storage/in-memory.d.ts +2 -1
  95. package/dist/lake-storage/in-memory.d.ts.map +1 -1
  96. package/dist/lake-storage/index.d.ts +2 -1
  97. package/dist/lake-storage/index.d.ts.map +1 -1
  98. package/dist/lake-storage/index.js +4 -2
  99. package/dist/lake-storage/index.js.map +1 -1
  100. package/dist/lake-storage/types.d.ts +36 -0
  101. package/dist/lake-storage/types.d.ts.map +1 -1
  102. package/dist/logging/internal.js +1 -1
  103. package/dist/materialization/index.d.ts +1 -1
  104. package/dist/materialization/index.d.ts.map +1 -1
  105. package/dist/materialization/model.d.ts +12 -1
  106. package/dist/materialization/model.d.ts.map +1 -1
  107. package/dist/materializer/index.d.ts +1 -1
  108. package/dist/materializer/index.d.ts.map +1 -1
  109. package/dist/materializer/index.js +5 -5
  110. package/dist/materializer/materializer.d.ts +5 -1
  111. package/dist/materializer/materializer.d.ts.map +1 -1
  112. package/dist/materializer/projections/entry-validator.d.ts +2 -1
  113. package/dist/materializer/projections/entry-validator.d.ts.map +1 -1
  114. package/dist/materializer/projections/replace.d.ts.map +1 -1
  115. package/dist/materializer/projections/source-materialization.d.ts +4 -3
  116. package/dist/materializer/projections/source-materialization.d.ts.map +1 -1
  117. package/dist/materializer/shared/normalize.d.ts +2 -1
  118. package/dist/materializer/shared/normalize.d.ts.map +1 -1
  119. package/dist/models/execution/index.js +5 -5
  120. package/dist/objects/index.js +9 -9
  121. package/dist/objects/query/index.js +5 -5
  122. package/dist/pipelines/index.js +6 -6
  123. package/dist/pipelines/run-dispatch.d.ts.map +1 -1
  124. package/dist/projections/internal.js +6 -8
  125. package/dist/projections/internal.js.map +1 -1
  126. package/dist/projections/run-dispatch.d.ts.map +1 -1
  127. package/dist/queues/in-memory.d.ts +7 -3
  128. package/dist/queues/in-memory.d.ts.map +1 -1
  129. package/dist/queues/index.js +1 -1
  130. package/dist/queues/indexed-heap.d.ts +15 -0
  131. package/dist/queues/indexed-heap.d.ts.map +1 -0
  132. package/dist/runtime/host.d.ts.map +1 -1
  133. package/dist/runtime/internal.js +1 -1
  134. package/dist/runtime/ontology-mutations.d.ts +3 -1
  135. package/dist/runtime/ontology-mutations.d.ts.map +1 -1
  136. package/dist/shares/internal.js +4 -4
  137. package/dist/storage/connector-connections/in-memory.d.ts.map +1 -1
  138. package/dist/storage/connector-connections/provider/runs.d.ts.map +1 -1
  139. package/dist/storage/connector-connections/provider.js +2 -2
  140. package/dist/storage/connector-connections/provider.js.map +3 -3
  141. package/dist/storage/connector-connections/types.d.ts +3 -2
  142. package/dist/storage/connector-connections/types.d.ts.map +1 -1
  143. package/dist/storage/index.d.ts +1 -1
  144. package/dist/storage/index.d.ts.map +1 -1
  145. package/dist/storage/index.js +4 -4
  146. package/dist/storage/internal.js +1 -1
  147. package/dist/storage/ontology/in-memory/materializations-state.d.ts.map +1 -1
  148. package/dist/storage/ontology/in-memory/materializations.d.ts +1 -1
  149. package/dist/storage/ontology/in-memory/materializations.d.ts.map +1 -1
  150. package/dist/storage/ontology/in-memory/shared-state.d.ts +15 -2
  151. package/dist/storage/ontology/in-memory/shared-state.d.ts.map +1 -1
  152. package/dist/storage/ontology/in-memory/source-roots.d.ts +6 -0
  153. package/dist/storage/ontology/in-memory/source-roots.d.ts.map +1 -0
  154. package/dist/storage/ontology/in-memory/sources.d.ts.map +1 -1
  155. package/dist/storage/ontology/index.d.ts +1 -1
  156. package/dist/storage/ontology/index.d.ts.map +1 -1
  157. package/dist/storage/ontology/provider.d.ts +16 -0
  158. package/dist/storage/ontology/provider.d.ts.map +1 -1
  159. package/dist/storage/ontology/provider.js +4 -2
  160. package/dist/storage/ontology/provider.js.map +1 -1
  161. package/dist/storage/ontology/sources.d.ts +13 -4
  162. package/dist/storage/ontology/sources.d.ts.map +1 -1
  163. package/dist/storage/projection-runs/lifecycle.d.ts.map +1 -1
  164. package/dist/storage/projection-runs/provider.js +2 -2
  165. package/dist/storage/projection-runs/types.d.ts +2 -0
  166. package/dist/storage/projection-runs/types.d.ts.map +1 -1
  167. package/dist/syncs/index.js +8 -7
  168. package/dist/syncs/index.js.map +1 -1
  169. package/dist/syncs/run-dispatch.d.ts.map +1 -1
  170. package/dist/testing/broker-contract.d.ts.map +1 -1
  171. package/dist/testing/connector-connection-storage-contract.d.ts.map +1 -1
  172. package/dist/testing/index.d.ts +1 -0
  173. package/dist/testing/index.d.ts.map +1 -1
  174. package/dist/testing/index.js +1459 -951
  175. package/dist/testing/index.js.map +9 -8
  176. package/dist/testing/lake-changes-contract.d.ts +6 -0
  177. package/dist/testing/lake-changes-contract.d.ts.map +1 -0
  178. package/dist/testing/materializer-storage-contract.d.ts.map +1 -1
  179. package/dist/testing/ontology-storage-contract.d.ts.map +1 -1
  180. package/dist/testing/projection-run-storage-contract.d.ts.map +1 -1
  181. package/dist/webhooks/connections.d.ts +6 -0
  182. package/dist/webhooks/connections.d.ts.map +1 -0
  183. package/dist/webhooks/index.d.ts +1 -1
  184. package/dist/webhooks/index.d.ts.map +1 -1
  185. package/dist/webhooks/internal.d.ts +2 -0
  186. package/dist/webhooks/internal.d.ts.map +1 -1
  187. package/dist/webhooks/internal.js +54 -9
  188. package/dist/webhooks/internal.js.map +4 -3
  189. package/dist/webhooks/registry.d.ts +1 -1
  190. package/dist/webhooks/registry.d.ts.map +1 -1
  191. package/dist/webhooks/types.d.ts +15 -4
  192. package/dist/webhooks/types.d.ts.map +1 -1
  193. package/dist/workflows/index.js +17 -16
  194. package/dist/workflows/index.js.map +1 -1
  195. package/dist/workflows/run-dispatch.d.ts +3 -1
  196. package/dist/workflows/run-dispatch.d.ts.map +1 -1
  197. package/package.json +1 -1
  198. package/src/broker/in-memory.ts +104 -39
  199. package/src/broker/index.ts +4 -1
  200. package/src/broker/retention.ts +28 -0
  201. package/src/broker/subscriber.ts +21 -0
  202. package/src/broker/types.ts +15 -1
  203. package/src/connectors/builders.ts +0 -6
  204. package/src/connectors/connections/authorizations.ts +11 -5
  205. package/src/connectors/connections/contracts.ts +3 -0
  206. package/src/connectors/connections/request.ts +56 -22
  207. package/src/connectors/connections/runs.ts +1 -0
  208. package/src/connectors/connections/tokens.ts +4 -0
  209. package/src/connectors/connections/validation.ts +91 -5
  210. package/src/connectors/types.ts +19 -4
  211. package/src/events/execution.ts +2 -2
  212. package/src/events/service.ts +3 -3
  213. package/src/index.ts +10 -2
  214. package/src/lake-storage/changes.ts +43 -0
  215. package/src/lake-storage/in-memory.ts +73 -0
  216. package/src/lake-storage/index.ts +4 -0
  217. package/src/lake-storage/types.ts +40 -0
  218. package/src/logging/service.ts +1 -1
  219. package/src/materialization/index.ts +2 -0
  220. package/src/materialization/model.ts +14 -1
  221. package/src/materializer/index.ts +2 -0
  222. package/src/materializer/materializer.ts +10 -0
  223. package/src/materializer/projections/entry-validator.ts +19 -1
  224. package/src/materializer/projections/replace.ts +36 -1
  225. package/src/materializer/projections/source-materialization.ts +26 -6
  226. package/src/materializer/shared/normalize.ts +1 -1
  227. package/src/pipelines/run-dispatch.ts +21 -1
  228. package/src/projections/run-dispatch.ts +30 -68
  229. package/src/queues/in-memory.ts +105 -62
  230. package/src/queues/indexed-heap.ts +74 -0
  231. package/src/runtime/host.ts +0 -14
  232. package/src/runtime/ontology-mutations.ts +4 -0
  233. package/src/storage/connector-connections/in-memory.ts +3 -1
  234. package/src/storage/connector-connections/provider/runs.ts +3 -1
  235. package/src/storage/connector-connections/types.ts +3 -2
  236. package/src/storage/index.ts +1 -0
  237. package/src/storage/ontology/in-memory/materializations-state.ts +5 -18
  238. package/src/storage/ontology/in-memory/materializations.ts +73 -92
  239. package/src/storage/ontology/in-memory/shared-state.ts +26 -1
  240. package/src/storage/ontology/in-memory/source-roots.ts +123 -0
  241. package/src/storage/ontology/in-memory/sources.ts +78 -23
  242. package/src/storage/ontology/index.ts +1 -0
  243. package/src/storage/ontology/provider.ts +70 -0
  244. package/src/storage/ontology/sources.ts +14 -3
  245. package/src/storage/projection-runs/lifecycle.ts +5 -2
  246. package/src/storage/projection-runs/types.ts +4 -3
  247. package/src/syncs/run-dispatch.ts +19 -1
  248. package/src/testing/broker-contract.ts +95 -0
  249. package/src/testing/connector-connection-storage-contract.ts +18 -1
  250. package/src/testing/index.ts +1 -0
  251. package/src/testing/lake-changes-contract.ts +240 -0
  252. package/src/testing/materializer-storage-contract.ts +172 -1
  253. package/src/testing/ontology-storage-contract.ts +7 -0
  254. package/src/testing/projection-run-storage-contract.ts +34 -0
  255. package/src/webhooks/connections.ts +56 -0
  256. package/src/webhooks/index.ts +2 -0
  257. package/src/webhooks/internal.ts +2 -0
  258. package/src/webhooks/registry.ts +3 -4
  259. package/src/webhooks/types.ts +17 -4
  260. package/src/workflows/run-dispatch.ts +42 -29
  261. package/dist/chunk-0rdrecsb.js.map +0 -11
  262. package/dist/chunk-3f7veca0.js.map +0 -10
  263. package/dist/chunk-3frsnm7x.js.map +0 -11
  264. package/dist/chunk-52hbv0ak.js.map +0 -36
  265. package/dist/chunk-emyyk7dx.js.map +0 -14
  266. package/dist/chunk-jjy0fhrn.js.map +0 -12
  267. package/dist/chunk-k3cqkkn2.js.map +0 -12
  268. package/dist/chunk-ppytyr6w.js.map +0 -12
  269. package/dist/chunk-qhcpj9wc.js.map +0 -14
  270. package/dist/chunk-r29fxdhn.js.map +0 -11
  271. package/dist/chunk-zcnq6105.js.map +0 -10
  272. /package/dist/{chunk-h1gfhhbh.js.map → chunk-28ja2cjs.js.map} +0 -0
  273. /package/dist/{chunk-1enhpacz.js.map → chunk-976rxdpx.js.map} +0 -0
  274. /package/dist/{chunk-r4tfx1sf.js.map → chunk-agbbygvh.js.map} +0 -0
  275. /package/dist/{chunk-0w35wavj.js.map → chunk-g7knyczr.js.map} +0 -0
  276. /package/dist/{chunk-ypbtykv6.js.map → chunk-nmbxcaqx.js.map} +0 -0
  277. /package/dist/{chunk-vf691g1r.js.map → chunk-p5vcsw9c.js.map} +0 -0
  278. /package/dist/{chunk-xyr8kppc.js.map → chunk-rtk71jzp.js.map} +0 -0
  279. /package/dist/{chunk-m3z3x9v0.js.map → chunk-t1f4f1ve.js.map} +0 -0
@@ -0,0 +1,240 @@
1
+ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
2
+ import { col, defineDataset } from "../datasets"
3
+ import type { DatasetChanges, DatasetRow, DatasetRowChange, LakeStorage } from "../lake-storage"
4
+
5
+ export function runLakeChangesContractSuite<T extends LakeStorage>(
6
+ name: string,
7
+ harness: {
8
+ readonly createStorage: () => T | Promise<T>
9
+ readonly teardown?: (storage: T) => void | Promise<void>
10
+ }
11
+ ): void {
12
+ describe(name, () => {
13
+ let storage: T
14
+ const dataset = defineDataset("delta.contract", {
15
+ schema: [
16
+ col("id", "string"),
17
+ col("part", "string"),
18
+ col("value", "string", { nullable: true }),
19
+ col("ignored", "string"),
20
+ ],
21
+ })
22
+ const row = (id: string, value: string | null, part = "x", ignored = "unused"): DatasetRow => ({
23
+ id,
24
+ part,
25
+ value,
26
+ ignored,
27
+ })
28
+ const columns = ["id", "part", "value"]
29
+ beforeEach(async () => {
30
+ storage = await harness.createStorage()
31
+ await storage.createDataset(dataset)
32
+ })
33
+ afterEach(async () => {
34
+ if (harness.teardown) await harness.teardown(storage)
35
+ else await storage.close?.()
36
+ })
37
+ async function write(rows: readonly DatasetRow[]) {
38
+ const writer = await storage.beginWrite({ dataset })
39
+ await writer.writeRows(rows)
40
+ return (await writer.commit()).versionId
41
+ }
42
+ async function read(fromVersionId: string, toVersionId: string, keyColumns = ["id"]) {
43
+ if (!storage.readChanges) throw new Error("Provider does not implement readChanges")
44
+ return storage.readChanges({
45
+ datasetId: dataset.id,
46
+ fromVersionId,
47
+ toVersionId,
48
+ columns,
49
+ keyColumns,
50
+ })
51
+ }
52
+ async function collect(reader: DatasetChanges | null) {
53
+ if (!reader) throw new Error("Expected a complete delta")
54
+ const changes: DatasetRowChange[] = []
55
+ try {
56
+ for await (const change of reader.changes) changes.push(change)
57
+ } finally {
58
+ await reader.close()
59
+ }
60
+ expect(changes).toHaveLength(reader.changeCount)
61
+ return changes.sort((a, b) =>
62
+ JSON.stringify(a.after ?? a.before).localeCompare(JSON.stringify(b.after ?? b.before))
63
+ )
64
+ }
65
+
66
+ // Red proof: remove the provider's readChanges implementation; this suite must fail.
67
+ test("compares pinned states across skipped versions, including removals and key changes", async () => {
68
+ const first = await write([row("A", "one"), row("B", "two"), row("C", "three")])
69
+ await write([row("A", "temporary"), row("B", "two"), row("C", "three")])
70
+ const last = await write([row("A", "one"), row("B", "new"), row("D", "three")])
71
+ const reader = await read(first, last)
72
+ expect(reader?.fromRowCount).toBe(3)
73
+ expect(reader?.toRowCount).toBe(3)
74
+ expect(await collect(reader)).toEqual([
75
+ {
76
+ before: { id: "B", part: "x", value: "two" },
77
+ after: { id: "B", part: "x", value: "new" },
78
+ },
79
+ { before: { id: "C", part: "x", value: "three" }, after: null },
80
+ { before: null, after: { id: "D", part: "x", value: "three" } },
81
+ ])
82
+ })
83
+ test("ignores row order and unrequested columns", async () => {
84
+ const first = await write([row("A", "one"), row("B", "two")])
85
+ const last = await write([row("B", "two", "x", "changed"), row("A", "one")])
86
+ expect(await collect(await read(first, last))).toEqual([])
87
+ expect(await collect(await read(last, last))).toEqual([])
88
+ })
89
+ test("distinguishes null from a value and handles an empty target", async () => {
90
+ const first = await write([row("A", null)])
91
+ const second = await write([row("A", "value")])
92
+ expect(await collect(await read(first, second))).toEqual([
93
+ {
94
+ before: { id: "A", part: "x", value: null },
95
+ after: { id: "A", part: "x", value: "value" },
96
+ },
97
+ ])
98
+ const empty = await write([])
99
+ expect(await collect(await read(second, empty))).toEqual([
100
+ { before: { id: "A", part: "x", value: "value" }, after: null },
101
+ ])
102
+ expect(await collect(await read(empty, second))).toEqual([
103
+ { before: null, after: { id: "A", part: "x", value: "value" } },
104
+ ])
105
+ })
106
+ test("supports composite keys without concatenation collisions", async () => {
107
+ const first = await write([row("a:b", "one", "c"), row("a", "two", "b:c")])
108
+ const last = await write([row("a:b", "new", "c"), row("a", "two", "b:c")])
109
+ expect(await collect(await read(first, last, ["id", "part"]))).toEqual([
110
+ {
111
+ before: { id: "a:b", part: "c", value: "one" },
112
+ after: { id: "a:b", part: "c", value: "new" },
113
+ },
114
+ ])
115
+ })
116
+ test("refuses duplicate keys in either snapshot rather than losing rows", async () => {
117
+ const first = await write([row("A", "one"), row("A", "two")])
118
+ const last = await write([row("A", "new")])
119
+ expect(await read(first, last)).toBeNull()
120
+ expect(await read(last, first)).toBeNull()
121
+ })
122
+ test("refuses blank identities and invalid column requests", async () => {
123
+ const first = await write([row(" ", "one")])
124
+ expect(await read(first, first)).toBeNull()
125
+ await expect(read(first, first, [])).rejects.toThrow("distinct columns")
126
+ await expect(read(first, first, ["missing"])).rejects.toThrow("distinct columns")
127
+ })
128
+ test("releases the connection between yielded changes and closes after early exit", async () => {
129
+ const first = await write([row("A", "one"), row("B", "two")])
130
+ const last = await write([row("A", "new"), row("B", "new")])
131
+ const reader = await read(first, last)
132
+ expect(reader).not.toBeNull()
133
+ try {
134
+ for await (const _change of reader!.changes) {
135
+ expect((await storage.getLatestVersion(dataset.id))?.versionId).toBe(last)
136
+ break
137
+ }
138
+ } finally {
139
+ await reader!.close()
140
+ }
141
+ expect((await storage.getLatestVersion(dataset.id))?.versionId).toBe(last)
142
+ })
143
+ test("pages a complete pinned delta while a newer snapshot is published", async () => {
144
+ const count = 5_007
145
+ const first = await write(Array.from({ length: count }, (_, i) => row(String(i), "old")))
146
+ const last = await write(Array.from({ length: count }, (_, i) => row(String(i), "new")))
147
+ const reader = await read(first, last)
148
+ expect(reader?.changeCount).toBe(count)
149
+ await write([row("later", "unrelated")])
150
+ const changes = await collect(reader)
151
+ expect(new Set(changes.map((change) => change.after?.id)).size).toBe(count)
152
+ expect(
153
+ changes.every((change) => change.before?.value === "old" && change.after?.value === "new")
154
+ ).toBe(true)
155
+ })
156
+ test("returns complete typed images without exposing mutable stored values", async () => {
157
+ const typed = defineDataset("delta.typed", {
158
+ schema: [
159
+ col("id", "string"),
160
+ col("payload", "json"),
161
+ col("at", "timestamp"),
162
+ col("amount", "int64"),
163
+ ],
164
+ })
165
+ await storage.createDataset(typed)
166
+ const writeTyped = async (value: number) => {
167
+ const writer = await storage.beginWrite({ dataset: typed })
168
+ await writer.writeRows([
169
+ {
170
+ id: "a",
171
+ payload: { nested: [value] },
172
+ at: "2026-01-01T00:00:00.000Z",
173
+ amount: "9007199254740993",
174
+ },
175
+ ])
176
+ return (await writer.commit()).versionId
177
+ }
178
+ const first = await writeTyped(1)
179
+ const last = await writeTyped(2)
180
+ const reader = await storage.readChanges!({
181
+ datasetId: typed.id,
182
+ fromVersionId: first,
183
+ toVersionId: last,
184
+ keyColumns: ["id"],
185
+ columns: ["id", "payload", "at", "amount"],
186
+ })
187
+ const changes = await collect(reader)
188
+ expect(changes).toEqual([
189
+ {
190
+ before: {
191
+ id: "a",
192
+ payload: { nested: [1] },
193
+ at: "2026-01-01T00:00:00.000Z",
194
+ amount: "9007199254740993",
195
+ },
196
+ after: {
197
+ id: "a",
198
+ payload: { nested: [2] },
199
+ at: "2026-01-01T00:00:00.000Z",
200
+ amount: "9007199254740993",
201
+ },
202
+ },
203
+ ])
204
+ ;(changes[0]!.after!.payload as { nested: number[] }).nested.push(3)
205
+ const rows: DatasetRow[] = []
206
+ for await (const value of storage.readRows({ datasetId: typed.id, versionId: last }))
207
+ rows.push(value)
208
+ expect(rows[0]?.payload).toEqual({ nested: [2] })
209
+ })
210
+ test("returns unavailable for a version belonging to another dataset", async () => {
211
+ const first = await write([row("A", "one")])
212
+ const other = defineDataset("delta.other", { schema: [col("id", "string")] })
213
+ await storage.createDataset(other)
214
+ const writer = await storage.beginWrite({ dataset: other })
215
+ await writer.writeRows([{ id: "other" }])
216
+ const otherVersion = await writer.commit()
217
+ expect(await read(otherVersion.versionId, first)).toBeNull()
218
+ expect(await read(first, otherVersion.versionId)).toBeNull()
219
+ })
220
+ test("can close an unread delta and observes cancellation", async () => {
221
+ const first = await write([row("A", "one")])
222
+ const last = await write([row("A", "new")])
223
+ const reader = await read(first, last)
224
+ await reader!.close()
225
+ await reader!.close()
226
+ const controller = new AbortController()
227
+ controller.abort(new Error("cancelled delta"))
228
+ await expect(
229
+ storage.readChanges!({
230
+ datasetId: dataset.id,
231
+ fromVersionId: first,
232
+ toVersionId: last,
233
+ columns,
234
+ keyColumns: ["id"],
235
+ signal: controller.signal,
236
+ })
237
+ ).rejects.toThrow("cancelled delta")
238
+ })
239
+ })
240
+ }
@@ -19,6 +19,8 @@ import {
19
19
  createOntologyMaterializer,
20
20
  type OntologyEditOperation,
21
21
  ProjectionRegistry,
22
+ type ProjectionSourceBase,
23
+ type ProjectionSourceDeletion,
22
24
  type ProjectionSourceEntry,
23
25
  } from "../materializer"
24
26
  import type { ActionRunStorage, ProjectionRunStorage } from "../storage"
@@ -125,6 +127,175 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
125
127
  name: string,
126
128
  provider: MaterializerStorageContractProvider<TStorage>
127
129
  ): void {
130
+ // Regression: replacing previousSourceRows with only the latest manifest loses "a" at v3.
131
+ // Removing root retirement checks from cleanup deletes "a" when v1 becomes terminal.
132
+ test(`${name} retains current roots across deltas, cleanup, overrides and full replacement`, async () => {
133
+ const createdStorage = await provider.createStorage()
134
+ const storage = requireContractStorage(createdStorage)
135
+ let ordinal = 0
136
+ const projectId = "materializer-storage-contract"
137
+ const source = { projectionId: devices.id }
138
+ const materializer = createOntologyMaterializer({
139
+ projectId,
140
+ ontology,
141
+ projections,
142
+ storage,
143
+ dependencies: { batching: { sourceStageRows: 1, statePageRows: 1, planChunkRows: 1 } },
144
+ })
145
+ const active = async (): Promise<ProjectionSourceBase> => {
146
+ const current = await materializer.projections.getActive(source)
147
+ if (!current?.lastCommitId) throw new Error("Missing source head")
148
+ return { materializationId: current.materializationId, lastCommitId: current.lastCommitId }
149
+ }
150
+ const replace = async (
151
+ values: readonly (ProjectionSourceEntry | ProjectionSourceDeletion)[],
152
+ base?: ProjectionSourceBase,
153
+ interleave?: { readonly version: number; readonly beforeSeal?: () => Promise<void> }
154
+ ) => {
155
+ const version = interleave?.version ?? ++ordinal
156
+ const datasetVersion = {
157
+ datasetId: devices.id,
158
+ versionId: `delta-v${version}`,
159
+ createdAt: `2026-01-${String(version).padStart(2, "0")}T00:00:00.000Z`,
160
+ }
161
+ const execution = await claim(storage, {
162
+ runId: `delta-run-${version}`,
163
+ projectionId: devices.id,
164
+ protocol: "replacement",
165
+ datasetVersion,
166
+ })
167
+ const bound = await projectionMaterializer(
168
+ materializer,
169
+ storage,
170
+ devices.id,
171
+ execution.projectionRunId
172
+ )
173
+ return bound.projections.replace({
174
+ source,
175
+ datasetVersion,
176
+ execution,
177
+ entries: (async function* () {
178
+ yield* entries(values)
179
+ await interleave?.beforeSeal?.()
180
+ })(),
181
+ ...(base ? { base } : {}),
182
+ })
183
+ }
184
+ const ref = (primaryId: string) => ({ objectTypeId: Device.id, primaryId })
185
+ const object = (id: string) => storage.objects.getByPrimaryId({ projectId, ...ref(id) })
186
+ const remove = (id: string): ProjectionSourceDeletion => ({
187
+ root: { kind: "object", ref: ref(id) },
188
+ deleted: true,
189
+ })
190
+ try {
191
+ await replace([sourceEntry("a", "A"), sourceEntry("b", "B", "a"), sourceEntry("c", "C")])
192
+ const stale = await active()
193
+ const initialA = await object("a")
194
+ const initialB = await object("b")
195
+ const transact = storage.transaction.bind(storage)
196
+ // Fail after finalize has moved the source pointers, written the commit and its outbox.
197
+ // Removing the enclosing transaction rollback must fail these head/value assertions.
198
+ storage.transaction = (run, options) =>
199
+ transact(async (tx) => {
200
+ const materializations = new Proxy(tx.ontology.materializations, {
201
+ get(target, key) {
202
+ if (key !== "finalize") return Reflect.get(target, key)
203
+ return async (input: Parameters<typeof target.finalize>[0]) => {
204
+ await target.finalize(input)
205
+ throw new Error("injected after source activation")
206
+ }
207
+ },
208
+ })
209
+ const ontology = new Proxy(tx.ontology, {
210
+ get(target, key) {
211
+ return key === "materializations" ? materializations : Reflect.get(target, key)
212
+ },
213
+ })
214
+ return run(
215
+ new Proxy(tx, {
216
+ get(target, key) {
217
+ return key === "ontology" ? ontology : Reflect.get(target, key)
218
+ },
219
+ })
220
+ )
221
+ }, options)
222
+ try {
223
+ await expect(replace([remove("a"), sourceEntry("b", "Failed B")], stale)).rejects.toThrow(
224
+ "injected after source activation"
225
+ )
226
+ } finally {
227
+ storage.transaction = transact
228
+ }
229
+ expect(await active()).toEqual(stale)
230
+ expect(await object("a")).toEqual(initialA)
231
+ expect(await object("b")).toEqual(initialB)
232
+ await replace([sourceEntry("b", "B2", "c")], await active())
233
+ await replace([sourceEntry("c", "C2")], await active())
234
+ expect(await object("a")).toEqual(initialA)
235
+ expect(await object("b")).toMatchObject({ properties: { name: "B2" } })
236
+ for (let pass = 0; pass < 20; pass++) {
237
+ const result = await storage.ontology.sources.cleanupTerminal({
238
+ projectId,
239
+ terminalBefore: "2100-01-01T00:00:00.000Z",
240
+ limit: 2,
241
+ })
242
+ if (result.rowsDeleted + result.materializationsDeleted === 0) break
243
+ }
244
+ const scoped = materializer.withScope(runtimeScope())
245
+ const edit = (operations: readonly OntologyEditOperation[]) =>
246
+ scoped.edits.commit({
247
+ mode: "atomic",
248
+ source: { kind: "runtime", requestId: `delta-edit-${++ordinal}` },
249
+ operations,
250
+ expectedObjects: [],
251
+ expectedLinks: [],
252
+ expectedLinkScopes: [],
253
+ })
254
+ await edit([
255
+ { id: "edit-a", kind: "object.upsert", ref: ref("a"), properties: { name: "Edited A" } },
256
+ ])
257
+ await edit([
258
+ { id: "reset-a", kind: "object.patch", ref: ref("a"), set: {}, unset: [], reset: ["name"] },
259
+ ])
260
+ expect(await object("a")).toMatchObject({ properties: { name: "A" } })
261
+ await replace([], await active())
262
+ expect(await object("a")).toMatchObject({ properties: { name: "A" } })
263
+ await expect(replace([remove("a")], stale)).rejects.toMatchObject({
264
+ kind: "source-materialization",
265
+ })
266
+ expect(await object("a")).not.toBeNull()
267
+ await replace([remove("a")], await active())
268
+ expect(await object("a")).toBeNull()
269
+ expect(await object("b")).toMatchObject({ properties: { name: "B2" } })
270
+ await replace([sourceEntry("b", "B3")])
271
+ expect(await object("b")).toMatchObject({ properties: { name: "B3" } })
272
+ expect(await object("c")).toBeNull()
273
+ await replace([], await active())
274
+ expect(await object("b")).toMatchObject({ properties: { name: "B3" } })
275
+ await replace([])
276
+ expect(await object("b")).toBeNull()
277
+
278
+ // A different run commits after base validation but before this candidate is sealed.
279
+ // Only checking the base before reading the delta would allow a stale publication here.
280
+ const base = await active()
281
+ const earlier = ++ordinal
282
+ const later = ++ordinal
283
+ await expect(
284
+ replace([sourceEntry("a", "Stale delta")], base, {
285
+ version: later,
286
+ beforeSeal: async () => {
287
+ await replace([sourceEntry("a", "Concurrent value")], base, { version: earlier })
288
+ },
289
+ })
290
+ ).rejects.toMatchObject({ kind: "source-materialization" })
291
+ expect(await object("a")).toMatchObject({ properties: { name: "Concurrent value" } })
292
+ await replace([sourceEntry("a", "Fresh delta")], await active())
293
+ expect(await object("a")).toMatchObject({ properties: { name: "Fresh delta" } })
294
+ } finally {
295
+ await provider.cleanup?.(createdStorage)
296
+ }
297
+ })
298
+
128
299
  test(`${name} persists replacement, Action, and telemetry commits atomically`, async () => {
129
300
  const createdStorage = await provider.createStorage()
130
301
  const storage = requireContractStorage(createdStorage)
@@ -811,7 +982,7 @@ function sourceEntry(id: string, name: string, parentId?: string): ProjectionSou
811
982
  }
812
983
  }
813
984
 
814
- async function* entries(values: readonly ProjectionSourceEntry[]) {
985
+ async function* entries(values: readonly (ProjectionSourceEntry | ProjectionSourceDeletion)[]) {
815
986
  for (const value of values) yield value
816
987
  }
817
988
 
@@ -383,6 +383,13 @@ export function runOntologyStorageContractSuite<TStorage extends OntologyStorage
383
383
  limit: 1,
384
384
  })
385
385
  ).toEqual({ rowsDeleted: 1, materializationsDeleted: 0 })
386
+ expect(
387
+ await storage.ontology.sources.cleanupTerminal({
388
+ projectId: "contract-project",
389
+ terminalBefore: "2026-02-01T00:00:00.000Z",
390
+ limit: 1,
391
+ })
392
+ ).toEqual({ rowsDeleted: 1, materializationsDeleted: 0 })
386
393
  expect(
387
394
  await storage.ontology.sources.cleanupTerminal({
388
395
  projectId: "contract-project",
@@ -256,6 +256,40 @@ export function runProjectionRunStorageContractSuite<TStorage extends Projection
256
256
  })
257
257
  })
258
258
 
259
+ test("persists independent incremental progress across reclaim without a physical-row floor", async () => {
260
+ await withStorage(async (context) => {
261
+ const storage = context.projectionRuns
262
+ await admitProjectionRun(context, replacementInput("delta-progress"))
263
+ const first = await storage.startOrReclaim(replacementInput("delta-progress"))
264
+ const updated = await storage.update({
265
+ ...executionInput(first),
266
+ progress: { sourceChangesRead: 12 },
267
+ })
268
+ expect(updated.progress).toEqual({
269
+ sourceRowsRead: 0,
270
+ sourceRowsSkipped: 0,
271
+ sourceChangesRead: 12,
272
+ })
273
+ const second = await storage.startOrReclaim(replacementInput("delta-progress"))
274
+ expect(second.run.progress).toEqual(updated.progress)
275
+ await expect(
276
+ storage.update({ ...executionInput(first), progress: { sourceChangesRead: 13 } })
277
+ ).rejects.toThrow()
278
+ await expect(
279
+ storage.update({ ...executionInput(second), progress: { sourceChangesRead: 11 } })
280
+ ).rejects.toThrow("must not decrease")
281
+ const fallback = await storage.update({
282
+ ...executionInput(second),
283
+ progress: { sourceRowsRead: 2 },
284
+ })
285
+ expect(fallback.progress).toEqual({
286
+ sourceRowsRead: 2,
287
+ sourceRowsSkipped: 0,
288
+ sourceChangesRead: 12,
289
+ })
290
+ })
291
+ })
292
+
259
293
  test("rejects immutable dataset-version metadata reuse", async () => {
260
294
  await withStorage(async (context) => {
261
295
  await admitProjectionRun(context, replacementInput("dataset-metadata-first"))
@@ -0,0 +1,56 @@
1
+ import { createConnectorCodedError } from "../connectors/errors"
2
+ import { type ConnectorRuntime, getConnectorExecutionSourceResolver } from "../connectors/execution"
3
+ import { type ConnectorDefinition, isOAuthConnectorDefinition } from "../connectors/types"
4
+ import type { WebhookConnections } from "./types"
5
+
6
+ /** Lookup stays inside the admitted execution's project and registered connector. */
7
+ export function createWebhookConnections(
8
+ runtime: ConnectorRuntime,
9
+ definition: ConnectorDefinition
10
+ ): WebhookConnections {
11
+ return {
12
+ async forAccount(accountId) {
13
+ if (!isOAuthConnectorDefinition(definition)) {
14
+ throw createConnectorCodedError(
15
+ "connector.configuration_invalid",
16
+ "Webhook connections.forAccount() requires a managed OAuth connector."
17
+ )
18
+ }
19
+ if (typeof accountId !== "string" || !accountId.trim()) {
20
+ throw createConnectorCodedError(
21
+ "connector.configuration_invalid",
22
+ "Webhook account ID must be a non-empty string."
23
+ )
24
+ }
25
+ const resolver = getConnectorExecutionSourceResolver(runtime)
26
+ const sources = await resolver.list(definition)
27
+ return sources.flatMap((source) => {
28
+ const connection = source.connection
29
+ if (!connection || connection.account.id !== accountId) return []
30
+ const connectionId = connection.id
31
+ let clientPromise: Promise<unknown> | undefined
32
+ return [
33
+ {
34
+ connection,
35
+ client() {
36
+ clientPromise ??= (async () => {
37
+ const current = (await resolver.list(definition)).find(
38
+ (candidate) =>
39
+ candidate.connection?.id === connectionId &&
40
+ candidate.connection.account.id === accountId
41
+ )
42
+ if (!current)
43
+ throw createConnectorCodedError(
44
+ "connector.not_found",
45
+ "Webhook connection is no longer active for this account."
46
+ )
47
+ return current.connect(new AbortController().signal)
48
+ })()
49
+ return clientPromise
50
+ },
51
+ },
52
+ ]
53
+ })
54
+ },
55
+ }
56
+ }
@@ -9,6 +9,8 @@ export type {
9
9
  WebhookBodyFormat,
10
10
  WebhookBodyParser,
11
11
  WebhookBodySchema,
12
+ WebhookConnection,
13
+ WebhookConnections,
12
14
  WebhookDefinition,
13
15
  WebhookHandlerContext,
14
16
  WebhookHandlerResult,
@@ -1,3 +1,5 @@
1
+ export { isStaticConnectorDefinition } from "../connectors/types"
2
+ export { createWebhookConnections } from "./connections"
1
3
  export {
2
4
  type AdmitWebhookRunInput,
3
5
  type AdmitWebhookRunResult,
@@ -1,6 +1,6 @@
1
- import { type ConnectorDefinition, isStaticConnectorDefinition } from "../connectors/types"
1
+ import type { ConnectorDefinition } from "../connectors/types"
2
2
  import { WebhookValidationError } from "./errors"
3
- import type { RegisteredWebhook, WebhookDefinition } from "./types"
3
+ import type { RegisteredWebhook } from "./types"
4
4
 
5
5
  /** Read-only access to connector webhooks registered by a host. */
6
6
  export interface WebhookCatalog {
@@ -21,7 +21,6 @@ export class WebhookRegistry implements WebhookCatalog {
21
21
  const registered: RegisteredWebhook[] = []
22
22
 
23
23
  for (const connector of options.connectors) {
24
- if (!isStaticConnectorDefinition(connector)) continue
25
24
  const webhooks = connector.adapter.webhooks ?? []
26
25
  if (!Array.isArray(webhooks)) {
27
26
  throw new WebhookValidationError(
@@ -82,7 +81,7 @@ export function webhookRoute(connectorId: string, webhookId: string): string {
82
81
  function assertValidWebhook(
83
82
  connectorId: string,
84
83
  webhook: unknown
85
- ): asserts webhook is WebhookDefinition {
84
+ ): asserts webhook is RegisteredWebhook["webhook"] {
86
85
  if (!isRecord(webhook)) {
87
86
  throw new WebhookValidationError(`[Sixb] Connector '${connectorId}' has an invalid webhook.`)
88
87
  }
@@ -1,4 +1,4 @@
1
- import type { StaticConnectorDefinition } from "../connectors/types"
1
+ import type { ConnectorConnectionMetadata, ConnectorDefinition } from "../connectors/types"
2
2
  import type { Logger } from "../logging"
3
3
  import type { OntologySource, Sixb } from "../runtime"
4
4
 
@@ -49,7 +49,7 @@ export interface WebhookMetadata {
49
49
 
50
50
  /** Pre-admission input. It intentionally has no execution SDK or run-scoped logger. */
51
51
  export interface WebhookVerifyContext {
52
- readonly connector: StaticConnectorDefinition
52
+ readonly connector: ConnectorDefinition
53
53
  readonly webhook: WebhookMetadata
54
54
  readonly request: Request
55
55
  readonly rawBody: Uint8Array
@@ -60,7 +60,9 @@ export interface WebhookHandlerContext<TBody, TClient> extends WebhookVerifyCont
60
60
  readonly sixb: Sixb<readonly OntologySource[]>
61
61
  readonly logger: Logger
62
62
  readonly body: TBody
63
- /** Lazily resolve the connector client only when the handler needs it. */
63
+ /** Resolve selected managed connections for an external account in this connector. */
64
+ readonly connections: WebhookConnections<TClient>
65
+ /** Resolve a static connector's client. For OAuth, use connections.forAccount(). */
64
66
  client(): Promise<TClient>
65
67
  }
66
68
 
@@ -93,9 +95,20 @@ export type WebhookHandlerResult = WebhookResponse | void
93
95
  export interface RegisteredWebhook<
94
96
  TBody = unknown,
95
97
  TClient = unknown,
96
- TConnector extends StaticConnectorDefinition = StaticConnectorDefinition,
98
+ TConnector extends ConnectorDefinition = ConnectorDefinition,
97
99
  > {
98
100
  readonly connector: TConnector
99
101
  readonly webhook: WebhookDefinition<TBody, TClient>
100
102
  readonly route: string
101
103
  }
104
+
105
+ export interface WebhookConnection<TClient = unknown> {
106
+ readonly connection: ConnectorConnectionMetadata
107
+ /** First call rechecks the connection and creates its client; later calls reuse that client. */
108
+ client(): Promise<TClient>
109
+ }
110
+
111
+ export interface WebhookConnections<TClient = unknown> {
112
+ /** Returns all matching selected connections; an unknown account returns an empty array. */
113
+ forAccount(accountId: string): Promise<readonly WebhookConnection<TClient>[]>
114
+ }