@sixb/core 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/index.js +9 -9
- package/dist/agents/index.js +17 -16
- package/dist/agents/index.js.map +1 -1
- package/dist/blob-storage/types.d.ts +4 -1
- package/dist/blob-storage/types.d.ts.map +1 -1
- package/dist/bootstrap/index.js +21 -20
- package/dist/bootstrap/index.js.map +1 -1
- package/dist/broker/in-memory.d.ts +8 -2
- package/dist/broker/in-memory.d.ts.map +1 -1
- package/dist/broker/index.d.ts +4 -2
- package/dist/broker/index.d.ts.map +1 -1
- package/dist/broker/index.js +7 -3
- package/dist/broker/index.js.map +1 -1
- package/dist/broker/retention.d.ts +4 -0
- package/dist/broker/retention.d.ts.map +1 -0
- package/dist/broker/subscriber.d.ts +10 -0
- package/dist/broker/subscriber.d.ts.map +1 -0
- package/dist/broker/types.d.ts +14 -1
- package/dist/broker/types.d.ts.map +1 -1
- package/dist/{chunk-ypbtykv6.js → chunk-01kj2pbh.js} +1 -1
- package/dist/{chunk-k3cqkkn2.js → chunk-1824gssz.js} +30 -65
- package/dist/chunk-1824gssz.js.map +12 -0
- package/dist/{chunk-xyr8kppc.js → chunk-1yefycb1.js} +1 -1
- package/dist/chunk-2c4j899j.js.map +1 -1
- package/dist/{chunk-ppytyr6w.js → chunk-358caqf5.js} +4 -4
- package/dist/chunk-358caqf5.js.map +12 -0
- package/dist/{chunk-m3z3x9v0.js → chunk-3ckk0xfj.js} +2 -2
- package/dist/{chunk-kxwa347f.js → chunk-3qms4nax.js} +2 -2
- package/dist/{chunk-kxwa347f.js.map → chunk-3qms4nax.js.map} +3 -3
- package/dist/{chunk-52hbv0ak.js → chunk-4zhmdsx6.js} +141 -55
- package/dist/chunk-4zhmdsx6.js.map +36 -0
- package/dist/{chunk-zcnq6105.js → chunk-85cd7e8c.js} +2 -1
- package/dist/chunk-85cd7e8c.js.map +10 -0
- package/dist/{chunk-3f7veca0.js → chunk-9x8dybsb.js} +3 -3
- package/dist/chunk-9x8dybsb.js.map +10 -0
- package/dist/{chunk-0w35wavj.js → chunk-a0e6x5hd.js} +1 -1
- package/dist/{chunk-r29fxdhn.js → chunk-aa6z4a19.js} +160 -44
- package/dist/chunk-aa6z4a19.js.map +12 -0
- package/dist/{chunk-emyyk7dx.js → chunk-cfvakeax.js} +22 -145
- package/dist/chunk-cfvakeax.js.map +12 -0
- package/dist/chunk-d62kjtbg.js.map +1 -1
- package/dist/{chunk-h1gfhhbh.js → chunk-d63nnr5b.js} +2 -2
- package/dist/{chunk-r4tfx1sf.js → chunk-fh8kgex1.js} +5 -5
- package/dist/{chunk-an5tt9y0.js → chunk-ftv470j8.js} +42 -4
- package/dist/{chunk-an5tt9y0.js.map → chunk-ftv470j8.js.map} +7 -7
- package/dist/{chunk-jjy0fhrn.js → chunk-g91qxjj0.js} +20 -3
- package/dist/chunk-g91qxjj0.js.map +12 -0
- package/dist/{chunk-hpr1r6av.js → chunk-gq8vhakp.js} +51 -40
- package/dist/{chunk-hpr1r6av.js.map → chunk-gq8vhakp.js.map} +6 -6
- package/dist/{chunk-x57ycxq0.js → chunk-gqh45d2n.js} +42 -3
- package/dist/{chunk-x57ycxq0.js.map → chunk-gqh45d2n.js.map} +3 -3
- package/dist/{chunk-mjbk2djy.js → chunk-gqm76kse.js} +231 -125
- package/dist/{chunk-mjbk2djy.js.map → chunk-gqm76kse.js.map} +9 -8
- package/dist/{chunk-e0d1rzef.js → chunk-jsy70480.js} +16 -3
- package/dist/chunk-jsy70480.js.map +15 -0
- package/dist/{chunk-0rdrecsb.js → chunk-kfee1h52.js} +116 -32
- package/dist/chunk-kfee1h52.js.map +13 -0
- package/dist/{chunk-3frsnm7x.js → chunk-qma80ych.js} +4 -6
- package/dist/chunk-qma80ych.js.map +11 -0
- package/dist/{chunk-vf691g1r.js → chunk-sepyxgpc.js} +3 -3
- package/dist/chunk-v2vcg0w6.js +151 -0
- package/dist/chunk-v2vcg0w6.js.map +11 -0
- package/dist/{chunk-3tgwrcyh.js → chunk-wf88w4kp.js} +17 -23
- package/dist/chunk-wf88w4kp.js.map +14 -0
- package/dist/{chunk-xfbydgzy.js → chunk-x9m9x2eq.js} +7 -8
- package/dist/{chunk-xfbydgzy.js.map → chunk-x9m9x2eq.js.map} +3 -3
- package/dist/{chunk-qhcpj9wc.js → chunk-z0271qkp.js} +98 -2
- package/dist/chunk-z0271qkp.js.map +15 -0
- package/dist/{chunk-1enhpacz.js → chunk-z0s4axeh.js} +6 -6
- package/dist/connectors/connections/authorizations.d.ts.map +1 -1
- package/dist/connectors/connections/command-context.d.ts +2 -0
- package/dist/connectors/connections/command-context.d.ts.map +1 -1
- package/dist/connectors/connections/contracts.d.ts +4 -1
- package/dist/connectors/connections/contracts.d.ts.map +1 -1
- package/dist/connectors/connections/execution.d.ts +1 -0
- package/dist/connectors/connections/execution.d.ts.map +1 -1
- package/dist/connectors/connections/internal.js +1 -1
- package/dist/connectors/connections/request.d.ts +3 -1
- package/dist/connectors/connections/request.d.ts.map +1 -1
- package/dist/connectors/connections/runs.d.ts +1 -0
- package/dist/connectors/connections/runs.d.ts.map +1 -1
- package/dist/connectors/connections/service.d.ts +2 -0
- package/dist/connectors/connections/service.d.ts.map +1 -1
- package/dist/connectors/connections/tokens.d.ts.map +1 -1
- package/dist/connectors/connections/validation.d.ts +5 -1
- package/dist/connectors/connections/validation.d.ts.map +1 -1
- package/dist/connectors/types.d.ts +19 -4
- package/dist/connectors/types.d.ts.map +1 -1
- package/dist/events/execution.d.ts +1 -1
- package/dist/events/execution.d.ts.map +1 -1
- package/dist/events/index.js +1 -1
- package/dist/events/outbox-dispatcher.d.ts +2 -2
- package/dist/events/outbox-dispatcher.d.ts.map +1 -1
- package/dist/events/service.d.ts +2 -2
- package/dist/events/service.d.ts.map +1 -1
- package/dist/execution/agent.js +17 -16
- package/dist/execution/agent.js.map +1 -1
- package/dist/execution/primitive.js +17 -16
- package/dist/execution/primitive.js.map +2 -2
- package/dist/execution/request.js +17 -16
- package/dist/execution/request.js.map +2 -2
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +31 -29
- package/dist/index.js.map +1 -1
- package/dist/lake-storage/changes.d.ts +5 -0
- package/dist/lake-storage/changes.d.ts.map +1 -0
- package/dist/lake-storage/in-memory.d.ts +2 -1
- package/dist/lake-storage/in-memory.d.ts.map +1 -1
- package/dist/lake-storage/index.d.ts +2 -1
- package/dist/lake-storage/index.d.ts.map +1 -1
- package/dist/lake-storage/index.js +4 -2
- package/dist/lake-storage/index.js.map +1 -1
- package/dist/lake-storage/types.d.ts +38 -0
- package/dist/lake-storage/types.d.ts.map +1 -1
- package/dist/logging/internal.js +1 -1
- package/dist/maintenance/ontology-maintenance.d.ts.map +1 -1
- package/dist/maintenance/types.d.ts +1 -1
- package/dist/maintenance/types.d.ts.map +1 -1
- package/dist/materialization/index.d.ts +1 -1
- package/dist/materialization/index.d.ts.map +1 -1
- package/dist/materialization/model.d.ts +12 -1
- package/dist/materialization/model.d.ts.map +1 -1
- package/dist/materializer/index.d.ts +1 -1
- package/dist/materializer/index.d.ts.map +1 -1
- package/dist/materializer/index.js +6 -6
- package/dist/materializer/materializer.d.ts +5 -1
- package/dist/materializer/materializer.d.ts.map +1 -1
- package/dist/materializer/projections/entry-validator.d.ts +2 -1
- package/dist/materializer/projections/entry-validator.d.ts.map +1 -1
- package/dist/materializer/projections/replace.d.ts.map +1 -1
- package/dist/materializer/projections/source-materialization.d.ts +4 -3
- package/dist/materializer/projections/source-materialization.d.ts.map +1 -1
- package/dist/materializer/shared/normalize.d.ts +2 -1
- package/dist/materializer/shared/normalize.d.ts.map +1 -1
- package/dist/models/definitions.d.ts +7 -0
- package/dist/models/definitions.d.ts.map +1 -1
- package/dist/models/execution/index.js +6 -6
- package/dist/models/index.js +1 -1
- package/dist/objects/index.js +10 -10
- package/dist/objects/query/index.js +6 -6
- package/dist/pipelines/index.js +7 -7
- package/dist/pipelines/run-dispatch.d.ts.map +1 -1
- package/dist/projections/internal.js +7 -9
- package/dist/projections/internal.js.map +1 -1
- package/dist/projections/run-dispatch.d.ts.map +1 -1
- package/dist/queues/in-memory.d.ts +7 -3
- package/dist/queues/in-memory.d.ts.map +1 -1
- package/dist/queues/index.js +1 -1
- package/dist/queues/indexed-heap.d.ts +15 -0
- package/dist/queues/indexed-heap.d.ts.map +1 -0
- package/dist/runtime/host.d.ts.map +1 -1
- package/dist/runtime/internal.js +1 -1
- package/dist/runtime/ontology-mutations.d.ts +3 -1
- package/dist/runtime/ontology-mutations.d.ts.map +1 -1
- package/dist/shares/internal.js +5 -5
- package/dist/storage/agents/provider.js +1 -1
- package/dist/storage/connector-connections/in-memory.d.ts +1 -0
- package/dist/storage/connector-connections/in-memory.d.ts.map +1 -1
- package/dist/storage/connector-connections/provider/connections.d.ts.map +1 -1
- package/dist/storage/connector-connections/provider/runs.d.ts +1 -0
- package/dist/storage/connector-connections/provider/runs.d.ts.map +1 -1
- package/dist/storage/connector-connections/provider/shared.d.ts +1 -0
- package/dist/storage/connector-connections/provider/shared.d.ts.map +1 -1
- package/dist/storage/connector-connections/provider.d.ts +1 -0
- package/dist/storage/connector-connections/provider.d.ts.map +1 -1
- package/dist/storage/connector-connections/provider.js +11 -17
- package/dist/storage/connector-connections/provider.js.map +6 -6
- package/dist/storage/connector-connections/types.d.ts +6 -3
- package/dist/storage/connector-connections/types.d.ts.map +1 -1
- package/dist/storage/index.d.ts +1 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +5 -5
- package/dist/storage/internal.js +1 -1
- package/dist/storage/ontology/in-memory/materializations-state.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/materializations.d.ts +1 -1
- package/dist/storage/ontology/in-memory/materializations.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/shared-state.d.ts +15 -2
- package/dist/storage/ontology/in-memory/shared-state.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/source-roots.d.ts +6 -0
- package/dist/storage/ontology/in-memory/source-roots.d.ts.map +1 -0
- package/dist/storage/ontology/in-memory/sources.d.ts.map +1 -1
- package/dist/storage/ontology/index.d.ts +1 -1
- package/dist/storage/ontology/index.d.ts.map +1 -1
- package/dist/storage/ontology/outbox.d.ts +1 -0
- package/dist/storage/ontology/outbox.d.ts.map +1 -1
- package/dist/storage/ontology/provider.d.ts +16 -0
- package/dist/storage/ontology/provider.d.ts.map +1 -1
- package/dist/storage/ontology/provider.js +4 -2
- package/dist/storage/ontology/provider.js.map +1 -1
- package/dist/storage/ontology/sources.d.ts +13 -4
- package/dist/storage/ontology/sources.d.ts.map +1 -1
- package/dist/storage/projection-runs/lifecycle.d.ts.map +1 -1
- package/dist/storage/projection-runs/provider.js +2 -2
- package/dist/storage/projection-runs/types.d.ts +2 -0
- package/dist/storage/projection-runs/types.d.ts.map +1 -1
- package/dist/syncs/index.js +9 -8
- package/dist/syncs/index.js.map +1 -1
- package/dist/syncs/run-dispatch.d.ts.map +1 -1
- package/dist/testing/broker-contract.d.ts.map +1 -1
- package/dist/testing/connector-connection-storage-contract.d.ts.map +1 -1
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +1499 -963
- package/dist/testing/index.js.map +10 -9
- package/dist/testing/lake-changes-contract.d.ts +6 -0
- package/dist/testing/lake-changes-contract.d.ts.map +1 -0
- package/dist/testing/lake-storage-contract.d.ts.map +1 -1
- package/dist/testing/materializer-storage-contract.d.ts.map +1 -1
- package/dist/testing/ontology-storage-contract.d.ts.map +1 -1
- package/dist/testing/projection-run-storage-contract.d.ts.map +1 -1
- package/dist/webhooks/connections.d.ts +6 -0
- package/dist/webhooks/connections.d.ts.map +1 -0
- package/dist/webhooks/index.d.ts +1 -1
- package/dist/webhooks/index.d.ts.map +1 -1
- package/dist/webhooks/internal.d.ts +2 -0
- package/dist/webhooks/internal.d.ts.map +1 -1
- package/dist/webhooks/internal.js +55 -10
- package/dist/webhooks/internal.js.map +4 -3
- package/dist/webhooks/registry.d.ts +1 -1
- package/dist/webhooks/registry.d.ts.map +1 -1
- package/dist/webhooks/types.d.ts +15 -4
- package/dist/webhooks/types.d.ts.map +1 -1
- package/dist/workflows/index.js +18 -17
- package/dist/workflows/index.js.map +1 -1
- package/dist/workflows/run-dispatch.d.ts +3 -1
- package/dist/workflows/run-dispatch.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/agents/model-loop.ts +7 -0
- package/src/blob-storage/types.ts +4 -1
- package/src/broker/in-memory.ts +104 -39
- package/src/broker/index.ts +4 -1
- package/src/broker/retention.ts +28 -0
- package/src/broker/subscriber.ts +21 -0
- package/src/broker/types.ts +15 -1
- package/src/connectors/builders.ts +0 -6
- package/src/connectors/connections/authorizations.ts +11 -5
- package/src/connectors/connections/command-context.ts +21 -4
- package/src/connectors/connections/contracts.ts +7 -1
- package/src/connectors/connections/execution.ts +3 -0
- package/src/connectors/connections/request.ts +56 -22
- package/src/connectors/connections/runs.ts +39 -1
- package/src/connectors/connections/service.ts +6 -0
- package/src/connectors/connections/tokens.ts +4 -0
- package/src/connectors/connections/validation.ts +91 -5
- package/src/connectors/types.ts +19 -4
- package/src/events/execution.ts +2 -2
- package/src/events/outbox-dispatcher.ts +20 -26
- package/src/events/service.ts +3 -3
- package/src/index.ts +10 -2
- package/src/lake-storage/changes.ts +43 -0
- package/src/lake-storage/in-memory.ts +75 -0
- package/src/lake-storage/index.ts +4 -0
- package/src/lake-storage/types.ts +42 -0
- package/src/logging/service.ts +1 -1
- package/src/maintenance/ontology-maintenance.ts +6 -2
- package/src/maintenance/types.ts +1 -1
- package/src/materialization/index.ts +2 -0
- package/src/materialization/model.ts +14 -1
- package/src/materializer/README.md +1 -1
- package/src/materializer/index.ts +2 -0
- package/src/materializer/materializer.ts +10 -0
- package/src/materializer/projections/entry-validator.ts +19 -1
- package/src/materializer/projections/replace.ts +36 -1
- package/src/materializer/projections/source-materialization.ts +26 -6
- package/src/materializer/shared/normalize.ts +1 -1
- package/src/models/definitions.ts +19 -0
- package/src/models/pricing.ts +6 -2
- package/src/pipelines/run-dispatch.ts +21 -1
- package/src/projections/run-dispatch.ts +30 -68
- package/src/queues/in-memory.ts +105 -62
- package/src/queues/indexed-heap.ts +74 -0
- package/src/runtime/host.ts +0 -14
- package/src/runtime/ontology-mutations.ts +4 -0
- package/src/storage/connector-connections/in-memory.ts +19 -19
- package/src/storage/connector-connections/provider/connections.ts +1 -15
- package/src/storage/connector-connections/provider/runs.ts +9 -1
- package/src/storage/connector-connections/provider/shared.ts +2 -3
- package/src/storage/connector-connections/provider.ts +6 -0
- package/src/storage/connector-connections/types.ts +8 -3
- package/src/storage/index.ts +1 -0
- package/src/storage/ontology/in-memory/materializations-state.ts +5 -18
- package/src/storage/ontology/in-memory/materializations.ts +73 -92
- package/src/storage/ontology/in-memory/shared-state.ts +36 -4
- package/src/storage/ontology/in-memory/source-roots.ts +123 -0
- package/src/storage/ontology/in-memory/sources.ts +78 -23
- package/src/storage/ontology/index.ts +1 -0
- package/src/storage/ontology/outbox.ts +1 -0
- package/src/storage/ontology/provider.ts +70 -0
- package/src/storage/ontology/sources.ts +14 -3
- package/src/storage/projection-runs/lifecycle.ts +5 -2
- package/src/storage/projection-runs/types.ts +4 -3
- package/src/syncs/run-dispatch.ts +19 -1
- package/src/testing/broker-contract.ts +95 -0
- package/src/testing/connector-connection-storage-contract.ts +29 -12
- package/src/testing/index.ts +1 -0
- package/src/testing/lake-changes-contract.ts +240 -0
- package/src/testing/lake-storage-contract.ts +29 -0
- package/src/testing/materializer-storage-contract.ts +172 -1
- package/src/testing/ontology-storage-contract.ts +15 -3
- package/src/testing/projection-run-storage-contract.ts +34 -0
- package/src/webhooks/connections.ts +56 -0
- package/src/webhooks/index.ts +2 -0
- package/src/webhooks/internal.ts +2 -0
- package/src/webhooks/registry.ts +3 -4
- package/src/webhooks/types.ts +17 -4
- package/src/workflows/run-dispatch.ts +42 -29
- package/dist/chunk-0rdrecsb.js.map +0 -11
- package/dist/chunk-3f7veca0.js.map +0 -10
- package/dist/chunk-3frsnm7x.js.map +0 -11
- package/dist/chunk-3tgwrcyh.js.map +0 -14
- package/dist/chunk-52hbv0ak.js.map +0 -36
- package/dist/chunk-e0d1rzef.js.map +0 -15
- package/dist/chunk-emyyk7dx.js.map +0 -14
- package/dist/chunk-jjy0fhrn.js.map +0 -12
- package/dist/chunk-k3cqkkn2.js.map +0 -12
- package/dist/chunk-ppytyr6w.js.map +0 -12
- package/dist/chunk-qhcpj9wc.js.map +0 -14
- package/dist/chunk-r29fxdhn.js.map +0 -11
- package/dist/chunk-zcnq6105.js.map +0 -10
- /package/dist/{chunk-ypbtykv6.js.map → chunk-01kj2pbh.js.map} +0 -0
- /package/dist/{chunk-xyr8kppc.js.map → chunk-1yefycb1.js.map} +0 -0
- /package/dist/{chunk-m3z3x9v0.js.map → chunk-3ckk0xfj.js.map} +0 -0
- /package/dist/{chunk-0w35wavj.js.map → chunk-a0e6x5hd.js.map} +0 -0
- /package/dist/{chunk-h1gfhhbh.js.map → chunk-d63nnr5b.js.map} +0 -0
- /package/dist/{chunk-r4tfx1sf.js.map → chunk-fh8kgex1.js.map} +0 -0
- /package/dist/{chunk-vf691g1r.js.map → chunk-sepyxgpc.js.map} +0 -0
- /package/dist/{chunk-1enhpacz.js.map → chunk-z0s4axeh.js.map} +0 -0
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
|
|
2
|
+
import { col, defineDataset } from "../datasets"
|
|
3
|
+
import type { DatasetChanges, DatasetRow, DatasetRowChange, LakeStorage } from "../lake-storage"
|
|
4
|
+
|
|
5
|
+
export function runLakeChangesContractSuite<T extends LakeStorage>(
|
|
6
|
+
name: string,
|
|
7
|
+
harness: {
|
|
8
|
+
readonly createStorage: () => T | Promise<T>
|
|
9
|
+
readonly teardown?: (storage: T) => void | Promise<void>
|
|
10
|
+
}
|
|
11
|
+
): void {
|
|
12
|
+
describe(name, () => {
|
|
13
|
+
let storage: T
|
|
14
|
+
const dataset = defineDataset("delta.contract", {
|
|
15
|
+
schema: [
|
|
16
|
+
col("id", "string"),
|
|
17
|
+
col("part", "string"),
|
|
18
|
+
col("value", "string", { nullable: true }),
|
|
19
|
+
col("ignored", "string"),
|
|
20
|
+
],
|
|
21
|
+
})
|
|
22
|
+
const row = (id: string, value: string | null, part = "x", ignored = "unused"): DatasetRow => ({
|
|
23
|
+
id,
|
|
24
|
+
part,
|
|
25
|
+
value,
|
|
26
|
+
ignored,
|
|
27
|
+
})
|
|
28
|
+
const columns = ["id", "part", "value"]
|
|
29
|
+
beforeEach(async () => {
|
|
30
|
+
storage = await harness.createStorage()
|
|
31
|
+
await storage.createDataset(dataset)
|
|
32
|
+
})
|
|
33
|
+
afterEach(async () => {
|
|
34
|
+
if (harness.teardown) await harness.teardown(storage)
|
|
35
|
+
else await storage.close?.()
|
|
36
|
+
})
|
|
37
|
+
async function write(rows: readonly DatasetRow[]) {
|
|
38
|
+
const writer = await storage.beginWrite({ dataset })
|
|
39
|
+
await writer.writeRows(rows)
|
|
40
|
+
return (await writer.commit()).versionId
|
|
41
|
+
}
|
|
42
|
+
async function read(fromVersionId: string, toVersionId: string, keyColumns = ["id"]) {
|
|
43
|
+
if (!storage.readChanges) throw new Error("Provider does not implement readChanges")
|
|
44
|
+
return storage.readChanges({
|
|
45
|
+
datasetId: dataset.id,
|
|
46
|
+
fromVersionId,
|
|
47
|
+
toVersionId,
|
|
48
|
+
columns,
|
|
49
|
+
keyColumns,
|
|
50
|
+
})
|
|
51
|
+
}
|
|
52
|
+
async function collect(reader: DatasetChanges | null) {
|
|
53
|
+
if (!reader) throw new Error("Expected a complete delta")
|
|
54
|
+
const changes: DatasetRowChange[] = []
|
|
55
|
+
try {
|
|
56
|
+
for await (const change of reader.changes) changes.push(change)
|
|
57
|
+
} finally {
|
|
58
|
+
await reader.close()
|
|
59
|
+
}
|
|
60
|
+
expect(changes).toHaveLength(reader.changeCount)
|
|
61
|
+
return changes.sort((a, b) =>
|
|
62
|
+
JSON.stringify(a.after ?? a.before).localeCompare(JSON.stringify(b.after ?? b.before))
|
|
63
|
+
)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Red proof: remove the provider's readChanges implementation; this suite must fail.
|
|
67
|
+
test("compares pinned states across skipped versions, including removals and key changes", async () => {
|
|
68
|
+
const first = await write([row("A", "one"), row("B", "two"), row("C", "three")])
|
|
69
|
+
await write([row("A", "temporary"), row("B", "two"), row("C", "three")])
|
|
70
|
+
const last = await write([row("A", "one"), row("B", "new"), row("D", "three")])
|
|
71
|
+
const reader = await read(first, last)
|
|
72
|
+
expect(reader?.fromRowCount).toBe(3)
|
|
73
|
+
expect(reader?.toRowCount).toBe(3)
|
|
74
|
+
expect(await collect(reader)).toEqual([
|
|
75
|
+
{
|
|
76
|
+
before: { id: "B", part: "x", value: "two" },
|
|
77
|
+
after: { id: "B", part: "x", value: "new" },
|
|
78
|
+
},
|
|
79
|
+
{ before: { id: "C", part: "x", value: "three" }, after: null },
|
|
80
|
+
{ before: null, after: { id: "D", part: "x", value: "three" } },
|
|
81
|
+
])
|
|
82
|
+
})
|
|
83
|
+
test("ignores row order and unrequested columns", async () => {
|
|
84
|
+
const first = await write([row("A", "one"), row("B", "two")])
|
|
85
|
+
const last = await write([row("B", "two", "x", "changed"), row("A", "one")])
|
|
86
|
+
expect(await collect(await read(first, last))).toEqual([])
|
|
87
|
+
expect(await collect(await read(last, last))).toEqual([])
|
|
88
|
+
})
|
|
89
|
+
test("distinguishes null from a value and handles an empty target", async () => {
|
|
90
|
+
const first = await write([row("A", null)])
|
|
91
|
+
const second = await write([row("A", "value")])
|
|
92
|
+
expect(await collect(await read(first, second))).toEqual([
|
|
93
|
+
{
|
|
94
|
+
before: { id: "A", part: "x", value: null },
|
|
95
|
+
after: { id: "A", part: "x", value: "value" },
|
|
96
|
+
},
|
|
97
|
+
])
|
|
98
|
+
const empty = await write([])
|
|
99
|
+
expect(await collect(await read(second, empty))).toEqual([
|
|
100
|
+
{ before: { id: "A", part: "x", value: "value" }, after: null },
|
|
101
|
+
])
|
|
102
|
+
expect(await collect(await read(empty, second))).toEqual([
|
|
103
|
+
{ before: null, after: { id: "A", part: "x", value: "value" } },
|
|
104
|
+
])
|
|
105
|
+
})
|
|
106
|
+
test("supports composite keys without concatenation collisions", async () => {
|
|
107
|
+
const first = await write([row("a:b", "one", "c"), row("a", "two", "b:c")])
|
|
108
|
+
const last = await write([row("a:b", "new", "c"), row("a", "two", "b:c")])
|
|
109
|
+
expect(await collect(await read(first, last, ["id", "part"]))).toEqual([
|
|
110
|
+
{
|
|
111
|
+
before: { id: "a:b", part: "c", value: "one" },
|
|
112
|
+
after: { id: "a:b", part: "c", value: "new" },
|
|
113
|
+
},
|
|
114
|
+
])
|
|
115
|
+
})
|
|
116
|
+
test("refuses duplicate keys in either snapshot rather than losing rows", async () => {
|
|
117
|
+
const first = await write([row("A", "one"), row("A", "two")])
|
|
118
|
+
const last = await write([row("A", "new")])
|
|
119
|
+
expect(await read(first, last)).toBeNull()
|
|
120
|
+
expect(await read(last, first)).toBeNull()
|
|
121
|
+
})
|
|
122
|
+
test("refuses blank identities and invalid column requests", async () => {
|
|
123
|
+
const first = await write([row(" ", "one")])
|
|
124
|
+
expect(await read(first, first)).toBeNull()
|
|
125
|
+
await expect(read(first, first, [])).rejects.toThrow("distinct columns")
|
|
126
|
+
await expect(read(first, first, ["missing"])).rejects.toThrow("distinct columns")
|
|
127
|
+
})
|
|
128
|
+
test("releases the connection between yielded changes and closes after early exit", async () => {
|
|
129
|
+
const first = await write([row("A", "one"), row("B", "two")])
|
|
130
|
+
const last = await write([row("A", "new"), row("B", "new")])
|
|
131
|
+
const reader = await read(first, last)
|
|
132
|
+
expect(reader).not.toBeNull()
|
|
133
|
+
try {
|
|
134
|
+
for await (const _change of reader!.changes) {
|
|
135
|
+
expect((await storage.getLatestVersion(dataset.id))?.versionId).toBe(last)
|
|
136
|
+
break
|
|
137
|
+
}
|
|
138
|
+
} finally {
|
|
139
|
+
await reader!.close()
|
|
140
|
+
}
|
|
141
|
+
expect((await storage.getLatestVersion(dataset.id))?.versionId).toBe(last)
|
|
142
|
+
})
|
|
143
|
+
test("pages a complete pinned delta while a newer snapshot is published", async () => {
|
|
144
|
+
const count = 5_007
|
|
145
|
+
const first = await write(Array.from({ length: count }, (_, i) => row(String(i), "old")))
|
|
146
|
+
const last = await write(Array.from({ length: count }, (_, i) => row(String(i), "new")))
|
|
147
|
+
const reader = await read(first, last)
|
|
148
|
+
expect(reader?.changeCount).toBe(count)
|
|
149
|
+
await write([row("later", "unrelated")])
|
|
150
|
+
const changes = await collect(reader)
|
|
151
|
+
expect(new Set(changes.map((change) => change.after?.id)).size).toBe(count)
|
|
152
|
+
expect(
|
|
153
|
+
changes.every((change) => change.before?.value === "old" && change.after?.value === "new")
|
|
154
|
+
).toBe(true)
|
|
155
|
+
})
|
|
156
|
+
test("returns complete typed images without exposing mutable stored values", async () => {
|
|
157
|
+
const typed = defineDataset("delta.typed", {
|
|
158
|
+
schema: [
|
|
159
|
+
col("id", "string"),
|
|
160
|
+
col("payload", "json"),
|
|
161
|
+
col("at", "timestamp"),
|
|
162
|
+
col("amount", "int64"),
|
|
163
|
+
],
|
|
164
|
+
})
|
|
165
|
+
await storage.createDataset(typed)
|
|
166
|
+
const writeTyped = async (value: number) => {
|
|
167
|
+
const writer = await storage.beginWrite({ dataset: typed })
|
|
168
|
+
await writer.writeRows([
|
|
169
|
+
{
|
|
170
|
+
id: "a",
|
|
171
|
+
payload: { nested: [value] },
|
|
172
|
+
at: "2026-01-01T00:00:00.000Z",
|
|
173
|
+
amount: "9007199254740993",
|
|
174
|
+
},
|
|
175
|
+
])
|
|
176
|
+
return (await writer.commit()).versionId
|
|
177
|
+
}
|
|
178
|
+
const first = await writeTyped(1)
|
|
179
|
+
const last = await writeTyped(2)
|
|
180
|
+
const reader = await storage.readChanges!({
|
|
181
|
+
datasetId: typed.id,
|
|
182
|
+
fromVersionId: first,
|
|
183
|
+
toVersionId: last,
|
|
184
|
+
keyColumns: ["id"],
|
|
185
|
+
columns: ["id", "payload", "at", "amount"],
|
|
186
|
+
})
|
|
187
|
+
const changes = await collect(reader)
|
|
188
|
+
expect(changes).toEqual([
|
|
189
|
+
{
|
|
190
|
+
before: {
|
|
191
|
+
id: "a",
|
|
192
|
+
payload: { nested: [1] },
|
|
193
|
+
at: "2026-01-01T00:00:00.000Z",
|
|
194
|
+
amount: "9007199254740993",
|
|
195
|
+
},
|
|
196
|
+
after: {
|
|
197
|
+
id: "a",
|
|
198
|
+
payload: { nested: [2] },
|
|
199
|
+
at: "2026-01-01T00:00:00.000Z",
|
|
200
|
+
amount: "9007199254740993",
|
|
201
|
+
},
|
|
202
|
+
},
|
|
203
|
+
])
|
|
204
|
+
;(changes[0]!.after!.payload as { nested: number[] }).nested.push(3)
|
|
205
|
+
const rows: DatasetRow[] = []
|
|
206
|
+
for await (const value of storage.readRows({ datasetId: typed.id, versionId: last }))
|
|
207
|
+
rows.push(value)
|
|
208
|
+
expect(rows[0]?.payload).toEqual({ nested: [2] })
|
|
209
|
+
})
|
|
210
|
+
test("returns unavailable for a version belonging to another dataset", async () => {
|
|
211
|
+
const first = await write([row("A", "one")])
|
|
212
|
+
const other = defineDataset("delta.other", { schema: [col("id", "string")] })
|
|
213
|
+
await storage.createDataset(other)
|
|
214
|
+
const writer = await storage.beginWrite({ dataset: other })
|
|
215
|
+
await writer.writeRows([{ id: "other" }])
|
|
216
|
+
const otherVersion = await writer.commit()
|
|
217
|
+
expect(await read(otherVersion.versionId, first)).toBeNull()
|
|
218
|
+
expect(await read(first, otherVersion.versionId)).toBeNull()
|
|
219
|
+
})
|
|
220
|
+
test("can close an unread delta and observes cancellation", async () => {
|
|
221
|
+
const first = await write([row("A", "one")])
|
|
222
|
+
const last = await write([row("A", "new")])
|
|
223
|
+
const reader = await read(first, last)
|
|
224
|
+
await reader!.close()
|
|
225
|
+
await reader!.close()
|
|
226
|
+
const controller = new AbortController()
|
|
227
|
+
controller.abort(new Error("cancelled delta"))
|
|
228
|
+
await expect(
|
|
229
|
+
storage.readChanges!({
|
|
230
|
+
datasetId: dataset.id,
|
|
231
|
+
fromVersionId: first,
|
|
232
|
+
toVersionId: last,
|
|
233
|
+
columns,
|
|
234
|
+
keyColumns: ["id"],
|
|
235
|
+
signal: controller.signal,
|
|
236
|
+
})
|
|
237
|
+
).rejects.toThrow("cancelled delta")
|
|
238
|
+
})
|
|
239
|
+
})
|
|
240
|
+
}
|
|
@@ -285,6 +285,35 @@ export function runLakeStorageContractSuite<TStorage extends LakeStorage>(
|
|
|
285
285
|
})
|
|
286
286
|
|
|
287
287
|
describe("writes and reads", () => {
|
|
288
|
+
test("cancels reads before starting and between rows", async () => {
|
|
289
|
+
// Red check: remove readRows' signal checks in the provider under test.
|
|
290
|
+
await withStorage(async (storage) => {
|
|
291
|
+
await storage.createDataset(writeDataset)
|
|
292
|
+
const write = await storage.beginWrite({ dataset: writeDataset, mode: "snapshot" })
|
|
293
|
+
await write.writeRows([
|
|
294
|
+
{ orderId: "1", customerName: "first" },
|
|
295
|
+
{ orderId: "2", customerName: "second" },
|
|
296
|
+
])
|
|
297
|
+
await write.commit()
|
|
298
|
+
const controller = new AbortController()
|
|
299
|
+
const rows = storage
|
|
300
|
+
.readRows({ datasetId: writeDataset.id, signal: controller.signal })
|
|
301
|
+
[Symbol.asyncIterator]()
|
|
302
|
+
try {
|
|
303
|
+
expect((await rows.next()).done).toBe(false)
|
|
304
|
+
controller.abort(new Error("cancel dataset read"))
|
|
305
|
+
await expect(rows.next()).rejects.toThrow("cancel dataset read")
|
|
306
|
+
await expect(
|
|
307
|
+
collectRows(
|
|
308
|
+
storage.readRows({ datasetId: writeDataset.id, signal: controller.signal })
|
|
309
|
+
)
|
|
310
|
+
).rejects.toThrow("cancel dataset read")
|
|
311
|
+
} finally {
|
|
312
|
+
await rows.return?.()
|
|
313
|
+
}
|
|
314
|
+
})
|
|
315
|
+
})
|
|
316
|
+
|
|
288
317
|
test("keeps immutable-version row order and offsets stable", async () => {
|
|
289
318
|
await withStorage(async (storage) => {
|
|
290
319
|
await storage.createDataset(writeDataset)
|
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
createOntologyMaterializer,
|
|
20
20
|
type OntologyEditOperation,
|
|
21
21
|
ProjectionRegistry,
|
|
22
|
+
type ProjectionSourceBase,
|
|
23
|
+
type ProjectionSourceDeletion,
|
|
22
24
|
type ProjectionSourceEntry,
|
|
23
25
|
} from "../materializer"
|
|
24
26
|
import type { ActionRunStorage, ProjectionRunStorage } from "../storage"
|
|
@@ -125,6 +127,175 @@ export function runMaterializerStorageContractSuite<TStorage extends Storage>(
|
|
|
125
127
|
name: string,
|
|
126
128
|
provider: MaterializerStorageContractProvider<TStorage>
|
|
127
129
|
): void {
|
|
130
|
+
// Regression: replacing previousSourceRows with only the latest manifest loses "a" at v3.
|
|
131
|
+
// Removing root retirement checks from cleanup deletes "a" when v1 becomes terminal.
|
|
132
|
+
test(`${name} retains current roots across deltas, cleanup, overrides and full replacement`, async () => {
|
|
133
|
+
const createdStorage = await provider.createStorage()
|
|
134
|
+
const storage = requireContractStorage(createdStorage)
|
|
135
|
+
let ordinal = 0
|
|
136
|
+
const projectId = "materializer-storage-contract"
|
|
137
|
+
const source = { projectionId: devices.id }
|
|
138
|
+
const materializer = createOntologyMaterializer({
|
|
139
|
+
projectId,
|
|
140
|
+
ontology,
|
|
141
|
+
projections,
|
|
142
|
+
storage,
|
|
143
|
+
dependencies: { batching: { sourceStageRows: 1, statePageRows: 1, planChunkRows: 1 } },
|
|
144
|
+
})
|
|
145
|
+
const active = async (): Promise<ProjectionSourceBase> => {
|
|
146
|
+
const current = await materializer.projections.getActive(source)
|
|
147
|
+
if (!current?.lastCommitId) throw new Error("Missing source head")
|
|
148
|
+
return { materializationId: current.materializationId, lastCommitId: current.lastCommitId }
|
|
149
|
+
}
|
|
150
|
+
const replace = async (
|
|
151
|
+
values: readonly (ProjectionSourceEntry | ProjectionSourceDeletion)[],
|
|
152
|
+
base?: ProjectionSourceBase,
|
|
153
|
+
interleave?: { readonly version: number; readonly beforeSeal?: () => Promise<void> }
|
|
154
|
+
) => {
|
|
155
|
+
const version = interleave?.version ?? ++ordinal
|
|
156
|
+
const datasetVersion = {
|
|
157
|
+
datasetId: devices.id,
|
|
158
|
+
versionId: `delta-v${version}`,
|
|
159
|
+
createdAt: `2026-01-${String(version).padStart(2, "0")}T00:00:00.000Z`,
|
|
160
|
+
}
|
|
161
|
+
const execution = await claim(storage, {
|
|
162
|
+
runId: `delta-run-${version}`,
|
|
163
|
+
projectionId: devices.id,
|
|
164
|
+
protocol: "replacement",
|
|
165
|
+
datasetVersion,
|
|
166
|
+
})
|
|
167
|
+
const bound = await projectionMaterializer(
|
|
168
|
+
materializer,
|
|
169
|
+
storage,
|
|
170
|
+
devices.id,
|
|
171
|
+
execution.projectionRunId
|
|
172
|
+
)
|
|
173
|
+
return bound.projections.replace({
|
|
174
|
+
source,
|
|
175
|
+
datasetVersion,
|
|
176
|
+
execution,
|
|
177
|
+
entries: (async function* () {
|
|
178
|
+
yield* entries(values)
|
|
179
|
+
await interleave?.beforeSeal?.()
|
|
180
|
+
})(),
|
|
181
|
+
...(base ? { base } : {}),
|
|
182
|
+
})
|
|
183
|
+
}
|
|
184
|
+
const ref = (primaryId: string) => ({ objectTypeId: Device.id, primaryId })
|
|
185
|
+
const object = (id: string) => storage.objects.getByPrimaryId({ projectId, ...ref(id) })
|
|
186
|
+
const remove = (id: string): ProjectionSourceDeletion => ({
|
|
187
|
+
root: { kind: "object", ref: ref(id) },
|
|
188
|
+
deleted: true,
|
|
189
|
+
})
|
|
190
|
+
try {
|
|
191
|
+
await replace([sourceEntry("a", "A"), sourceEntry("b", "B", "a"), sourceEntry("c", "C")])
|
|
192
|
+
const stale = await active()
|
|
193
|
+
const initialA = await object("a")
|
|
194
|
+
const initialB = await object("b")
|
|
195
|
+
const transact = storage.transaction.bind(storage)
|
|
196
|
+
// Fail after finalize has moved the source pointers, written the commit and its outbox.
|
|
197
|
+
// Removing the enclosing transaction rollback must fail these head/value assertions.
|
|
198
|
+
storage.transaction = (run, options) =>
|
|
199
|
+
transact(async (tx) => {
|
|
200
|
+
const materializations = new Proxy(tx.ontology.materializations, {
|
|
201
|
+
get(target, key) {
|
|
202
|
+
if (key !== "finalize") return Reflect.get(target, key)
|
|
203
|
+
return async (input: Parameters<typeof target.finalize>[0]) => {
|
|
204
|
+
await target.finalize(input)
|
|
205
|
+
throw new Error("injected after source activation")
|
|
206
|
+
}
|
|
207
|
+
},
|
|
208
|
+
})
|
|
209
|
+
const ontology = new Proxy(tx.ontology, {
|
|
210
|
+
get(target, key) {
|
|
211
|
+
return key === "materializations" ? materializations : Reflect.get(target, key)
|
|
212
|
+
},
|
|
213
|
+
})
|
|
214
|
+
return run(
|
|
215
|
+
new Proxy(tx, {
|
|
216
|
+
get(target, key) {
|
|
217
|
+
return key === "ontology" ? ontology : Reflect.get(target, key)
|
|
218
|
+
},
|
|
219
|
+
})
|
|
220
|
+
)
|
|
221
|
+
}, options)
|
|
222
|
+
try {
|
|
223
|
+
await expect(replace([remove("a"), sourceEntry("b", "Failed B")], stale)).rejects.toThrow(
|
|
224
|
+
"injected after source activation"
|
|
225
|
+
)
|
|
226
|
+
} finally {
|
|
227
|
+
storage.transaction = transact
|
|
228
|
+
}
|
|
229
|
+
expect(await active()).toEqual(stale)
|
|
230
|
+
expect(await object("a")).toEqual(initialA)
|
|
231
|
+
expect(await object("b")).toEqual(initialB)
|
|
232
|
+
await replace([sourceEntry("b", "B2", "c")], await active())
|
|
233
|
+
await replace([sourceEntry("c", "C2")], await active())
|
|
234
|
+
expect(await object("a")).toEqual(initialA)
|
|
235
|
+
expect(await object("b")).toMatchObject({ properties: { name: "B2" } })
|
|
236
|
+
for (let pass = 0; pass < 20; pass++) {
|
|
237
|
+
const result = await storage.ontology.sources.cleanupTerminal({
|
|
238
|
+
projectId,
|
|
239
|
+
terminalBefore: "2100-01-01T00:00:00.000Z",
|
|
240
|
+
limit: 2,
|
|
241
|
+
})
|
|
242
|
+
if (result.rowsDeleted + result.materializationsDeleted === 0) break
|
|
243
|
+
}
|
|
244
|
+
const scoped = materializer.withScope(runtimeScope())
|
|
245
|
+
const edit = (operations: readonly OntologyEditOperation[]) =>
|
|
246
|
+
scoped.edits.commit({
|
|
247
|
+
mode: "atomic",
|
|
248
|
+
source: { kind: "runtime", requestId: `delta-edit-${++ordinal}` },
|
|
249
|
+
operations,
|
|
250
|
+
expectedObjects: [],
|
|
251
|
+
expectedLinks: [],
|
|
252
|
+
expectedLinkScopes: [],
|
|
253
|
+
})
|
|
254
|
+
await edit([
|
|
255
|
+
{ id: "edit-a", kind: "object.upsert", ref: ref("a"), properties: { name: "Edited A" } },
|
|
256
|
+
])
|
|
257
|
+
await edit([
|
|
258
|
+
{ id: "reset-a", kind: "object.patch", ref: ref("a"), set: {}, unset: [], reset: ["name"] },
|
|
259
|
+
])
|
|
260
|
+
expect(await object("a")).toMatchObject({ properties: { name: "A" } })
|
|
261
|
+
await replace([], await active())
|
|
262
|
+
expect(await object("a")).toMatchObject({ properties: { name: "A" } })
|
|
263
|
+
await expect(replace([remove("a")], stale)).rejects.toMatchObject({
|
|
264
|
+
kind: "source-materialization",
|
|
265
|
+
})
|
|
266
|
+
expect(await object("a")).not.toBeNull()
|
|
267
|
+
await replace([remove("a")], await active())
|
|
268
|
+
expect(await object("a")).toBeNull()
|
|
269
|
+
expect(await object("b")).toMatchObject({ properties: { name: "B2" } })
|
|
270
|
+
await replace([sourceEntry("b", "B3")])
|
|
271
|
+
expect(await object("b")).toMatchObject({ properties: { name: "B3" } })
|
|
272
|
+
expect(await object("c")).toBeNull()
|
|
273
|
+
await replace([], await active())
|
|
274
|
+
expect(await object("b")).toMatchObject({ properties: { name: "B3" } })
|
|
275
|
+
await replace([])
|
|
276
|
+
expect(await object("b")).toBeNull()
|
|
277
|
+
|
|
278
|
+
// A different run commits after base validation but before this candidate is sealed.
|
|
279
|
+
// Only checking the base before reading the delta would allow a stale publication here.
|
|
280
|
+
const base = await active()
|
|
281
|
+
const earlier = ++ordinal
|
|
282
|
+
const later = ++ordinal
|
|
283
|
+
await expect(
|
|
284
|
+
replace([sourceEntry("a", "Stale delta")], base, {
|
|
285
|
+
version: later,
|
|
286
|
+
beforeSeal: async () => {
|
|
287
|
+
await replace([sourceEntry("a", "Concurrent value")], base, { version: earlier })
|
|
288
|
+
},
|
|
289
|
+
})
|
|
290
|
+
).rejects.toMatchObject({ kind: "source-materialization" })
|
|
291
|
+
expect(await object("a")).toMatchObject({ properties: { name: "Concurrent value" } })
|
|
292
|
+
await replace([sourceEntry("a", "Fresh delta")], await active())
|
|
293
|
+
expect(await object("a")).toMatchObject({ properties: { name: "Fresh delta" } })
|
|
294
|
+
} finally {
|
|
295
|
+
await provider.cleanup?.(createdStorage)
|
|
296
|
+
}
|
|
297
|
+
})
|
|
298
|
+
|
|
128
299
|
test(`${name} persists replacement, Action, and telemetry commits atomically`, async () => {
|
|
129
300
|
const createdStorage = await provider.createStorage()
|
|
130
301
|
const storage = requireContractStorage(createdStorage)
|
|
@@ -811,7 +982,7 @@ function sourceEntry(id: string, name: string, parentId?: string): ProjectionSou
|
|
|
811
982
|
}
|
|
812
983
|
}
|
|
813
984
|
|
|
814
|
-
async function* entries(values: readonly ProjectionSourceEntry[]) {
|
|
985
|
+
async function* entries(values: readonly (ProjectionSourceEntry | ProjectionSourceDeletion)[]) {
|
|
815
986
|
for (const value of values) yield value
|
|
816
987
|
}
|
|
817
988
|
|
|
@@ -383,6 +383,13 @@ export function runOntologyStorageContractSuite<TStorage extends OntologyStorage
|
|
|
383
383
|
limit: 1,
|
|
384
384
|
})
|
|
385
385
|
).toEqual({ rowsDeleted: 1, materializationsDeleted: 0 })
|
|
386
|
+
expect(
|
|
387
|
+
await storage.ontology.sources.cleanupTerminal({
|
|
388
|
+
projectId: "contract-project",
|
|
389
|
+
terminalBefore: "2026-02-01T00:00:00.000Z",
|
|
390
|
+
limit: 1,
|
|
391
|
+
})
|
|
392
|
+
).toEqual({ rowsDeleted: 1, materializationsDeleted: 0 })
|
|
386
393
|
expect(
|
|
387
394
|
await storage.ontology.sources.cleanupTerminal({
|
|
388
395
|
projectId: "contract-project",
|
|
@@ -670,7 +677,10 @@ export function runOntologyStorageContractSuite<TStorage extends OntologyStorage
|
|
|
670
677
|
})
|
|
671
678
|
})
|
|
672
679
|
|
|
673
|
-
test(
|
|
680
|
+
test.each([
|
|
681
|
+
"root",
|
|
682
|
+
"transaction",
|
|
683
|
+
] as const)("keeps %s batch outbox settlement atomic when callers catch a stale lease", async (boundary) => {
|
|
674
684
|
await withStorage(async (storage) => {
|
|
675
685
|
await commitExactObject(storage, "lease-atomic-a")
|
|
676
686
|
await commitExactObject(storage, "lease-atomic-b")
|
|
@@ -692,7 +702,7 @@ export function runOntologyStorageContractSuite<TStorage extends OntologyStorage
|
|
|
692
702
|
const untouched = claimed.find((row) => row.envelope.id !== reclaimed!.envelope.id)!
|
|
693
703
|
|
|
694
704
|
let conflict: unknown
|
|
695
|
-
|
|
705
|
+
const settle = async (tx: OntologyContractStorage) => {
|
|
696
706
|
try {
|
|
697
707
|
await tx.ontology.outbox.markPublished({
|
|
698
708
|
projectId: "contract-project",
|
|
@@ -703,7 +713,9 @@ export function runOntologyStorageContractSuite<TStorage extends OntologyStorage
|
|
|
703
713
|
} catch (error) {
|
|
704
714
|
conflict = error
|
|
705
715
|
}
|
|
706
|
-
}
|
|
716
|
+
}
|
|
717
|
+
if (boundary === "transaction") await storage.transaction(settle)
|
|
718
|
+
else await settle(storage)
|
|
707
719
|
expect(conflict).toMatchObject({ kind: "outbox-lease" })
|
|
708
720
|
|
|
709
721
|
await storage.ontology.outbox.markPublished({
|
|
@@ -256,6 +256,40 @@ export function runProjectionRunStorageContractSuite<TStorage extends Projection
|
|
|
256
256
|
})
|
|
257
257
|
})
|
|
258
258
|
|
|
259
|
+
test("persists independent incremental progress across reclaim without a physical-row floor", async () => {
|
|
260
|
+
await withStorage(async (context) => {
|
|
261
|
+
const storage = context.projectionRuns
|
|
262
|
+
await admitProjectionRun(context, replacementInput("delta-progress"))
|
|
263
|
+
const first = await storage.startOrReclaim(replacementInput("delta-progress"))
|
|
264
|
+
const updated = await storage.update({
|
|
265
|
+
...executionInput(first),
|
|
266
|
+
progress: { sourceChangesRead: 12 },
|
|
267
|
+
})
|
|
268
|
+
expect(updated.progress).toEqual({
|
|
269
|
+
sourceRowsRead: 0,
|
|
270
|
+
sourceRowsSkipped: 0,
|
|
271
|
+
sourceChangesRead: 12,
|
|
272
|
+
})
|
|
273
|
+
const second = await storage.startOrReclaim(replacementInput("delta-progress"))
|
|
274
|
+
expect(second.run.progress).toEqual(updated.progress)
|
|
275
|
+
await expect(
|
|
276
|
+
storage.update({ ...executionInput(first), progress: { sourceChangesRead: 13 } })
|
|
277
|
+
).rejects.toThrow()
|
|
278
|
+
await expect(
|
|
279
|
+
storage.update({ ...executionInput(second), progress: { sourceChangesRead: 11 } })
|
|
280
|
+
).rejects.toThrow("must not decrease")
|
|
281
|
+
const fallback = await storage.update({
|
|
282
|
+
...executionInput(second),
|
|
283
|
+
progress: { sourceRowsRead: 2 },
|
|
284
|
+
})
|
|
285
|
+
expect(fallback.progress).toEqual({
|
|
286
|
+
sourceRowsRead: 2,
|
|
287
|
+
sourceRowsSkipped: 0,
|
|
288
|
+
sourceChangesRead: 12,
|
|
289
|
+
})
|
|
290
|
+
})
|
|
291
|
+
})
|
|
292
|
+
|
|
259
293
|
test("rejects immutable dataset-version metadata reuse", async () => {
|
|
260
294
|
await withStorage(async (context) => {
|
|
261
295
|
await admitProjectionRun(context, replacementInput("dataset-metadata-first"))
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { createConnectorCodedError } from "../connectors/errors"
|
|
2
|
+
import { type ConnectorRuntime, getConnectorExecutionSourceResolver } from "../connectors/execution"
|
|
3
|
+
import { type ConnectorDefinition, isOAuthConnectorDefinition } from "../connectors/types"
|
|
4
|
+
import type { WebhookConnections } from "./types"
|
|
5
|
+
|
|
6
|
+
/** Lookup stays inside the admitted execution's project and registered connector. */
|
|
7
|
+
export function createWebhookConnections(
|
|
8
|
+
runtime: ConnectorRuntime,
|
|
9
|
+
definition: ConnectorDefinition
|
|
10
|
+
): WebhookConnections {
|
|
11
|
+
return {
|
|
12
|
+
async forAccount(accountId) {
|
|
13
|
+
if (!isOAuthConnectorDefinition(definition)) {
|
|
14
|
+
throw createConnectorCodedError(
|
|
15
|
+
"connector.configuration_invalid",
|
|
16
|
+
"Webhook connections.forAccount() requires a managed OAuth connector."
|
|
17
|
+
)
|
|
18
|
+
}
|
|
19
|
+
if (typeof accountId !== "string" || !accountId.trim()) {
|
|
20
|
+
throw createConnectorCodedError(
|
|
21
|
+
"connector.configuration_invalid",
|
|
22
|
+
"Webhook account ID must be a non-empty string."
|
|
23
|
+
)
|
|
24
|
+
}
|
|
25
|
+
const resolver = getConnectorExecutionSourceResolver(runtime)
|
|
26
|
+
const sources = await resolver.list(definition)
|
|
27
|
+
return sources.flatMap((source) => {
|
|
28
|
+
const connection = source.connection
|
|
29
|
+
if (!connection || connection.account.id !== accountId) return []
|
|
30
|
+
const connectionId = connection.id
|
|
31
|
+
let clientPromise: Promise<unknown> | undefined
|
|
32
|
+
return [
|
|
33
|
+
{
|
|
34
|
+
connection,
|
|
35
|
+
client() {
|
|
36
|
+
clientPromise ??= (async () => {
|
|
37
|
+
const current = (await resolver.list(definition)).find(
|
|
38
|
+
(candidate) =>
|
|
39
|
+
candidate.connection?.id === connectionId &&
|
|
40
|
+
candidate.connection.account.id === accountId
|
|
41
|
+
)
|
|
42
|
+
if (!current)
|
|
43
|
+
throw createConnectorCodedError(
|
|
44
|
+
"connector.not_found",
|
|
45
|
+
"Webhook connection is no longer active for this account."
|
|
46
|
+
)
|
|
47
|
+
return current.connect(new AbortController().signal)
|
|
48
|
+
})()
|
|
49
|
+
return clientPromise
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
]
|
|
53
|
+
})
|
|
54
|
+
},
|
|
55
|
+
}
|
|
56
|
+
}
|
package/src/webhooks/index.ts
CHANGED
package/src/webhooks/internal.ts
CHANGED
package/src/webhooks/registry.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { ConnectorDefinition } from "../connectors/types"
|
|
2
2
|
import { WebhookValidationError } from "./errors"
|
|
3
|
-
import type { RegisteredWebhook
|
|
3
|
+
import type { RegisteredWebhook } from "./types"
|
|
4
4
|
|
|
5
5
|
/** Read-only access to connector webhooks registered by a host. */
|
|
6
6
|
export interface WebhookCatalog {
|
|
@@ -21,7 +21,6 @@ export class WebhookRegistry implements WebhookCatalog {
|
|
|
21
21
|
const registered: RegisteredWebhook[] = []
|
|
22
22
|
|
|
23
23
|
for (const connector of options.connectors) {
|
|
24
|
-
if (!isStaticConnectorDefinition(connector)) continue
|
|
25
24
|
const webhooks = connector.adapter.webhooks ?? []
|
|
26
25
|
if (!Array.isArray(webhooks)) {
|
|
27
26
|
throw new WebhookValidationError(
|
|
@@ -82,7 +81,7 @@ export function webhookRoute(connectorId: string, webhookId: string): string {
|
|
|
82
81
|
function assertValidWebhook(
|
|
83
82
|
connectorId: string,
|
|
84
83
|
webhook: unknown
|
|
85
|
-
): asserts webhook is
|
|
84
|
+
): asserts webhook is RegisteredWebhook["webhook"] {
|
|
86
85
|
if (!isRecord(webhook)) {
|
|
87
86
|
throw new WebhookValidationError(`[Sixb] Connector '${connectorId}' has an invalid webhook.`)
|
|
88
87
|
}
|