@reventlessdev/reventless-seed-aws 1.0.0-alpha.2 → 1.0.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/lib/bs/.compiler.log +2 -2
- package/lib/bs/.sourcedirs.json +1 -0
- package/lib/bs/compiler-info.json +2 -2
- package/lib/bs/src/ReventlessSeedAws.ast +0 -0
- package/lib/bs/src/ReventlessSeedAws.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws.cmj +0 -0
- package/lib/bs/src/ReventlessSeedAws.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws.res +87 -75
- package/lib/bs/src/ReventlessSeedAws.res.mjs +34 -30
- package/lib/bs/src/ReventlessSeedAws_Quiesce.ast +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.cmj +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.res +287 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.res.mjs +205 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.ast +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.cmj +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.res +385 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.res.mjs +318 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.ast +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.cmj +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.res +1098 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.res.mjs +991 -0
- package/lib/bs/tests/EndpointResolutionTest.ast +0 -0
- package/lib/bs/tests/EndpointResolutionTest.cmi +0 -0
- package/lib/bs/tests/EndpointResolutionTest.cmj +0 -0
- package/lib/bs/tests/EndpointResolutionTest.cmt +0 -0
- package/lib/bs/tests/EndpointResolutionTest.res +83 -0
- package/lib/bs/tests/EndpointResolutionTest.res.mjs +115 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.ast +0 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.cmi +0 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.cmj +0 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.cmt +0 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.res +84 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.res.mjs +102 -0
- package/lib/bs/tests/ObjectStoreResetTest.ast +0 -0
- package/lib/bs/tests/ObjectStoreResetTest.cmi +0 -0
- package/lib/bs/tests/ObjectStoreResetTest.cmj +0 -0
- package/lib/bs/tests/ObjectStoreResetTest.cmt +0 -0
- package/lib/bs/tests/ObjectStoreResetTest.res +170 -0
- package/lib/bs/tests/ObjectStoreResetTest.res.mjs +211 -0
- package/lib/bs/tests/QuiesceResetTest.ast +0 -0
- package/lib/bs/tests/QuiesceResetTest.cmi +0 -0
- package/lib/bs/tests/QuiesceResetTest.cmj +0 -0
- package/lib/bs/tests/QuiesceResetTest.cmt +0 -0
- package/lib/bs/tests/QuiesceResetTest.res +133 -0
- package/lib/bs/tests/QuiesceResetTest.res.mjs +141 -0
- package/lib/ocaml/.compiler.log +2 -2
- package/lib/ocaml/EndpointResolutionTest.ast +0 -0
- package/lib/ocaml/EndpointResolutionTest.cmi +0 -0
- package/lib/ocaml/EndpointResolutionTest.cmj +0 -0
- package/lib/ocaml/EndpointResolutionTest.cmt +0 -0
- package/lib/ocaml/EndpointResolutionTest.res +83 -0
- package/lib/ocaml/ObjectStoreReconcileTest.ast +0 -0
- package/lib/ocaml/ObjectStoreReconcileTest.cmi +0 -0
- package/lib/ocaml/ObjectStoreReconcileTest.cmj +0 -0
- package/lib/ocaml/ObjectStoreReconcileTest.cmt +0 -0
- package/lib/ocaml/ObjectStoreReconcileTest.res +84 -0
- package/lib/ocaml/ObjectStoreResetTest.ast +0 -0
- package/lib/ocaml/ObjectStoreResetTest.cmi +0 -0
- package/lib/ocaml/ObjectStoreResetTest.cmj +0 -0
- package/lib/ocaml/ObjectStoreResetTest.cmt +0 -0
- package/lib/ocaml/ObjectStoreResetTest.res +170 -0
- package/lib/ocaml/QuiesceResetTest.ast +0 -0
- package/lib/ocaml/QuiesceResetTest.cmi +0 -0
- package/lib/ocaml/QuiesceResetTest.cmj +0 -0
- package/lib/ocaml/QuiesceResetTest.cmt +0 -0
- package/lib/ocaml/QuiesceResetTest.res +133 -0
- package/lib/ocaml/ReventlessSeedAws.ast +0 -0
- package/lib/ocaml/ReventlessSeedAws.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws.cmj +0 -0
- package/lib/ocaml/ReventlessSeedAws.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws.res +87 -75
- package/lib/ocaml/ReventlessSeedAws_Quiesce.ast +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.cmj +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.res +287 -0
- package/lib/ocaml/ReventlessSeedAws_Reconcile.ast +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reconcile.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reconcile.cmj +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reconcile.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reconcile.res +385 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.ast +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.cmj +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.res +1098 -0
- package/package.json +7 -4
- package/rescript.json +9 -1
- package/src/ReventlessSeedAws.res +87 -75
- package/src/ReventlessSeedAws.res.mjs +34 -30
- package/src/ReventlessSeedAws_Quiesce.res +287 -0
- package/src/ReventlessSeedAws_Quiesce.res.mjs +205 -0
- package/src/ReventlessSeedAws_Reconcile.res +385 -0
- package/src/ReventlessSeedAws_Reconcile.res.mjs +318 -0
- package/src/ReventlessSeedAws_Reset.res +1098 -0
- package/src/ReventlessSeedAws_Reset.res.mjs +991 -0
- package/tests/EndpointResolutionTest.res +83 -0
- package/tests/EndpointResolutionTest.res.mjs +115 -0
- package/tests/ObjectStoreReconcileTest.res +84 -0
- package/tests/ObjectStoreReconcileTest.res.mjs +102 -0
- package/tests/ObjectStoreResetTest.res +170 -0
- package/tests/ObjectStoreResetTest.res.mjs +211 -0
- package/tests/QuiesceResetTest.res +133 -0
- package/tests/QuiesceResetTest.res.mjs +141 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// The half of reconciliation that decides whether an object is referenced.
|
|
2
|
+
//
|
|
3
|
+
// Everything else in the report is counting; `collectRefs` is the judgement, and
|
|
4
|
+
// it is judgement in one direction only. A ref it misses turns a live object
|
|
5
|
+
// into an "unreferenced, tagged pending" row — which is the row an operator
|
|
6
|
+
// reads as permission to enable an expiry rule. A ref it collects too eagerly
|
|
7
|
+
// only ever leaves an object alive.
|
|
8
|
+
//
|
|
9
|
+
// So these tests are about reach: nesting, arrays, records, and the prefix
|
|
10
|
+
// boundary that separates one store from another.
|
|
11
|
+
|
|
12
|
+
open JestGlobals
|
|
13
|
+
|
|
14
|
+
module Reconcile = ReventlessSeedAws_Reconcile
|
|
15
|
+
|
|
16
|
+
let prefix = "Catalog/productImages"
|
|
17
|
+
let refIn = "/Catalog/productImages/user-1/uuid/photo.png"
|
|
18
|
+
let refOther = "/Ordering/receipts/user-1/uuid/r.pdf"
|
|
19
|
+
|
|
20
|
+
let collect = (json: JSON.t): array<string> => {
|
|
21
|
+
let out = Set.make()
|
|
22
|
+
Reconcile.collectRefs(~prefix, json, out)
|
|
23
|
+
out->Set.toArray
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let obj = (fields: array<(string, JSON.t)>) => fields->Dict.fromArray->JSON.Encode.object
|
|
27
|
+
|
|
28
|
+
describe("ReventlessSeedAws_Reconcile.collectRefs", () => {
|
|
29
|
+
testSync("finds a ref on a top-level field", () =>
|
|
30
|
+
expect(collect(obj([("imageUrl", JSON.Encode.string(refIn))])))->toEqual([refIn])
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
// Deliberately not declaration-driven: the claim component reads `@storageRef`
|
|
34
|
+
// annotations, so a report reading the same list would inherit the same blind
|
|
35
|
+
// spot and agree precisely where both are wrong. An unannotated field holding a
|
|
36
|
+
// ref has to show up here, as the thing to fix before expiry goes on.
|
|
37
|
+
testSync("finds a ref on a field no annotation would have named", () =>
|
|
38
|
+
expect(collect(obj([("someUndeclaredField", JSON.Encode.string(refIn))])))->toEqual([refIn])
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
testSync("finds refs inside an array", () =>
|
|
42
|
+
expect(collect(obj([("images", JSON.Encode.array([JSON.Encode.string(refIn)]))])))->toEqual([
|
|
43
|
+
refIn,
|
|
44
|
+
])
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
testSync("finds a ref nested inside a record", () =>
|
|
48
|
+
expect(collect(obj([("variant", obj([("photo", JSON.Encode.string(refIn))]))])))->toEqual([refIn])
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
// Store isolation: one bucket holds several stores under distinct prefixes, so
|
|
52
|
+
// a reconciliation scoped to one store must not count another's refs — that
|
|
53
|
+
// would mark a genuinely abandoned object as referenced and hide it forever.
|
|
54
|
+
testSync("ignores a ref belonging to another store", () =>
|
|
55
|
+
expect(collect(obj([("receipt", JSON.Encode.string(refOther))])))->toEqual([])
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
// The prefix boundary is a path segment, not a string prefix.
|
|
59
|
+
testSync("ignores a store whose name merely starts the same way", () =>
|
|
60
|
+
expect(
|
|
61
|
+
collect(obj([("x", JSON.Encode.string("/Catalog/productImagesArchive/u/i/p.png"))])),
|
|
62
|
+
)->toEqual([])
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
testSync("ignores strings that are not refs at all", () =>
|
|
66
|
+
expect(
|
|
67
|
+
collect(
|
|
68
|
+
obj([
|
|
69
|
+
("name", JSON.Encode.string("A product")),
|
|
70
|
+
("external", JSON.Encode.string("https://example.com/Catalog/productImages/p.png")),
|
|
71
|
+
("count", JSON.Encode.int(3)),
|
|
72
|
+
]),
|
|
73
|
+
),
|
|
74
|
+
)->toEqual([])
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
testSync("collects each distinct ref once", () => {
|
|
78
|
+
let json = obj([
|
|
79
|
+
("a", JSON.Encode.string(refIn)),
|
|
80
|
+
("b", JSON.Encode.string(refIn)),
|
|
81
|
+
])
|
|
82
|
+
expect(collect(json)->Array.length)->toBe(1)
|
|
83
|
+
})
|
|
84
|
+
})
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// How the reset learns which uploaded objects belong to which plugin, and what
|
|
2
|
+
// it refuses.
|
|
3
|
+
//
|
|
4
|
+
// A declared store's bucket is built by the PLATFORM deploy, so its tags say
|
|
5
|
+
// which project built it, not whose data is in it. Ownership therefore comes
|
|
6
|
+
// from the platform's `objectStores` stack output, and these pin the pure half
|
|
7
|
+
// of that: parsing the output, and refusing any store set a prefix-scoped wipe
|
|
8
|
+
// could not separate. The impure half is a `pulumi stack output` subprocess and
|
|
9
|
+
// the S3 calls, and the decisions under test are in neither.
|
|
10
|
+
|
|
11
|
+
open JestGlobals
|
|
12
|
+
|
|
13
|
+
module Reset = ReventlessSeedAws_Reset
|
|
14
|
+
|
|
15
|
+
let obj = entries => JSON.Encode.object(entries->Dict.fromArray)
|
|
16
|
+
let str = JSON.Encode.string
|
|
17
|
+
|
|
18
|
+
let storeEntry = (~bucket, ~prefix) =>
|
|
19
|
+
obj([("bucketName", str(bucket)), ("keyPrefix", str(prefix))])
|
|
20
|
+
|
|
21
|
+
let output = entries => Some(obj(entries))
|
|
22
|
+
|
|
23
|
+
let store = (~qualified, ~bucket, ~prefix): Reset.objectStore => {
|
|
24
|
+
let (plugin, name) = Reset.splitQualified(qualified)->Option.getOr(("", qualified))
|
|
25
|
+
{qualified, plugin, store: name, bucketName: bucket, keyPrefix: prefix}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe("parseObjectStores", () => {
|
|
29
|
+
testSync("splits the qualified key into plugin and store", () =>
|
|
30
|
+
switch Reset.parseObjectStores(
|
|
31
|
+
output([("Catalog.productImages", storeEntry(~bucket="alpha-stores", ~prefix="productImages"))]),
|
|
32
|
+
) {
|
|
33
|
+
| Ok([s]) =>
|
|
34
|
+
expect((s.plugin, s.store, s.bucketName, s.keyPrefix))->toEqual((
|
|
35
|
+
"Catalog",
|
|
36
|
+
"productImages",
|
|
37
|
+
"alpha-stores",
|
|
38
|
+
"productImages",
|
|
39
|
+
))
|
|
40
|
+
| other => fail(`expected one parsed store, got ${other->JSON.stringifyAny->Option.getOr("?")}`)
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
// A store name may contain a dot; a registered plugin name may not — so the
|
|
45
|
+
// split is at the first one, matching how the key is composed.
|
|
46
|
+
testSync("splits at the first dot only", () =>
|
|
47
|
+
switch Reset.splitQualified("Catalog.product.images") {
|
|
48
|
+
| Some((plugin, name)) => expect((plugin, name))->toEqual(("Catalog", "product.images"))
|
|
49
|
+
| None => fail("expected a split")
|
|
50
|
+
}
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
// A platform that declares no stores is ordinary, not an error.
|
|
54
|
+
testSync("absent output yields no stores", () =>
|
|
55
|
+
switch Reset.parseObjectStores(None) {
|
|
56
|
+
| Ok(stores) => expect(stores->Array.length)->toBe(0)
|
|
57
|
+
| Error(message) => fail(message)
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
// A store the reset cannot read is a store it would silently leave behind, so
|
|
62
|
+
// a malformed entry fails the run rather than being skipped.
|
|
63
|
+
testSync("a malformed entry is an error, not a skip", () =>
|
|
64
|
+
switch Reset.parseObjectStores(output([("Catalog.productImages", obj([("keyPrefix", str("x"))]))])) {
|
|
65
|
+
| Ok(_) => fail("expected a malformed entry to be refused")
|
|
66
|
+
| Error(message) => expect(message->String.includes("Catalog.productImages"))->toBe(true)
|
|
67
|
+
}
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
testSync("an unqualified key is an error", () =>
|
|
71
|
+
switch Reset.parseObjectStores(
|
|
72
|
+
output([("productImages", storeEntry(~bucket="alpha-stores", ~prefix="productImages"))]),
|
|
73
|
+
) {
|
|
74
|
+
| Ok(_) => fail("expected an unqualified key to be refused")
|
|
75
|
+
| Error(message) => expect(message->String.includes("productImages"))->toBe(true)
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
describe("validateStores", () => {
|
|
81
|
+
testSync("distinct prefixes in one bucket are fine", () =>
|
|
82
|
+
expect(
|
|
83
|
+
Reset.validateStores([
|
|
84
|
+
store(~qualified="Catalog.productImages", ~bucket="alpha-stores", ~prefix="productImages"),
|
|
85
|
+
store(~qualified="Ordering.labels", ~bucket="alpha-stores", ~prefix="labels"),
|
|
86
|
+
]),
|
|
87
|
+
)->toEqual(Ok())
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
// The cross-plugin collision: one prefix, two owners, nothing to tell their
|
|
91
|
+
// objects apart.
|
|
92
|
+
testSync("two plugins on one prefix are refused", () =>
|
|
93
|
+
switch Reset.validateStores([
|
|
94
|
+
store(~qualified="Catalog.productImages", ~bucket="alpha-stores", ~prefix="productImages"),
|
|
95
|
+
store(~qualified="Ordering.productImages", ~bucket="alpha-stores", ~prefix="productImages"),
|
|
96
|
+
]) {
|
|
97
|
+
| Ok() => fail("expected a collision to be refused")
|
|
98
|
+
| Error(message) =>
|
|
99
|
+
expect(
|
|
100
|
+
message->String.includes("Catalog.productImages") &&
|
|
101
|
+
message->String.includes("Ordering.productImages"),
|
|
102
|
+
)->toBe(true)
|
|
103
|
+
}
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
// The same shape one level up: `Catalog` encloses `Catalog/productImages`, so
|
|
107
|
+
// wiping the first would delete the second's objects.
|
|
108
|
+
testSync("an enclosing prefix is refused", () =>
|
|
109
|
+
switch Reset.validateStores([
|
|
110
|
+
store(~qualified="Legacy.catalog", ~bucket="alpha-stores", ~prefix="Catalog"),
|
|
111
|
+
store(~qualified="Catalog.productImages", ~bucket="alpha-stores", ~prefix="Catalog/productImages"),
|
|
112
|
+
]) {
|
|
113
|
+
| Ok() => fail("expected an enclosing prefix to be refused")
|
|
114
|
+
| Error(message) => expect(message->String.includes("encloses"))->toBe(true)
|
|
115
|
+
}
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
// Different buckets cannot reach each other, so an equal prefix there is not a
|
|
119
|
+
// collision — which is what the per-store layout produces in production.
|
|
120
|
+
testSync("one prefix in two different buckets is fine", () =>
|
|
121
|
+
expect(
|
|
122
|
+
Reset.validateStores([
|
|
123
|
+
store(~qualified="Catalog.productImages", ~bucket="catalog-productImages", ~prefix="productImages"),
|
|
124
|
+
store(~qualified="Ordering.productImages", ~bucket="ordering-productImages", ~prefix="productImages"),
|
|
125
|
+
]),
|
|
126
|
+
)->toEqual(Ok())
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
// A shared prefix that merely starts with another's characters is not nesting:
|
|
130
|
+
// the delete prefix carries a trailing slash, so `images` cannot reach
|
|
131
|
+
// `images-archive`.
|
|
132
|
+
testSync("a sibling prefix sharing a character run is fine", () =>
|
|
133
|
+
expect(
|
|
134
|
+
Reset.validateStores([
|
|
135
|
+
store(~qualified="Catalog.images", ~bucket="alpha-stores", ~prefix="images"),
|
|
136
|
+
store(~qualified="Catalog.imagesArchive", ~bucket="alpha-stores", ~prefix="images-archive"),
|
|
137
|
+
]),
|
|
138
|
+
)->toEqual(Ok())
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
testSync("an empty or slash-bearing store name is refused", () => {
|
|
142
|
+
switch Reset.validateStores([
|
|
143
|
+
store(~qualified="Catalog.productImages", ~bucket="alpha-stores", ~prefix=""),
|
|
144
|
+
]) {
|
|
145
|
+
| Ok() => fail("expected an empty prefix to be refused")
|
|
146
|
+
| Error(_) => ()
|
|
147
|
+
}
|
|
148
|
+
switch Reset.validateStores([
|
|
149
|
+
store(~qualified="Catalog.a/b", ~bucket="alpha-stores", ~prefix="a/b"),
|
|
150
|
+
]) {
|
|
151
|
+
| Ok() => fail("expected a store name containing a slash to be refused")
|
|
152
|
+
| Error(_) => ()
|
|
153
|
+
}
|
|
154
|
+
})
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
describe("pluginOf", () => {
|
|
158
|
+
// The `objectStores` keys carry the REGISTERED plugin name; the menu carries
|
|
159
|
+
// the operator-facing label. Attribution uses the declared name so the reset
|
|
160
|
+
// never has to guess the relation between the two.
|
|
161
|
+
testSync("prefers the declared plugin name over the label", () =>
|
|
162
|
+
expect(
|
|
163
|
+
Reset.pluginOf({projectDir: "../catalog-aws", label: "catalog", group: Domain, plugin: "Catalog"}),
|
|
164
|
+
)->toBe("Catalog")
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
testSync("falls back to the label when no plugin name is declared", () =>
|
|
168
|
+
expect(Reset.pluginOf({projectDir: ".", label: "platform", group: Platform}))->toBe("platform")
|
|
169
|
+
)
|
|
170
|
+
})
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// Holding a stack's runtimes still for the length of a wipe: what gets held, and
|
|
2
|
+
// what the reset says when a store comes back anyway.
|
|
3
|
+
//
|
|
4
|
+
// A truncate is not durable while the runtimes that own the data are running — a
|
|
5
|
+
// slice runtime keeps its TODO list in memory for the life of its execution
|
|
6
|
+
// environment and re-saves every row it holds at the end of each invocation,
|
|
7
|
+
// including the scheduled sweep that carries no events. So the reset now
|
|
8
|
+
// discovers the stack's Lambda functions alongside its stores and holds them at
|
|
9
|
+
// zero concurrency across the wipe.
|
|
10
|
+
//
|
|
11
|
+
// The pure half of that is what these pin: which ARNs resolve to a function to
|
|
12
|
+
// hold, the bounded fan-out the control-plane calls go through, and the failure
|
|
13
|
+
// message — which used to say "re-run to finish" for a condition re-running
|
|
14
|
+
// could not clear. The impure half is the Lambda control-plane calls themselves.
|
|
15
|
+
|
|
16
|
+
open JestGlobals
|
|
17
|
+
|
|
18
|
+
module Reset = ReventlessSeedAws_Reset
|
|
19
|
+
module Quiesce = ReventlessSeedAws_Quiesce
|
|
20
|
+
|
|
21
|
+
describe("classify", () => {
|
|
22
|
+
testSync("reads the function name out of a Lambda ARN", () =>
|
|
23
|
+
expect(Reset.classify("arn:aws:lambda:eu-west-1:123456789012:function:AllAutomationSlices-28b116c"))
|
|
24
|
+
->toEqual(Reset.Function("AllAutomationSlices-28b116c"))
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
// The tagging API returns unqualified ARNs, but a qualified one must still
|
|
28
|
+
// yield the function name — every Lambda control-plane call the hold makes
|
|
29
|
+
// takes a name, and "Name:1" is not one.
|
|
30
|
+
testSync("drops a version or alias qualifier", () =>
|
|
31
|
+
expect(Reset.classify("arn:aws:lambda:eu-west-1:123456789012:function:AllReadModels-0287438:42"))
|
|
32
|
+
->toEqual(Reset.Function("AllReadModels-0287438"))
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
testSync("still classifies tables and buckets", () => {
|
|
36
|
+
expect(Reset.classify("arn:aws:dynamodb:eu-west-1:123456789012:table/Products-0ba6849"))
|
|
37
|
+
->toEqual(Reset.Table("Products-0ba6849"))
|
|
38
|
+
expect(Reset.classify("arn:aws:s3:::alpha-product-images"))->toEqual(
|
|
39
|
+
Reset.Bucket("alpha-product-images"),
|
|
40
|
+
)
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
testSync("ignores anything else the tag scope returns", () =>
|
|
44
|
+
expect(Reset.classify("arn:aws:sqs:eu-west-1:123456789012:OrderingDcbCmdTopic-4b3658c"))
|
|
45
|
+
->toEqual(Reset.Other)
|
|
46
|
+
)
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
describe("mapBounded", () => {
|
|
50
|
+
// Results must line up with their inputs regardless of which worker finished
|
|
51
|
+
// first: the hold pairs each result with the function it came from, and a
|
|
52
|
+
// shuffled array would restore the wrong concurrency to the wrong function.
|
|
53
|
+
test("keeps results in input order under out-of-order completion", async () => {
|
|
54
|
+
let delays = [40, 0, 30, 10, 20, 5]
|
|
55
|
+
let out = await delays->Quiesce.mapBounded(~limit=3, async ms => {
|
|
56
|
+
await ReventlessSeed.Seed.Client.sleep(ms)
|
|
57
|
+
ms
|
|
58
|
+
})
|
|
59
|
+
expect(out)->toEqual(delays)
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
test("never runs more than the limit at once", async () => {
|
|
63
|
+
let inFlight = ref(0)
|
|
64
|
+
let peak = ref(0)
|
|
65
|
+
let _ = await Array.make(~length=12, 0)->Quiesce.mapBounded(~limit=4, async _ => {
|
|
66
|
+
inFlight := inFlight.contents + 1
|
|
67
|
+
peak := Math.Int.max(peak.contents, inFlight.contents)
|
|
68
|
+
await ReventlessSeed.Seed.Client.sleep(5)
|
|
69
|
+
inFlight := inFlight.contents - 1
|
|
70
|
+
})
|
|
71
|
+
expect(peak.contents)->toBe(4)
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
test("an empty list spawns no workers and returns nothing", async () => {
|
|
75
|
+
let out = await []->Quiesce.mapBounded(~limit=4, async x => x)
|
|
76
|
+
expect(out->Array.length)->toBe(0)
|
|
77
|
+
})
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
describe("refillMessage", () => {
|
|
81
|
+
// The whole point of the change: the operator has to be able to see WHICH
|
|
82
|
+
// store came back, not just a total.
|
|
83
|
+
testSync("names every store that came back, with its count", () => {
|
|
84
|
+
let message = Reset.refillMessage(
|
|
85
|
+
~refilled=[("AutoShipOrderTodo-61f39a2", 12), ("GeocodeCustomerAddressTodo-50d9488", 1)],
|
|
86
|
+
~heldCount=41,
|
|
87
|
+
)
|
|
88
|
+
expect(message->String.includes("AutoShipOrderTodo-61f39a2"))->toBe(true)
|
|
89
|
+
expect(message->String.includes("GeocodeCustomerAddressTodo-50d9488"))->toBe(true)
|
|
90
|
+
expect(message->String.startsWith("13 item(s)/object(s) came back"))->toBe(true)
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
// Under a hold nothing could START during the wipe, so the only writer left is
|
|
94
|
+
// an invocation already running when the hold landed — and that one really
|
|
95
|
+
// does go away. This is the only case where "re-run" is honest advice.
|
|
96
|
+
testSync("advises re-running only when the runtimes were held", () => {
|
|
97
|
+
let message = Reset.refillMessage(~refilled=[("AutoShipOrderTodo-61f39a2", 12)], ~heldCount=41)
|
|
98
|
+
expect(message->String.includes("already in flight"))->toBe(true)
|
|
99
|
+
expect(message->String.includes("Re-running clears it"))->toBe(true)
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
// Without a hold, the writer holds the rows in memory and restores them every
|
|
103
|
+
// invocation — the exact condition the old "re-run to finish" sent operators
|
|
104
|
+
// round in circles on. It must not appear here.
|
|
105
|
+
testSync("refuses to advise re-running when the stack was not held", () => {
|
|
106
|
+
let message = Reset.refillMessage(~refilled=[("AutoShipOrderTodo-61f39a2", 12)], ~heldCount=0)
|
|
107
|
+
expect(message->String.includes("re-running will not help"->String.toLowerCase))->toBe(true)
|
|
108
|
+
expect(message->String.includes("SEED_RESET_NO_QUIESCE"))->toBe(true)
|
|
109
|
+
})
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
describe("noQuiesce", () => {
|
|
113
|
+
let set = v => NodeProcess.env->Dict.set("SEED_RESET_NO_QUIESCE", v)
|
|
114
|
+
|
|
115
|
+
// Fail-closed: the hold is what makes the wipe durable, so anything other than
|
|
116
|
+
// a deliberate opt-out keeps it on.
|
|
117
|
+
testSync("defaults to holding the platform", () => {
|
|
118
|
+
set("")
|
|
119
|
+
expect(Reset.noQuiesce())->toBe(false)
|
|
120
|
+
set("0")
|
|
121
|
+
expect(Reset.noQuiesce())->toBe(false)
|
|
122
|
+
set("no")
|
|
123
|
+
expect(Reset.noQuiesce())->toBe(false)
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
testSync("accepts the spellings an operator would actually type", () => {
|
|
127
|
+
["1", "true", "TRUE", " yes "]->Array.forEach(v => {
|
|
128
|
+
set(v)
|
|
129
|
+
expect(Reset.noQuiesce())->toBe(true)
|
|
130
|
+
})
|
|
131
|
+
set("")
|
|
132
|
+
})
|
|
133
|
+
})
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -14,11 +14,6 @@ open ReventlessSeed
|
|
|
14
14
|
|
|
15
15
|
// ── Node bindings ─────────────────────────────────────────────────────────────
|
|
16
16
|
|
|
17
|
-
@val @scope("process") external processEnv: dict<string> = "env"
|
|
18
|
-
|
|
19
|
-
type execOptions = {cwd: string, encoding: string, env?: dict<string>}
|
|
20
|
-
@module("node:child_process")
|
|
21
|
-
external execFileSync: (string, array<string>, execOptions) => string = "execFileSync"
|
|
22
17
|
|
|
23
18
|
type fetchInit = {method: string, headers: dict<string>, body?: string}
|
|
24
19
|
type response
|
|
@@ -53,13 +48,17 @@ let envForBackend = (backend: option<string>): option<dict<string>> =>
|
|
|
53
48
|
switch backend {
|
|
54
49
|
| None => None
|
|
55
50
|
| Some(url) =>
|
|
56
|
-
let copy =
|
|
51
|
+
let copy = NodeProcess.env->Dict.toArray->Dict.fromArray
|
|
57
52
|
copy->Dict.set("PULUMI_BACKEND_URL", url)
|
|
58
53
|
Some(copy)
|
|
59
54
|
}
|
|
60
55
|
|
|
61
56
|
let pulumi = (~projectDir: string, ~backend: option<string>, args: array<string>): string =>
|
|
62
|
-
execFileSync(
|
|
57
|
+
NodeChildProcess.execFileSync(
|
|
58
|
+
"pulumi",
|
|
59
|
+
args,
|
|
60
|
+
{cwd: projectDir, encoding: "utf8", env: ?envForBackend(backend)},
|
|
61
|
+
)
|
|
63
62
|
|
|
64
63
|
// Stacks that actually exist in the Pulumi backend for this project — the names
|
|
65
64
|
// `pulumi stack output --stack <name>` accepts. A `Pulumi.<stack>.yaml` config
|
|
@@ -147,70 +146,88 @@ let resolveField = (~envKey: string, ~fromSource: option<string>, ~human: string
|
|
|
147
146
|
}
|
|
148
147
|
}
|
|
149
148
|
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
149
|
+
// Which document publishes the deployment's endpoints. A stack that serves a
|
|
150
|
+
// host shell publishes them in the shell's `config.json` under the client's key
|
|
151
|
+
// names; one that does not publishes them as stack outputs under Pulumi's. The
|
|
152
|
+
// two arms name different keys for the same values, which is why this is a
|
|
153
|
+
// variant and not a merged lookup.
|
|
154
|
+
type source = HostShellConfig(JSON.t) | StackOutputs(JSON.t)
|
|
155
|
+
|
|
156
|
+
type endpoints = {
|
|
157
|
+
graphql: string,
|
|
158
|
+
cognitoRegion: string,
|
|
159
|
+
cognitoClientId: string,
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The endpoints a source document publishes — pure, so the branch selection and
|
|
164
|
+
* every key it reads can be exercised with synthetic documents.
|
|
165
|
+
*
|
|
166
|
+
* Upload endpoints are no longer resolved here: under route B the seed mints through
|
|
167
|
+
* the domain API's `Upload_Presign` mutation on `graphql`, passing the store per asset,
|
|
168
|
+
* so there is no per-store URL to discover.
|
|
169
|
+
*/
|
|
170
|
+
let endpointsFrom = (~stack: string, source: source): endpoints =>
|
|
171
|
+
switch source {
|
|
172
|
+
| HostShellConfig(cfg) =>
|
|
163
173
|
let fromCfg = key => cfg->field(key)->Option.flatMap(asString)
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
~human="cognitoClientId",
|
|
183
|
-
)
|
|
184
|
-
(endpoint, uploadEndpoint, region, clientId)
|
|
185
|
-
| None =>
|
|
174
|
+
{
|
|
175
|
+
graphql: resolveField(
|
|
176
|
+
~envKey="REVENTLESS_GRAPHQL_ENDPOINT",
|
|
177
|
+
~fromSource=fromCfg("apiEndpoint"),
|
|
178
|
+
~human="apiEndpoint",
|
|
179
|
+
),
|
|
180
|
+
cognitoRegion: resolveField(
|
|
181
|
+
~envKey="AWS_REGION",
|
|
182
|
+
~fromSource=fromCfg("region"),
|
|
183
|
+
~human="region",
|
|
184
|
+
),
|
|
185
|
+
cognitoClientId: resolveField(
|
|
186
|
+
~envKey="COGNITO_CLIENT_ID",
|
|
187
|
+
~fromSource=fromCfg("cognitoClientId"),
|
|
188
|
+
~human="cognitoClientId",
|
|
189
|
+
),
|
|
190
|
+
}
|
|
191
|
+
| StackOutputs(outputs) =>
|
|
186
192
|
let out = key => outputs->field(key)->Option.flatMap(asString)
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
193
|
+
{
|
|
194
|
+
graphql: switch (
|
|
195
|
+
Seed.Prompt.envValue("REVENTLESS_GRAPHQL_ENDPOINT"),
|
|
196
|
+
out("domainMergedApiEndpoint"),
|
|
197
|
+
out("domainApiEndpoint"),
|
|
198
|
+
) {
|
|
199
|
+
| (Some(v), _, _) | (_, Some(v), _) | (_, _, Some(v)) => v
|
|
200
|
+
| _ =>
|
|
201
|
+
throw(
|
|
202
|
+
Seed.Failed(
|
|
203
|
+
`stack "${stack}" exports neither domainMergedApiEndpoint nor domainApiEndpoint`,
|
|
204
|
+
),
|
|
205
|
+
)
|
|
206
|
+
},
|
|
207
|
+
cognitoRegion: resolveField(
|
|
208
|
+
~envKey="AWS_REGION",
|
|
209
|
+
~fromSource=out("cognitoRegion"),
|
|
210
|
+
~human="cognitoRegion",
|
|
211
|
+
),
|
|
212
|
+
cognitoClientId: resolveField(
|
|
213
|
+
~envKey="COGNITO_CLIENT_ID",
|
|
214
|
+
~fromSource=out("cognitoUserPoolClientId"),
|
|
215
|
+
~human="cognitoUserPoolClientId",
|
|
216
|
+
),
|
|
199
217
|
}
|
|
200
|
-
let region = resolveField(
|
|
201
|
-
~envKey="AWS_REGION",
|
|
202
|
-
~fromSource=out("cognitoRegion"),
|
|
203
|
-
~human="cognitoRegion",
|
|
204
|
-
)
|
|
205
|
-
let clientId = resolveField(
|
|
206
|
-
~envKey="COGNITO_CLIENT_ID",
|
|
207
|
-
~fromSource=out("cognitoUserPoolClientId"),
|
|
208
|
-
~human="cognitoUserPoolClientId",
|
|
209
|
-
)
|
|
210
|
-
// Absent → "" → the data set skips its upload phase.
|
|
211
|
-
let uploadEndpoint = Seed.Prompt.envValue("REVENTLESS_UPLOAD_ENDPOINT")->Option.getOr("")
|
|
212
|
-
(endpoint, uploadEndpoint, region, clientId)
|
|
213
218
|
}
|
|
219
|
+
|
|
220
|
+
let resolveEndpoints = async (
|
|
221
|
+
~projectDir: string,
|
|
222
|
+
~backend: option<string>,
|
|
223
|
+
~stack: string,
|
|
224
|
+
): endpoints => {
|
|
225
|
+
let outputs = stackOutputs(~projectDir, ~backend, stack)
|
|
226
|
+
let source = switch outputs->field("hostShellUrl")->Option.flatMap(asString) {
|
|
227
|
+
| Some(hostShellUrl) => HostShellConfig(await fetchConfig(hostShellUrl))
|
|
228
|
+
| None => StackOutputs(outputs)
|
|
229
|
+
}
|
|
230
|
+
endpointsFrom(~stack, source)
|
|
214
231
|
}
|
|
215
232
|
|
|
216
233
|
// ── Cognito login ─────────────────────────────────────────────────────────────
|
|
@@ -302,15 +319,10 @@ let connect = (~projectDir: string=".", ~stack=?, ~backend=?, ()): (
|
|
|
302
319
|
| None => backend
|
|
303
320
|
}
|
|
304
321
|
let stackName = await resolveStack(~projectDir, ~backend, ~stack)
|
|
305
|
-
let
|
|
306
|
-
~projectDir,
|
|
307
|
-
~backend,
|
|
308
|
-
~stack=stackName,
|
|
309
|
-
)
|
|
322
|
+
let eps = await resolveEndpoints(~projectDir, ~backend, ~stack=stackName)
|
|
310
323
|
await Seed.Connect.make(
|
|
311
324
|
~label=stackName,
|
|
312
|
-
~endpoint,
|
|
313
|
-
~
|
|
314
|
-
~login=cognito(~region, ~clientId),
|
|
325
|
+
~endpoint=eps.graphql,
|
|
326
|
+
~login=cognito(~region=eps.cognitoRegion, ~clientId=eps.cognitoClientId),
|
|
315
327
|
)
|
|
316
328
|
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|