@reventlessdev/reventless-aws 3.0.0-alpha.253 → 3.0.0-alpha.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/package.json +9 -9
- package/src/Platform.res +134 -20
- package/src/Platform.res.mjs +95 -6
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res +5 -0
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res.mjs +3 -0
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res +5 -0
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res.mjs +3 -0
- package/src/adapter/Runtime/RuntimeEnvironment_Lambda.res +66 -8
- package/src/adapter/Runtime/RuntimeEnvironment_Lambda.res.mjs +16 -3
- package/src/adapter/Runtime/StateChangeSliceRuntime_Builder_Single.res +5 -0
- package/src/adapter/Runtime/StateChangeSliceRuntime_Builder_Single.res.mjs +3 -0
- package/src/adapter/Upload/Upload_Claim_S3.res +361 -0
- package/src/adapter/Upload/Upload_Claim_S3.res.mjs +203 -0
- package/src/adapter/Upload/Upload_Claim_S3_Ops.res +279 -0
- package/src/adapter/Upload/Upload_Claim_S3_Ops.res.mjs +261 -0
- package/src/adapter/Upload/Upload_PendingTag.res +41 -0
- package/src/adapter/Upload/Upload_PendingTag.res.mjs +15 -0
- package/src/adapter/Upload/Upload_Presign_S3.res +12 -2
- package/src/adapter/Upload/Upload_Presign_S3.res.mjs +1 -0
- package/src/adapter/Upload/Upload_Presign_S3_Ops.res +13 -1
- package/src/adapter/Upload/Upload_Presign_S3_Ops.res.mjs +3 -1
- package/src/capability/Capability_ObjectStore_S3.res +42 -1
- package/src/capability/Capability_ObjectStore_S3.res.mjs +19 -1
- package/src/components/Api/AppSync_Adapter.res +36 -0
- package/src/components/Api/AppSync_Adapter.res.mjs +14 -0
- package/src/util/Util_LogRetention.res +86 -0
- package/src/util/Util_LogRetention.res.mjs +43 -0
- package/src/util/Util_StoreLayout.res +37 -0
- package/src/util/Util_StoreLayout.res.mjs +18 -0
- package/tests/Upload_ClaimTest.res +132 -0
- package/tests/Upload_ClaimTest.res.mjs +101 -0
- package/tests/Util_LogRetentionTest.res +122 -0
- package/tests/Util_LogRetentionTest.res.mjs +95 -0
- package/tests/Util_StoreLayoutTest.res +63 -0
- package/tests/Util_StoreLayoutTest.res.mjs +42 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// Runtime handler for the upload claim component — compiled, type-checked and
|
|
2
|
+
// Pulumi-free so it ships as an EntryPoint module (`Upload_Claim_S3` bundles it
|
|
3
|
+
// and attaches it to every ref-bearing event log's stream).
|
|
4
|
+
//
|
|
5
|
+
// It answers one question per committed event: has anything now referenced this
|
|
6
|
+
// uploaded object? If so the object stops being provisional, and the pending
|
|
7
|
+
// tag the mint side wrote comes off. An object nobody ever references keeps its
|
|
8
|
+
// tag, and a lifecycle rule — opt-in, and only once reconciliation has confirmed
|
|
9
|
+
// the tagged set is the unreferenced set — expires it.
|
|
10
|
+
//
|
|
11
|
+
// Triggered by DynamoDB streams over the event log tables, one shared Lambda for
|
|
12
|
+
// the whole platform: routing is per record via `CLAIM_REF_FIELDS`, keyed by the
|
|
13
|
+
// table name in `eventSourceARN`, exactly as `StateTopic_AppSync_Ops` routes its
|
|
14
|
+
// own shared Lambda. The event log row is `{event: "<Type>", data: {…}}`, so a
|
|
15
|
+
// row without an `event` attribute — a snapshot, a DCB fence row — is not an
|
|
16
|
+
// event and drops out before anything else runs.
|
|
17
|
+
//
|
|
18
|
+
// Four properties this needs, and one it deliberately does not:
|
|
19
|
+
//
|
|
20
|
+
// Idempotent untagging an untagged object, or one that is gone, is
|
|
21
|
+
// success. Replays and at-least-once delivery are non-events.
|
|
22
|
+
// Scoped it only ever touches keys under a declared store's served
|
|
23
|
+
// prefix; a ref pointing anywhere else is refused, not guessed.
|
|
24
|
+
// Observable lag is the one failure mode that deletes data, so the ESM's
|
|
25
|
+
// `IteratorAge` is alarmed at deploy time (see `Upload_Claim_S3`).
|
|
26
|
+
// Narrow its input is only event logs with a declared ref field. It is
|
|
27
|
+
// not a projection and must not grow into one.
|
|
28
|
+
// Not ordered the object exists before the event that references it — the
|
|
29
|
+
// PUT precedes the command — so there is no race to design for,
|
|
30
|
+
// and no consistent read is needed.
|
|
31
|
+
//
|
|
32
|
+
// Failure direction, deliberately: anything this handler cannot do leaves the
|
|
33
|
+
// tag on. A tag left behind delays nothing except an eventual expiry that is
|
|
34
|
+
// off by default; a tag wrongly removed is only ever a missed cleanup. The one
|
|
35
|
+
// unsafe direction — a claim that never runs while the rule is on — is what the
|
|
36
|
+
// lag alarm and the opt-in sequencing exist for.
|
|
37
|
+
|
|
38
|
+
module S3 = AwsSdk.S3
|
|
39
|
+
|
|
40
|
+
// ── Environment ─────────────────────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
let getEnv = (k: string): option<string> =>
|
|
43
|
+
switch NodeProcess.env->Dict.get(k) {
|
|
44
|
+
| Some("") | None => None
|
|
45
|
+
| Some(v) => Some(v)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
let parseEnvObject = (k: string): dict<JSON.t> =>
|
|
49
|
+
switch getEnv(k)->Option.map(s => JSON.parseOrThrow(s)) {
|
|
50
|
+
| Some(Object(obj)) => obj
|
|
51
|
+
| _ => Dict.make()
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** A store the claimer may untag in: its physical bucket and the prefix its keys
|
|
55
|
+
are rooted at. Same `UPLOAD_STORES` shape the presign service reads, keyed by
|
|
56
|
+
the qualified `{plugin}.{store}` name a declaration resolves to. */
|
|
57
|
+
type storeConfig = {
|
|
58
|
+
bucket: string,
|
|
59
|
+
prefix: string,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
let decodeStore = (json: JSON.t): option<storeConfig> =>
|
|
63
|
+
switch json {
|
|
64
|
+
| Object(obj) =>
|
|
65
|
+
switch (
|
|
66
|
+
obj->Dict.get("bucket")->Option.flatMap(JSON.Decode.string),
|
|
67
|
+
obj->Dict.get("prefix")->Option.flatMap(JSON.Decode.string),
|
|
68
|
+
) {
|
|
69
|
+
| (Some(bucket), Some(prefix)) => Some({bucket, prefix})
|
|
70
|
+
| _ => None
|
|
71
|
+
}
|
|
72
|
+
| _ => None
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
let stores: dict<storeConfig> =
|
|
76
|
+
parseEnvObject("UPLOAD_STORES")
|
|
77
|
+
->Dict.toArray
|
|
78
|
+
->Array.filterMap(((k, v)) => decodeStore(v)->Option.map(c => (k, c)))
|
|
79
|
+
->Dict.fromArray
|
|
80
|
+
|
|
81
|
+
/** One declared ref-bearing field, as `StorageRefFields.toJson` wrote it. */
|
|
82
|
+
type refField = {
|
|
83
|
+
field: string,
|
|
84
|
+
many: bool,
|
|
85
|
+
store: string,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
let decodeRefField = (json: JSON.t): option<refField> =>
|
|
89
|
+
switch json {
|
|
90
|
+
| Object(obj) =>
|
|
91
|
+
switch (
|
|
92
|
+
obj->Dict.get("field")->Option.flatMap(JSON.Decode.string),
|
|
93
|
+
obj->Dict.get("store")->Option.flatMap(JSON.Decode.string),
|
|
94
|
+
) {
|
|
95
|
+
| (Some(field), Some(store)) =>
|
|
96
|
+
Some({
|
|
97
|
+
field,
|
|
98
|
+
many: obj->Dict.get("arity")->Option.flatMap(JSON.Decode.string) == Some("many"),
|
|
99
|
+
store,
|
|
100
|
+
})
|
|
101
|
+
| _ => None
|
|
102
|
+
}
|
|
103
|
+
| _ => None
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
let decodeEventFields = (json: JSON.t): dict<array<refField>> =>
|
|
107
|
+
switch json {
|
|
108
|
+
| Object(obj) =>
|
|
109
|
+
obj
|
|
110
|
+
->Dict.toArray
|
|
111
|
+
->Array.map(((eventType, v)) => (
|
|
112
|
+
eventType,
|
|
113
|
+
switch v {
|
|
114
|
+
| Array(items) => items->Array.filterMap(decodeRefField)
|
|
115
|
+
| _ => []
|
|
116
|
+
},
|
|
117
|
+
))
|
|
118
|
+
->Dict.fromArray
|
|
119
|
+
| _ => Dict.make()
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** `{ "<eventLogTableName>": { "<eventType>": [refField] } }`, baked at deploy
|
|
123
|
+
time from the plugins' `@storageRef` declarations. Its keys are the whole of
|
|
124
|
+
this component's input: a table absent from the map is a table it never
|
|
125
|
+
reads a record from. */
|
|
126
|
+
let refFieldsByTable: dict<dict<array<refField>>> =
|
|
127
|
+
parseEnvObject("CLAIM_REF_FIELDS")
|
|
128
|
+
->Dict.toArray
|
|
129
|
+
->Array.map(((table, v)) => (table, decodeEventFields(v)))
|
|
130
|
+
->Dict.fromArray
|
|
131
|
+
|
|
132
|
+
// ── Pure derivations ────────────────────────────────────────────────────────
|
|
133
|
+
|
|
134
|
+
/** Table name out of a stream ARN — `…:table/<TableName>/stream/<ts>`. */
|
|
135
|
+
let tableNameFromEventSourceArn = (arn: string): option<string> => {
|
|
136
|
+
let parts = arn->String.split("/")
|
|
137
|
+
switch (parts->Array.get(0), parts->Array.get(1), parts->Array.get(2)) {
|
|
138
|
+
| (Some(prefix), Some(tableName), Some("stream")) if prefix->String.endsWith(":table") =>
|
|
139
|
+
Some(tableName)
|
|
140
|
+
| _ => None
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** The ref strings a declared field holds in one event payload.
|
|
145
|
+
|
|
146
|
+
Non-string values and the `""` sentinel (which `StorageRef` admits to mean
|
|
147
|
+
"no object") yield nothing, so a field that is present but empty costs no
|
|
148
|
+
S3 call. */
|
|
149
|
+
let refsOfField = (~data: dict<JSON.t>, field: refField): array<string> =>
|
|
150
|
+
switch data->Dict.get(field.field) {
|
|
151
|
+
| Some(String(ref)) if !field.many && ref != "" => [ref]
|
|
152
|
+
| Some(Array(items)) if field.many =>
|
|
153
|
+
items->Array.filterMap(v =>
|
|
154
|
+
switch v {
|
|
155
|
+
| String(ref) if ref != "" => Some(ref)
|
|
156
|
+
| _ => None
|
|
157
|
+
}
|
|
158
|
+
)
|
|
159
|
+
| _ => []
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Strip the leading `/` a ref carries (`/{prefix}/{key}`) to recover the S3
|
|
163
|
+
object key. Mirrors the presign service, which mints `/${key}`. */
|
|
164
|
+
let keyOfRef = (storageRef: string): string =>
|
|
165
|
+
storageRef->String.startsWith("/")
|
|
166
|
+
? storageRef->String.slice(~start=1, ~end=storageRef->String.length)
|
|
167
|
+
: storageRef
|
|
168
|
+
|
|
169
|
+
/** An object this claim may touch: a declared store, and a key that actually
|
|
170
|
+
sits under that store's served prefix.
|
|
171
|
+
|
|
172
|
+
The prefix check is the same one the release rule makes, for the same
|
|
173
|
+
reason — a ref that resolves outside the store it names is refused rather
|
|
174
|
+
than acted on, so a malformed or hostile ref cannot steer an untag at an
|
|
175
|
+
object the declaration does not cover. */
|
|
176
|
+
type target = {
|
|
177
|
+
bucket: string,
|
|
178
|
+
key: string,
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
let resolveTarget = (~store: string, ~storageRef: string): result<target, string> =>
|
|
182
|
+
switch stores->Dict.get(store) {
|
|
183
|
+
| None => Error(`unknown_store ${store}`)
|
|
184
|
+
| Some({bucket, prefix}) =>
|
|
185
|
+
let key = storageRef->keyOfRef
|
|
186
|
+
key->String.startsWith(`${prefix}/`)
|
|
187
|
+
? Ok({bucket, key})
|
|
188
|
+
: Error(`not_in_store ${store} ${storageRef}`)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// ── The claim ───────────────────────────────────────────────────────────────
|
|
192
|
+
|
|
193
|
+
/** Remove the pending tag, keeping every other tag the object carries.
|
|
194
|
+
Read-then-write rather than `DeleteObjectTagging`, which would take a
|
|
195
|
+
deployment's own tags with it.
|
|
196
|
+
|
|
197
|
+
Returns without writing when the tag is already absent — the replay case,
|
|
198
|
+
and the common one — so a redelivered batch costs one read per object and
|
|
199
|
+
no write. A missing object is success for the same reason a missing object
|
|
200
|
+
is a successful release: the outcome asked for has already happened. */
|
|
201
|
+
let claim = async (~bucket: string, ~key: string): unit =>
|
|
202
|
+
try {
|
|
203
|
+
let current = await S3.GetObjectTaggingCommand.send(
|
|
204
|
+
S3.GetObjectTaggingCommand.make({bucket, key}),
|
|
205
|
+
)
|
|
206
|
+
let tagSet = current.tagSet->Option.getOr([])
|
|
207
|
+
let remaining = tagSet->Array.filter(t => t.key != Upload_PendingTag.key)
|
|
208
|
+
if remaining->Array.length != tagSet->Array.length {
|
|
209
|
+
let _ = await S3.PutObjectTaggingCommand.send(
|
|
210
|
+
S3.PutObjectTaggingCommand.make({bucket, key, tagging: {tagSet: remaining}}),
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
} catch {
|
|
214
|
+
| exn =>
|
|
215
|
+
switch exn->JsExn.fromException->Option.flatMap(JsExn.name) {
|
|
216
|
+
| Some("NoSuchKey") | Some("NotFound") => ()
|
|
217
|
+
| _ => throw(exn)
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// ── DynamoDB stream event (only the fields this handler reads) ──────────────
|
|
222
|
+
|
|
223
|
+
type attributeValue = AwsSdk.DynamoDb.Util.attributeValue
|
|
224
|
+
type streamRecord = {@as("NewImage") newImage?: dict<attributeValue>}
|
|
225
|
+
type record = {
|
|
226
|
+
eventName: string,
|
|
227
|
+
eventSourceARN: string,
|
|
228
|
+
dynamodb?: streamRecord,
|
|
229
|
+
}
|
|
230
|
+
type event = {@as("Records") records: array<record>}
|
|
231
|
+
|
|
232
|
+
/** The refs one stream record claims, already resolved to a bucket and key.
|
|
233
|
+
|
|
234
|
+
Pure and separately testable: everything between "a row appeared" and "an
|
|
235
|
+
S3 call happens" is decided here, so the untag itself has no branching left
|
|
236
|
+
in it. */
|
|
237
|
+
let targetsOfRecord = (record: record): array<target> =>
|
|
238
|
+
switch (
|
|
239
|
+
record.eventSourceARN->tableNameFromEventSourceArn->Option.flatMap(t => refFieldsByTable->Dict.get(t)),
|
|
240
|
+
record.dynamodb->Option.flatMap(d => d.newImage),
|
|
241
|
+
) {
|
|
242
|
+
| (Some(byEventType), Some(image)) =>
|
|
243
|
+
let row = AwsSdk.DynamoDb_Util_Helpers.unmarshallDict(image)
|
|
244
|
+
// No `event` attribute → not an event row (a snapshot, a DCB fence row).
|
|
245
|
+
switch (row->Dict.get("event")->Option.flatMap(JSON.Decode.string), row->Dict.get("data")) {
|
|
246
|
+
| (Some(eventType), Some(JSON.Object(data))) =>
|
|
247
|
+
byEventType
|
|
248
|
+
->Dict.get(eventType)
|
|
249
|
+
->Option.getOr([])
|
|
250
|
+
->Array.flatMap(field =>
|
|
251
|
+
refsOfField(~data, field)->Array.filterMap(storageRef =>
|
|
252
|
+
switch resolveTarget(~store=field.store, ~storageRef) {
|
|
253
|
+
| Ok(target) => Some(target)
|
|
254
|
+
| Error(why) =>
|
|
255
|
+
// Refused, not acted on — and said out loud, because a ref that
|
|
256
|
+
// does not resolve is a declaration and a deploy disagreeing.
|
|
257
|
+
Console.error2("Upload_Claim: refusing ref outside a declared store —", why)
|
|
258
|
+
None
|
|
259
|
+
}
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
| _ => []
|
|
263
|
+
}
|
|
264
|
+
| _ => []
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ── Runtime handler ─────────────────────────────────────────────────────────
|
|
268
|
+
|
|
269
|
+
let handler = async (event: event): unit => {
|
|
270
|
+
// An append-only log only ever inserts events; MODIFY rows are fence and
|
|
271
|
+
// snapshot updates, which `targetsOfRecord` would drop anyway.
|
|
272
|
+
let targets = event.records->Array.filter(r => r.eventName == "INSERT")->Array.flatMap(targetsOfRecord)
|
|
273
|
+
// Sequential: a batch carries a handful of refs at most, and a failure must
|
|
274
|
+
// reach the ESM as a throw so the record is retried rather than silently
|
|
275
|
+
// leaving an object tagged.
|
|
276
|
+
await targets->Array.reduce(Promise.resolve(), (acc, {bucket, key}) =>
|
|
277
|
+
acc->Promise.then(_ => claim(~bucket, ~key))
|
|
278
|
+
)
|
|
279
|
+
}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as S3$AwsSdk from "@reventlessdev/rescript-aws-sdk/src/S3.res.mjs";
|
|
4
|
+
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
5
|
+
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
6
|
+
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
7
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
8
|
+
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
9
|
+
import * as ClientS3 from "@aws-sdk/client-s3";
|
|
10
|
+
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
11
|
+
import * as DynamoDb_Util_Helpers$AwsSdk from "@reventlessdev/rescript-aws-sdk/src/DynamoDb_Util_Helpers.res.mjs";
|
|
12
|
+
import * as Upload_PendingTag$ReventlessAws from "./Upload_PendingTag.res.mjs";
|
|
13
|
+
|
|
14
|
+
function getEnv(k) {
|
|
15
|
+
let v = process.env[k];
|
|
16
|
+
if (v !== undefined && v !== "") {
|
|
17
|
+
return v;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function parseEnvObject(k) {
|
|
22
|
+
let match = Stdlib_Option.map(getEnv(k), s => JSON.parse(s));
|
|
23
|
+
if (match !== undefined) {
|
|
24
|
+
if (typeof match === "object" && match !== null && !Array.isArray(match)) {
|
|
25
|
+
return match;
|
|
26
|
+
} else {
|
|
27
|
+
return {};
|
|
28
|
+
}
|
|
29
|
+
} else {
|
|
30
|
+
return {};
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function decodeStore(json) {
|
|
35
|
+
if (typeof json !== "object" || json === null || Array.isArray(json)) {
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
let match = Stdlib_Option.flatMap(json["bucket"], Stdlib_JSON.Decode.string);
|
|
39
|
+
let match$1 = Stdlib_Option.flatMap(json["prefix"], Stdlib_JSON.Decode.string);
|
|
40
|
+
if (match !== undefined && match$1 !== undefined) {
|
|
41
|
+
return {
|
|
42
|
+
bucket: match,
|
|
43
|
+
prefix: match$1
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
let stores = Object.fromEntries(Stdlib_Array.filterMap(Object.entries(parseEnvObject("UPLOAD_STORES")), param => {
|
|
49
|
+
let k = param[0];
|
|
50
|
+
return Stdlib_Option.map(decodeStore(param[1]), c => [
|
|
51
|
+
k,
|
|
52
|
+
c
|
|
53
|
+
]);
|
|
54
|
+
}));
|
|
55
|
+
|
|
56
|
+
function decodeRefField(json) {
|
|
57
|
+
if (typeof json !== "object" || json === null || Array.isArray(json)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
let match = Stdlib_Option.flatMap(json["field"], Stdlib_JSON.Decode.string);
|
|
61
|
+
let match$1 = Stdlib_Option.flatMap(json["store"], Stdlib_JSON.Decode.string);
|
|
62
|
+
if (match !== undefined && match$1 !== undefined) {
|
|
63
|
+
return {
|
|
64
|
+
field: match,
|
|
65
|
+
many: Primitive_object.equal(Stdlib_Option.flatMap(json["arity"], Stdlib_JSON.Decode.string), "many"),
|
|
66
|
+
store: match$1
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function decodeEventFields(json) {
|
|
72
|
+
if (typeof json === "object" && json !== null && !Array.isArray(json)) {
|
|
73
|
+
return Object.fromEntries(Object.entries(json).map(param => {
|
|
74
|
+
let v = param[1];
|
|
75
|
+
let tmp;
|
|
76
|
+
tmp = Array.isArray(v) ? Stdlib_Array.filterMap(v, decodeRefField) : [];
|
|
77
|
+
return [
|
|
78
|
+
param[0],
|
|
79
|
+
tmp
|
|
80
|
+
];
|
|
81
|
+
}));
|
|
82
|
+
} else {
|
|
83
|
+
return {};
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
let refFieldsByTable = Object.fromEntries(Object.entries(parseEnvObject("CLAIM_REF_FIELDS")).map(param => [
|
|
88
|
+
param[0],
|
|
89
|
+
decodeEventFields(param[1])
|
|
90
|
+
]));
|
|
91
|
+
|
|
92
|
+
function tableNameFromEventSourceArn(arn) {
|
|
93
|
+
let parts = arn.split("/");
|
|
94
|
+
let match = parts[0];
|
|
95
|
+
let match$1 = parts[1];
|
|
96
|
+
let match$2 = parts[2];
|
|
97
|
+
if (match !== undefined && match$1 !== undefined && match$2 !== undefined && match$2 === "stream" && match.endsWith(":table")) {
|
|
98
|
+
return match$1;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function refsOfField(data, field) {
|
|
103
|
+
let match = data[field.field];
|
|
104
|
+
if (match === undefined) {
|
|
105
|
+
return [];
|
|
106
|
+
}
|
|
107
|
+
if (Array.isArray(match)) {
|
|
108
|
+
if (field.many) {
|
|
109
|
+
return Stdlib_Array.filterMap(match, v => {
|
|
110
|
+
if (typeof v === "string" && v !== "") {
|
|
111
|
+
return v;
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
} else {
|
|
115
|
+
return [];
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
switch (typeof match) {
|
|
119
|
+
case "string" :
|
|
120
|
+
if (!field.many && match !== "") {
|
|
121
|
+
return [match];
|
|
122
|
+
} else {
|
|
123
|
+
return [];
|
|
124
|
+
}
|
|
125
|
+
default:
|
|
126
|
+
return [];
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function keyOfRef(storageRef) {
|
|
131
|
+
if (storageRef.startsWith("/")) {
|
|
132
|
+
return storageRef.slice(1, storageRef.length);
|
|
133
|
+
} else {
|
|
134
|
+
return storageRef;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function resolveTarget(store, storageRef) {
|
|
139
|
+
let match = stores[store];
|
|
140
|
+
if (match === undefined) {
|
|
141
|
+
return {
|
|
142
|
+
TAG: "Error",
|
|
143
|
+
_0: `unknown_store ` + store
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
let key = keyOfRef(storageRef);
|
|
147
|
+
if (key.startsWith(match.prefix + `/`)) {
|
|
148
|
+
return {
|
|
149
|
+
TAG: "Ok",
|
|
150
|
+
_0: {
|
|
151
|
+
bucket: match.bucket,
|
|
152
|
+
key: key
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
} else {
|
|
156
|
+
return {
|
|
157
|
+
TAG: "Error",
|
|
158
|
+
_0: `not_in_store ` + store + ` ` + storageRef
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function claim(bucket, key) {
|
|
164
|
+
try {
|
|
165
|
+
let current = await S3$AwsSdk.GetObjectTaggingCommand.send(new ClientS3.GetObjectTaggingCommand({
|
|
166
|
+
Bucket: bucket,
|
|
167
|
+
Key: key
|
|
168
|
+
}));
|
|
169
|
+
let tagSet = Stdlib_Option.getOr(current.TagSet, []);
|
|
170
|
+
let remaining = tagSet.filter(t => t.Key !== Upload_PendingTag$ReventlessAws.key);
|
|
171
|
+
if (remaining.length !== tagSet.length) {
|
|
172
|
+
await S3$AwsSdk.PutObjectTaggingCommand.send(new ClientS3.PutObjectTaggingCommand({
|
|
173
|
+
Bucket: bucket,
|
|
174
|
+
Key: key,
|
|
175
|
+
Tagging: {
|
|
176
|
+
TagSet: remaining
|
|
177
|
+
}
|
|
178
|
+
}));
|
|
179
|
+
return;
|
|
180
|
+
} else {
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
} catch (raw_exn) {
|
|
184
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
185
|
+
let match = Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.name);
|
|
186
|
+
if (match !== undefined) {
|
|
187
|
+
switch (match) {
|
|
188
|
+
case "NoSuchKey" :
|
|
189
|
+
case "NotFound" :
|
|
190
|
+
return;
|
|
191
|
+
default:
|
|
192
|
+
throw exn;
|
|
193
|
+
}
|
|
194
|
+
} else {
|
|
195
|
+
throw exn;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function targetsOfRecord(record) {
|
|
201
|
+
let match = Stdlib_Option.flatMap(tableNameFromEventSourceArn(record.eventSourceARN), t => refFieldsByTable[t]);
|
|
202
|
+
let match$1 = Stdlib_Option.flatMap(record.dynamodb, d => d.NewImage);
|
|
203
|
+
if (match === undefined) {
|
|
204
|
+
return [];
|
|
205
|
+
}
|
|
206
|
+
if (match$1 === undefined) {
|
|
207
|
+
return [];
|
|
208
|
+
}
|
|
209
|
+
let row = DynamoDb_Util_Helpers$AwsSdk.unmarshallDict(undefined, match$1);
|
|
210
|
+
let match$2 = Stdlib_Option.flatMap(row["event"], Stdlib_JSON.Decode.string);
|
|
211
|
+
let match$3 = row["data"];
|
|
212
|
+
if (match$2 !== undefined) {
|
|
213
|
+
if (match$3 !== undefined) {
|
|
214
|
+
if (typeof match$3 === "object" && match$3 !== null && !Array.isArray(match$3)) {
|
|
215
|
+
return Stdlib_Option.getOr(match[match$2], []).flatMap(field => Stdlib_Array.filterMap(refsOfField(match$3, field), storageRef => {
|
|
216
|
+
let target = resolveTarget(field.store, storageRef);
|
|
217
|
+
if (target.TAG === "Ok") {
|
|
218
|
+
return target._0;
|
|
219
|
+
}
|
|
220
|
+
console.error("Upload_Claim: refusing ref outside a declared store —", target._0);
|
|
221
|
+
}));
|
|
222
|
+
} else {
|
|
223
|
+
return [];
|
|
224
|
+
}
|
|
225
|
+
} else {
|
|
226
|
+
return [];
|
|
227
|
+
}
|
|
228
|
+
} else {
|
|
229
|
+
return [];
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async function handler(event) {
|
|
234
|
+
let targets = event.Records.filter(r => r.eventName === "INSERT").flatMap(targetsOfRecord);
|
|
235
|
+
return await Stdlib_Array.reduce(targets, Promise.resolve(), (acc, param) => {
|
|
236
|
+
let key = param.key;
|
|
237
|
+
let bucket = param.bucket;
|
|
238
|
+
return acc.then(() => claim(bucket, key));
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
let S3;
|
|
243
|
+
|
|
244
|
+
export {
|
|
245
|
+
S3,
|
|
246
|
+
getEnv,
|
|
247
|
+
parseEnvObject,
|
|
248
|
+
decodeStore,
|
|
249
|
+
stores,
|
|
250
|
+
decodeRefField,
|
|
251
|
+
decodeEventFields,
|
|
252
|
+
refFieldsByTable,
|
|
253
|
+
tableNameFromEventSourceArn,
|
|
254
|
+
refsOfField,
|
|
255
|
+
keyOfRef,
|
|
256
|
+
resolveTarget,
|
|
257
|
+
claim,
|
|
258
|
+
targetsOfRecord,
|
|
259
|
+
handler,
|
|
260
|
+
}
|
|
261
|
+
/* stores Not a pure module */
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// The tag that makes "nobody has committed a reference to this object yet"
|
|
2
|
+
// visible to S3 itself.
|
|
3
|
+
//
|
|
4
|
+
// One definition, three readers, and they must agree exactly or the mechanism
|
|
5
|
+
// deletes live data:
|
|
6
|
+
// - the mint side writes it on the presigned PUT (`Upload_Presign_S3_Ops`),
|
|
7
|
+
// - the claim side removes it when a committed event carries the ref
|
|
8
|
+
// (`Upload_Claim_S3_Ops`),
|
|
9
|
+
// - the bucket's lifecycle rule expires what still carries it
|
|
10
|
+
// (`Capability_ObjectStore_S3`).
|
|
11
|
+
// A rule filtered on a key the mint side never writes matches *nothing*, which
|
|
12
|
+
// fails safe; a claim side that strips a different key leaves every object
|
|
13
|
+
// tagged, which does not. Hence one module rather than three string literals.
|
|
14
|
+
//
|
|
15
|
+
// Runtime-pure and Pulumi-free on purpose: two of the three readers are Lambda
|
|
16
|
+
// handlers shipped as EntryPoint modules, so a deploy-time import here would
|
|
17
|
+
// leak `@pulumi/pulumi` into their cold-start graph.
|
|
18
|
+
//
|
|
19
|
+
// The tag's meaning is deliberately "not yet claimed", never "safe to delete".
|
|
20
|
+
// Nothing reads it as permission; the lifecycle rule adds an age condition of
|
|
21
|
+
// its own, and it is opt-in per store.
|
|
22
|
+
|
|
23
|
+
/** Tag key. `:` is a legal S3 tag-key character, and the `reventless:` prefix
|
|
24
|
+
keeps the framework's tags from colliding with a deployment's own. */
|
|
25
|
+
let key = "reventless:pending"
|
|
26
|
+
|
|
27
|
+
/** Tag value. A single-valued tag: the key's presence is the fact, and the
|
|
28
|
+
value exists only because S3 tags are pairs. */
|
|
29
|
+
let value = "true"
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
The `Tagging` parameter of a `PutObject`, in the URL-encoded query-string form
|
|
33
|
+
S3 specifies for that header.
|
|
34
|
+
|
|
35
|
+
The colon is percent-encoded here because the value is parsed as a query string
|
|
36
|
+
by S3, so its key/value separators are the only characters that may appear raw.
|
|
37
|
+
The SDK hoists this into the presigned URL's query string (verified against a
|
|
38
|
+
live bucket: a PUT that sends nothing but `Content-Type` still lands the tag),
|
|
39
|
+
so the caller needs no cooperation and cannot opt out.
|
|
40
|
+
*/
|
|
41
|
+
let putObjectTagging = `${key->encodeURIComponent}=${value->encodeURIComponent}`
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
let key = "reventless:pending";
|
|
5
|
+
|
|
6
|
+
let value = "true";
|
|
7
|
+
|
|
8
|
+
let putObjectTagging = encodeURIComponent(key) + `=` + encodeURIComponent(value);
|
|
9
|
+
|
|
10
|
+
export {
|
|
11
|
+
key,
|
|
12
|
+
value,
|
|
13
|
+
putObjectTagging,
|
|
14
|
+
}
|
|
15
|
+
/* putObjectTagging Not a pure module */
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// There is no Function URL and no anonymous surface: authentication is the platform
|
|
12
12
|
// API's Cognito authorizer, and the verified caller identity reaches the handler in
|
|
13
13
|
// the resolver payload (see the JS resolver code below). This is route B of
|
|
14
|
-
// [docs/plans/upload-release-path.md]; the anonymous Function URL and the unverified
|
|
14
|
+
// [docs/plans/done/upload-release-path.md]; the anonymous Function URL and the unverified
|
|
15
15
|
// `decodeJwtSub` the mint side used to carry are gone.
|
|
16
16
|
//
|
|
17
17
|
// Why an EntryPoint and not a Pulumi `CallbackFunction`: the handler needs the AWS
|
|
@@ -106,6 +106,11 @@ let make = (
|
|
|
106
106
|
// `GetObject` backs the `HeadObject` the age check reads. One Lambda covers every
|
|
107
107
|
// declared store (route B1) — but its reach is still the union of those prefixes,
|
|
108
108
|
// not a wildcard, so it cannot touch a key outside a declared store.
|
|
109
|
+
//
|
|
110
|
+
// `PutObjectTagging` is what lets the presigned PUT carry the pending tag: a
|
|
111
|
+
// presigned request runs with the *signer's* permissions, so tag-on-put fails
|
|
112
|
+
// with AccessDenied without it — and an untagged object is one the sweep can
|
|
113
|
+
// never reach.
|
|
109
114
|
let _policy =
|
|
110
115
|
resolvedStores->Pulumi.Output.apply(list => {
|
|
111
116
|
let arns = list->Array.map(((_, bucket, prefix)) => `arn:aws:s3:::${bucket}/${prefix}/*`)
|
|
@@ -124,7 +129,12 @@ let make = (
|
|
|
124
129
|
{
|
|
125
130
|
sid: "AllowUploadObjectAccess",
|
|
126
131
|
effect: Allow,
|
|
127
|
-
actions: Actions([
|
|
132
|
+
actions: Actions([
|
|
133
|
+
"s3:PutObject",
|
|
134
|
+
"s3:PutObjectTagging",
|
|
135
|
+
"s3:DeleteObject",
|
|
136
|
+
"s3:GetObject",
|
|
137
|
+
]),
|
|
128
138
|
resources: Resources(arns),
|
|
129
139
|
},
|
|
130
140
|
],
|
|
@@ -25,6 +25,7 @@ type putObjectInput = {
|
|
|
25
25
|
@as("Bucket") bucket: string,
|
|
26
26
|
@as("Key") key: string,
|
|
27
27
|
@as("ContentType") contentType?: string,
|
|
28
|
+
@as("Tagging") tagging?: string,
|
|
28
29
|
}
|
|
29
30
|
type putObjectCommand
|
|
30
31
|
|
|
@@ -225,7 +226,18 @@ let handlePresign = async (
|
|
|
225
226
|
// served prefix so the CloudFront `{prefix}/*` behavior fronts it, and namespaced
|
|
226
227
|
// by the verified `sub` so the release rule can tell one caller's objects apart.
|
|
227
228
|
let key = `${servedPrefix}/${sub}/${NodeCrypto.randomUUID()}/${fileName}`
|
|
228
|
-
|
|
229
|
+
// The object is provisional from the moment the bytes land: it carries the
|
|
230
|
+
// pending tag until a committed event referencing it strips it (see
|
|
231
|
+
// `Upload_Claim_S3_Ops`). Written into the signature here, not asked of the
|
|
232
|
+
// caller — the SDK hoists `x-amz-tagging` into the presigned URL's query
|
|
233
|
+
// string, so a client that PUTs exactly as it always has still gets tagged
|
|
234
|
+
// bytes, and cannot opt out.
|
|
235
|
+
let command = makePutObjectCommand({
|
|
236
|
+
bucket,
|
|
237
|
+
key,
|
|
238
|
+
contentType: ?args.contentType,
|
|
239
|
+
tagging: Upload_PendingTag.putObjectTagging,
|
|
240
|
+
})
|
|
229
241
|
let uploadUrl = await getSignedUrl(client, command, {expiresIn: 300})
|
|
230
242
|
// Same-origin relative ref `/{key}`: the served bucket is fronted read-only by the
|
|
231
243
|
// UI's own CloudFront distribution under `{prefix}/*`, so a command stores this
|
|
@@ -10,6 +10,7 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
|
10
10
|
import * as ClientS3 from "@aws-sdk/client-s3";
|
|
11
11
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
12
12
|
import * as S3RequestPresigner from "@aws-sdk/s3-request-presigner";
|
|
13
|
+
import * as Upload_PendingTag$ReventlessAws from "./Upload_PendingTag.res.mjs";
|
|
13
14
|
|
|
14
15
|
function getEnv(k) {
|
|
15
16
|
let v = process.env[k];
|
|
@@ -175,7 +176,8 @@ async function handlePresign(client, bucket, servedPrefix, sub, args) {
|
|
|
175
176
|
let command = new ClientS3.PutObjectCommand({
|
|
176
177
|
Bucket: bucket,
|
|
177
178
|
Key: key,
|
|
178
|
-
ContentType: args.contentType
|
|
179
|
+
ContentType: args.contentType,
|
|
180
|
+
Tagging: Upload_PendingTag$ReventlessAws.putObjectTagging
|
|
179
181
|
});
|
|
180
182
|
let uploadUrl = await S3RequestPresigner.getSignedUrl(client, command, {
|
|
181
183
|
expiresIn: 300
|