@farukada/aws-langgraph-dynamodb-ts 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/checkpointer/actions/list.d.ts.map +1 -1
- package/dist/checkpointer/actions/list.js +7 -0
- package/dist/checkpointer/actions/list.js.map +1 -1
- package/dist/checkpointer/actions/put-writes.d.ts.map +1 -1
- package/dist/checkpointer/actions/put-writes.js +2 -1
- package/dist/checkpointer/actions/put-writes.js.map +1 -1
- package/dist/checkpointer/internal/item-reader.d.ts +14 -0
- package/dist/checkpointer/internal/item-reader.d.ts.map +1 -1
- package/dist/checkpointer/internal/item-reader.js +15 -1
- package/dist/checkpointer/internal/item-reader.js.map +1 -1
- package/dist/checkpointer/internal/item-writer.d.ts.map +1 -1
- package/dist/checkpointer/internal/item-writer.js +2 -1
- package/dist/checkpointer/internal/item-writer.js.map +1 -1
- package/dist/checkpointer/internal/special-write-cas.d.ts +30 -0
- package/dist/checkpointer/internal/special-write-cas.d.ts.map +1 -0
- package/dist/checkpointer/internal/special-write-cas.js +104 -0
- package/dist/checkpointer/internal/special-write-cas.js.map +1 -0
- package/dist/checkpointer/internal/special-write-cleanup.d.ts +18 -12
- package/dist/checkpointer/internal/special-write-cleanup.d.ts.map +1 -1
- package/dist/checkpointer/internal/special-write-cleanup.js +31 -86
- package/dist/checkpointer/internal/special-write-cleanup.js.map +1 -1
- package/dist/checkpointer/internal/special-write-verify.d.ts +54 -0
- package/dist/checkpointer/internal/special-write-verify.d.ts.map +1 -0
- package/dist/checkpointer/internal/special-write-verify.js +65 -0
- package/dist/checkpointer/internal/special-write-verify.js.map +1 -0
- package/dist/checkpointer/internal/write-guard.d.ts +0 -8
- package/dist/checkpointer/internal/write-guard.d.ts.map +1 -1
- package/dist/checkpointer/internal/write-guard.js +0 -9
- package/dist/checkpointer/internal/write-guard.js.map +1 -1
- package/dist/checkpointer/internal/write-index.d.ts +8 -0
- package/dist/checkpointer/internal/write-index.d.ts.map +1 -1
- package/dist/checkpointer/internal/write-index.js +6 -2
- package/dist/checkpointer/internal/write-index.js.map +1 -1
- package/dist/checkpointer/types.d.ts +6 -0
- package/dist/checkpointer/types.d.ts.map +1 -1
- package/dist/shared/constants.d.ts +15 -0
- package/dist/shared/constants.d.ts.map +1 -1
- package/dist/shared/constants.js +16 -1
- package/dist/shared/constants.js.map +1 -1
- package/dist/shared/dynamodb/conditional-put.d.ts +51 -0
- package/dist/shared/dynamodb/conditional-put.d.ts.map +1 -0
- package/dist/shared/dynamodb/conditional-put.js +59 -0
- package/dist/shared/dynamodb/conditional-put.js.map +1 -0
- package/dist/shared/logging/redaction.d.ts +5 -1
- package/dist/shared/logging/redaction.d.ts.map +1 -1
- package/dist/shared/logging/redaction.js.map +1 -1
- package/dist/shared/logging/secret-patterns.d.ts +33 -0
- package/dist/shared/logging/secret-patterns.d.ts.map +1 -1
- package/dist/shared/logging/secret-patterns.js +35 -2
- package/dist/shared/logging/secret-patterns.js.map +1 -1
- package/dist/store/actions/put.d.ts.map +1 -1
- package/dist/store/actions/put.js +4 -16
- package/dist/store/actions/put.js.map +1 -1
- package/dist/store/internal/backend-search.d.ts.map +1 -1
- package/dist/store/internal/backend-search.js +2 -1
- package/dist/store/internal/backend-search.js.map +1 -1
- package/dist/store/internal/filter.d.ts.map +1 -1
- package/dist/store/internal/filter.js +25 -5
- package/dist/store/internal/filter.js.map +1 -1
- package/dist/store/internal/index-reconcile.d.ts.map +1 -1
- package/dist/store/internal/index-reconcile.js +8 -2
- package/dist/store/internal/index-reconcile.js.map +1 -1
- package/dist/store/internal/item-mapper.d.ts +1 -0
- package/dist/store/internal/item-mapper.d.ts.map +1 -1
- package/dist/store/internal/item-mapper.js +2 -0
- package/dist/store/internal/item-mapper.js.map +1 -1
- package/dist/store/internal/overwrite-swap.d.ts +33 -0
- package/dist/store/internal/overwrite-swap.d.ts.map +1 -0
- package/dist/store/internal/overwrite-swap.js +62 -0
- package/dist/store/internal/overwrite-swap.js.map +1 -0
- package/dist/store/internal/persist.d.ts +22 -6
- package/dist/store/internal/persist.d.ts.map +1 -1
- package/dist/store/internal/persist.js +42 -17
- package/dist/store/internal/persist.js.map +1 -1
- package/dist/store/internal/read-existing.d.ts +19 -0
- package/dist/store/internal/read-existing.d.ts.map +1 -0
- package/dist/store/internal/read-existing.js +29 -0
- package/dist/store/internal/read-existing.js.map +1 -0
- package/dist/store/internal/score-direction.d.ts +32 -0
- package/dist/store/internal/score-direction.d.ts.map +1 -0
- package/dist/store/internal/score-direction.js +39 -0
- package/dist/store/internal/score-direction.js.map +1 -0
- package/dist/store/internal/setup.d.ts +2 -0
- package/dist/store/internal/setup.d.ts.map +1 -1
- package/dist/store/internal/setup.js +41 -1
- package/dist/store/internal/setup.js.map +1 -1
- package/dist/store/internal/write-verify.d.ts +20 -7
- package/dist/store/internal/write-verify.d.ts.map +1 -1
- package/dist/store/internal/write-verify.js +22 -9
- package/dist/store/internal/write-verify.js.map +1 -1
- package/dist/store/types.d.ts +16 -0
- package/dist/store/types.d.ts.map +1 -1
- package/dist/store/vector-backend.d.ts +6 -6
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -207,7 +207,7 @@ When `keyPrefix` is omitted, each adapter defaults to its own sub-prefix under t
|
|
|
207
207
|
|
|
208
208
|
**Gzip compression** — set `compression: { enabled: true }`. Payloads at or above `minSizeBytes` (default 1 KB) are gzipped transparently; decompression auto-detects on read and is guarded against decompression-bomb expansion (`maxDecompressedBytes`, default 50 MiB).
|
|
209
209
|
|
|
210
|
-
**S3 offloading** — set `s3: { bucketName }`. Any serialized payload at or above `thresholdBytes` (default 350 KB) is written to S3, with only a reference stored in DynamoDB; reads rehydrate transparently. Requires the optional `@aws-sdk/client-s3` peer. Deleting a checkpoint thread / chat session also best-effort deletes its offloaded objects. When a `ttl` is also configured, call `ensureS3LifecycleRule()` once (e.g. during deployment) to best-effort install a matching S3 lifecycle expiration rule (logged, never fatal) — this is opt-in rather than automatic, since it requires the broader `s3:PutLifecycleConfiguration` bucket-level permission and is not safe to fire on every adapter construction. If you configure `ttl` + `s3` but never call it, nothing reclaims objects that best-effort cleanup misses
|
|
210
|
+
**S3 offloading** — set `s3: { bucketName }`. Any serialized payload at or above `thresholdBytes` (default 350 KB) is written to S3, with only a reference stored in DynamoDB; reads rehydrate transparently. Requires the optional `@aws-sdk/client-s3` peer. Deleting a checkpoint thread / chat session also best-effort deletes its offloaded objects. When a `ttl` is also configured, call `ensureS3LifecycleRule()` once (e.g. during deployment) to best-effort install a matching S3 lifecycle expiration rule (logged, never fatal) — this is opt-in rather than automatic, since it requires the broader `s3:PutLifecycleConfiguration` bucket-level permission and is not safe to fire on every adapter construction. If you configure `ttl` + `s3` but never call it, nothing reclaims objects that best-effort cleanup misses — they stay in the bucket until you remove them or add a lifecycle rule yourself. Both the store's concurrent-`put` overwrite race and the checkpointer's *special*-write overwrite race (`__error__`, `__interrupt__`, `__resume__`, `__scheduled__`) are now **prevented** by a compare-and-swap: each overwrite pins the previous descriptor it observed and re-reads on rejection, so it deletes exactly the payload it actually superseded instead of racing another writer for the same one. A leak from either path is now possible only in these residual cases, still backstopped by `ensureS3LifecycleRule()`: the bounded compare-and-swap (3 attempts) is exhausted under pathological contention, which falls back to an unconditional overwrite and logs a `warn`; a best-effort delete genuinely fails; or one double-fault interleaving — a write that loses the swap and then exhausts its transient-error retries on an attempt that actually landed — leaves cleanup targeting the stale descriptor rather than the one it truly superseded, orphaning one object (it never deletes a live object). Separately, and unchanged by any of the above, the checkpointer's *regular* (non-special) writes still resolve a genuine race first-write-wins with no compare-and-swap, so the loser's own upload there remains an orphan reclaimed only by best-effort cleanup and `ensureS3LifecycleRule()`.
|
|
211
211
|
|
|
212
212
|
**TTL expiry** — set `ttl: { days }` or `ttl: { seconds }`. The `ttl` attribute is written as a Unix-epoch-seconds timestamp; enable DynamoDB TTL on the `ttl` attribute for automatic deletion. Chat history anchors a single **uniform whole-conversation TTL** on the session's metadata row, shared by every message: normally it's set once, at session creation, via `if_not_exists`; but if the previously-stored anchor is ever found missing or already expired (DynamoDB's own TTL sweep can lag up to ~48h), the next append heals it with a plain overwrite instead of staying stuck. Every message written at any point in time shares whatever the current anchor is; expired messages are also filtered out on read. If the append that triggers a stale-anchor heal is itself later rolled back (a later chunk in the same call failed), the healed ttl is not reverted — the session simply keeps the fresher, never-shorter expiry rather than risk regressing a value a concurrent legitimate extension may have since written; this is a deliberate, self-healing tradeoff, not a bug.
|
|
213
213
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/actions/list.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/actions/list.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAU9F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAyC7D;;;;GAIG;AACH,wBAAuB,eAAe,CACpC,OAAO,EAAE,mBAAmB,EAC5B,MAAM,EAAE,cAAc,EACtB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,cAAc,CAAC,eAAe,CAAC,CAgDjC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.listCheckpoints = listCheckpoints;
|
|
4
|
+
const constants_1 = require("../../shared/constants");
|
|
4
5
|
const paginate_1 = require("../../shared/dynamodb/paginate");
|
|
5
6
|
const assemble_1 = require("../internal/assemble");
|
|
6
7
|
const configurable_1 = require("../internal/configurable");
|
|
@@ -43,6 +44,7 @@ async function* listCheckpoints(context, config, options) {
|
|
|
43
44
|
const { before, filter, limit } = readListOptions(options);
|
|
44
45
|
const params = (0, query_1.beginsWithQuery)(context.tableName, (0, keys_1.partitionKey)(threadId), (0, keys_1.metaSortKeyPrefix)(checkpointNs));
|
|
45
46
|
let yielded = 0;
|
|
47
|
+
let scanned = 0;
|
|
46
48
|
/**
|
|
47
49
|
* Unbounded: this is a generator that streams and never accumulates, and
|
|
48
50
|
* `limit` already returns early, so the caller's own bound is what stops the
|
|
@@ -57,6 +59,11 @@ async function* listCheckpoints(context, config, options) {
|
|
|
57
59
|
maxItems: Number.POSITIVE_INFINITY,
|
|
58
60
|
maxIterations: Number.POSITIVE_INFINITY,
|
|
59
61
|
})) {
|
|
62
|
+
scanned += 1;
|
|
63
|
+
if (scanned === constants_1.LIST_SCAN_WARN_THRESHOLD) {
|
|
64
|
+
context.logger.warn('list: scanned a large number of rows without the caller stopping; this read is ' +
|
|
65
|
+
'deliberately unbounded, so narrow the filter or pass options.limit', { threadId, checkpointNs, scanned });
|
|
66
|
+
}
|
|
60
67
|
if (limit !== undefined && yielded >= limit)
|
|
61
68
|
return;
|
|
62
69
|
const meta = (0, item_reader_1.narrowMetaItem)(raw);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"list.js","sourceRoot":"","sources":["../../../src/checkpointer/actions/list.ts"],"names":[],"mappings":";;;AAGA,6DAA+D;AAC/D,mDAAqD;AACrD,2DAA4D;AAC5D,2DAA2E;AAC3E,yDAAuE;AACvE,2CAAmE;AACnE,6CAAoD;AAIpD,kDAAkD;AAClD,SAAS,eAAe,CAAC,OAA+B;IAKtD,OAAO;QACL,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,aAAmC;QAC1E,MAAM,EAAE,OAAO,EAAE,MAAiD;QAClE,KAAK,EAAE,OAAO,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CACvB,IAAwB,EACxB,YAAgC,EAChC,MAA0B;IAE1B,IAAI,YAAY,KAAK,SAAS,IAAI,IAAI,CAAC,YAAY,KAAK,YAAY;QAAE,OAAO,KAAK,CAAC;IACnF,OAAO,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;AAC5D,CAAC;AAED,uEAAuE;AACvE,KAAK,UAAU,oBAAoB,CACjC,OAA4B,EAC5B,IAAwB,EACxB,MAA+C;IAE/C,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAA,0BAAY,EAAC,OAAO,EAAE,IAAI,CAAC,CAAgC,CAAC;IACpF,OAAO,IAAA,4BAAa,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACI,KAAK,SAAS,CAAC,iBACpB,OAA4B,EAC5B,MAAsB,EACtB,OAA+B;IAE/B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,IAAA,+BAAgB,EAAC,MAAM,CAAC,CAAC;IAC1E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAA,uBAAe,EAC5B,OAAO,CAAC,SAAS,EACjB,IAAA,mBAAY,EAAC,QAAQ,CAAC,EACtB,IAAA,wBAAiB,EAAC,YAAY,CAAC,CAChC,CAAC;IACF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB;;;;;;;OAOG;IACH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAA,wBAAa,EAAC;QACpC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,iBAAiB;QAClC,aAAa,EAAE,MAAM,CAAC,iBAAiB;KACxC,CAAC,EAAE,CAAC;QACH,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,KAAK;YAAE,OAAO;QACpD,MAAM,IAAI,GAAG,IAAA,4BAAc,EAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE;gBAC5E,OAAO,EAAE,GAAG,CAAC,EAAY;aAC1B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC;YAAE,SAAS;QAC5D,IAAI,CAAC,CAAC,MAAM,oBAAoB,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YAAE,SAAS;QACnE,MAAM,KAAK,GAAG,MAAM,IAAA,wBAAa,EAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QACzE,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"list.js","sourceRoot":"","sources":["../../../src/checkpointer/actions/list.ts"],"names":[],"mappings":";;;AAGA,sDAAkE;AAClE,6DAA+D;AAC/D,mDAAqD;AACrD,2DAA4D;AAC5D,2DAA2E;AAC3E,yDAAuE;AACvE,2CAAmE;AACnE,6CAAoD;AAIpD,kDAAkD;AAClD,SAAS,eAAe,CAAC,OAA+B;IAKtD,OAAO;QACL,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,aAAmC;QAC1E,MAAM,EAAE,OAAO,EAAE,MAAiD;QAClE,KAAK,EAAE,OAAO,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CACvB,IAAwB,EACxB,YAAgC,EAChC,MAA0B;IAE1B,IAAI,YAAY,KAAK,SAAS,IAAI,IAAI,CAAC,YAAY,KAAK,YAAY;QAAE,OAAO,KAAK,CAAC;IACnF,OAAO,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;AAC5D,CAAC;AAED,uEAAuE;AACvE,KAAK,UAAU,oBAAoB,CACjC,OAA4B,EAC5B,IAAwB,EACxB,MAA+C;IAE/C,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAA,0BAAY,EAAC,OAAO,EAAE,IAAI,CAAC,CAAgC,CAAC;IACpF,OAAO,IAAA,4BAAa,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACI,KAAK,SAAS,CAAC,iBACpB,OAA4B,EAC5B,MAAsB,EACtB,OAA+B;IAE/B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,IAAA,+BAAgB,EAAC,MAAM,CAAC,CAAC;IAC1E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAA,uBAAe,EAC5B,OAAO,CAAC,SAAS,EACjB,IAAA,mBAAY,EAAC,QAAQ,CAAC,EACtB,IAAA,wBAAiB,EAAC,YAAY,CAAC,CAChC,CAAC;IACF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB;;;;;;;OAOG;IACH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAA,wBAAa,EAAC;QACpC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,iBAAiB;QAClC,aAAa,EAAE,MAAM,CAAC,iBAAiB;KACxC,CAAC,EAAE,CAAC;QACH,OAAO,IAAI,CAAC,CAAC;QACb,IAAI,OAAO,KAAK,oCAAwB,EAAE,CAAC;YACzC,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,iFAAiF;gBAC/E,oEAAoE,EACtE,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CACpC,CAAC;QACJ,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,KAAK;YAAE,OAAO;QACpD,MAAM,IAAI,GAAG,IAAA,4BAAc,EAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE;gBAC5E,OAAO,EAAE,GAAG,CAAC,EAAY;aAC1B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC;YAAE,SAAS;QAC5D,IAAI,CAAC,CAAC,MAAM,oBAAoB,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YAAE,SAAS;QACnE,MAAM,KAAK,GAAG,MAAM,IAAA,wBAAa,EAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QACzE,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"put-writes.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/actions/put-writes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"put-writes.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/actions/put-writes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAWpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AA2E7D;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,mBAAmB,EAC5B,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,YAAY,EAAE,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CA4Bf"}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.putWrites = putWrites;
|
|
4
4
|
const descriptor_keys_1 = require("../../shared/codec/descriptor-keys");
|
|
5
5
|
const orphans_1 = require("../../shared/codec/s3/orphans");
|
|
6
|
+
const conditional_put_1 = require("../../shared/dynamodb/conditional-put");
|
|
6
7
|
const retry_1 = require("../../shared/dynamodb/retry");
|
|
7
8
|
const errors_1 = require("../../shared/errors/errors");
|
|
8
9
|
const ulid_1 = require("../../shared/ulid");
|
|
@@ -46,7 +47,7 @@ async function writeRegularItems(context, items) {
|
|
|
46
47
|
if (result.status === 'fulfilled')
|
|
47
48
|
return;
|
|
48
49
|
const reason = result.reason;
|
|
49
|
-
if ((0,
|
|
50
|
+
if ((0, conditional_put_1.isConditionalCheckFailed)(reason)) {
|
|
50
51
|
(0, write_guard_1.reportGuardRejection)(context, items[index], reason);
|
|
51
52
|
return;
|
|
52
53
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"put-writes.js","sourceRoot":"","sources":["../../../src/checkpointer/actions/put-writes.ts"],"names":[],"mappings":";;;AAGA,wEAAmE;AACnE,2DAAiE;AACjE,uDAAgE;AAChE,uDAA6D;AAC7D,4CAAsD;AACtD,qDAAoE;AACpE,2DAA4D;AAC5D,yDAA0D;AAE1D,6EAAiF;AACjF,uDAAwD;AACxD,
|
|
1
|
+
{"version":3,"file":"put-writes.js","sourceRoot":"","sources":["../../../src/checkpointer/actions/put-writes.ts"],"names":[],"mappings":";;;AAGA,wEAAmE;AACnE,2DAAiE;AACjE,2EAAiF;AACjF,uDAAgE;AAChE,uDAA6D;AAC7D,4CAAsD;AACtD,qDAAoE;AACpE,2DAA4D;AAC5D,yDAA0D;AAE1D,6EAAiF;AACjF,uDAAwD;AACxD,yDAA+D;AAG/D;;;;;;;;GAQG;AACH,MAAM,cAAc,GAAG,IAAA,wBAAiB,GAAE,CAAC;AAE3C,uFAAuF;AACvF,KAAK,UAAU,YAAY,CACzB,OAA4B,EAC5B,KAA4B;IAE5B,IAAI,CAAC,OAAO,CAAC,SAAS;QAAE,OAAO;IAC/B,MAAM,IAAA,0BAAgB,EACpB,OAAO,CAAC,SAAS,EACjB,IAAA,+BAAa,EAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAC9C,WAAW,EACX,OAAO,CAAC,MAAM,CACf,CAAC;AACJ,CAAC;AAWD;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAC9B,OAA4B,EAC5B,KAA4B;IAE5B,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,IAAI,KAAwB,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,IAAA,yBAAiB,EAAC,GAAG,EAAE,CACrB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QACjB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,IAAI,EAAE,IAAI;QACV,mBAAmB,EAAE,0BAA0B;QAC/C,mCAAmC,EAAE,SAAS;KAC/C,CAAC,CACH,CACF,CACF,CAAC;IACF,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO;QAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAe,CAAC;QACtC,IAAI,IAAA,0CAAwB,EAAC,MAAM,CAAC,EAAE,CAAC;YACrC,IAAA,kCAAoB,EAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;YACpD,OAAO;QACT,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1B,KAAK,GAAG,KAAK,IAAI,MAAM,CAAC;IAC1B,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AACD;;;;;;;;;GASG;AACI,KAAK,oBACV,OAA4B,EAC5B,MAAsB,EACtB,MAAsB,EACtB,MAAc;IAEd,IAAA,2BAAc,EAAC,MAAM,CAAC,CAAC;IACvB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,IAAA,+BAAgB,EAAC,MAAM,CAAC,CAAC;IAC1E,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,wBAAe,CAAC,2CAA2C,EAAE,eAAe,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAChC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAA,2BAAqB,EAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClF,MAAM,KAAK,GAAG,MAAM,IAAA,6BAAe,EACjC,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,MAAM,EACN,MAAM,EACN,cAAc,EAAE,EAChB,YAAY,CACb,CAAC;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACxD,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACvD,IAAA,oDAA4B,EAAC,OAAO,EAAE,OAAO,CAAC;QAC9C,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC;KACpC,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,YAAY,IAAI,cAAc,CAAC,KAAK,CAAC;IACxD,IAAI,CAAC,UAAU;QAAE,OAAO;IACxB,MAAM,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,UAAU,CAAC;AACnB,CAAC"}
|
|
@@ -34,6 +34,20 @@ export declare function readMetadata(context: CheckpointerContext, item: Checkpo
|
|
|
34
34
|
* original row and would hand back the later call's value instead. The
|
|
35
35
|
* superseded row stays in DynamoDB and is removed by `deleteThread` or TTL
|
|
36
36
|
* like any other.
|
|
37
|
+
*
|
|
38
|
+
* Identity is `(task, channel, occurrence)`, not `(task, channel)`. Keying on
|
|
39
|
+
* the channel alone treated *any* second row for a channel as a superseding
|
|
40
|
+
* duplicate — including one a retry added at an occurrence no earlier call had
|
|
41
|
+
* ever written, which the write-side guard accepted cleanly. That row was then
|
|
42
|
+
* discarded on read, silently returning fewer values than were written, with
|
|
43
|
+
* `putWrites()` having reported success. For that specific shape — a retry
|
|
44
|
+
* that legitimately emits a channel *more* often than the original call —
|
|
45
|
+
* keeping the occurrence restores the upstream outcome: `MemorySaver` keys
|
|
46
|
+
* first-write-wins on `(taskId, idx)`, so a grown retry keeps both values
|
|
47
|
+
* there too. That is not a claim of parity in general: a retry that reorders
|
|
48
|
+
* or shrinks its writes is still resolved to the earliest call per `(task,
|
|
49
|
+
* channel, occurrence)` as above, which is what keeps an accumulating channel
|
|
50
|
+
* from being double-counted — a case `MemorySaver` does not have to handle.
|
|
37
51
|
*/
|
|
38
52
|
export declare function dropSupersededWrites(items: CheckpointWriteItem[]): CheckpointWriteItem[];
|
|
39
53
|
/** Decode WRITE items into `[taskId, channel, value]` pending-write tuples. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-reader.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,iCAAiC,CAAC;AAGzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEnD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,kBAAkB,GAAG,SAAS,CAM3E;AAED,sDAAsD;AACtD,wBAAsB,cAAc,CAClC,OAAO,EAAE,mBAAmB,EAC5B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAErB;AAED,iDAAiD;AACjD,wBAAsB,YAAY,CAChC,OAAO,EAAE,mBAAmB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,kBAAkB,CAAC,CAE7B;AAED
|
|
1
|
+
{"version":3,"file":"item-reader.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,iCAAiC,CAAC;AAGzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEnD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,kBAAkB,GAAG,SAAS,CAM3E;AAED,sDAAsD;AACtD,wBAAsB,cAAc,CAClC,OAAO,EAAE,mBAAmB,EAC5B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAErB;AAED,iDAAiD;AACjD,wBAAsB,YAAY,CAChC,OAAO,EAAE,mBAAmB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,kBAAkB,CAAC,CAE7B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,mBAAmB,EAAE,GAAG,mBAAmB,EAAE,CAUxF;AAED,+EAA+E;AAC/E,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,KAAK,EAAE,mBAAmB,EAAE,GAC3B,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAQnC"}
|
|
@@ -49,9 +49,23 @@ async function readMetadata(context, item) {
|
|
|
49
49
|
* original row and would hand back the later call's value instead. The
|
|
50
50
|
* superseded row stays in DynamoDB and is removed by `deleteThread` or TTL
|
|
51
51
|
* like any other.
|
|
52
|
+
*
|
|
53
|
+
* Identity is `(task, channel, occurrence)`, not `(task, channel)`. Keying on
|
|
54
|
+
* the channel alone treated *any* second row for a channel as a superseding
|
|
55
|
+
* duplicate — including one a retry added at an occurrence no earlier call had
|
|
56
|
+
* ever written, which the write-side guard accepted cleanly. That row was then
|
|
57
|
+
* discarded on read, silently returning fewer values than were written, with
|
|
58
|
+
* `putWrites()` having reported success. For that specific shape — a retry
|
|
59
|
+
* that legitimately emits a channel *more* often than the original call —
|
|
60
|
+
* keeping the occurrence restores the upstream outcome: `MemorySaver` keys
|
|
61
|
+
* first-write-wins on `(taskId, idx)`, so a grown retry keeps both values
|
|
62
|
+
* there too. That is not a claim of parity in general: a retry that reorders
|
|
63
|
+
* or shrinks its writes is still resolved to the earliest call per `(task,
|
|
64
|
+
* channel, occurrence)` as above, which is what keeps an accumulating channel
|
|
65
|
+
* from being double-counted — a case `MemorySaver` does not have to handle.
|
|
52
66
|
*/
|
|
53
67
|
function dropSupersededWrites(items) {
|
|
54
|
-
const identity = (item) =>
|
|
68
|
+
const identity = (item) => JSON.stringify([item.taskId, item.channel, item.occurrence ?? 0]);
|
|
55
69
|
const earliestGroup = new Map();
|
|
56
70
|
for (const item of items) {
|
|
57
71
|
const id = identity(item);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-reader.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-reader.ts"],"names":[],"mappings":";;;;;;;AAMA,oDAAyD;AAGzD,+CAA0C;AAG1C;;;;GAIG;AACH,wBAA+B,GAAY;IACzC,OAAO,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ;QACzC,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ;QACpC,GAAG,CAAC,QAAQ,KAAK,SAAS;QAC1B,CAAC,CAAE,GAA0B;QAC7B,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,sDAAsD;AAC/C,KAAK,yBACV,OAA4B,EAC5B,IAA2B;IAE3B,OAAO,IAAA,qBAAa,EAAa,IAAI,CAAC,UAAU,EAAE,IAAA,uBAAS,EAAC,OAAO,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,iDAAiD;AAC1C,KAAK,uBACV,OAA4B,EAC5B,IAAwB;IAExB,OAAO,IAAA,qBAAa,EAAqB,IAAI,CAAC,QAAQ,EAAE,IAAA,uBAAS,EAAC,OAAO,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"item-reader.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-reader.ts"],"names":[],"mappings":";;;;;;;AAMA,oDAAyD;AAGzD,+CAA0C;AAG1C;;;;GAIG;AACH,wBAA+B,GAAY;IACzC,OAAO,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ;QACzC,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ;QACpC,GAAG,CAAC,QAAQ,KAAK,SAAS;QAC1B,CAAC,CAAE,GAA0B;QAC7B,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,sDAAsD;AAC/C,KAAK,yBACV,OAA4B,EAC5B,IAA2B;IAE3B,OAAO,IAAA,qBAAa,EAAa,IAAI,CAAC,UAAU,EAAE,IAAA,uBAAS,EAAC,OAAO,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,iDAAiD;AAC1C,KAAK,uBACV,OAA4B,EAC5B,IAAwB;IAExB,OAAO,IAAA,qBAAa,EAAqB,IAAI,CAAC,QAAQ,EAAE,IAAA,uBAAS,EAAC,OAAO,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,8BAAqC,KAA4B;IAC/D,MAAM,QAAQ,GAAG,CAAC,IAAyB,EAAU,EAAE,CACrD,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI;YAAE,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,CAAC;AACvF,CAAC;AAED,+EAA+E;AACxE,KAAK,0BACV,OAA4B,EAC5B,KAA4B;IAE5B,MAAM,IAAI,GAAG,IAAA,uBAAS,EAAC,OAAO,CAAC,CAAC;IAChC,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,MAAM,IAAA,qBAAa,EAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-writer.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAEpG,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAGnD,gDAAgD;AAChD,wBAAgB,SAAS,CAAC,OAAO,EAAE,mBAAmB,GAAG,SAAS,CAEjE;AAOD,sEAAsE;AACtE,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,mBAAmB,EAC5B,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,kBAAkB,EAC5B,kBAAkB,CAAC,EAAE,MAAM,EAC3B,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,qBAAqB,CAAA;CAAE,CAAC,CAwBvE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,YAAY,EAAE,EACtB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,mBAAmB,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"item-writer.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAEpG,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAGnD,gDAAgD;AAChD,wBAAgB,SAAS,CAAC,OAAO,EAAE,mBAAmB,GAAG,SAAS,CAEjE;AAOD,sEAAsE;AACtE,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,mBAAmB,EAC5B,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,kBAAkB,EAC5B,kBAAkB,CAAC,EAAE,MAAM,EAC3B,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,qBAAqB,CAAA;CAAE,CAAC,CAwBvE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,YAAY,EAAE,EACtB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAiChC"}
|
|
@@ -59,7 +59,7 @@ async function buildWriteItems(context, threadId, checkpointNs, checkpointId, ta
|
|
|
59
59
|
const deps = codecDeps(context);
|
|
60
60
|
const pk = (0, keys_1.partitionKey)(threadId);
|
|
61
61
|
const items = [];
|
|
62
|
-
for (const { channel, value, index } of (0, write_index_1.resolveWriteIndices)(writes)) {
|
|
62
|
+
for (const { channel, value, index, occurrence } of (0, write_index_1.resolveWriteIndices)(writes)) {
|
|
63
63
|
/**
|
|
64
64
|
* `channel` is part of the key as well as the index: two channels can
|
|
65
65
|
* share an index (each channel's first occurrence is 0), so without it
|
|
@@ -82,6 +82,7 @@ async function buildWriteItems(context, threadId, checkpointNs, checkpointId, ta
|
|
|
82
82
|
* call legitimately wrote more than once.
|
|
83
83
|
*/
|
|
84
84
|
writeGroup: nonce,
|
|
85
|
+
occurrence,
|
|
85
86
|
value: descriptor,
|
|
86
87
|
};
|
|
87
88
|
items.push(withTtl(item, ttlTimestamp));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"item-writer.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-writer.ts"],"names":[],"mappings":";;;;;AAEA,oDAAyE;AAEzE,iCAAiF;AAEjF,+CAAoD;AAEpD,gDAAgD;AAChD,mBAA0B,OAA4B;IACpD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;AAClG,CAAC;AAED,SAAS,OAAO,CAA6B,IAAO,EAAE,YAAqB;IACzE,IAAI,YAAY,KAAK,SAAS;QAAE,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC;IACxD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sEAAsE;AAC/D,KAAK,+BACV,OAA4B,EAC5B,QAAgB,EAChB,YAAoB,EACpB,UAAsB,EACtB,QAA4B,EAC5B,kBAA2B,EAC3B,YAAqB;IAErB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,IAAA,mBAAY,EAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,oBAAoB,GAAG,MAAM,IAAA,qBAAa,EAAC,UAAU,EAAE,IAAI,EAAE;QACjE,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,YAAY,CAAC;KAChE,CAAC,CAAC;IACH,MAAM,kBAAkB,GAAG,MAAM,IAAA,qBAAa,EAAC,QAAQ,EAAE,IAAI,EAAE;QAC7D,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC;KAC9D,CAAC,CAAC;IACH,MAAM,IAAI,GAAuB;QAC/B,EAAE,EAAE,EAAE;QACN,EAAE,EAAE,IAAA,kBAAW,EAAC,YAAY,EAAE,UAAU,CAAC,EAAE,CAAC;QAC5C,QAAQ;QACR,YAAY;QACZ,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,QAAQ,EAAE,kBAAkB;KAC7B,CAAC;IACF,IAAI,kBAAkB,KAAK,SAAS;QAAE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IACnF,MAAM,OAAO,GAA0B;QACrC,EAAE,EAAE,EAAE;QACN,EAAE,EAAE,IAAA,qBAAc,EAAC,YAAY,EAAE,UAAU,CAAC,EAAE,CAAC;QAC/C,UAAU,EAAE,oBAAoB;KACjC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,0BACV,OAA4B,EAC5B,QAAgB,EAChB,YAAoB,EACpB,YAAoB,EACpB,MAAc,EACd,MAAsB,EACtB,KAAa,EACb,YAAqB;IAErB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,IAAA,mBAAY,EAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,KAAK,GAA0B,EAAE,CAAC;IACxC,KAAK,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,IAAA,iCAAmB,EAAC,MAAM,CAAC,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"item-writer.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/item-writer.ts"],"names":[],"mappings":";;;;;AAEA,oDAAyE;AAEzE,iCAAiF;AAEjF,+CAAoD;AAEpD,gDAAgD;AAChD,mBAA0B,OAA4B;IACpD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;AAClG,CAAC;AAED,SAAS,OAAO,CAA6B,IAAO,EAAE,YAAqB;IACzE,IAAI,YAAY,KAAK,SAAS;QAAE,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC;IACxD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sEAAsE;AAC/D,KAAK,+BACV,OAA4B,EAC5B,QAAgB,EAChB,YAAoB,EACpB,UAAsB,EACtB,QAA4B,EAC5B,kBAA2B,EAC3B,YAAqB;IAErB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,IAAA,mBAAY,EAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,oBAAoB,GAAG,MAAM,IAAA,qBAAa,EAAC,UAAU,EAAE,IAAI,EAAE;QACjE,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,YAAY,CAAC;KAChE,CAAC,CAAC;IACH,MAAM,kBAAkB,GAAG,MAAM,IAAA,qBAAa,EAAC,QAAQ,EAAE,IAAI,EAAE;QAC7D,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC;KAC9D,CAAC,CAAC;IACH,MAAM,IAAI,GAAuB;QAC/B,EAAE,EAAE,EAAE;QACN,EAAE,EAAE,IAAA,kBAAW,EAAC,YAAY,EAAE,UAAU,CAAC,EAAE,CAAC;QAC5C,QAAQ;QACR,YAAY;QACZ,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,QAAQ,EAAE,kBAAkB;KAC7B,CAAC;IACF,IAAI,kBAAkB,KAAK,SAAS;QAAE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IACnF,MAAM,OAAO,GAA0B;QACrC,EAAE,EAAE,EAAE;QACN,EAAE,EAAE,IAAA,qBAAc,EAAC,YAAY,EAAE,UAAU,CAAC,EAAE,CAAC;QAC/C,UAAU,EAAE,oBAAoB;KACjC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,0BACV,OAA4B,EAC5B,QAAgB,EAChB,YAAoB,EACpB,YAAoB,EACpB,MAAc,EACd,MAAsB,EACtB,KAAa,EACb,YAAqB;IAErB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,IAAA,mBAAY,EAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,KAAK,GAA0B,EAAE,CAAC;IACxC,KAAK,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,IAAA,iCAAmB,EAAC,MAAM,CAAC,EAAE,CAAC;QAChF;;;;WAIG;QACH,MAAM,UAAU,GAAG,MAAM,IAAA,qBAAa,EAAC,KAAK,EAAE,IAAI,EAAE;YAClD,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC;SAC3F,CAAC,CAAC;QACH,MAAM,IAAI,GAAwB;YAChC,EAAE,EAAE,EAAE;YACN,EAAE,EAAE,IAAA,mBAAY,EAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC;YACpE,MAAM;YACN,KAAK;YACL,OAAO;YACP;;;;;;eAMG;YACH,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,KAAK,EAAE,UAAU;SAClB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { CheckpointWriteItem } from '../types';
|
|
2
|
+
import type { CheckpointerContext } from './setup';
|
|
3
|
+
import { type SpecialWriteOutcome } from './special-write-verify';
|
|
4
|
+
/**
|
|
5
|
+
* Overwrite one special row, pinned to the `writeGroup` this call observed, and
|
|
6
|
+
* report the descriptor it superseded.
|
|
7
|
+
*
|
|
8
|
+
* Overwriting is correct for special channels — every reference implementation
|
|
9
|
+
* does it — but two concurrent calls to the same channel both read the same
|
|
10
|
+
* previous descriptor and both delete it, orphaning the loser's upload. Pinning
|
|
11
|
+
* the observed `writeGroup` and re-reading on rejection makes each call
|
|
12
|
+
* supersede exactly one payload.
|
|
13
|
+
*
|
|
14
|
+
* `BatchWriteItem` cannot carry conditions, which is why this path issues
|
|
15
|
+
* individual puts; a call holds at most one row per special channel, so that is
|
|
16
|
+
* four writes at worst.
|
|
17
|
+
*
|
|
18
|
+
* The compare-and-swap runs only when an offloader is configured — matching
|
|
19
|
+
* `store/internal/persist.ts` — because without one there is no S3 object to
|
|
20
|
+
* orphan, so a plain unconditional put stays correct and costs no extra
|
|
21
|
+
* ConsistentRead or write capacity. That shortcut is also why the surviving
|
|
22
|
+
* `catch` may report `committed: false` without verifying: with no offloader
|
|
23
|
+
* there is no object for the caller to delete on the strength of it.
|
|
24
|
+
*
|
|
25
|
+
* Never rejects: the caller runs this concurrently with the regular writes
|
|
26
|
+
* under `Promise.all`, whose own cleanup depends on every branch resolving
|
|
27
|
+
* rather than short-circuiting.
|
|
28
|
+
*/
|
|
29
|
+
export declare function writeSpecialItem(context: CheckpointerContext, item: CheckpointWriteItem): Promise<SpecialWriteOutcome>;
|
|
30
|
+
//# sourceMappingURL=special-write-cas.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"special-write-cas.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/special-write-cas.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAIL,KAAK,mBAAmB,EAEzB,MAAM,wBAAwB,CAAC;AAsEhC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,mBAAmB,EAC5B,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,mBAAmB,CAAC,CAoB9B"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.writeSpecialItem = writeSpecialItem;
|
|
4
|
+
const conditional_put_1 = require("../../shared/dynamodb/conditional-put");
|
|
5
|
+
const retry_1 = require("../../shared/dynamodb/retry");
|
|
6
|
+
const special_write_verify_1 = require("./special-write-verify");
|
|
7
|
+
/**
|
|
8
|
+
* Retry a conditional put up to {@link OVERWRITE_CAS_MAX_ATTEMPTS} times,
|
|
9
|
+
* re-reading the row each time a racer's write invalidates the pinned
|
|
10
|
+
* `writeGroup`. Extracted from {@link writeSpecialItem} to keep both
|
|
11
|
+
* functions under the repo's block-nesting limit.
|
|
12
|
+
*
|
|
13
|
+
* A rejection is not proof a competitor won: `withDynamoDBRetry` retries
|
|
14
|
+
* transient errors, so an attempt can commit server-side, its response can be
|
|
15
|
+
* lost, and the retried put can hit the row it just wrote and fail the same
|
|
16
|
+
* guard — indistinguishable from a competitor's win by the rejection alone.
|
|
17
|
+
* Each attempt's pinned observation is captured in `attempted` before the
|
|
18
|
+
* put, so that when {@link verifyAfterFailure} finds the row already holding
|
|
19
|
+
* *this item's own* `writeGroup`, the outcome reports having superseded
|
|
20
|
+
* whatever `attempted` held — never the item's own just-committed payload,
|
|
21
|
+
* which would strand the live row pointing at a deleted object.
|
|
22
|
+
*
|
|
23
|
+
* Only a rejection whose re-read proves some *other* writer holds the row is
|
|
24
|
+
* retried; every other failure is already settled by the verification.
|
|
25
|
+
*/
|
|
26
|
+
async function attemptCasWrites(context, item, initial) {
|
|
27
|
+
let observed = initial;
|
|
28
|
+
for (let attempt = 1; attempt <= conditional_put_1.OVERWRITE_CAS_MAX_ATTEMPTS; attempt++) {
|
|
29
|
+
const attempted = observed;
|
|
30
|
+
try {
|
|
31
|
+
await (0, retry_1.withDynamoDBRetry)(() => context.client.put({
|
|
32
|
+
TableName: context.tableName,
|
|
33
|
+
Item: item,
|
|
34
|
+
...(0, conditional_put_1.revisionGuard)(special_write_verify_1.SPECIAL_REVISION_ATTRIBUTE, attempted),
|
|
35
|
+
}));
|
|
36
|
+
return { done: true, outcome: { committed: true, superseded: attempted.value } };
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
const verified = await (0, special_write_verify_1.verifyAfterFailure)(context, item, attempted, error);
|
|
40
|
+
if (!verified.observed || !(0, conditional_put_1.isConditionalCheckFailed)(error)) {
|
|
41
|
+
return { done: true, outcome: verified.outcome };
|
|
42
|
+
}
|
|
43
|
+
observed = verified.observed;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return { done: false, observed };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Overwrite the row unconditionally once the compare-and-swap budget is spent,
|
|
50
|
+
* verifying rather than assuming if that put fails too.
|
|
51
|
+
*/
|
|
52
|
+
async function overwriteUnconditionally(context, item, observed) {
|
|
53
|
+
try {
|
|
54
|
+
await (0, retry_1.withDynamoDBRetry)(() => context.client.put({ TableName: context.tableName, Item: item }));
|
|
55
|
+
return { committed: true, superseded: observed.value };
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
return (await (0, special_write_verify_1.verifyAfterFailure)(context, item, observed, error)).outcome;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Overwrite one special row, pinned to the `writeGroup` this call observed, and
|
|
63
|
+
* report the descriptor it superseded.
|
|
64
|
+
*
|
|
65
|
+
* Overwriting is correct for special channels — every reference implementation
|
|
66
|
+
* does it — but two concurrent calls to the same channel both read the same
|
|
67
|
+
* previous descriptor and both delete it, orphaning the loser's upload. Pinning
|
|
68
|
+
* the observed `writeGroup` and re-reading on rejection makes each call
|
|
69
|
+
* supersede exactly one payload.
|
|
70
|
+
*
|
|
71
|
+
* `BatchWriteItem` cannot carry conditions, which is why this path issues
|
|
72
|
+
* individual puts; a call holds at most one row per special channel, so that is
|
|
73
|
+
* four writes at worst.
|
|
74
|
+
*
|
|
75
|
+
* The compare-and-swap runs only when an offloader is configured — matching
|
|
76
|
+
* `store/internal/persist.ts` — because without one there is no S3 object to
|
|
77
|
+
* orphan, so a plain unconditional put stays correct and costs no extra
|
|
78
|
+
* ConsistentRead or write capacity. That shortcut is also why the surviving
|
|
79
|
+
* `catch` may report `committed: false` without verifying: with no offloader
|
|
80
|
+
* there is no object for the caller to delete on the strength of it.
|
|
81
|
+
*
|
|
82
|
+
* Never rejects: the caller runs this concurrently with the regular writes
|
|
83
|
+
* under `Promise.all`, whose own cleanup depends on every branch resolving
|
|
84
|
+
* rather than short-circuiting.
|
|
85
|
+
*/
|
|
86
|
+
async function writeSpecialItem(context, item) {
|
|
87
|
+
try {
|
|
88
|
+
if (!context.offloader) {
|
|
89
|
+
await (0, retry_1.withDynamoDBRetry)(() => context.client.put({ TableName: context.tableName, Item: item }));
|
|
90
|
+
return { committed: true };
|
|
91
|
+
}
|
|
92
|
+
const initial = await (0, special_write_verify_1.readSpecialRow)(context, item);
|
|
93
|
+
const attempt = await attemptCasWrites(context, item, initial);
|
|
94
|
+
if (attempt.done)
|
|
95
|
+
return attempt.outcome;
|
|
96
|
+
context.logger.warn('putWrites: special-write compare-and-swap exhausted; overwriting unconditionally, which ' +
|
|
97
|
+
'can orphan one S3 object under a concurrent call (reclaimed by ensureS3LifecycleRule)', { sortKey: item.SK, channel: item.channel, attempts: conditional_put_1.OVERWRITE_CAS_MAX_ATTEMPTS });
|
|
98
|
+
return await overwriteUnconditionally(context, item, attempt.observed);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
return { committed: false, error: error };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=special-write-cas.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"special-write-cas.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/special-write-cas.ts"],"names":[],"mappings":";;;AAAA,2EAI+C;AAC/C,uDAAgE;AAGhE,iEAMgC;AAMhC;;;;;;;;;;;;;;;;;;GAkBG;AACH,KAAK,UAAU,gBAAgB,CAC7B,OAA4B,EAC5B,IAAyB,EACzB,OAAwB;IAExB,IAAI,QAAQ,GAAG,OAAO,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,4CAA0B,EAAE,OAAO,EAAE,EAAE,CAAC;QACvE,MAAM,SAAS,GAAG,QAAQ,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,IAAA,yBAAiB,EAAC,GAAG,EAAE,CAC3B,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,IAAI,EAAE,IAAI;gBACV,GAAG,IAAA,+BAAa,EAAC,iDAA0B,EAAE,SAAS,CAAC;aACxD,CAAC,CACH,CAAC;YACF,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;QACnF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,IAAA,yCAAkB,EAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAc,CAAC,CAAC;YACpF,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,CAAC,IAAA,0CAAwB,EAAC,KAAc,CAAC,EAAE,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnD,CAAC;YACD,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,wBAAwB,CACrC,OAA4B,EAC5B,IAAyB,EACzB,QAAyB;IAEzB,IAAI,CAAC;QACH,MAAM,IAAA,yBAAiB,EAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAChG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,IAAA,yCAAkB,EAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAc,CAAC,CAAC,CAAC,OAAO,CAAC;IACrF,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,KAAK,2BACV,OAA4B,EAC5B,IAAyB;IAEzB,IAAI,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAA,yBAAiB,EAAC,GAAG,EAAE,CAC3B,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CACjE,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC7B,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAA,qCAAc,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,OAAO,CAAC,IAAI;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,0FAA0F;YACxF,uFAAuF,EACzF,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,4CAA0B,EAAE,CAClF,CAAC;QACF,OAAO,MAAM,wBAAwB,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,KAAc,EAAE,CAAC;IACrD,CAAC;AACH,CAAC"}
|
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
import type { CheckpointWriteItem } from '../types';
|
|
2
2
|
import type { CheckpointerContext } from './setup';
|
|
3
3
|
/**
|
|
4
|
-
* Write special (negative-index) items
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
4
|
+
* Write special (negative-index) items, then clean up the correct side of each.
|
|
5
|
+
*
|
|
6
|
+
* Overwrite is correct here, matching every reference checkpointer. Each item
|
|
7
|
+
* is written with a compare-and-swap on its row's `writeGroup` (see
|
|
8
|
+
* {@link writeSpecialItem}) so a concurrent call to the same special channel
|
|
9
|
+
* cannot make both callers delete the same superseded object and orphan one
|
|
10
|
+
* upload. A committed item cleans up the payload it actually superseded; an
|
|
11
|
+
* item confirmed never to have committed cleans up its own new upload.
|
|
12
|
+
*
|
|
13
|
+
* "Confirmed" is load-bearing, and {@link writeSpecialItem} is what earns it:
|
|
14
|
+
* an ambiguous failure is verified against the row and reported as committed
|
|
15
|
+
* unless the read proves otherwise. Deleting on *unknown* would strand a live
|
|
16
|
+
* row pointing at a deleted object; leaking one object instead is recoverable.
|
|
17
|
+
*
|
|
18
|
+
* Never rejects — a failure is reported via the return value, because the
|
|
19
|
+
* caller runs this concurrently with `writeRegularItems` under `Promise.all`,
|
|
20
|
+
* whose own cleanup depends on every branch resolving rather than
|
|
21
|
+
* short-circuiting.
|
|
16
22
|
*/
|
|
17
23
|
export declare function writeSpecialItemsWithCleanup(context: CheckpointerContext, items: CheckpointWriteItem[]): Promise<Error | undefined>;
|
|
18
24
|
//# sourceMappingURL=special-write-cleanup.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"special-write-cleanup.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/special-write-cleanup.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"special-write-cleanup.d.ts","sourceRoot":"","sources":["../../../src/checkpointer/internal/special-write-cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAgBnD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,mBAAmB,EAC5B,KAAK,EAAE,mBAAmB,EAAE,GAC3B,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,CAmB5B"}
|
|
@@ -3,100 +3,45 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.writeSpecialItemsWithCleanup = writeSpecialItemsWithCleanup;
|
|
4
4
|
const descriptor_keys_1 = require("../../shared/codec/descriptor-keys");
|
|
5
5
|
const orphans_1 = require("../../shared/codec/s3/orphans");
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Read each special item's stored descriptor before it's overwritten —
|
|
10
|
-
* mirrors store/actions/put.ts's readExisting. Keyed by SK (unique per
|
|
11
|
-
* special channel per call).
|
|
12
|
-
*/
|
|
13
|
-
async function readPreviousDescriptors(context, items) {
|
|
14
|
-
const previous = new Map();
|
|
15
|
-
for (const item of items) {
|
|
16
|
-
const result = await (0, retry_1.withDynamoDBRetry)(() => context.client.get({
|
|
17
|
-
TableName: context.tableName,
|
|
18
|
-
Key: { PK: item.PK, SK: item.SK },
|
|
19
|
-
ConsistentRead: true,
|
|
20
|
-
ProjectionExpression: '#v',
|
|
21
|
-
ExpressionAttributeNames: { '#v': 'value' },
|
|
22
|
-
}));
|
|
23
|
-
previous.set(item.SK, result.Item?.value);
|
|
24
|
-
}
|
|
25
|
-
return previous;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Split `items` by outcome after a failed batch write (always a
|
|
29
|
-
* BatchWriteAllIncompleteError — batchWriteAll's only throw, and every one of
|
|
30
|
-
* its own failedChunks is in turn always a BatchWriteIncompleteError with an
|
|
31
|
-
* accurate `unprocessed`, whichever of drainUnprocessedWrites's exit paths
|
|
32
|
-
* produced it — see drain-unprocessed.ts): never-committed items are safe to
|
|
33
|
-
* clean up their own upload, every other item its previous one. Uses `.name`,
|
|
34
|
-
* not `instanceof` (banned repo-wide, see base-error.ts).
|
|
35
|
-
*/
|
|
36
|
-
function splitSpecialOutcome(items, error) {
|
|
37
|
-
const neverCommittedSks = new Set();
|
|
38
|
-
for (const failedChunk of error.failedChunks) {
|
|
39
|
-
for (const request of failedChunk.unprocessed) {
|
|
40
|
-
/** Always a PutRequest echoing this submission — special writes never send DeleteRequests. */
|
|
41
|
-
const { SK } = request.PutRequest.Item;
|
|
42
|
-
neverCommittedSks.add(SK);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return {
|
|
46
|
-
committed: items.filter((item) => !neverCommittedSks.has(item.SK)),
|
|
47
|
-
neverCommitted: items.filter((item) => neverCommittedSks.has(item.SK)),
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
/** Best-effort delete S3 objects backing `descriptors`, if an offloader is configured. */
|
|
6
|
+
const special_write_cas_1 = require("./special-write-cas");
|
|
7
|
+
/** Best-effort delete the S3 objects backing `descriptors`, if offloading is on. */
|
|
51
8
|
async function deleteDescriptors(context, descriptors, label) {
|
|
52
9
|
if (!context.offloader)
|
|
53
10
|
return;
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
const descriptors = items
|
|
59
|
-
.map((item) => previous.get(item.SK))
|
|
60
|
-
.filter((descriptor) => descriptor !== undefined);
|
|
61
|
-
await deleteDescriptors(context, descriptors, 'putWrites.special.previous');
|
|
11
|
+
const present = descriptors.filter((d) => d !== undefined);
|
|
12
|
+
if (present.length === 0)
|
|
13
|
+
return;
|
|
14
|
+
await (0, orphans_1.cleanUpS3Orphans)(context.offloader, (0, descriptor_keys_1.collectS3Keys)(present), label, context.logger);
|
|
62
15
|
}
|
|
63
16
|
/**
|
|
64
|
-
* Write special (negative-index) items
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
17
|
+
* Write special (negative-index) items, then clean up the correct side of each.
|
|
18
|
+
*
|
|
19
|
+
* Overwrite is correct here, matching every reference checkpointer. Each item
|
|
20
|
+
* is written with a compare-and-swap on its row's `writeGroup` (see
|
|
21
|
+
* {@link writeSpecialItem}) so a concurrent call to the same special channel
|
|
22
|
+
* cannot make both callers delete the same superseded object and orphan one
|
|
23
|
+
* upload. A committed item cleans up the payload it actually superseded; an
|
|
24
|
+
* item confirmed never to have committed cleans up its own new upload.
|
|
25
|
+
*
|
|
26
|
+
* "Confirmed" is load-bearing, and {@link writeSpecialItem} is what earns it:
|
|
27
|
+
* an ambiguous failure is verified against the row and reported as committed
|
|
28
|
+
* unless the read proves otherwise. Deleting on *unknown* would strand a live
|
|
29
|
+
* row pointing at a deleted object; leaking one object instead is recoverable.
|
|
30
|
+
*
|
|
31
|
+
* Never rejects — a failure is reported via the return value, because the
|
|
32
|
+
* caller runs this concurrently with `writeRegularItems` under `Promise.all`,
|
|
33
|
+
* whose own cleanup depends on every branch resolving rather than
|
|
34
|
+
* short-circuiting.
|
|
76
35
|
*/
|
|
77
36
|
async function writeSpecialItemsWithCleanup(context, items) {
|
|
78
37
|
if (items.length === 0)
|
|
79
38
|
return undefined;
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
return error;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
try {
|
|
91
|
-
await (0, batch_write_1.batchWriteAll)(context.client, context.tableName, items.map((item) => ({ PutRequest: { Item: item } })));
|
|
92
|
-
}
|
|
93
|
-
catch (error) {
|
|
94
|
-
const { committed, neverCommitted } = splitSpecialOutcome(items, error);
|
|
95
|
-
await deleteDescriptors(context, neverCommitted.map((item) => item.value), 'putWrites.special.newUpload');
|
|
96
|
-
await cleanUpPrevious(context, committed, previous);
|
|
97
|
-
return error;
|
|
98
|
-
}
|
|
99
|
-
await cleanUpPrevious(context, items, previous);
|
|
100
|
-
return undefined;
|
|
39
|
+
const outcomes = await Promise.all(items.map(async (item) => [
|
|
40
|
+
item,
|
|
41
|
+
await (0, special_write_cas_1.writeSpecialItem)(context, item),
|
|
42
|
+
]));
|
|
43
|
+
await deleteDescriptors(context, outcomes.filter(([, o]) => o.committed).map(([, o]) => o.superseded), 'putWrites.special.previous');
|
|
44
|
+
await deleteDescriptors(context, outcomes.filter(([, o]) => !o.committed).map(([item]) => item.value), 'putWrites.special.newUpload');
|
|
45
|
+
return outcomes.find(([, o]) => o.error)?.[1].error;
|
|
101
46
|
}
|
|
102
47
|
//# sourceMappingURL=special-write-cleanup.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"special-write-cleanup.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/special-write-cleanup.ts"],"names":[],"mappings":";;;AACA,wEAAmE;AACnE,2DAAiE;
|
|
1
|
+
{"version":3,"file":"special-write-cleanup.js","sourceRoot":"","sources":["../../../src/checkpointer/internal/special-write-cleanup.ts"],"names":[],"mappings":";;;AACA,wEAAmE;AACnE,2DAAiE;AAGjE,2DAAuD;AAGvD,oFAAoF;AACpF,KAAK,UAAU,iBAAiB,CAC9B,OAA4B,EAC5B,WAA8C,EAC9C,KAAa;IAEb,IAAI,CAAC,OAAO,CAAC,SAAS;QAAE,OAAO;IAC/B,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAA0B,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IACnF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACjC,MAAM,IAAA,0BAAgB,EAAC,OAAO,CAAC,SAAS,EAAE,IAAA,+BAAa,EAAC,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC3F,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACI,KAAK,uCACV,OAA4B,EAC5B,KAA4B;IAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAuD,EAAE,CAAC;QAC7E,IAAI;QACJ,MAAM,IAAA,oCAAgB,EAAC,OAAO,EAAE,IAAI,CAAC;KACtC,CAAC,CACH,CAAC;IACF,MAAM,iBAAiB,CACrB,OAAO,EACP,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EACpE,4BAA4B,CAC7B,CAAC;IACF,MAAM,iBAAiB,CACrB,OAAO,EACP,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EACpE,6BAA6B,CAC9B,CAAC;IACF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACtD,CAAC"}
|