@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.1 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1621 -288
- package/dist/backfill/backfill.d.ts +168 -0
- package/dist/backfill/backfill.js +393 -0
- package/dist/checkpointer/actions/delete-thread.d.ts +44 -4
- package/dist/checkpointer/actions/delete-thread.js +56 -22
- package/dist/checkpointer/actions/get-tuple.d.ts +29 -5
- package/dist/checkpointer/actions/get-tuple.js +41 -13
- package/dist/checkpointer/actions/list.d.ts +42 -10
- package/dist/checkpointer/actions/list.js +86 -57
- package/dist/checkpointer/actions/put-writes.d.ts +41 -9
- package/dist/checkpointer/actions/put-writes.js +62 -48
- package/dist/checkpointer/actions/put.d.ts +82 -10
- package/dist/checkpointer/actions/put.js +166 -44
- package/dist/checkpointer/internal/delta-history.d.ts +112 -0
- package/dist/checkpointer/internal/delta-history.js +252 -0
- package/dist/checkpointer/internal/listing.d.ts +149 -0
- package/dist/checkpointer/internal/listing.js +245 -0
- package/dist/checkpointer/internal/parse.d.ts +262 -0
- package/dist/checkpointer/internal/parse.js +372 -0
- package/dist/checkpointer/internal/pending-writes.d.ts +275 -0
- package/dist/checkpointer/internal/pending-writes.js +588 -0
- package/dist/checkpointer/internal/read.d.ts +130 -0
- package/dist/checkpointer/internal/read.js +264 -0
- package/dist/checkpointer/internal/rows.d.ts +571 -0
- package/dist/checkpointer/internal/rows.js +834 -0
- package/dist/checkpointer/internal/setup.d.ts +42 -21
- package/dist/checkpointer/internal/setup.js +65 -31
- package/dist/checkpointer/saver.d.ts +242 -42
- package/dist/checkpointer/saver.js +257 -53
- package/dist/checkpointer/types.d.ts +37 -44
- package/dist/checkpointer/types.js +10 -0
- package/dist/factory/factory.d.ts +120 -11
- package/dist/factory/factory.js +195 -16
- package/dist/factory/types.d.ts +23 -4
- package/dist/factory/types.js +8 -0
- package/dist/history/actions/add-messages.d.ts +30 -2
- package/dist/history/actions/add-messages.js +37 -57
- package/dist/history/actions/clear.d.ts +46 -4
- package/dist/history/actions/clear.js +64 -15
- package/dist/history/actions/get-messages.d.ts +41 -0
- package/dist/history/actions/get-messages.js +94 -32
- package/dist/history/actions/list-sessions.d.ts +52 -10
- package/dist/history/actions/list-sessions.js +141 -38
- package/dist/history/actions/reconcile-count.d.ts +41 -8
- package/dist/history/actions/reconcile-count.js +44 -57
- package/dist/history/chat-message-history.d.ts +200 -38
- package/dist/history/chat-message-history.js +218 -51
- package/dist/history/internal/append.d.ts +212 -0
- package/dist/history/internal/append.js +500 -0
- package/dist/history/internal/message-read.d.ts +84 -0
- package/dist/history/internal/message-read.js +204 -0
- package/dist/history/internal/parse.d.ts +153 -0
- package/dist/history/internal/parse.js +252 -0
- package/dist/history/internal/rows.d.ts +195 -0
- package/dist/history/internal/rows.js +250 -0
- package/dist/history/internal/session.d.ts +331 -0
- package/dist/history/internal/session.js +628 -0
- package/dist/history/internal/setup.d.ts +52 -19
- package/dist/history/internal/setup.js +90 -27
- package/dist/history/session-adapter.d.ts +95 -6
- package/dist/history/session-adapter.js +99 -8
- package/dist/history/types.d.ts +61 -31
- package/dist/history/types.js +10 -0
- package/dist/index.d.ts +38 -14
- package/dist/index.js +31 -12
- package/dist/shared/adapter.d.ts +135 -0
- package/dist/shared/adapter.js +143 -0
- package/dist/shared/clock.d.ts +48 -4
- package/dist/shared/clock.js +51 -4
- package/dist/shared/codec/codec.d.ts +275 -9
- package/dist/shared/codec/codec.js +394 -19
- package/dist/shared/codec/compression.d.ts +43 -6
- package/dist/shared/codec/compression.js +52 -11
- package/dist/shared/codec/json-serde.d.ts +76 -3
- package/dist/shared/codec/json-serde.js +181 -7
- package/dist/shared/codec/s3/client-types.d.ts +20 -1
- package/dist/shared/codec/s3/client-types.js +20 -1
- package/dist/shared/codec/s3/client.d.ts +35 -8
- package/dist/shared/codec/s3/client.js +61 -10
- package/dist/shared/codec/s3/config.d.ts +226 -22
- package/dist/shared/codec/s3/config.js +274 -24
- package/dist/shared/codec/s3/lifecycle.d.ts +163 -11
- package/dist/shared/codec/s3/lifecycle.js +328 -36
- package/dist/shared/codec/s3/offloader.d.ts +390 -19
- package/dist/shared/codec/s3/offloader.js +590 -50
- package/dist/shared/concurrency.d.ts +40 -3
- package/dist/shared/concurrency.js +55 -7
- package/dist/shared/dynamodb/abort.d.ts +45 -7
- package/dist/shared/dynamodb/abort.js +46 -14
- package/dist/shared/dynamodb/batch-write.d.ts +77 -13
- package/dist/shared/dynamodb/batch-write.js +146 -26
- package/dist/shared/dynamodb/cancellation.d.ts +121 -3
- package/dist/shared/dynamodb/cancellation.js +147 -2
- package/dist/shared/dynamodb/client.d.ts +159 -12
- package/dist/shared/dynamodb/client.js +135 -10
- package/dist/shared/dynamodb/idempotent-write.d.ts +551 -0
- package/dist/shared/dynamodb/idempotent-write.js +593 -0
- package/dist/shared/dynamodb/paginate.d.ts +105 -8
- package/dist/shared/dynamodb/paginate.js +175 -6
- package/dist/shared/dynamodb/partition-delete.d.ts +181 -16
- package/dist/shared/dynamodb/partition-delete.js +312 -43
- package/dist/shared/dynamodb/recency-index.d.ts +231 -0
- package/dist/shared/dynamodb/recency-index.js +377 -0
- package/dist/shared/dynamodb/retry.d.ts +265 -10
- package/dist/shared/dynamodb/retry.js +426 -20
- package/dist/shared/dynamodb/table-schema.d.ts +190 -0
- package/dist/shared/dynamodb/table-schema.js +209 -0
- package/dist/shared/errors/base-error.d.ts +174 -8
- package/dist/shared/errors/base-error.js +154 -7
- package/dist/shared/errors/boundary.d.ts +63 -10
- package/dist/shared/errors/boundary.js +110 -13
- package/dist/shared/errors/classify.d.ts +97 -0
- package/dist/shared/errors/classify.js +257 -0
- package/dist/shared/errors/error-code.d.ts +76 -1
- package/dist/shared/errors/error-code.js +83 -1
- package/dist/shared/errors/errors.d.ts +157 -58
- package/dist/shared/errors/errors.js +218 -93
- package/dist/shared/logging/logger.d.ts +63 -1
- package/dist/shared/logging/logger.js +97 -2
- package/dist/shared/logging/redaction.d.ts +86 -6
- package/dist/shared/logging/redaction.js +265 -24
- package/dist/shared/logging/secret-patterns.d.ts +136 -24
- package/dist/shared/logging/secret-patterns.js +165 -30
- package/dist/shared/logging/truncate.d.ts +197 -0
- package/dist/shared/logging/truncate.js +231 -0
- package/dist/shared/options.d.ts +52 -5
- package/dist/shared/options.js +9 -0
- package/dist/shared/ulid.d.ts +71 -14
- package/dist/shared/ulid.js +77 -14
- package/dist/shared/validation/collaborators.d.ts +141 -0
- package/dist/shared/validation/collaborators.js +188 -0
- package/dist/shared/validation/option-shape.d.ts +89 -0
- package/dist/shared/validation/option-shape.js +113 -0
- package/dist/shared/validation/options.d.ts +142 -5
- package/dist/shared/validation/options.js +284 -46
- package/dist/shared/validation/primitives.d.ts +287 -24
- package/dist/shared/validation/primitives.js +347 -42
- package/dist/shared/validation/ttl.d.ts +61 -12
- package/dist/shared/validation/ttl.js +102 -22
- package/dist/store/actions/list-namespaces.d.ts +76 -5
- package/dist/store/actions/list-namespaces.js +160 -22
- package/dist/store/actions/put.d.ts +33 -7
- package/dist/store/actions/put.js +53 -71
- package/dist/store/actions/reconcile-vector-index.d.ts +28 -8
- package/dist/store/actions/reconcile-vector-index.js +33 -13
- package/dist/store/actions/search.d.ts +34 -5
- package/dist/store/actions/search.js +52 -20
- package/dist/store/internal/batch-plan.d.ts +24 -9
- package/dist/store/internal/batch-plan.js +91 -34
- package/dist/store/internal/filter.d.ts +36 -2
- package/dist/store/internal/filter.js +66 -14
- package/dist/store/internal/get-item.d.ts +45 -0
- package/dist/store/internal/get-item.js +115 -0
- package/dist/store/internal/item-write.d.ts +230 -0
- package/dist/store/internal/item-write.js +463 -0
- package/dist/store/internal/parse.d.ts +225 -0
- package/dist/store/internal/parse.js +350 -0
- package/dist/store/internal/rows.d.ts +355 -0
- package/dist/store/internal/rows.js +447 -0
- package/dist/store/internal/semantic-search.d.ts +146 -12
- package/dist/store/internal/semantic-search.js +317 -15
- package/dist/store/internal/setup.d.ts +86 -21
- package/dist/store/internal/setup.js +209 -26
- package/dist/store/internal/table-search.d.ts +100 -0
- package/dist/store/internal/table-search.js +213 -0
- package/dist/store/internal/vector-index.d.ts +247 -0
- package/dist/store/internal/vector-index.js +546 -0
- package/dist/store/store.d.ts +258 -33
- package/dist/store/store.js +309 -53
- package/dist/store/types.d.ts +74 -28
- package/dist/store/types.js +13 -0
- package/dist/store/vector-backend.d.ts +64 -3
- package/dist/store/vector-backend.js +15 -0
- package/package.json +41 -25
- package/dist/checkpointer/internal/assemble.d.ts +0 -17
- package/dist/checkpointer/internal/assemble.js +0 -38
- package/dist/checkpointer/internal/checkpoint-write-verify.d.ts +0 -24
- package/dist/checkpointer/internal/checkpoint-write-verify.js +0 -67
- package/dist/checkpointer/internal/configurable.d.ts +0 -12
- package/dist/checkpointer/internal/configurable.js +0 -24
- package/dist/checkpointer/internal/fetch.d.ts +0 -19
- package/dist/checkpointer/internal/fetch.js +0 -88
- package/dist/checkpointer/internal/filter-match.d.ts +0 -13
- package/dist/checkpointer/internal/filter-match.js +0 -15
- package/dist/checkpointer/internal/item-reader.d.ts +0 -66
- package/dist/checkpointer/internal/item-reader.js +0 -113
- package/dist/checkpointer/internal/item-writer.d.ts +0 -35
- package/dist/checkpointer/internal/item-writer.js +0 -107
- package/dist/checkpointer/internal/keys.d.ts +0 -41
- package/dist/checkpointer/internal/keys.js +0 -117
- package/dist/checkpointer/internal/list-scope.d.ts +0 -61
- package/dist/checkpointer/internal/list-scope.js +0 -108
- package/dist/checkpointer/internal/pending-sends.d.ts +0 -13
- package/dist/checkpointer/internal/pending-sends.js +0 -29
- package/dist/checkpointer/internal/query.d.ts +0 -21
- package/dist/checkpointer/internal/query.js +0 -42
- package/dist/checkpointer/internal/regular-write.d.ts +0 -28
- package/dist/checkpointer/internal/regular-write.js +0 -66
- package/dist/checkpointer/internal/special-write-cas.d.ts +0 -29
- package/dist/checkpointer/internal/special-write-cas.js +0 -104
- package/dist/checkpointer/internal/special-write-cleanup.d.ts +0 -23
- package/dist/checkpointer/internal/special-write-cleanup.js +0 -50
- package/dist/checkpointer/internal/special-write-verify.d.ts +0 -53
- package/dist/checkpointer/internal/special-write-verify.js +0 -80
- package/dist/checkpointer/internal/stored-channels.d.ts +0 -31
- package/dist/checkpointer/internal/stored-channels.js +0 -70
- package/dist/checkpointer/internal/validation.d.ts +0 -18
- package/dist/checkpointer/internal/validation.js +0 -40
- package/dist/checkpointer/internal/write-guard.d.ts +0 -20
- package/dist/checkpointer/internal/write-guard.js +0 -48
- package/dist/checkpointer/internal/write-index.d.ts +0 -36
- package/dist/checkpointer/internal/write-index.js +0 -41
- package/dist/history/internal/append-saga.d.ts +0 -25
- package/dist/history/internal/append-saga.js +0 -79
- package/dist/history/internal/compensation.d.ts +0 -25
- package/dist/history/internal/compensation.js +0 -92
- package/dist/history/internal/item-mapper.d.ts +0 -9
- package/dist/history/internal/item-mapper.js +0 -27
- package/dist/history/internal/keys.d.ts +0 -16
- package/dist/history/internal/keys.js +0 -48
- package/dist/history/internal/message-chunker.d.ts +0 -13
- package/dist/history/internal/message-chunker.js +0 -67
- package/dist/history/internal/message-transaction.d.ts +0 -25
- package/dist/history/internal/message-transaction.js +0 -61
- package/dist/history/internal/message-window.d.ts +0 -17
- package/dist/history/internal/message-window.js +0 -49
- package/dist/history/internal/query.d.ts +0 -23
- package/dist/history/internal/query.js +0 -39
- package/dist/history/internal/session-count.d.ts +0 -47
- package/dist/history/internal/session-count.js +0 -115
- package/dist/history/internal/session-title.d.ts +0 -19
- package/dist/history/internal/session-title.js +0 -43
- package/dist/history/internal/session-update.d.ts +0 -27
- package/dist/history/internal/session-update.js +0 -69
- package/dist/history/internal/title-generator.d.ts +0 -12
- package/dist/history/internal/title-generator.js +0 -41
- package/dist/history/internal/ttl-anchor.d.ts +0 -24
- package/dist/history/internal/ttl-anchor.js +0 -38
- package/dist/history/internal/validation.d.ts +0 -25
- package/dist/history/internal/validation.js +0 -53
- package/dist/shared/codec/descriptor-keys.d.ts +0 -12
- package/dist/shared/codec/descriptor-keys.js +0 -14
- package/dist/shared/codec/encode.d.ts +0 -13
- package/dist/shared/codec/encode.js +0 -43
- package/dist/shared/codec/payload-loss.d.ts +0 -12
- package/dist/shared/codec/payload-loss.js +0 -38
- package/dist/shared/codec/s3/adapter-config.d.ts +0 -13
- package/dist/shared/codec/s3/adapter-config.js +0 -22
- package/dist/shared/codec/s3/bounded-body.d.ts +0 -14
- package/dist/shared/codec/s3/bounded-body.js +0 -47
- package/dist/shared/codec/s3/delete.d.ts +0 -7
- package/dist/shared/codec/s3/delete.js +0 -28
- package/dist/shared/codec/s3/key-scope.d.ts +0 -23
- package/dist/shared/codec/s3/key-scope.js +0 -44
- package/dist/shared/codec/s3/orphans.d.ts +0 -23
- package/dist/shared/codec/s3/orphans.js +0 -71
- package/dist/shared/codec/s3/read-write.d.ts +0 -18
- package/dist/shared/codec/s3/read-write.js +0 -51
- package/dist/shared/codec/s3/retry.d.ts +0 -5
- package/dist/shared/codec/s3/retry.js +0 -22
- package/dist/shared/constants.d.ts +0 -112
- package/dist/shared/constants.js +0 -115
- package/dist/shared/dynamodb/backoff.d.ts +0 -15
- package/dist/shared/dynamodb/backoff.js +0 -48
- package/dist/shared/dynamodb/conditional-put.d.ts +0 -67
- package/dist/shared/dynamodb/conditional-put.js +0 -76
- package/dist/shared/dynamodb/drain-unprocessed.d.ts +0 -21
- package/dist/shared/dynamodb/drain-unprocessed.js +0 -43
- package/dist/shared/dynamodb/expiry.d.ts +0 -16
- package/dist/shared/dynamodb/expiry.js +0 -29
- package/dist/shared/dynamodb/paginate-core.d.ts +0 -24
- package/dist/shared/dynamodb/paginate-core.js +0 -82
- package/dist/shared/dynamodb/retry-classifier.d.ts +0 -17
- package/dist/shared/dynamodb/retry-classifier.js +0 -118
- package/dist/shared/dynamodb/retry-policy.d.ts +0 -26
- package/dist/shared/dynamodb/retry-policy.js +0 -22
- package/dist/shared/dynamodb/scan.d.ts +0 -14
- package/dist/shared/dynamodb/scan.js +0 -19
- package/dist/shared/dynamodb/types.d.ts +0 -23
- package/dist/shared/dynamodb/types.js +0 -2
- package/dist/shared/errors/upstream-error.d.ts +0 -16
- package/dist/shared/errors/upstream-error.js +0 -27
- package/dist/shared/errors/wrap-error.d.ts +0 -7
- package/dist/shared/errors/wrap-error.js +0 -15
- package/dist/shared/logging/redaction-walk.d.ts +0 -23
- package/dist/shared/logging/redaction-walk.js +0 -112
- package/dist/store/actions/get.d.ts +0 -13
- package/dist/store/actions/get.js +0 -73
- package/dist/store/internal/backend-search.d.ts +0 -4
- package/dist/store/internal/backend-search.js +0 -74
- package/dist/store/internal/candidates.d.ts +0 -23
- package/dist/store/internal/candidates.js +0 -94
- package/dist/store/internal/index-reconcile.d.ts +0 -21
- package/dist/store/internal/index-reconcile.js +0 -118
- package/dist/store/internal/index-sync.d.ts +0 -10
- package/dist/store/internal/index-sync.js +0 -25
- package/dist/store/internal/item-mapper.d.ts +0 -27
- package/dist/store/internal/item-mapper.js +0 -61
- package/dist/store/internal/keys.d.ts +0 -17
- package/dist/store/internal/keys.js +0 -41
- package/dist/store/internal/namespace-match.d.ts +0 -11
- package/dist/store/internal/namespace-match.js +0 -40
- package/dist/store/internal/option-validation.d.ts +0 -13
- package/dist/store/internal/option-validation.js +0 -77
- package/dist/store/internal/overwrite-swap.d.ts +0 -32
- package/dist/store/internal/overwrite-swap.js +0 -68
- package/dist/store/internal/persist.d.ts +0 -28
- package/dist/store/internal/persist.js +0 -64
- package/dist/store/internal/query.d.ts +0 -11
- package/dist/store/internal/query.js +0 -48
- package/dist/store/internal/ranker.d.ts +0 -18
- package/dist/store/internal/ranker.js +0 -39
- package/dist/store/internal/read-existing.d.ts +0 -23
- package/dist/store/internal/read-existing.js +0 -41
- package/dist/store/internal/score-direction.d.ts +0 -31
- package/dist/store/internal/score-direction.js +0 -38
- package/dist/store/internal/search-filter.d.ts +0 -3
- package/dist/store/internal/search-filter.js +0 -10
- package/dist/store/internal/text-path.d.ts +0 -17
- package/dist/store/internal/text-path.js +0 -147
- package/dist/store/internal/validation.d.ts +0 -18
- package/dist/store/internal/validation.js +0 -51
- package/dist/store/internal/write-verify.d.ts +0 -42
- package/dist/store/internal/write-verify.js +0 -71
|
@@ -1,96 +1,78 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Hides the order a put happens in.
|
|
4
|
+
*
|
|
5
|
+
* A put reads the row it replaces for its `createdAt`, embeds once — a vector
|
|
6
|
+
* per extracted path onto the row, or one vector for the `vectorBackend`,
|
|
7
|
+
* never both — writes the row, and only then syncs the backend, best-effort.
|
|
8
|
+
* A `null` value takes the delete path instead. A caller hands over an item;
|
|
9
|
+
* that the table commits first and the vector copy follows is decided here.
|
|
10
|
+
*/
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.putItem = putItem;
|
|
4
13
|
const node_crypto_1 = require("node:crypto");
|
|
5
14
|
const clock_1 = require("../../shared/clock");
|
|
6
|
-
const descriptor_keys_1 = require("../../shared/codec/descriptor-keys");
|
|
7
|
-
const orphans_1 = require("../../shared/codec/s3/orphans");
|
|
8
|
-
const retry_1 = require("../../shared/dynamodb/retry");
|
|
9
15
|
const ttl_1 = require("../../shared/validation/ttl");
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const keys_1 = require("../internal/keys");
|
|
13
|
-
const persist_1 = require("../internal/persist");
|
|
14
|
-
const read_existing_1 = require("../internal/read-existing");
|
|
16
|
+
const item_write_1 = require("../internal/item-write");
|
|
17
|
+
const rows_1 = require("../internal/rows");
|
|
15
18
|
const semantic_search_1 = require("../internal/semantic-search");
|
|
16
|
-
const
|
|
17
|
-
const write_verify_1 = require("../internal/write-verify");
|
|
19
|
+
const vector_index_1 = require("../internal/vector-index");
|
|
18
20
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* so S3 cleanup targets the descriptor that was actually removed rather than
|
|
23
|
-
* one read a moment earlier — a concurrent put between a pre-read and the
|
|
24
|
-
* delete used to leave its just-written object orphaned — and the delete costs
|
|
25
|
-
* one request instead of two.
|
|
26
|
-
*
|
|
27
|
-
* A retry-exhausted failure is *ambiguous*: the delete may have landed
|
|
28
|
-
* server-side with only its acknowledgement lost. Mirroring `persistRecord`,
|
|
29
|
-
* that case is resolved with a strongly-consistent read — if the row is gone
|
|
30
|
-
* the delete succeeded and the vector-backend delete must still run. The
|
|
31
|
-
* removed descriptor travelled with the lost response, so that one object, if
|
|
32
|
-
* any, is left to the lifecycle rule. Any other failure, or a row still
|
|
33
|
-
* present, propagates unchanged.
|
|
21
|
+
* The vectors a put stores on the row: one per extracted path, scored by best
|
|
22
|
+
* match on read. Not computed when a `vectorBackend` holds the vectors, which
|
|
23
|
+
* takes a single vector per item instead (see {@link itemVector}).
|
|
34
24
|
*/
|
|
35
|
-
async function
|
|
36
|
-
let removed;
|
|
37
|
-
try {
|
|
38
|
-
const result = await (0, retry_1.withDynamoDBRetry)(() => context.client.delete({
|
|
39
|
-
TableName: context.tableName,
|
|
40
|
-
Key: { PK: pk, SK: sk },
|
|
41
|
-
ReturnValues: 'ALL_OLD',
|
|
42
|
-
}), context.retry);
|
|
43
|
-
removed = result.Attributes?.value;
|
|
44
|
-
}
|
|
45
|
-
catch (error) {
|
|
46
|
-
const landed = (0, write_verify_1.isRetryExhausted)(error) && (await (0, write_verify_1.rowIsAbsent)(context, { PK: pk, SK: sk }));
|
|
47
|
-
if (!landed)
|
|
48
|
-
throw error;
|
|
49
|
-
}
|
|
50
|
-
if (context.offloader && removed) {
|
|
51
|
-
await (0, orphans_1.cleanUpS3Orphans)(context.offloader, (0, descriptor_keys_1.collectS3Keys)([removed]), 'store.delete', context.logger, {
|
|
52
|
-
scope: [...op.namespace, op.key],
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
|
-
if (context.vectorBackend) {
|
|
56
|
-
await (0, index_sync_1.syncVectorIndex)(context.vectorBackend, op.namespace, op.key, undefined, context.logger);
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
/** Compute the embedding for a put, honoring `op.index` (false disables it). */
|
|
60
|
-
async function resolveEmbedding(context, op, value) {
|
|
25
|
+
async function resolvePassages(context, op, value) {
|
|
61
26
|
if (op.index === false)
|
|
62
27
|
return undefined;
|
|
63
|
-
return (0, semantic_search_1.
|
|
28
|
+
return (0, semantic_search_1.embedPassages)(context, value, op.index);
|
|
64
29
|
}
|
|
65
30
|
/**
|
|
66
|
-
* Store, update
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
31
|
+
* Store, update or delete an item.
|
|
32
|
+
*
|
|
33
|
+
* Accepts: `op` — parsed; a `ParsedDelete` removes the item, a `ParsedPut` is
|
|
34
|
+
* stored, its value encoded with optional compression and S3 offload under
|
|
35
|
+
* this row's own path, in an object named by this put's own id. `op.index` —
|
|
36
|
+
* `false` stores the item without indexing it and clears any vector it had, an
|
|
37
|
+
* array overrides the configured fields for this put, and absent uses the
|
|
38
|
+
* store's configuration.
|
|
39
|
+
*
|
|
40
|
+
* Returns: nothing. Deleting an item that is not there is not an error.
|
|
41
|
+
*
|
|
42
|
+
* Throws: `VALIDATION` naming `value` for a value that JSON cannot represent —
|
|
43
|
+
* refused at the write rather than stored as a row that can never be read back;
|
|
44
|
+
* `S3_OFFLOAD_FAILED`; whatever the write throws.
|
|
45
|
+
*
|
|
46
|
+
* Guarantees: DynamoDB holds the canonical item — the vector index is synced
|
|
47
|
+
* afterwards and best-effort, so a backend outage never fails a put or leaves a
|
|
48
|
+
* half-written item. `createdAt` survives every update. The superseded payload
|
|
49
|
+
* is deleted only once the new row is committed, and this put's own upload
|
|
50
|
+
* only once a read proves its write did not land. Neither release reads the
|
|
51
|
+
* row first: each put uploads under a key ending in an id of its own, so the
|
|
52
|
+
* object a put uploads is named only by that put's own rows.
|
|
71
53
|
*/
|
|
72
54
|
async function putItem(context, op) {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const sk = (0, keys_1.sortKey)(op.namespace, op.key);
|
|
76
|
-
if (op.value === null) {
|
|
77
|
-
await deleteStoreItem(context, op, pk, sk);
|
|
55
|
+
if (op.kind === 'delete') {
|
|
56
|
+
await (0, item_write_1.deleteStoreItem)(context, op.address);
|
|
78
57
|
return;
|
|
79
58
|
}
|
|
59
|
+
const { namespace, key } = op.address;
|
|
80
60
|
const value = op.value;
|
|
81
61
|
const timestamp = (0, clock_1.nowIso)();
|
|
82
|
-
const existing = await (0,
|
|
83
|
-
|
|
62
|
+
const existing = await (0, rows_1.readExisting)(context, (0, rows_1.itemRowKey)(op.address));
|
|
63
|
+
// The two indexing modes are exclusive, so only one of them embeds: the row
|
|
64
|
+
// carries a vector per extracted path, while a configured backend takes one
|
|
65
|
+
// vector per item because that is what its `upsert` contract addresses.
|
|
66
|
+
const embedding = await (0, vector_index_1.itemVector)(context, op);
|
|
67
|
+
const embeddings = context.vectorBackend ? undefined : await resolvePassages(context, op, value);
|
|
84
68
|
const ttlTimestamp = context.ttl ? (0, ttl_1.calculateTtlTimestamp)(context.ttl) : undefined;
|
|
85
|
-
const record = await (0,
|
|
69
|
+
const record = await (0, rows_1.buildStoreRow)(context, { namespace, key }, value, {
|
|
86
70
|
createdAt: existing.createdAt ?? timestamp,
|
|
87
71
|
updatedAt: timestamp,
|
|
88
|
-
|
|
72
|
+
embeddings,
|
|
89
73
|
ttlTimestamp,
|
|
90
|
-
|
|
74
|
+
rev: (0, node_crypto_1.randomUUID)(),
|
|
91
75
|
});
|
|
92
|
-
await (0,
|
|
93
|
-
|
|
94
|
-
await (0, index_sync_1.syncVectorIndex)(context.vectorBackend, op.namespace, op.key, embedding, context.logger);
|
|
95
|
-
}
|
|
76
|
+
await (0, item_write_1.persistRow)(context, record, existing);
|
|
77
|
+
await (0, vector_index_1.syncItemVector)(context, op.address, embedding);
|
|
96
78
|
}
|
|
@@ -1,17 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hides what a reconcile refuses before it touches the backend.
|
|
3
|
+
*
|
|
4
|
+
* A repair is scoped to one partition's items, so its prefix must be a
|
|
5
|
+
* non-empty namespace, and without an `index` and a `vectorBackend` there is
|
|
6
|
+
* nothing to repair; both are refused here, before any read. How the backend
|
|
7
|
+
* is then made to agree with the table belongs to the vector index, so the
|
|
8
|
+
* store's method knows neither half.
|
|
9
|
+
*/
|
|
1
10
|
import type { StoreContext } from '../internal/setup';
|
|
2
|
-
|
|
3
|
-
export interface VectorReconcileResult {
|
|
4
|
-
upserted: number;
|
|
5
|
-
pruned: number;
|
|
6
|
-
}
|
|
11
|
+
import type { VectorReconcileResult } from '../types';
|
|
7
12
|
/**
|
|
8
13
|
* Repair the vector backend against the canonical DynamoDB items under
|
|
9
14
|
* `namespacePrefix`: re-push every live embedding, and — when the backend
|
|
10
15
|
* implements {@link VectorBackend.listKeys} — prune vectors with no canonical
|
|
11
16
|
* item. A maintenance tool for backend drift; run it when the namespace is
|
|
12
|
-
* idle.
|
|
13
|
-
*
|
|
14
|
-
* `
|
|
17
|
+
* idle.
|
|
18
|
+
*
|
|
19
|
+
* Accepts: `namespacePrefix` — a non-empty namespace, since the repair is
|
|
20
|
+
* scoped to one partition's items. `options.signal` — aborts between pages.
|
|
21
|
+
*
|
|
22
|
+
* Returns: how many vectors were upserted and how many pruned.
|
|
23
|
+
*
|
|
24
|
+
* Throws: `VALIDATION` naming `namespacePrefix` or `namespacePrefix element`
|
|
25
|
+
* for an empty or malformed prefix, as `search` names it, and `vectorBackend`
|
|
26
|
+
* when the store has no index or backend to reconcile;
|
|
27
|
+
* `RESULT_TRUNCATED` past `maxScanItems`, since repairing from a
|
|
28
|
+
* partial view would prune live vectors; whatever the reads, the embeddings
|
|
29
|
+
* model and the backend throw.
|
|
30
|
+
*
|
|
31
|
+
* Guarantees: re-embeds with the store's configured fields, so a per-put
|
|
32
|
+
* `index` field override is not reproduced — a reconcile makes the backend
|
|
33
|
+
* agree with the store's configuration, not with each item's write-time one.
|
|
34
|
+
* DynamoDB is never written: only the backend is repaired.
|
|
15
35
|
*/
|
|
16
36
|
export declare function reconcileVectorIndex(context: StoreContext, namespacePrefix: string[], options?: {
|
|
17
37
|
signal?: AbortSignal;
|
|
@@ -1,26 +1,46 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Hides what a reconcile refuses before it touches the backend.
|
|
4
|
+
*
|
|
5
|
+
* A repair is scoped to one partition's items, so its prefix must be a
|
|
6
|
+
* non-empty namespace, and without an `index` and a `vectorBackend` there is
|
|
7
|
+
* nothing to repair; both are refused here, before any read. How the backend
|
|
8
|
+
* is then made to agree with the table belongs to the vector index, so the
|
|
9
|
+
* store's method knows neither half.
|
|
10
|
+
*/
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.reconcileVectorIndex = reconcileVectorIndex;
|
|
4
13
|
const errors_1 = require("../../shared/errors/errors");
|
|
5
|
-
const
|
|
6
|
-
const
|
|
14
|
+
const parse_1 = require("../internal/parse");
|
|
15
|
+
const vector_index_1 = require("../internal/vector-index");
|
|
7
16
|
/**
|
|
8
17
|
* Repair the vector backend against the canonical DynamoDB items under
|
|
9
18
|
* `namespacePrefix`: re-push every live embedding, and — when the backend
|
|
10
19
|
* implements {@link VectorBackend.listKeys} — prune vectors with no canonical
|
|
11
20
|
* item. A maintenance tool for backend drift; run it when the namespace is
|
|
12
|
-
* idle.
|
|
13
|
-
*
|
|
14
|
-
* `
|
|
21
|
+
* idle.
|
|
22
|
+
*
|
|
23
|
+
* Accepts: `namespacePrefix` — a non-empty namespace, since the repair is
|
|
24
|
+
* scoped to one partition's items. `options.signal` — aborts between pages.
|
|
25
|
+
*
|
|
26
|
+
* Returns: how many vectors were upserted and how many pruned.
|
|
27
|
+
*
|
|
28
|
+
* Throws: `VALIDATION` naming `namespacePrefix` or `namespacePrefix element`
|
|
29
|
+
* for an empty or malformed prefix, as `search` names it, and `vectorBackend`
|
|
30
|
+
* when the store has no index or backend to reconcile;
|
|
31
|
+
* `RESULT_TRUNCATED` past `maxScanItems`, since repairing from a
|
|
32
|
+
* partial view would prune live vectors; whatever the reads, the embeddings
|
|
33
|
+
* model and the backend throw.
|
|
34
|
+
*
|
|
35
|
+
* Guarantees: re-embeds with the store's configured fields, so a per-put
|
|
36
|
+
* `index` field override is not reproduced — a reconcile makes the backend
|
|
37
|
+
* agree with the store's configuration, not with each item's write-time one.
|
|
38
|
+
* DynamoDB is never written: only the backend is repaired.
|
|
15
39
|
*/
|
|
16
40
|
async function reconcileVectorIndex(context, namespacePrefix, options = {}) {
|
|
17
|
-
(0,
|
|
18
|
-
if (!
|
|
19
|
-
throw
|
|
41
|
+
const prefix = (0, parse_1.parseNamespace)(namespacePrefix, 'namespacePrefix');
|
|
42
|
+
if (!(0, vector_index_1.hasVectorBackend)(context)) {
|
|
43
|
+
throw (0, errors_1.validationError)('reconcileVectorIndex requires a configured index and vectorBackend', 'vectorBackend');
|
|
20
44
|
}
|
|
21
|
-
|
|
22
|
-
const targets = await (0, index_reconcile_1.collectReconcileTargets)(context, namespacePrefix, options.signal);
|
|
23
|
-
const upserted = await (0, index_reconcile_1.pushEmbeddings)(backend, targets);
|
|
24
|
-
const pruned = await (0, index_reconcile_1.pruneOrphans)(context, backend, namespacePrefix, targets);
|
|
25
|
-
return { upserted, pruned };
|
|
45
|
+
return (0, vector_index_1.reconcileVectors)(context, prefix, options.signal);
|
|
26
46
|
}
|
|
@@ -1,9 +1,38 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Hides which of three paths serves a search.
|
|
3
|
+
*
|
|
4
|
+
* A query with a `vectorBackend` is answered by the backend; no query, or no
|
|
5
|
+
* `index` to embed one with, is a page read from the table that stops once
|
|
6
|
+
* full; anything else is ranked in memory over at most `maxSearchCandidates`.
|
|
7
|
+
* A page of zero is answered before any of them. A caller gets the same shape
|
|
8
|
+
* of page whichever path ran, scored exactly when a query could be ranked.
|
|
9
|
+
*/
|
|
10
|
+
import type { SearchItem } from '@langchain/langgraph-checkpoint';
|
|
11
|
+
import type { ParsedSearch } from '../internal/parse';
|
|
2
12
|
import type { StoreContext } from '../internal/setup';
|
|
3
13
|
/**
|
|
4
14
|
* Search items under a namespace prefix: metadata filtering plus optional
|
|
5
|
-
* semantic ranking.
|
|
6
|
-
*
|
|
7
|
-
*
|
|
15
|
+
* semantic ranking.
|
|
16
|
+
*
|
|
17
|
+
* Accepts: `search` — already parsed, so `namespacePrefix`, `offset` and
|
|
18
|
+
* `limit` are resolved; `search.query` absent or empty
|
|
19
|
+
* (which is absent: there is no query to embed) ranks nothing and returns the
|
|
20
|
+
* page as read, which is what the reference store does
|
|
21
|
+
* (`@langchain/langgraph-checkpoint@1.1.5` `dist/store/memory.js:70-80`, where a
|
|
22
|
+
* falsy query takes the unscored path). A query without a configured `index`
|
|
23
|
+
* does the same, since there is nothing to embed it with.
|
|
24
|
+
*
|
|
25
|
+
* Returns: at most `limit` items from `offset`. With a query and an index every
|
|
26
|
+
* item carries a `score`; without one none does. Scores rank best-first; an item
|
|
27
|
+
* that cannot be scored ranks last rather than being dropped.
|
|
28
|
+
*
|
|
29
|
+
* Throws: `VALIDATION` naming `maxSearchCandidates` or `index.dims`; whatever
|
|
30
|
+
* the reads, decodes and the embeddings model throw.
|
|
31
|
+
*
|
|
32
|
+
* Guarantees: a page of zero costs no request at all, and otherwise only the
|
|
33
|
+
* page's own items are decoded on the unranked path — the
|
|
34
|
+
* read stops as soon as it is full. A semantic search must read every candidate
|
|
35
|
+
* to rank it, which is why it is capped and why a large corpus belongs in a
|
|
36
|
+
* `vectorBackend`.
|
|
8
37
|
*/
|
|
9
|
-
export declare function searchItems(context: StoreContext,
|
|
38
|
+
export declare function searchItems(context: StoreContext, search: ParsedSearch, signal?: AbortSignal): Promise<SearchItem[]>;
|
|
@@ -1,34 +1,66 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Hides which of three paths serves a search.
|
|
4
|
+
*
|
|
5
|
+
* A query with a `vectorBackend` is answered by the backend; no query, or no
|
|
6
|
+
* `index` to embed one with, is a page read from the table that stops once
|
|
7
|
+
* full; anything else is ranked in memory over at most `maxSearchCandidates`.
|
|
8
|
+
* A page of zero is answered before any of them. A caller gets the same shape
|
|
9
|
+
* of page whichever path ran, scored exactly when a query could be ranked.
|
|
10
|
+
*/
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.searchItems = searchItems;
|
|
4
|
-
const
|
|
5
|
-
const candidates_1 = require("../internal/candidates");
|
|
6
|
-
const ranker_1 = require("../internal/ranker");
|
|
13
|
+
const truncate_1 = require("../../shared/logging/truncate");
|
|
7
14
|
const semantic_search_1 = require("../internal/semantic-search");
|
|
8
|
-
const
|
|
9
|
-
const
|
|
15
|
+
const table_search_1 = require("../internal/table-search");
|
|
16
|
+
const vector_index_1 = require("../internal/vector-index");
|
|
10
17
|
/**
|
|
11
18
|
* Search items under a namespace prefix: metadata filtering plus optional
|
|
12
|
-
* semantic ranking.
|
|
13
|
-
*
|
|
14
|
-
*
|
|
19
|
+
* semantic ranking.
|
|
20
|
+
*
|
|
21
|
+
* Accepts: `search` — already parsed, so `namespacePrefix`, `offset` and
|
|
22
|
+
* `limit` are resolved; `search.query` absent or empty
|
|
23
|
+
* (which is absent: there is no query to embed) ranks nothing and returns the
|
|
24
|
+
* page as read, which is what the reference store does
|
|
25
|
+
* (`@langchain/langgraph-checkpoint@1.1.5` `dist/store/memory.js:70-80`, where a
|
|
26
|
+
* falsy query takes the unscored path). A query without a configured `index`
|
|
27
|
+
* does the same, since there is nothing to embed it with.
|
|
28
|
+
*
|
|
29
|
+
* Returns: at most `limit` items from `offset`. With a query and an index every
|
|
30
|
+
* item carries a `score`; without one none does. Scores rank best-first; an item
|
|
31
|
+
* that cannot be scored ranks last rather than being dropped.
|
|
32
|
+
*
|
|
33
|
+
* Throws: `VALIDATION` naming `maxSearchCandidates` or `index.dims`; whatever
|
|
34
|
+
* the reads, decodes and the embeddings model throw.
|
|
35
|
+
*
|
|
36
|
+
* Guarantees: a page of zero costs no request at all, and otherwise only the
|
|
37
|
+
* page's own items are decoded on the unranked path — the
|
|
38
|
+
* read stops as soon as it is full. A semantic search must read every candidate
|
|
39
|
+
* to rank it, which is why it is capped and why a large corpus belongs in a
|
|
40
|
+
* `vectorBackend`.
|
|
15
41
|
*/
|
|
16
|
-
async function searchItems(context,
|
|
17
|
-
const offset
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
42
|
+
async function searchItems(context, search, signal) {
|
|
43
|
+
const { offset, limit } = search;
|
|
44
|
+
// A zero page is answered here, ahead of all three paths below, because each
|
|
45
|
+
// of them pays for it: `collectCandidates` pulls the first row out of the
|
|
46
|
+
// paginator before it tests its `offset + limit` bound, so even a page that
|
|
47
|
+
// needs nothing costs one Query or Scan, and the two ranked paths embed the
|
|
48
|
+
// query as well. Slicing the result to nothing afterwards hid the cost
|
|
49
|
+
// rather than avoiding it.
|
|
50
|
+
if (limit === 0)
|
|
51
|
+
return [];
|
|
52
|
+
if (search.query && (0, vector_index_1.hasVectorBackend)(context)) {
|
|
53
|
+
const ranked = await (0, vector_index_1.searchViaBackend)(context, search, signal);
|
|
22
54
|
return ranked.slice(offset, offset + limit);
|
|
23
55
|
}
|
|
24
|
-
if (!
|
|
25
|
-
const page = await (0,
|
|
56
|
+
if (!search.query || !context.index) {
|
|
57
|
+
const page = await (0, table_search_1.collectCandidates)(context, search, { kind: 'page', need: offset + limit }, signal);
|
|
26
58
|
return page.map(({ item }) => ({ ...item })).slice(offset, offset + limit);
|
|
27
59
|
}
|
|
28
|
-
const candidates = await (0,
|
|
29
|
-
const queryVector = await context.index.embeddings.embedQuery(
|
|
60
|
+
const candidates = await (0, table_search_1.collectCandidates)(context, search, { kind: 'semantic', cap: context.maxSearchCandidates }, signal);
|
|
61
|
+
const queryVector = await context.index.embeddings.embedQuery(search.query);
|
|
30
62
|
(0, semantic_search_1.assertVectorDims)(context.index, queryVector, 'query');
|
|
31
|
-
const ranked = (0,
|
|
32
|
-
'were ranked unscored; re-embed them with reconcileVectorIndex or a re-put', { namespacePrefix:
|
|
63
|
+
const ranked = (0, table_search_1.rankInMemory)(candidates, queryVector, context.maxSearchCandidates, (count) => context.logger.warn('search: some candidates carry an embedding of a different dimension than the query and ' +
|
|
64
|
+
'were ranked unscored; re-embed them with reconcileVectorIndex or a re-put', { namespacePrefix: (0, truncate_1.truncateLabelsForLog)(search.namespacePrefix), count }));
|
|
33
65
|
return ranked.slice(offset, offset + limit);
|
|
34
66
|
}
|
|
@@ -1,11 +1,26 @@
|
|
|
1
|
-
import type { Operation } from '@langchain/langgraph-checkpoint';
|
|
2
1
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
2
|
+
* Hides how a batch runs concurrently and still in the order it was written.
|
|
3
|
+
*
|
|
4
|
+
* Operations on different items, and reads of the same item, share a run and
|
|
5
|
+
* go in flight together; anything that could observe or overwrite an earlier
|
|
6
|
+
* operation's effect starts the next run. A caller sees the order it wrote —
|
|
7
|
+
* a `get` after a `put` of the same item sees it, a `search` sees every write
|
|
8
|
+
* before it — and how independence is judged can change without the store.
|
|
10
9
|
*/
|
|
11
|
-
|
|
10
|
+
import type { ParsedOperation } from './parse';
|
|
11
|
+
/**
|
|
12
|
+
* Run a batch: each run of independent operations concurrently, at most
|
|
13
|
+
* `limit` in flight, and the runs themselves in
|
|
14
|
+
* order. Results come back in operation order.
|
|
15
|
+
*
|
|
16
|
+
* Accepts: `operations` — in caller order, which is the order they are
|
|
17
|
+
* observed in; empty returns empty. `limit` — operations in flight within one
|
|
18
|
+
* run.
|
|
19
|
+
*
|
|
20
|
+
* Returns: the results in operation order, not completion order.
|
|
21
|
+
*
|
|
22
|
+
* Throws: the first failure. Any failure rejects the whole batch: no operation
|
|
23
|
+
* in a later run starts, the ones already in flight settle, and that first
|
|
24
|
+
* error is the one thrown.
|
|
25
|
+
*/
|
|
26
|
+
export declare function runBatch<R>(operations: readonly ParsedOperation[], dispatch: (operation: ParsedOperation) => Promise<R>, limit?: number): Promise<R[]>;
|
|
@@ -1,52 +1,109 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Hides how a batch runs concurrently and still in the order it was written.
|
|
4
|
+
*
|
|
5
|
+
* Operations on different items, and reads of the same item, share a run and
|
|
6
|
+
* go in flight together; anything that could observe or overwrite an earlier
|
|
7
|
+
* operation's effect starts the next run. A caller sees the order it wrote —
|
|
8
|
+
* a `get` after a `put` of the same item sees it, a `search` sees every write
|
|
9
|
+
* before it — and how independence is judged can change without the store.
|
|
10
|
+
*/
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.runBatch = runBatch;
|
|
4
13
|
const concurrency_1 = require("../../shared/concurrency");
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
14
|
+
function itemOf(address) {
|
|
15
|
+
return JSON.stringify([address.namespace, address.key]);
|
|
16
|
+
}
|
|
17
|
+
/** What `op` touches, decided by the kind the parser already assigned — the only place that asks. */
|
|
18
|
+
function touchOf(op) {
|
|
19
|
+
switch (op.kind) {
|
|
20
|
+
case 'put':
|
|
21
|
+
case 'delete':
|
|
22
|
+
return { kind: 'write', item: itemOf(op.address) };
|
|
23
|
+
case 'get':
|
|
24
|
+
return { kind: 'get', item: itemOf(op.address) };
|
|
25
|
+
default:
|
|
26
|
+
return { kind: 'broad' };
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* True when `touch` may not join `segment` — because some operation already in
|
|
31
|
+
* it addresses the same item, or because a write and a whole-store read would
|
|
32
|
+
* then be unordered with respect to each other.
|
|
33
|
+
*/
|
|
34
|
+
function conflicts(segment, touch) {
|
|
35
|
+
if (touch.kind === 'broad')
|
|
36
|
+
return segment.hasWrite;
|
|
37
|
+
if (touch.kind === 'get')
|
|
38
|
+
return segment.written.has(touch.item);
|
|
39
|
+
return segment.written.has(touch.item) || segment.read.has(touch.item) || segment.hasBroad;
|
|
40
|
+
}
|
|
41
|
+
function admit(segment, index, touch) {
|
|
42
|
+
segment.indices.push(index);
|
|
43
|
+
if (touch.kind === 'broad')
|
|
44
|
+
segment.hasBroad = true;
|
|
45
|
+
if (touch.kind === 'get')
|
|
46
|
+
segment.read.add(touch.item);
|
|
47
|
+
if (touch.kind === 'write') {
|
|
48
|
+
segment.written.add(touch.item);
|
|
49
|
+
segment.hasWrite = true;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
function emptySegment() {
|
|
53
|
+
return { indices: [], written: new Set(), read: new Set(), hasWrite: false, hasBroad: false };
|
|
8
54
|
}
|
|
9
55
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
56
|
+
* Split a batch into runs of mutually independent operations, in caller order.
|
|
57
|
+
*
|
|
58
|
+
* Operations that address different items, and reads that address the same
|
|
59
|
+
* item, are independent of each other and share a run. Anything that could
|
|
60
|
+
* observe or overwrite an earlier operation's effect starts a new one, so the
|
|
61
|
+
* order the caller wrote is the order the caller observes: a `get` after a
|
|
62
|
+
* `put` of the same item sees it, a `get` before one does not, and a `search`
|
|
63
|
+
* sees every write that precedes it and none that follow.
|
|
64
|
+
*
|
|
65
|
+
* Running every write before every read — the earlier behaviour — made a
|
|
66
|
+
* `[delete, get]` batch return nothing where the reference store returns the
|
|
67
|
+
* value, and a `[get, put]` batch return the new value where the reference
|
|
68
|
+
* returns null. `AsyncBatchedStore` coalesces everything enqueued in one tick
|
|
69
|
+
* into a single `batch()`, so that difference is reachable from ordinary code.
|
|
14
70
|
*/
|
|
15
71
|
function planBatch(operations) {
|
|
16
|
-
const
|
|
17
|
-
|
|
72
|
+
const runs = [];
|
|
73
|
+
let segment = emptySegment();
|
|
18
74
|
operations.forEach((op, index) => {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
75
|
+
const touch = touchOf(op);
|
|
76
|
+
if (segment.indices.length > 0 && conflicts(segment, touch)) {
|
|
77
|
+
runs.push(segment.indices);
|
|
78
|
+
segment = emptySegment();
|
|
22
79
|
}
|
|
23
|
-
|
|
24
|
-
const group = groups.get(item);
|
|
25
|
-
if (group)
|
|
26
|
-
group.push(index);
|
|
27
|
-
else
|
|
28
|
-
groups.set(item, [index]);
|
|
80
|
+
admit(segment, index, touch);
|
|
29
81
|
});
|
|
30
|
-
|
|
82
|
+
if (segment.indices.length > 0)
|
|
83
|
+
runs.push(segment.indices);
|
|
84
|
+
return runs;
|
|
31
85
|
}
|
|
32
86
|
/**
|
|
33
|
-
* Run a batch:
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
87
|
+
* Run a batch: each run of independent operations concurrently, at most
|
|
88
|
+
* `limit` in flight, and the runs themselves in
|
|
89
|
+
* order. Results come back in operation order.
|
|
90
|
+
*
|
|
91
|
+
* Accepts: `operations` — in caller order, which is the order they are
|
|
92
|
+
* observed in; empty returns empty. `limit` — operations in flight within one
|
|
93
|
+
* run.
|
|
94
|
+
*
|
|
95
|
+
* Returns: the results in operation order, not completion order.
|
|
96
|
+
*
|
|
97
|
+
* Throws: the first failure. Any failure rejects the whole batch: no operation
|
|
98
|
+
* in a later run starts, the ones already in flight settle, and that first
|
|
99
|
+
* error is the one thrown.
|
|
40
100
|
*/
|
|
41
|
-
async function runBatch(operations, dispatch) {
|
|
42
|
-
const plan = planBatch(operations);
|
|
101
|
+
async function runBatch(operations, dispatch, limit = concurrency_1.DEFAULT_READ_CONCURRENCY) {
|
|
43
102
|
const results = [];
|
|
44
|
-
|
|
45
|
-
|
|
103
|
+
for (const run of planBatch(operations)) {
|
|
104
|
+
await (0, concurrency_1.mapWithConcurrency)(run, limit, async (index) => {
|
|
46
105
|
results[index] = await dispatch(operations[index]);
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
results[index] = await dispatch(operations[index]);
|
|
50
|
-
});
|
|
106
|
+
});
|
|
107
|
+
}
|
|
51
108
|
return results;
|
|
52
109
|
}
|
|
@@ -1,9 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hides the store's filter grammar.
|
|
3
|
+
*
|
|
4
|
+
* Which operators a search filter accepts, how each compares a stored field —
|
|
5
|
+
* ordered comparisons only between like-typed values, an absent field matching
|
|
6
|
+
* nothing — and whether an item passes a search's optional filter are decided
|
|
7
|
+
* here, the same way whichever search path runs.
|
|
8
|
+
*/
|
|
9
|
+
import type { Item } from '@langchain/langgraph-checkpoint';
|
|
1
10
|
/** A JSON value stored in an item or supplied in a filter. */
|
|
2
11
|
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
3
12
|
[key: string]: JsonValue;
|
|
4
13
|
};
|
|
5
14
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
15
|
+
* Whether `value` satisfies every field condition in `filter`.
|
|
16
|
+
*
|
|
17
|
+
* Accepts: `value` — a stored item's decoded value. Declared as a record, but a
|
|
18
|
+
* row holds whatever its writer stored, including `null` and a scalar; such a
|
|
19
|
+
* value has no own properties and so satisfies no condition. `filter` — a plain
|
|
20
|
+
* value per field is exact match, an operator object (`{ $gt: 4 }`) applies
|
|
21
|
+
* comparisons, and `{}` constrains nothing.
|
|
22
|
+
*
|
|
23
|
+
* Returns: whether every condition holds.
|
|
24
|
+
*
|
|
25
|
+
* Throws: nothing. `search` walks every candidate row, so one row whose value
|
|
26
|
+
* is not an object must not fail the search.
|
|
27
|
+
*
|
|
28
|
+
* Guarantees: own properties only — `value['toString']` would otherwise resolve
|
|
29
|
+
* up the prototype chain and be compared as if it were stored data.
|
|
8
30
|
*/
|
|
9
31
|
export declare function matchesStoreFilter(value: Record<string, JsonValue>, filter: Record<string, JsonValue>): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Whether `item` satisfies a search's optional metadata filter.
|
|
34
|
+
*
|
|
35
|
+
* Accepts: `filter` — a parsed search's filter; absent passes every item.
|
|
36
|
+
* `item.value` — whatever the item's writer stored; a value that is not
|
|
37
|
+
* an object satisfies no condition (see `matchesStoreFilter`).
|
|
38
|
+
*
|
|
39
|
+
* Returns: whether the item belongs in the result.
|
|
40
|
+
*
|
|
41
|
+
* Throws: nothing, so one unusual row cannot fail a search over many.
|
|
42
|
+
*/
|
|
43
|
+
export declare function passesFilter(item: Item, filter: Record<string, JsonValue> | undefined): boolean;
|