@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.2 → 1.0.0

package/dist/esm/checkpointer/internal/rows.js

@@ -0,0 +1,323 @@
+import { WRITES_IDX_MAP } from '@langchain/langgraph-checkpoint';
+import { nowIso } from '../../shared/clock.js';
+import { codecDepsOf, collectS3Keys, decodePayload, encodePayload, } from '../../shared/codec/codec.js';
+import { cleanUpS3Orphans } from '../../shared/codec/s3/offloader.js';
+import { DEFAULT_READ_CONCURRENCY, mapWithConcurrency } from '../../shared/concurrency.js';
+import { namedDescriptor } from '../../shared/dynamodb/partition-delete.js';
+import { BACKFILLED_AT, DEFAULT_INDEX_SHARDS, indexKeys, } from '../../shared/dynamodb/recency-index.js';
+import { ADAPTER_TAGS, assertReadableRow, KEY_SEPARATOR, PARTITION_KEY_ATTRIBUTE, ROW_FORMAT_VERSION, SORT_KEY_ATTRIBUTE, } from '../../shared/dynamodb/table-schema.js';
+import { validationError } from '../../shared/errors/errors.js';
+import { truncateForLog } from '../../shared/logging/truncate.js';
+import { createUlidFactory } from '../../shared/ulid.js';
+export const WRITE_GROUP_ATTRIBUTE = 'writeGroup';
+const WRITE_INDEX_PAD_WIDTH = 10;
+const WRITE_INDEX_OFFSET = 8;
+export const MIN_ENCODABLE_WRITE_INDEX = -WRITE_INDEX_OFFSET;
+var CheckpointRowKind;
+(function (CheckpointRowKind) {
+    CheckpointRowKind["META"] = "META";
+    CheckpointRowKind["PAYLOAD"] = "PAYLOAD";
+    CheckpointRowKind["WRITE"] = "WRITE";
+})(CheckpointRowKind || (CheckpointRowKind = {}));
+const ADAPTER_PARTITION_PREFIX = `${ADAPTER_TAGS.checkpointer}${KEY_SEPARATOR}`;
+export function checkpointerPartitionPrefix() {
+    return ADAPTER_PARTITION_PREFIX;
+}
+export function partitionKey(threadId) {
+    return `${ADAPTER_PARTITION_PREFIX}${threadId}`;
+}
+export function metaSortKey(checkpointNs, checkpointId) {
+    return `${CheckpointRowKind.META}${KEY_SEPARATOR}${checkpointNs}${KEY_SEPARATOR}${checkpointId}`;
+}
+export function metaSortKeyPrefix(checkpointNs) {
+    return `${CheckpointRowKind.META}${KEY_SEPARATOR}${checkpointNs}${KEY_SEPARATOR}`;
+}
+export function metaAnyNamespacePrefix() {
+    return `${CheckpointRowKind.META}${KEY_SEPARATOR}`;
+}
+export function payloadSortKey(checkpointNs, checkpointId) {
+    return `${CheckpointRowKind.PAYLOAD}${KEY_SEPARATOR}${checkpointNs}${KEY_SEPARATOR}${checkpointId}`;
+}
+export function writeSortKey(at) {
+    const offsetIndex = at.index + WRITE_INDEX_OFFSET;
+    if (!Number.isInteger(offsetIndex) ||
+        offsetIndex < 0 ||
+        offsetIndex.toString().length > WRITE_INDEX_PAD_WIDTH) {
+        throw validationError(`write index ${at.index} is not an integer encodable at offset ${WRITE_INDEX_OFFSET} ` +
+            `with ${WRITE_INDEX_PAD_WIDTH} digits`, 'index');
+    }
+    const paddedIndex = offsetIndex.toString().padStart(WRITE_INDEX_PAD_WIDTH, '0');
+    return [
+        CheckpointRowKind.WRITE,
+        at.checkpointNs,
+        at.checkpointId,
+        at.taskId,
+        paddedIndex,
+        at.channel,
+    ].join(KEY_SEPARATOR);
+}
+export function writeSortKeyBytes(at) {
+    return Buffer.byteLength(writeSortKey({ ...at, index: 0 }), 'utf8');
+}
+export function isCheckpointerSortKey(sortKey) {
+    return Object.values(CheckpointRowKind).some((kind) => sortKey.startsWith(`${kind}${KEY_SEPARATOR}`));
+}
+export function writeSortKeyPrefix(checkpointNs, checkpointId) {
+    return `${CheckpointRowKind.WRITE}${KEY_SEPARATOR}${checkpointNs}${KEY_SEPARATOR}${checkpointId}${KEY_SEPARATOR}`;
+}
+export function metaRowKey(at) {
+    return { PK: partitionKey(at.threadId), SK: metaSortKey(at.checkpointNs, at.checkpointId) };
+}
+export function payloadRowKey(at) {
+    return { PK: partitionKey(at.threadId), SK: payloadSortKey(at.checkpointNs, at.checkpointId) };
+}
+export function partitionQuery(tableName, partition, options = {}) {
+    const params = {
+        TableName: tableName,
+        KeyConditionExpression: '#pk = :pk',
+        ExpressionAttributeNames: { '#pk': PARTITION_KEY_ATTRIBUTE },
+        ExpressionAttributeValues: { ':pk': partition },
+    };
+    if (options.consistent)
+        params.ConsistentRead = true;
+    return params;
+}
+export function beginsWithQuery(tableName, partition, skPrefix, options = {}) {
+    const bounded = options.beforeSortKey !== undefined;
+    const params = {
+        TableName: tableName,
+        KeyConditionExpression: bounded
+            ? '#pk = :pk AND #sk BETWEEN :skPrefix AND :before'
+            : '#pk = :pk AND begins_with(#sk, :skPrefix)',
+        ExpressionAttributeNames: { '#pk': PARTITION_KEY_ATTRIBUTE, '#sk': SORT_KEY_ATTRIBUTE },
+        ExpressionAttributeValues: {
+            ':pk': partition,
+            ':skPrefix': skPrefix,
+            ...(bounded ? { ':before': options.beforeSortKey } : {}),
+        },
+        ScanIndexForward: options.ascending ?? false,
+    };
+    if (options.limit !== undefined && options.limit >= 1)
+        params.Limit = options.limit;
+    if (options.consistent)
+        params.ConsistentRead = true;
+    return params;
+}
+function withTtl(item, ttlTimestamp) {
+    if (ttlTimestamp !== undefined)
+        item.ttl = ttlTimestamp;
+    return item;
+}
+async function releaseUploads(context, uploaded, operation) {
+    if (!context.offloader)
+        return;
+    await cleanUpS3Orphans(context.offloader, {
+        keys: collectS3Keys(uploaded),
+        operation,
+        logger: context.logger,
+    });
+}
+const nextPutObjectId = createUlidFactory();
+export async function buildCheckpointRows(context, request, ttlTimestamp) {
+    const { threadId, checkpointNs, checkpointId } = request.address;
+    const deps = codecDepsOf(context, request.signal);
+    const pk = partitionKey(threadId);
+    const objectId = nextPutObjectId();
+    const checkpointDescriptor = await encodePayload(request.checkpoint, deps, {
+        keyParts: [threadId, checkpointNs, checkpointId, 'checkpoint'],
+        objectId,
+        row: { pk, sk: payloadSortKey(checkpointNs, checkpointId) },
+    });
+    let metadataDescriptor;
+    try {
+        metadataDescriptor = await encodePayload(request.metadata, deps, {
+            keyParts: [threadId, checkpointNs, checkpointId, 'metadata'],
+            objectId,
+            row: { pk, sk: metaSortKey(checkpointNs, checkpointId) },
+        });
+    }
+    catch (error) {
+        await releaseUploads(context, [checkpointDescriptor], 'put.encode');
+        throw error;
+    }
+    const index = indexKeys('CHKPT', checkpointId, nowIso(), context.indexShards ?? DEFAULT_INDEX_SHARDS);
+    const meta = {
+        PK: pk,
+        SK: metaSortKey(checkpointNs, checkpointId),
+        v: ROW_FORMAT_VERSION,
+        ...index,
+        threadId,
+        checkpointNs,
+        checkpointId,
+        metadata: metadataDescriptor,
+    };
+    if (request.parentCheckpointId !== undefined)
+        meta.parentCheckpointId = request.parentCheckpointId;
+    const payload = {
+        PK: pk,
+        SK: payloadSortKey(checkpointNs, checkpointId),
+        v: ROW_FORMAT_VERSION,
+        checkpoint: checkpointDescriptor,
+    };
+    return { meta: withTtl(meta, ttlTimestamp), payload: withTtl(payload, ttlTimestamp) };
+}
+export async function buildWriteRows(context, request, writeGroup, ttlTimestamp) {
+    const { threadId, checkpointNs, checkpointId } = request.address;
+    const { taskId, taskPath } = request;
+    const deps = codecDepsOf(context, request.signal);
+    const pk = partitionKey(threadId);
+    const items = [];
+    try {
+        for (const { channel, value, index, occurrence } of resolveWriteIndices(request.writes)) {
+            const sk = writeSortKey({ checkpointNs, checkpointId, taskId, index, channel });
+            const descriptor = await encodePayload(value, deps, {
+                keyParts: [threadId, checkpointNs, checkpointId, taskId, `write-${index}`, channel],
+                objectId: writeGroup,
+                row: { pk, sk },
+            });
+            const item = {
+                PK: pk,
+                SK: sk,
+                v: ROW_FORMAT_VERSION,
+                taskId,
+                index,
+                channel,
+                ...(taskPath === undefined ? {} : { taskPath }),
+                writeGroup,
+                occurrence,
+                value: descriptor,
+            };
+            items.push(withTtl(item, ttlTimestamp));
+        }
+    }
+    catch (error) {
+        await releaseUploads(context, items.map((item) => item.value), 'putWrites.encode');
+        throw error;
+    }
+    return items;
+}
+export function parseMetaRow(raw) {
+    assertReadableRow(raw, 'checkpoint');
+    const isCheckpoint = typeof raw.threadId === 'string' &&
+        typeof raw.checkpointId === 'string' &&
+        typeof raw.checkpointNs === 'string' &&
+        typeof raw.metadata === 'object' &&
+        raw.metadata !== null;
+    if (!isCheckpoint)
+        return undefined;
+    const item = raw;
+    const consistent = item.PK === partitionKey(item.threadId) &&
+        item.SK === metaSortKey(item.checkpointNs, item.checkpointId);
+    return consistent ? item : undefined;
+}
+export function parseWriteRow(raw) {
+    assertReadableRow(raw, 'pending write');
+    const isWrite = typeof raw.taskId === 'string' &&
+        typeof raw.channel === 'string' &&
+        (raw.taskPath === undefined || typeof raw.taskPath === 'string') &&
+        typeof raw.value === 'object' &&
+        raw.value !== null;
+    return isWrite ? raw : undefined;
+}
+export function parseHeadRow(context, raw, reader) {
+    if (raw === undefined)
+        return undefined;
+    const meta = parseMetaRow(raw);
+    if (!meta) {
+        context.logger.warn(`${reader}: skipped a row that is not a checkpoint meta item`, {
+            sortKey: truncateForLog(raw.SK),
+        });
+    }
+    return meta;
+}
+export async function readCheckpoint(context, item, threadId, signal) {
+    return decodePayload(item.checkpoint, codecDepsOf(context, signal), [threadId]);
+}
+export async function readMetadata(context, item, threadId, signal) {
+    return decodePayload(item.metadata, codecDepsOf(context, signal), [threadId]);
+}
+export function compareByCodePoint(a, b) {
+    let i = 0;
+    let j = 0;
+    while (i < a.length && j < b.length) {
+        const left = a.codePointAt(i);
+        const right = b.codePointAt(j);
+        if (left !== right)
+            return left < right ? -1 : 1;
+        i += left > 0xffff ? 2 : 1;
+        j += right > 0xffff ? 2 : 1;
+    }
+    if (i >= a.length && j >= b.length)
+        return 0;
+    return i >= a.length ? -1 : 1;
+}
+export function compareWriteOrder(a, b) {
+    return (compareByCodePoint(a.taskPath ?? '', b.taskPath ?? '') ||
+        compareByCodePoint(a.taskId, b.taskId) ||
+        a.index - b.index ||
+        compareByCodePoint(a.channel, b.channel));
+}
+export async function toPendingWrites(context, items, threadId, signal) {
+    const deps = codecDepsOf(context, signal);
+    const live = [...dropSupersededWrites(items)].sort(compareWriteOrder);
+    const values = await mapWithConcurrency(live, context.readConcurrency ?? DEFAULT_READ_CONCURRENCY, (item) => decodePayload(item.value, deps, [threadId]));
+    return live.map((item, index) => [
+        item.taskId,
+        item.channel,
+        values[index],
+    ]);
+}
+export function resolveWriteIndices(writes) {
+    const bySpecialIndex = new Map();
+    const regular = [];
+    const occurrences = new Map();
+    writes.forEach(([channel, value], positional) => {
+        if (Object.hasOwn(WRITES_IDX_MAP, channel)) {
+            const index = WRITES_IDX_MAP[channel];
+            bySpecialIndex.set(index, { channel, value, index, occurrence: 0 });
+            return;
+        }
+        const occurrence = occurrences.get(channel) ?? 0;
+        occurrences.set(channel, occurrence + 1);
+        regular.push({ channel, value, index: positional, occurrence });
+    });
+    return [...bySpecialIndex.values(), ...regular];
+}
+export function dropSupersededWrites(items) {
+    const identity = (item) => JSON.stringify([item.taskId, item.channel, item.occurrence ?? 0]);
+    const groupOf = (item) => item.writeGroup ?? '';
+    const earliestGroup = new Map();
+    for (const item of items) {
+        const id = identity(item);
+        const seen = earliestGroup.get(id);
+        const group = groupOf(item);
+        if (seen === undefined || group < seen)
+            earliestGroup.set(id, group);
+    }
+    return items.filter((item) => earliestGroup.get(identity(item)) === groupOf(item));
+}
+const PAYLOAD_ATTRIBUTES = ['metadata', 'checkpoint', 'value'];
+export function checkpointRowDescriptors(row) {
+    const named = [];
+    for (const attribute of PAYLOAD_ATTRIBUTES) {
+        const entry = namedDescriptor(row, attribute);
+        if (entry !== undefined)
+            named.push(entry);
+    }
+    return named;
+}
+export function checkpointRowUnit(row) {
+    return row.SK.split(KEY_SEPARATOR).slice(1, 3).join(KEY_SEPARATOR);
+}
+export function checkpointRowKind(row) {
+    return row.SK.split(KEY_SEPARATOR)[0];
+}
+export function checkpointIndexTarget(row) {
+    const pk = typeof row.PK === 'string' ? row.PK : '';
+    const sk = typeof row.SK === 'string' ? row.SK : '';
+    if (!pk.startsWith(ADAPTER_PARTITION_PREFIX))
+        return undefined;
+    return sk.startsWith(metaAnyNamespacePrefix()) && typeof row.checkpointId === 'string'
+        ? { tag: 'CHKPT', id: row.checkpointId, at: BACKFILLED_AT }
+        : undefined;
+}

package/dist/esm/checkpointer/internal/setup.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Hides which options the saver accepts, and how the saver is assembled from
+ * them.
+ *
+ * The exhaustive key list of the constructor's options lives here,
+ * compiler-checked against its type, beside the code that resolves those
+ * options into the context every action receives. `list`'s and
+ * `getDeltaChannelHistory`'s options have no list: LangGraph defines them, so
+ * a key it adds is ignored rather than refused (decision record 28).
+ */
+import type { SerializerProtocol } from '@langchain/langgraph-checkpoint';
+import { type AdapterCore, type AdapterShell } from '../../shared/adapter.js';
+import type { DynamoDBSaverOptions } from '../types.js';
+/**
+ * The keys of the saver's constructor options, exhaustive in both directions:
+ * `allKeysOf<T>` makes omitting or inventing one a compile error, so the list
+ * cannot rot away from the type it guards. It lives with the feature because
+ * the type it is checked against does; `shared/` knows no feature.
+ */
+export declare const SAVER_KEYS: readonly string[];
+/** Resolved collaborators shared by every checkpointer action. */
+export interface CheckpointerContext extends AdapterCore {
+    serde: SerializerProtocol;
+}
+/** Result of wiring up a checkpointer from its options. */
+export interface CheckpointerSetup {
+    context: CheckpointerContext;
+    shell: AdapterShell;
+}
+/**
+ * Check the options, then resolve the client, offloader and serializer.
+ *
+ * Accepts: `options` — already held to the saver's key list by its constructor,
+ * which must read `serde` before anything else runs; checked here before
+ * anything is built, so no half-built saver exists when one is wrong. `serde` —
+ * the serializer the base class resolved, which is the caller's own serializer
+ * when they gave one.
+ *
+ * Returns: the context every action receives, and the shell that releases what
+ * it holds — a client the caller passed in is never destroyed by `destroy()`.
+ *
+ * Throws: `VALIDATION` for an option or collaborator that fails a check,
+ * naming the offending option.
+ *
+ * Guarantees: constructing a saver performs no I/O.
+ */
+export declare function setUpCheckpointer(options: DynamoDBSaverOptions, serde: SerializerProtocol): CheckpointerSetup;

package/dist/esm/checkpointer/internal/setup.js

@@ -0,0 +1,21 @@
+import { openAdapter } from '../../shared/adapter.js';
+import { allKeysOf } from '../../shared/validation/option-shape.js';
+export const SAVER_KEYS = allKeysOf({
+    tableName: 'tableName',
+    client: 'client',
+    clientConfig: 'clientConfig',
+    createClient: 'createClient',
+    ttl: 'ttl',
+    logger: 'logger',
+    retry: 'retry',
+    indexShards: 'indexShards',
+    indexName: 'indexName',
+    readConcurrency: 'readConcurrency',
+    compression: 'compression',
+    s3: 's3',
+    serde: 'serde',
+});
+export function setUpCheckpointer(options, serde) {
+    const shell = openAdapter(options, 'checkpointer');
+    return { shell, context: { ...shell.core, serde } };
+}