@farukada/aws-langgraph-dynamodb-ts 0.9.0 → 1.0.0-rc.2

package/dist/store/internal/semantic-search.js

@@ -1,10 +1,39 @@
 "use strict";
+/**
+ * Hides which text of a value is embedded, and how.
+ *
+ * The text an index field names is extracted exactly as LangGraph's in-memory
+ * store extracts it — dotted paths, `[n]` and `[*]`, `{a,b}` groups, pretty
+ * JSON for containers — then embedded in bounded batches, and every vector the
+ * model returns is held to the configured dimension.
+ */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.cosineSimilarity = cosineSimilarity;
+exports.extractTexts = extractTexts;
 exports.extractText = extractText;
+exports.assertVectorDims = assertVectorDims;
+exports.embedValues = embedValues;
 exports.embedValue = embedValue;
-const langgraph_checkpoint_1 = require("@langchain/langgraph-checkpoint");
-/** Cosine similarity of two equal-length vectors; 0 for zero/mismatched vectors. */
+exports.embedPassages = embedPassages;
+exports.tokenizePath = tokenizePath;
+exports.getTextAtPath = getTextAtPath;
+const errors_1 = require("../../shared/errors/errors");
+/** Texts per `embedDocuments` call; keeps provider request sizes bounded. */
+const EMBED_BATCH_SIZE = 100;
+/**
+ * Cosine similarity of two vectors.
+ *
+ * Accepts: any two vectors. Lengths that disagree, and a vector of all zeros,
+ * have no defined angle between them.
+ *
+ * Returns: the cosine, in [-1, 1]; `0` for the undefined cases, which ranks
+ * such a pair as unrelated rather than as opposed. A caller that must
+ * distinguish "unrelated" from "incomparable" checks the lengths itself —
+ * `rankInMemory` does, so it can report a model mismatch instead of ranking
+ * everything at zero.
+ *
+ * Throws: nothing.
+ */
 function cosineSimilarity(a, b) {
     if (a.length !== b.length)
         return 0;
@@ -20,28 +49,341 @@ function cosineSimilarity(a, b) {
         return 0;
     return dot / (Math.sqrt(normA) * Math.sqrt(normB));
 }
-/** Extract and join the indexable text of a value for the configured fields. */
+/**
+ * The indexable texts of a value, one per extracted path, in field order.
+ *
+ * Extraction is byte-for-byte what LangGraph's `InMemoryStore` does (see
+ * {@link getTextAtPath}), and so is keeping them apart: the reference embeds each
+ * extracted element separately and scores an item by its best-matching one.
+ * Joining them and embedding once averages a long document into a single
+ * vector, which ranks a document holding one strongly-matching section
+ * materially lower — a retrieval-quality difference, not an edge case.
+ *
+ * Accepts: `value` — a stored item's value. `fields` — JSON paths; a path that
+ * addresses nothing contributes nothing.
+ *
+ * Returns: one text per extracted path, in field order, with empty ones
+ * dropped: an empty string embeds to a vector that means nothing and would rank
+ * against every query.
+ *
+ * Throws: nothing.
+ */
+function extractTexts(value, fields) {
+    return fields.flatMap((field) => getTextAtPath(value, field)).filter((text) => text.length > 0);
+}
+/**
+ * The same texts joined into one, which is what a single-vector consumer needs.
+ *
+ * Accepts: as {@link extractTexts}.
+ *
+ * Returns: the texts joined by a space, and `''` when there are none — which is
+ * how a caller tells "nothing to index" from a text to embed.
+ *
+ * Throws: nothing.
+ *
+ * Used only for a configured `vectorBackend`, whose contract is one vector per
+ * `(namespace, key)`; the in-DynamoDB path embeds each text separately.
+ */
 function extractText(value, fields) {
-    const parts = [];
-    for (const field of fields) {
-        if (field === '$')
-            parts.push(JSON.stringify(value));
-        else
-            parts.push(...(0, langgraph_checkpoint_1.getTextAtPath)(value, field));
+    return extractTexts(value, fields).join(' ');
+}
+/**
+ * Reject a vector whose length disagrees with the configured `index.dims`. A
+ * mismatch means the embeddings model does not match the configuration, so
+ * every stored vector would be incomparable with every query; failing here
+ * surfaces that at the first put or search instead of ranking silently.
+ *
+ * Accepts: `index.dims` — the check is skipped when it is not a positive
+ * integer, since nothing else in this package reads it and a store configured
+ * without it must keep working. `vector` — a document or query vector the model
+ * just returned. `what` — names which of the two, for the message.
+ *
+ * Returns: nothing: `vector` is kept under its declared type, and this
+ * checks it.
+ *
+ * Throws: `VALIDATION` naming `index.dims`.
+ */
+function assertVectorDims(index, vector, what) {
+    const dims = index.dims;
+    if (!Number.isInteger(dims) || dims <= 0 || vector.length === dims)
+        return;
+    throw (0, errors_1.validationError)(`index.embeddings returned a ${vector.length}-dimensional ${what} vector but index.dims ` +
+        `is ${dims}; the embeddings model does not match the configured index`, 'index.dims');
+}
+/**
+ * Embed several values for indexing with `embedDocuments` — the document-side
+ * method; providers such as Titan and Cohere embed documents and queries with
+ * different task types, so embedding a document with `embedQuery` degrades
+ * retrieval.
+ *
+ * Accepts: `values` — any number, including none. `fieldsOverride` — a put's
+ * own `index` fields; absent uses the store's, and a store with no configured
+ * fields indexes the whole document (`'$'`), as the reference does.
+ *
+ * Returns: one entry per value, in input order: its vector, or `undefined` when
+ * the value has no indexable text or the store has no index at all.
+ *
+ * Throws: `VALIDATION` naming `index.dims` when the model's width disagrees
+ * with the configuration; whatever the model throws.
+ *
+ * Guarantees: values with no text are never sent to the model, and the rest go
+ * in batches of {@link EMBED_BATCH_SIZE}, so one call cannot turn into one
+ * provider request per item or a single unbounded one.
+ */
+async function embedValues(context, values, fieldsOverride) {
+    const index = context.index;
+    if (!index)
+        return values.map(() => undefined);
+    const fields = fieldsOverride ?? index.fields ?? ['$'];
+    const pending = [];
+    values.forEach((value, position) => {
+        const text = extractText(value, fields);
+        if (text.length > 0)
+            pending.push({ text, position });
+    });
+    const vectors = values.map(() => undefined);
+    for (let start = 0; start < pending.length; start += EMBED_BATCH_SIZE) {
+        const batch = pending.slice(start, start + EMBED_BATCH_SIZE);
+        const embedded = await index.embeddings.embedDocuments(batch.map((entry) => entry.text));
+        batch.forEach((entry, i) => {
+            assertVectorDims(index, embedded[i], 'document');
+            vectors[entry.position] = embedded[i];
+        });
     }
-    return parts.join(' ');
+    return vectors;
 }
 /**
- * Embed a value for indexing, or undefined when indexing is off or the text is
- * empty. `fieldsOverride` (from a put's `index` option) takes precedence over
- * the store's configured fields.
+ * Embed one value for indexing.
+ *
+ * Accepts: as {@link embedValues}, for one value; `fieldsOverride` (from a
+ * put's `index` option) takes precedence over the store's configured fields.
+ *
+ * Returns: the value's vector, or undefined when indexing is off or it has no
+ * indexable text.
+ *
+ * Throws: as {@link embedValues}.
  */
 async function embedValue(context, value, fieldsOverride) {
-    if (!context.index)
+    return (await embedValues(context, [value], fieldsOverride))[0];
+}
+/**
+ * One vector per extracted path, for the item's own row — the shape the
+ * in-DynamoDB ranker scores by best match, matching the reference store.
+ *
+ * Accepts: as {@link embedValues}, for one value.
+ *
+ * Returns: one vector per extracted text, in path order; `undefined` when
+ * indexing is off or the value yields no indexable text — which is what clears
+ * a stale vector on a re-put, since the row is written without the attribute.
+ *
+ * Throws: as {@link embedValues}.
+ *
+ * Guarantees: every text of one value goes in as few `embedDocuments` calls as
+ * {@link EMBED_BATCH_SIZE} allows, never one call per path.
+ */
+async function embedPassages(context, value, fieldsOverride) {
+    const index = context.index;
+    if (!index)
         return undefined;
-    const text = extractText(value, fieldsOverride ?? context.index.fields ?? ['$']);
-    if (text.length === 0)
+    const texts = extractTexts(value, fieldsOverride ?? index.fields ?? ['$']);
+    if (texts.length === 0)
         return undefined;
-    return context.index.embeddings.embedQuery(text);
+    const vectors = [];
+    for (let start = 0; start < texts.length; start += EMBED_BATCH_SIZE) {
+        const embedded = await index.embeddings.embedDocuments(texts.slice(start, start + EMBED_BATCH_SIZE));
+        for (const vector of embedded) {
+            assertVectorDims(index, vector, 'document');
+            vectors.push(vector);
+        }
+    }
+    return vectors;
+}
+/**
+ * Pretty JSON, exactly as `InMemoryStore` embeds non-scalar values, as a list so
+ * that a value with no JSON text contributes nothing.
+ *
+ * `JSON.stringify` answers "no text" in two ways: it *returns* undefined for a
+ * value JSON cannot represent (`undefined`, a function, a symbol) and it
+ * *throws* for one it refuses (a circular structure, a `BigInt`). Both mean the
+ * same thing here — there is no text to index — and neither may escape: the
+ * first would put a literal `undefined` into a `string[]` and crash the
+ * caller's `text.length` filter, and the second would surface a raw
+ * `TypeError` from inside the embedding step. A value JSON refuses outright is
+ * still refused, by the codec, at the write that follows, which names the
+ * offending field.
+ */
+function prettyText(value) {
+    let text;
+    try {
+        text = JSON.stringify(value, null, 2);
+    }
+    catch {
+        return [];
+    }
+    return text === undefined ? [] : [text];
+}
+function isScalar(value) {
+    return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';
+}
+/** Text for a leaf: scalars stringify, containers pretty-print, null yields nothing. */
+function leafText(value) {
+    if (isScalar(value))
+        return [String(value)];
+    if (value === null)
+        return [];
+    return prettyText(value);
+}
+/** Index just past the closer matching an opener at `start - 1`, honouring nesting. */
+function scanGroup(path, start, open, close) {
+    let depth = 1;
+    let i = start;
+    while (i < path.length && depth > 0) {
+        if (path[i] === open)
+            depth += 1;
+        else if (path[i] === close)
+            depth -= 1;
+        i += 1;
+    }
+    return i;
+}
+/**
+ * Split a JSON path into tokens the way LangGraph's `InMemoryStore` does: dots
+ * separate plain segments, while a `[…]` index and a `{…}` field group each
+ * become a token of their own (`tags[0]` → `['tags', '[0]']`).
+ *
+ * Accepts: any path string. An empty path, and a path of nothing but
+ * separators, tokenize to nothing. An **unterminated** `[` or `{` runs to the
+ * end of the path and becomes a token that no longer closes — `'tags[0'` →
+ * `['tags', '[0']` — so it is resolved as a plain member name and matches
+ * nothing unless the value at `tags` holds a field literally called `[0`. That is the reference
+ * tokenizer's own answer (`@langchain/langgraph-checkpoint@1.1.5`
+ * `dist/store/utils.js:23-30`), and matching it matters more than refusing the
+ * path: `index.fields` is validated where it is configured, and text extraction
+ * must stay byte-for-byte comparable with the reference store.
+ *
+ * Returns: the tokens, in path order.
+ *
+ * Throws: nothing.
+ */
+function tokenizePath(path) {
+    const tokens = [];
+    let current = '';
+    const flush = () => {
+        if (current.length > 0)
+            tokens.push(current);
+        current = '';
+    };
+    let i = 0;
+    while (i < path.length) {
+        const char = path[i];
+        if (char === '[' || char === '{') {
+            flush();
+            const end = scanGroup(path, i + 1, char, char === '[' ? ']' : '}');
+            tokens.push(path.slice(i, end));
+            i = end;
+            continue;
+        }
+        if (char === '.')
+            flush();
+        else
+            current += char;
+        i += 1;
+    }
+    flush();
+    return tokens;
+}
+/** Walk one `{…}`-group field by plain `in` lookups, as the reference does. */
+function groupFieldText(value, field) {
+    let current = value;
+    for (const token of tokenizePath(field)) {
+        if (current !== null && typeof current === 'object' && token in current) {
+            current = current[token];
+        }
+        else {
+            return [];
+        }
+    }
+    return isScalar(current) ? [String(current)] : prettyText(current);
+}
+/** Resolve a `[n]`, `[-n]` or `[*]` token against an array; anything else yields nothing. */
+function indexText(value, token, tokens, pos) {
+    if (!Array.isArray(value))
+        return [];
+    const index = token.slice(1, -1);
+    if (index === '*')
+        return value.flatMap((item) => extract(item, tokens, pos + 1));
+    let idx = Number.parseInt(index, 10);
+    if (Number.isNaN(idx))
+        return [];
+    if (idx < 0)
+        idx = value.length + idx;
+    return idx >= 0 && idx < value.length ? extract(value[idx], tokens, pos + 1) : [];
+}
+/** Resolve a `{a,b.c}` token: each listed field, walked from the current value. */
+function groupText(value, token) {
+    if (typeof value !== 'object' || value === null)
+        return [];
+    const results = [];
+    for (const field of token.slice(1, -1).split(',')) {
+        const trimmed = field.trim();
+        if (tokenizePath(trimmed).length > 0)
+            results.push(...groupFieldText(value, trimmed));
+    }
+    return results;
+}
+/** Resolve a bare `*` token: every array item or every object value. */
+function wildcardText(value, tokens, pos) {
+    if (Array.isArray(value))
+        return value.flatMap((item) => extract(item, tokens, pos + 1));
+    if (typeof value === 'object' && value !== null) {
+        return Object.values(value).flatMap((item) => extract(item, tokens, pos + 1));
+    }
+    return [];
+}
+/** Resolve a plain member token by an `in` lookup, exactly as the reference does. */
+function memberText(value, token, tokens, pos) {
+    if (typeof value !== 'object' || value === null || !(token in value))
+        return [];
+    return extract(value[token], tokens, pos + 1);
+}
+function extract(value, tokens, pos) {
+    if (pos >= tokens.length)
+        return leafText(value);
+    const token = tokens[pos];
+    if (token.startsWith('[') && token.endsWith(']'))
+        return indexText(value, token, tokens, pos);
+    const results = pos === 0 && token === '$' ? prettyText(value) : [];
+    if (token.startsWith('{') && token.endsWith('}'))
+        results.push(...groupText(value, token));
+    else if (token === '*')
+        results.push(...wildcardText(value, tokens, pos));
+    else
+        results.push(...memberText(value, token, tokens, pos));
+    return results;
+}
+/**
+ * Extract the indexable text of `value` at `path`, byte-for-byte as LangGraph's
+ * `InMemoryStore` does (`store/utils`), so an embedding computed here matches
+ * one computed by the reference store for the same document. Supports plain
+ * paths, `[n]`/`[-n]`/`[*]` indexing, a bare `*` wildcard, `{a,b.c}` field
+ * groups and `$` for the whole document. The package root's `getTextAtPath` is
+ * a string-only variant that returns nothing for numbers, booleans, objects
+ * and arrays and throws on a `null` intermediate; it must not be used here.
+ *
+ * Accepts: `value` — any stored value, including a scalar and `null`. `path` —
+ * empty or `$` asks for the whole document; anything else is tokenized. A path
+ * that addresses nothing present, a `[n]` against a non-array, a `{…}` group
+ * against a non-object: each yields no text rather than an error, because one
+ * unindexable field of one item must not fail a put or a search.
+ *
+ * Returns: the extracted texts, in path order — one per match, kept apart so
+ * each is embedded on its own (see `extractTexts`). Empty when the path
+ * addresses nothing, and empty for a leaf JSON cannot represent.
+ *
+ * Throws: nothing.
+ */
+function getTextAtPath(value, path) {
+    if (path === '' || path === '$')
+        return prettyText(value);
+    return extract(value, tokenizePath(path), 0);
 }
-//# sourceMappingURL=semantic-search.js.map

package/dist/store/internal/setup.d.ts

@@ -1,22 +1,36 @@
-import type { DynamoDBClient } from '@aws-sdk/client-dynamodb';
-import type { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
+/**
+ * Hides which options the store and its methods accept, and how the store is
+ * assembled from them.
+ *
+ * The exhaustive key list of every option bag — the constructor's, `search`'s,
+ * `listNamespaces`' — lives here, compiler-checked against its type. The
+ * store's own options are checked after the shared ones — the in-memory caps,
+ * that a vector backend comes with an index, that the index can embed, the
+ * score direction — and the context every operation receives is built with its
+ * defaults filled in.
+ */
 import type { IndexConfig, SerializerProtocol } from '@langchain/langgraph-checkpoint';
-import type { CompressionConfig } from '../../shared/codec/compression';
-import { S3Offloader } from '../../shared/codec/s3/offloader';
-import { type Logger } from '../../shared/logging/logger';
-import type { TtlOption } from '../../shared/validation/ttl';
+import { type AdapterCore, type AdapterShell } from '../../shared/adapter';
 import type { DynamoDBStoreOptions } from '../types';
-import type { VectorBackend } from '../vector-backend';
-import { type VectorScoreDirection } from './score-direction';
+import { type VectorBackend, type VectorScoreDirection } from '../vector-backend';
+/** Default cap on candidates the in-DB semantic ranker will score. */
+export declare const DEFAULT_MAX_SEARCH_CANDIDATES = 1000;
+/**
+ * Largest `maxSearchCandidates` an adapter accepts: this many decoded
+ * candidates are held and re-ranked in memory by one `search()` call, so an
+ * unbounded value lets a typo or hostile config hold an unbounded working set.
+ */
+export declare const MAX_SEARCH_CANDIDATES = 100000;
+/**
+ * Largest `maxScanItems` an adapter accepts: this many raw rows are collected
+ * into memory across one paginated scan/query before it errors, so — like
+ * {@link MAX_SEARCH_CANDIDATES} — an unbounded value lets a typo or hostile
+ * config hold an unbounded working set.
+ */
+export declare const MAX_SCAN_ITEMS = 1000000;
 /** Resolved collaborators shared by every store action. */
-export interface StoreContext {
-    client: DynamoDBDocument;
-    tableName: string;
+export interface StoreContext extends AdapterCore {
     serde: SerializerProtocol;
-    compression?: CompressionConfig;
-    offloader?: S3Offloader;
-    ttl?: TtlOption;
-    logger: Logger;
     index?: IndexConfig;
     vectorBackend?: VectorBackend;
     vectorScoreDirection: VectorScoreDirection;
@@ -26,11 +40,43 @@ export interface StoreContext {
 /** Result of wiring up a store from its options. */
 export interface StoreSetup {
     context: StoreContext;
-    ddbClient: DynamoDBClient | undefined;
-    ownsClient: boolean;
+    shell: AdapterShell;
 }
 /**
- * Resolve the client, optional S3 offloader, serializer, and index config.
+ * Validate the options, then resolve the client, offloader, serializer and index.
+ *
+ * Accepts: `options` — validated first, so no half-built store exists when one
+ * is wrong. Everything optional has a default here and nowhere else, which is
+ * what lets every action read `context.x` without re-deciding what absent means.
+ *
+ * Returns: the context every action shares, and the shell that releases what
+ * it holds — a client the caller passed in is never destroyed by `destroy()`.
+ *
+ * Throws: `VALIDATION` for any invalid option, naming the option.
+ *
+ * Guarantees: constructing a store performs no I/O. The stacked-retry check is
+ * deliberately not awaited: it is a warning about a caller-supplied client, not
+ * a precondition.
+ */
+export declare function setUpStore(options: DynamoDBStoreOptions): StoreSetup;
+/**
+ * The keys of each store option bag, exhaustive in both directions:
+ * `allKeysOf<T>` makes omitting or inventing one a compile error, so a list
+ * cannot rot away from the type it guards. They live with the feature because
+ * the types they are checked against do; `shared/` knows no feature.
+ */
+export declare const STORE_KEYS: readonly string[];
+/** See {@link STORE_KEYS}. */
+export declare const STORE_SEARCH_KEYS: readonly string[];
+/**
+ * See {@link STORE_KEYS}. `ListNamespacesOptions` is pinned equal to
+ * `BaseStore.listNamespaces`' own parameter type, so this list is checked
+ * against upstream's options through it.
+ */
+export declare const STORE_LIST_NAMESPACES_KEYS: readonly string[];
+/**
+ * Validate the store's own options at construction; the shared ones are
+ * `openAdapter`'s, which checks them before these.
  *
  * A `vectorBackend` without an `index` is rejected outright rather than
  * silently degrading: with no embeddings configured, every `put` would compute
@@ -39,6 +85,17 @@ export interface StoreSetup {
  * listing with no `.score` field and no error — a semantic query returning a
  * normal-looking but meaningless response. `reconcileVectorIndex` already
  * refused this exact misconfiguration.
+ *
+ * Accepts: every option the store takes. The types describe the intended
+ * shapes; this runs for the JavaScript caller the types never see, and for the
+ * combinations no type can express — a backend without an index, an `index`
+ * key `IndexConfig` does not declare, an `embeddings` object missing a method,
+ * a direction outside its union.
+ *
+ * Returns: nothing: the value is kept under its declared type, and this
+ * checks it.
+ *
+ * Throws: `VALIDATION` naming the offending option. Every failure is raised
+ * at construction, where the fix is, rather than at the first put or search.
  */
-export declare function setUpStore(options: DynamoDBStoreOptions): StoreSetup;
-//# sourceMappingURL=setup.d.ts.map
+export declare function assertStoreOptions(options: DynamoDBStoreOptions): void;