@resistdesign/voltra 3.0.0-alpha.2 → 3.0.0-alpha.21

package/api/Indexing/fulltext/FullTextMemoryBackend.d.ts

@@ -0,0 +1,93 @@
+import type { DocId, DocTokenBatchReader, DocTokenKey, IndexReader, IndexWriter, LossyPagingReader, LossyPostingsPage, LossyPostingsPageOptions, TokenStats } from "../Types";
+/**
+ * In-memory backend combining lossy and exact indexes.
+ */
+export declare class FullTextMemoryBackend implements IndexReader, IndexWriter, LossyPagingReader, DocTokenBatchReader {
+    private LossyIndex;
+    private ExactIndex;
+    private docTokenMembership;
+    private createMembershipKey;
+    /**
+     * Add a lossy posting for a token.
+     * @param token Token value to add.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @returns Promise resolved once posting is added.
+     */
+    addLossyPosting(token: string, indexField: string, docId: DocId): Promise<void>;
+    /**
+     * Remove a lossy posting for a token.
+     * @param token Token value to remove.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @returns Promise resolved once posting is removed.
+     */
+    removeLossyPosting(token: string, indexField: string, docId: DocId): Promise<void>;
+    /**
+     * Load lossy postings for a token.
+     * @param token Token value to load postings for.
+     * @param indexField Field name the token was indexed under.
+     * @returns Document ids containing the token.
+     */
+    loadLossyPostings(token: string, indexField: string): Promise<DocId[]>;
+    /**
+     * Query a page of lossy postings for a token.
+     * @param token Token value to query postings for.
+     * @param indexField Field name the token was indexed under.
+     * @param options Paging options for the query.
+     * @returns Postings page with optional cursor.
+     */
+    queryLossyPostingsPage(token: string, indexField: string, options?: LossyPostingsPageOptions): Promise<LossyPostingsPage>;
+    /**
+     * Add exact token positions for a document.
+     * @param token Token value to store positions for.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @param positions Token positions within the document.
+     * @returns Promise resolved once positions are stored.
+     */
+    addExactPositions(token: string, indexField: string, docId: DocId, positions: number[]): Promise<void>;
+    /**
+     * Remove exact token positions for a document.
+     * @param token Token value to remove positions for.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @returns Promise resolved once positions are removed.
+     */
+    removeExactPositions(token: string, indexField: string, docId: DocId): Promise<void>;
+    /**
+     * Load exact positions for a token in a document.
+     * @param token Token value to load positions for.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @returns Positions array or undefined when missing.
+     */
+    loadExactPositions(token: string, indexField: string, docId: DocId): Promise<number[] | undefined>;
+    /**
+     * Batch load exact positions for token keys.
+     * @param keys Token keys to load positions for.
+     * @returns Positions arrays aligned with the input keys.
+     */
+    batchLoadExactPositions(keys: DocTokenKey[]): Promise<(number[] | undefined)[]>;
+    /**
+     * Load token stats for a token.
+     * @param token Token value to load stats for.
+     * @param indexField Field name the token was indexed under.
+     * @returns Token stats or undefined when no postings exist.
+     */
+    loadTokenStats(token: string, indexField: string): Promise<TokenStats | undefined>;
+    /**
+     * Check whether a document contains a token.
+     * @param docId Document id to check.
+     * @param indexField Field name the token was indexed under.
+     * @param token Token value to check.
+     * @returns True when the document contains the token.
+     */
+    hasDocToken(docId: DocId, indexField: string, token: string): Promise<boolean>;
+    /**
+     * Batch check whether documents contain tokens.
+     * @param keys Token keys to check.
+     * @returns Booleans aligned with the input keys.
+     */
+    batchHasDocTokens(keys: DocTokenKey[]): Promise<boolean[]>;
+}

package/api/Indexing/fulltext/Schema.d.ts

@@ -0,0 +1,164 @@
+/**
+ * @packageDocumentation
+ *
+ * DynamoDB schema constants and key encoders for fulltext tables.
+ */
+export declare const fullTextKeyPrefixes: {
+    /**
+     * Prefix for index field values.
+     */
+    readonly field: "f#";
+    /**
+     * Prefix for token values.
+     */
+    readonly token: "t#";
+    /**
+     * Prefix for document ids.
+     */
+    readonly doc: "d#";
+    /**
+     * Prefix for token position values.
+     */
+    readonly position: "p#";
+};
+/**
+ * LossyPostings table
+ * PK: f#{indexField}#t#{token}
+ * SK: d#{docId}
+ */
+export declare const lossyPostingsSchema: {
+    /**
+     * Partition key attribute for lossy postings.
+     */
+    readonly partitionKey: "pk";
+    /**
+     * Sort key attribute for lossy postings.
+     */
+    readonly sortKey: "sk";
+};
+/**
+ * ExactPostings table
+ * PK: f#{indexField}#t#{token}
+ * SK: d#{docId}
+ */
+export declare const exactPostingsSchema: {
+    /**
+     * Partition key attribute for exact postings.
+     */
+    readonly partitionKey: "pk";
+    /**
+     * Sort key attribute for exact postings.
+     */
+    readonly sortKey: "sk";
+    /**
+     * Attribute name holding position arrays.
+     */
+    readonly positionsAttribute: "positions";
+};
+/**
+ * Optional FullTextDocMirror table
+ * PK: d#{docId}
+ */
+export declare const fullTextDocMirrorSchema: {
+    /**
+     * Partition key attribute for document mirrors.
+     */
+    readonly partitionKey: "pk";
+    /**
+     * Attribute name for stored normalized content.
+     */
+    readonly contentAttribute: "content";
+};
+/**
+ * FullTextTokenStats table
+ * PK: f#{indexField}#t#{token}
+ */
+export declare const fullTextTokenStatsSchema: {
+    /**
+     * Partition key attribute for token stats.
+     */
+    readonly partitionKey: "pk";
+    /**
+     * Attribute name for document frequency values.
+     */
+    readonly documentFrequencyAttribute: "df";
+};
+/**
+ * Optional DocTokens table
+ * PK: d#{docId}
+ * SK: f#{indexField}#t#{token}
+ */
+export declare const docTokensSchema: {
+    /**
+     * Partition key attribute for doc token membership.
+     */
+    readonly partitionKey: "pk";
+    /**
+     * Sort key attribute for doc token membership.
+     */
+    readonly sortKey: "sk";
+};
+/**
+ * Optional DocTokenPositions table
+ * PK: d#{docId}
+ * SK: f#{indexField}#t#{token}
+ */
+export declare const docTokenPositionsSchema: {
+    /**
+     * Partition key attribute for doc token positions.
+     */
+    readonly partitionKey: "pk";
+    /**
+     * Sort key attribute for doc token positions.
+     */
+    readonly sortKey: "sk";
+    /**
+     * Attribute name holding position arrays.
+     */
+    readonly positionsAttribute: "positions";
+};
+/**
+ * Encode a token key for token-based tables.
+ * @param indexField Field name the token was indexed under. Use a type-qualified
+ * field name (for example "Article.title") when multiple types share fields.
+ * @param token Token value.
+ * @returns Encoded token key.
+ */
+export declare function encodeTokenKey(indexField: string, token: string): string;
+/**
+ * Encode a document key for document-based tables.
+ * @param docId Document id to encode.
+ * @returns Encoded document key.
+ */
+export declare function encodeDocKey(docId: string | number): string;
+/**
+ * Encode the key used for the document mirror table.
+ * @param indexField Field name the document was indexed under. Use a
+ * type-qualified field name when multiple types share fields.
+ * @param docId Document id to encode.
+ * @returns Encoded document mirror key.
+ */
+export declare function encodeDocMirrorKey(indexField: string | number, docId: string | number): string;
+/**
+ * Encode sort key for token-to-document tables.
+ * @param docId Document id to encode.
+ * @returns Encoded sort key for token docs.
+ */
+export declare function encodeTokenDocSortKey(docId: string | number): string;
+/**
+ * Encode sort key for document-to-token tables.
+ * @param indexField Field name the token was indexed under. Use a
+ * type-qualified field name when multiple types share fields.
+ * @param token Token value.
+ * @returns Encoded sort key for doc tokens.
+ */
+export declare function encodeDocTokenSortKey(indexField: string, token: string): string;
+/**
+ * Encode sort key for token positions within a document.
+ * @param indexField Field name the token was indexed under. Use a
+ * type-qualified field name when multiple types share fields.
+ * @param token Token value.
+ * @param position Token position within the document.
+ * @returns Encoded sort key for token positions.
+ */
+export declare function encodeDocTokenPositionSortKey(indexField: string, token: string, position: number): string;

package/api/Indexing/hashUniversal.d.ts

@@ -0,0 +1,13 @@
+type HashEncoding = "hex";
+type Hash = {
+    update(value: string | ArrayBuffer | ArrayBufferView): Hash;
+    digest(): Promise<Uint8Array>;
+    digest(encoding: HashEncoding): Promise<string>;
+};
+/**
+ * Create a hashing helper compatible with Node-style update/digest flow.
+ * @param alg Hash algorithm name (only "sha256" is supported).
+ * @returns Hashing helper with update and digest methods.
+ */
+export declare function createHash(alg: "sha256"): Hash;
+export {};

package/api/Indexing/index.d.ts

@@ -0,0 +1,45 @@
+/**
+ * @packageDocumentation
+ *
+ * Indexing utilities and backends for exact, lossy, fulltext, relational, and
+ * structured search.
+ *
+ * Concepts:
+ * - Doc IDs can be strings or numbers; they are normalized for cursor and key
+ *   encoding when persisted or paginated.
+ * - Cursors are URL-safe tokens that encode paging state (lossy/exact/structured/rel).
+ * - Tokenization helpers normalize text for exact tokens or lossy trigrams.
+ * - Search traces capture metrics (token counts, batch calls, etc.) for observability.
+ *
+ * Quick usage:
+ * ```ts
+ * import { indexDocument, searchLossy } from "./Indexing";
+ *
+ * await indexDocument({ backend, indexField: "title", docId: "doc-1", text: "Hello world" });
+ * const page = await searchLossy({ backend, indexField: "title", query: "Hello", limit: 10 });
+ * ```
+ */
+export * from "./API";
+export * from "./Cursor";
+export * from "./fieldQualification";
+export * from "./ddb/AwsSdkV3Adapter";
+export * from "./ddb/Types";
+export * from "./exact/ExactDdb";
+export * from "./exact/ExactIndex";
+export * from "./exact/ExactS3";
+export * from "./fulltext/FullTextMemoryBackend";
+export { FullTextDdbBackend, FullTextDdbWriter, type BatchGetItemInput, type BatchGetItemOutput, type BatchWriteItemInput, type BatchWriteItemOutput, type FullTextTableNames, type GetItemInput, type GetItemOutput, type KeysAndAttributes, type QueryInput, type QueryOutput, } from "./fulltext/FullTextDdbBackend";
+export * from "./fulltext/Schema";
+export * from "./lossy/LossyDdb";
+export * from "./lossy/LossyIndex";
+export * from "./lossy/LossyS3";
+export * from "./rel/RelationalInMemoryBackend";
+export * from "./rel/RelationalDdb";
+export * from "./rel/Cursor";
+export { handler as relHandler, setRelationalHandlerDependencies, type EdgePutEvent, type EdgeRemoveEvent, type EdgeQueryEvent, type RelationalHandlerDependencies, type RelationalHandlerEvent, type LambdaResponse as RelLambdaResponse, } from "./rel/Handlers";
+export * from "./rel/Types";
+export * from "./structured/index";
+export * from "./tokenize";
+export * from "./Types";
+export * from "./Trace";
+export type { ResolvedSearchLimits } from "./Handler/Config";

package/api/Indexing/lossy/LossyDdb.d.ts

@@ -0,0 +1,43 @@
+/**
+ * @packageDocumentation
+ *
+ * DynamoDB schema helpers for the lossy postings table. The lossy index stores
+ * token -> docId mappings for recall-oriented search.
+ */
+import type { DocId } from "../Types";
+/**
+ * DynamoDB key shape for lossy postings items.
+ */
+export type LossyDdbKey = {
+    /**
+     * Token value stored in the lossy index.
+     */
+    token: string;
+    /**
+     * Field name the token was indexed under.
+     */
+    indexField: string;
+    /**
+     * Document id containing the token.
+     */
+    docId: DocId;
+};
+/**
+ * DynamoDB item shape for lossy postings entries.
+ */
+export type LossyDdbItem = LossyDdbKey;
+/**
+ * Schema metadata for the lossy postings table.
+ */
+export declare const lossyDdbSchema: {
+    readonly partitionKey: "token";
+    readonly sortKey: "docId";
+};
+/**
+ * Build the DynamoDB key for a lossy postings item.
+ * @param token Token value stored in the lossy index.
+ * @param indexField Field name the token was indexed under.
+ * @param docId Document id containing the token.
+ * @returns Lossy postings key for DynamoDB.
+ */
+export declare function buildLossyDdbKey(token: string, indexField: string, docId: DocId): LossyDdbKey;

package/api/Indexing/lossy/LossyIndex.d.ts

@@ -0,0 +1,72 @@
+/**
+ * @packageDocumentation
+ *
+ * In-memory lossy index for recall-heavy search. It stores token postings and
+ * supports paging with cursor-style doc IDs.
+ */
+import type { DocId } from "../Types";
+/**
+ * Paging options for lossy postings queries.
+ */
+export type LossyQueryOptions = {
+    /**
+     * Maximum number of postings to return.
+     */
+    limit?: number;
+    /**
+     * Resume cursor using the last processed doc id.
+     */
+    lastDocId?: DocId;
+};
+/**
+ * Results for lossy postings queries.
+ */
+export type LossyQueryResult = {
+    /**
+     * Document ids for the token postings.
+     */
+    docIds: DocId[];
+    /**
+     * Cursor to resume paging through postings.
+     */
+    nextCursor?: DocId;
+};
+/**
+ * In-memory lossy index for token postings.
+ */
+export declare class LossyIndex {
+    private postings;
+    private buildTokenKey;
+    /**
+     * Add a posting for a token.
+     * @param token Token value to add.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @returns Nothing.
+     */
+    addPosting(token: string, indexField: string, docId: DocId): void;
+    /**
+     * Remove a posting for a token.
+     * @param token Token value to remove.
+     * @param indexField Field name the token was indexed under.
+     * @param docId Document id containing the token.
+     * @returns Nothing.
+     */
+    removePosting(token: string, indexField: string, docId: DocId): void;
+    /**
+     * Add a document by inserting postings for each unique token.
+     * @param docId Document id to add.
+     * @param indexField Field name the tokens are indexed under.
+     * @param tokens Token list for the document.
+     * @returns Nothing.
+     */
+    addDocument(docId: DocId, indexField: string, tokens: string[]): void;
+    /**
+     * Get postings for a token with optional paging.
+     * @param token Token value to fetch postings for.
+     * @param indexField Field name the token was indexed under.
+     * @param options Paging options for the query.
+     * @returns Postings result with optional next cursor.
+     */
+    getPostings(token: string, indexField: string, options?: LossyQueryOptions): LossyQueryResult;
+}

package/api/Indexing/lossy/LossyS3.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @packageDocumentation
+ *
+ * S3 helpers for storing lossy postings. Each token is stored as a JSON list
+ * of doc IDs under a deterministic key.
+ */
+import type { DocId } from "../Types";
+/**
+ * Location of a lossy postings object in S3.
+ */
+export type LossyS3Pointer = {
+    /**
+     * Bucket name containing the lossy postings object.
+     */
+    bucket: string;
+    /**
+     * Object key for the lossy postings object.
+     */
+    key: string;
+};
+/**
+ * Build the S3 key for a lossy postings object.
+ * @param token Token value stored in the lossy index.
+ * @param indexField Field name the token was indexed under.
+ * @returns Object key for the lossy postings JSON.
+ */
+export declare function buildLossyS3Key(token: string, indexField: string): string;
+/**
+ * Store lossy postings for a pointer.
+ * @param pointer Bucket/key pair for the postings object.
+ * @param docIds Document ids to store for the token.
+ * @returns Promise resolved once postings are stored.
+ */
+export declare function storeLossyIndex(pointer: LossyS3Pointer, docIds: DocId[]): Promise<void>;
+/**
+ * Load lossy postings for a pointer.
+ * @param pointer Bucket/key pair for the postings object.
+ * @returns Document ids stored for the token.
+ */
+export declare function loadLossyIndex(pointer: LossyS3Pointer): Promise<DocId[]>;

package/api/Indexing/rel/Cursor.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Cursor state for relational paging.
+ */
+export type RelationalCursorState = {
+    /**
+     * Last entity id processed in the previous page.
+     */
+    lastId?: string;
+    /**
+     * Backend continuation token used to resume paging.
+     */
+    continuationToken?: string;
+};
+/**
+ * Encode relational cursor state into a URL-safe string.
+ * @param state Cursor state to encode.
+ * @returns URL-safe cursor string, or undefined when there is no state to encode.
+ */
+export declare function encodeRelationalCursor(state?: RelationalCursorState): string | undefined;
+/**
+ * Decode a relational cursor string back into state.
+ * @param cursor Cursor string to decode.
+ * @returns Parsed cursor state, or undefined when cursor is missing.
+ */
+export declare function decodeRelationalCursor(cursor?: string): RelationalCursorState | undefined;

package/api/Indexing/rel/Handlers.d.ts

@@ -0,0 +1,143 @@
+/**
+ * @packageDocumentation
+ *
+ * Lambda-style handler for relational edge operations (put/remove/query).
+ * Use {@link setRelationalHandlerDependencies} to supply a backend implementation.
+ */
+import type { Direction, Edge, EdgeKey, EdgePage, RelationalQueryOptions } from "./Types";
+type EdgeMetadata = Record<string, unknown>;
+type RelationalBackend<TMetadata extends EdgeMetadata = EdgeMetadata> = {
+    /**
+     * Insert or update an edge.
+     * @param edge Edge to store.
+     * @returns Promise resolved once stored.
+     */
+    putEdge(edge: Edge<TMetadata>): Promise<void> | void;
+    /**
+     * Remove an edge by key.
+     * @param key Edge key to remove.
+     * @returns Promise resolved once removed.
+     */
+    removeEdge(key: EdgeKey): Promise<void> | void;
+    /**
+     * Query outgoing edges for an entity and relation.
+     * @param fromId Source entity id.
+     * @param relation Relation name.
+     * @param options Optional paging options.
+     * @returns Page of outgoing edges.
+     */
+    getOutgoing(fromId: string, relation: string, options?: RelationalQueryOptions): Promise<EdgePage<TMetadata>> | EdgePage<TMetadata>;
+    /**
+     * Query incoming edges for an entity and relation.
+     * @param toId Target entity id.
+     * @param relation Relation name.
+     * @param options Optional paging options.
+     * @returns Page of incoming edges.
+     */
+    getIncoming(toId: string, relation: string, options?: RelationalQueryOptions): Promise<EdgePage<TMetadata>> | EdgePage<TMetadata>;
+};
+/**
+ * Event payload for inserting or updating an edge.
+ */
+export type EdgePutEvent<TMetadata extends EdgeMetadata = EdgeMetadata> = {
+    /**
+     * Action discriminator for edge inserts.
+     */
+    action: "edge/put";
+    /**
+     * Edge payload to insert.
+     */
+    edge: Edge<TMetadata>;
+};
+/**
+ * Event payload for removing an edge.
+ */
+export type EdgeRemoveEvent = {
+    /**
+     * Action discriminator for edge removals.
+     */
+    action: "edge/remove";
+    /**
+     * Edge key to remove.
+     */
+    key: EdgeKey;
+};
+/**
+ * Event payload for querying edges.
+ */
+export type EdgeQueryEvent = {
+    /**
+     * Action discriminator for edge queries.
+     */
+    action: "edge/query";
+    /**
+     * Direction of traversal (outgoing or incoming).
+     */
+    direction: Direction;
+    /**
+     * Entity id to query from.
+     */
+    entityId: string;
+    /**
+     * Relation name to filter by.
+     */
+    relation: string;
+    /**
+     * Optional maximum number of edges to return.
+     */
+    limit?: number;
+    /**
+     * Optional cursor string for pagination.
+     */
+    cursor?: string;
+};
+/**
+ * Union of relational handler event payloads.
+ */
+export type RelationalHandlerEvent<TMetadata extends EdgeMetadata = EdgeMetadata> = EdgePutEvent<TMetadata> | EdgeRemoveEvent | EdgeQueryEvent;
+/**
+ * Dependencies for the relational handler.
+ */
+export type RelationalHandlerDependencies<TMetadata extends EdgeMetadata = EdgeMetadata> = {
+    /**
+     * Relational backend implementation to execute operations.
+     */
+    backend: RelationalBackend<TMetadata>;
+};
+/**
+ * Lambda-style response payload.
+ */
+export type LambdaResponse = {
+    /**
+     * HTTP status code for the response.
+     */
+    statusCode: number;
+    /**
+     * Serialized response body.
+     */
+    body: string;
+};
+/**
+ * Configure the handler with a relational backend implementation.
+ *
+ * Example put event:
+ * { "action": "edge/put", "edge": { "key": { "from": "a", "to": "b", "relation": "owns" } } }
+ *
+ * Example remove event:
+ * { "action": "edge/remove", "key": { "from": "a", "to": "b", "relation": "owns" } }
+ *
+ * Example query event:
+ * { "action": "edge/query", "direction": "out", "entityId": "a", "relation": "owns", "limit": 25 }
+ */
+export declare function setRelationalHandlerDependencies(
+/**
+ * Backend dependencies used by the handler.
+ */
+value: RelationalHandlerDependencies): void;
+/**
+ * Handle relational edge events for a backend.
+ * @param event Handler event describing the operation to execute.
+ * @returns Lambda response with status and body payload.
+ */
+export declare function handler(event: RelationalHandlerEvent): Promise<LambdaResponse>;
+export {};