@infinityi/engine-lib 1.0.0 → 1.4.0

package/dist/retrieval/loaders.d.ts

@@ -0,0 +1,8 @@
+import type { EngineContext } from "../runtime/types";
+import type { DocumentLoader, DocumentLoaderOutput, LoadDocumentsResult, LoadedDocument } from "./types";
+/** Build a document loader from a host-supplied load function. */
+export declare function createDocumentLoader(name: string, load: (ctx?: EngineContext) => DocumentLoaderOutput | Promise<DocumentLoaderOutput>): DocumentLoader;
+/** A deterministic in-memory loader useful for examples and tests. */
+export declare function staticDocumentLoader(documents: readonly LoadedDocument[], name?: string): DocumentLoader;
+/** Load and normalize documents from all loaders in declaration order. */
+export declare function loadDocuments(loaders: readonly DocumentLoader[], ctx?: EngineContext): Promise<LoadDocumentsResult>;

package/dist/retrieval/memory.d.ts

@@ -0,0 +1,29 @@
+import type { EngineContext } from "../runtime/types";
+import type { VectorQuery, VectorRecord, VectorSearchResult, VectorStore, VectorStoreStats } from "./types";
+import { type VectorSimilarity } from "./vector-store";
+/** Options for {@link InMemoryVectorStore}. */
+export interface InMemoryVectorStoreOptions {
+    readonly name?: string;
+    readonly dimensions?: number;
+    readonly similarity?: VectorSimilarity;
+}
+/**
+ * Network-free vector store for tests, examples, and small in-memory corpora.
+ * Queries linearly scan every record, so use a dedicated vector database for
+ * large or latency-sensitive corpora.
+ */
+export declare class InMemoryVectorStore implements VectorStore {
+    readonly name: string;
+    private readonly records;
+    private readonly similarity;
+    private recordDimensions;
+    constructor(options?: InMemoryVectorStoreOptions);
+    get dimensions(): number | undefined;
+    private ensureDimensions;
+    private ensureQueryDimensions;
+    upsert(records: readonly VectorRecord[], ctx?: EngineContext): Promise<void>;
+    query(query: VectorQuery, ctx?: EngineContext): Promise<readonly VectorSearchResult[]>;
+    delete(ids: readonly string[], ctx?: EngineContext): Promise<void>;
+    clear(ctx?: EngineContext): Promise<void>;
+    stats(ctx?: EngineContext): Promise<VectorStoreStats>;
+}

package/dist/retrieval/retriever.d.ts

@@ -0,0 +1,8 @@
+import type { EngineContext } from "../runtime/types";
+import type { HybridRetrievalOptions, IndexDocumentsOptions, IndexDocumentsResult, KeywordRetrievalResult, RetrievalResult, Retriever, VectorRetrieverOptions } from "./types";
+/** Merge vector and keyword retrieval hits with score normalization and de-duplication. */
+export declare function mergeHybridResults(vectorResults: readonly RetrievalResult[], keywordResults: readonly KeywordRetrievalResult[], options?: HybridRetrievalOptions): RetrievalResult[];
+/** Load, chunk, embed, and upsert documents into a vector store. */
+export declare function indexDocuments(options: IndexDocumentsOptions, ctx?: EngineContext): Promise<IndexDocumentsResult>;
+/** Create a retriever that embeds the query and searches a vector store. */
+export declare function createVectorRetriever(options: VectorRetrieverOptions): Retriever;

package/dist/retrieval/types.d.ts

@@ -0,0 +1,253 @@
+/**
+ * Public contracts for document loading, chunking, embeddings, vector storage,
+ * retrievers, and context-provider integration.
+ *
+ * @module
+ */
+import type { ContextResolveContext, ContextProvider, TokenCounter } from "../context/types";
+import type { EngineContext } from "../runtime/types";
+/** Location of a document or chunk inside its source. */
+export interface SourceLocation {
+    readonly startOffset?: number;
+    readonly endOffset?: number;
+    readonly startLine?: number;
+    readonly endLine?: number;
+}
+/** Source metadata carried through chunks, vector records, and retrieval hits. */
+export interface SourceAttribution {
+    readonly id?: string;
+    readonly uri?: string;
+    readonly title?: string;
+    readonly mimeType?: string;
+    readonly location?: SourceLocation;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** A source document before chunking. */
+export interface LoadedDocument {
+    readonly id?: string;
+    readonly content: string;
+    readonly source?: SourceAttribution;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** The values a document loader may return. */
+export type DocumentLoaderOutput = Iterable<LoadedDocument> | AsyncIterable<LoadedDocument>;
+/** Loads source documents from a host-owned corpus. */
+export interface DocumentLoader {
+    readonly name: string;
+    load(ctx?: EngineContext): DocumentLoaderOutput | Promise<DocumentLoaderOutput>;
+}
+/** Result summary for loading documents from one or more loaders. */
+export interface LoadDocumentsResult {
+    readonly documents: readonly LoadedDocument[];
+    readonly loaders: number;
+}
+/** A source document unit after chunking. */
+export interface RetrievalChunk {
+    readonly id: string;
+    readonly text: string;
+    readonly source?: SourceAttribution;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** Context passed to chunkers while indexing documents. */
+export interface ChunkerContext {
+    readonly documentIndex: number;
+    readonly engine?: EngineContext;
+}
+/** Splits a loaded document into retrieval chunks. */
+export interface Chunker {
+    readonly name: string;
+    chunk(document: LoadedDocument, ctx?: ChunkerContext): readonly RetrievalChunk[] | Promise<readonly RetrievalChunk[]>;
+}
+/** Options for the built-in plain-text chunker. */
+export interface TextChunkerOptions {
+    readonly maxChars: number;
+    readonly overlapChars?: number;
+    readonly separators?: readonly string[];
+}
+/** Embedding vector. */
+export type EmbeddingVector = readonly number[];
+/**
+ * Request to an embedding provider.
+ *
+ * Providers should validate their own maximum batch size and throw a clear
+ * provider-specific error when `input.length` exceeds what the backing service
+ * can accept. `indexDocuments` batches calls with `batchSize` to help callers
+ * stay within those limits.
+ */
+export interface EmbeddingRequest {
+    readonly input: readonly string[];
+    readonly model?: string;
+    readonly metadata?: Readonly<Record<string, string>>;
+}
+/** Optional embedding token accounting. */
+export interface EmbeddingUsage {
+    readonly inputTokens?: number;
+    readonly totalTokens?: number;
+}
+/** Embedding provider response. */
+export interface EmbeddingResult {
+    readonly model: string;
+    readonly vectors: readonly EmbeddingVector[];
+    readonly dimensions?: number;
+    readonly usage?: EmbeddingUsage;
+    readonly raw?: unknown;
+}
+/**
+ * Host-supplied embedding provider contract.
+ *
+ * Implementations should document their maximum `input` batch size. Callers can
+ * tune `IndexDocumentsOptions.batchSize` for indexing, while query retrieval
+ * sends one input per embedding call.
+ */
+export interface EmbeddingProvider {
+    readonly name: string;
+    readonly defaultModel?: string;
+    readonly dimensions?: number;
+    embed(req: EmbeddingRequest, ctx?: EngineContext): Promise<EmbeddingResult>;
+}
+/** A vector-store record. */
+export interface VectorRecord {
+    readonly id: string;
+    readonly vector: EmbeddingVector;
+    readonly text: string;
+    readonly source?: SourceAttribution;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** Host-supplied vector record filter. */
+export type VectorRecordFilter = (record: VectorRecord) => boolean;
+/** Query against a vector store. */
+export interface VectorQuery {
+    readonly vector: EmbeddingVector;
+    readonly topK?: number;
+    readonly filter?: VectorRecordFilter;
+    readonly minScore?: number;
+    readonly includeVectors?: boolean;
+}
+/** A vector-store search hit. */
+export interface VectorSearchResult {
+    readonly id: string;
+    readonly score: number;
+    readonly text: string;
+    readonly source?: SourceAttribution;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+    readonly vector?: EmbeddingVector;
+}
+/** Vector-store statistics. */
+export interface VectorStoreStats {
+    readonly records: number;
+    readonly dimensions?: number;
+}
+/** Minimal vector-store contract. */
+export interface VectorStore {
+    readonly name: string;
+    readonly dimensions?: number;
+    upsert(records: readonly VectorRecord[], ctx?: EngineContext): Promise<void>;
+    query(query: VectorQuery, ctx?: EngineContext): Promise<readonly VectorSearchResult[]>;
+    delete?(ids: readonly string[], ctx?: EngineContext): Promise<void>;
+    clear?(ctx?: EngineContext): Promise<void>;
+    stats?(ctx?: EngineContext): Promise<VectorStoreStats>;
+}
+/** Query passed to retrievers. */
+export interface RetrievalQuery {
+    readonly query: string;
+    readonly topK?: number;
+    readonly filter?: VectorRecordFilter;
+    readonly minScore?: number;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** Per-channel retrieval scores. */
+export interface RetrievalScores {
+    readonly vector?: number;
+    readonly keyword?: number;
+    readonly combined?: number;
+}
+/** A retriever result ready for context rendering. */
+export interface RetrievalResult {
+    readonly id: string;
+    readonly rank: number;
+    readonly score: number;
+    readonly scores?: RetrievalScores;
+    readonly text: string;
+    readonly source?: SourceAttribution;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** Main retrieval contract. */
+export interface Retriever {
+    readonly name: string;
+    retrieve(query: RetrievalQuery, ctx?: EngineContext): Promise<readonly RetrievalResult[]>;
+}
+/** Keyword retrieval hit used for hybrid merging. */
+export interface KeywordRetrievalResult {
+    readonly id: string;
+    readonly score: number;
+    readonly text: string;
+    readonly source?: SourceAttribution;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/** Optional keyword retriever hook. */
+export interface KeywordRetriever {
+    readonly name?: string;
+    retrieve(query: RetrievalQuery, ctx?: EngineContext): Promise<readonly KeywordRetrievalResult[]>;
+}
+/** Hybrid merge settings for vector and keyword retrieval results. */
+export interface HybridRetrievalOptions {
+    readonly topK?: number;
+    readonly minScore?: number;
+    readonly vectorWeight?: number;
+    readonly keywordWeight?: number;
+}
+/** Options for constructing a vector retriever. */
+export interface VectorRetrieverOptions {
+    readonly name?: string;
+    readonly embeddings: EmbeddingProvider;
+    readonly store: VectorStore;
+    readonly model?: string;
+    readonly topK?: number;
+    readonly minScore?: number;
+    readonly filter?: VectorRecordFilter;
+    readonly keyword?: KeywordRetriever;
+    readonly hybrid?: HybridRetrievalOptions;
+}
+/** Options for indexing loaded documents into a vector store. */
+export interface IndexDocumentsOptions {
+    readonly loaders: readonly DocumentLoader[];
+    readonly chunker: Chunker;
+    readonly embeddings: EmbeddingProvider;
+    readonly store: VectorStore;
+    /** Number of chunks to embed per provider call. Defaults to 64; tune to provider limits. */
+    readonly batchSize?: number;
+    readonly embeddingModel?: string;
+}
+/** Summary returned by document indexing. */
+export interface IndexDocumentsResult {
+    readonly documents: number;
+    readonly chunks: number;
+    readonly records: number;
+    readonly model?: string;
+    readonly dimensions?: number;
+}
+/** Options that control retriever context budget selection. */
+export interface RetrieverContextBudgetOptions {
+    readonly maxContextTokens?: number;
+    readonly reserveTokens?: number;
+    /**
+     * Counts rendered context tokens for budget checks. Keep this local and
+     * synchronous; truncating an oversized retrieval result may call it multiple
+     * times to find the largest exact fit.
+     */
+    readonly countTokens?: TokenCounter;
+}
+/** Options for converting a retriever into a context provider. */
+export interface RetrieverContextOptions extends RetrieverContextBudgetOptions {
+    readonly retriever: Retriever;
+    readonly name?: string;
+    readonly title?: string;
+    readonly query?: string | ((ctx: EngineContext, run?: ContextResolveContext) => string | undefined | Promise<string | undefined>);
+    readonly topK?: number;
+    readonly minScore?: number;
+    /** Include numeric retrieval scores in rendered context. Defaults to false. */
+    readonly includeScores?: boolean;
+    readonly onResults?: (results: readonly RetrievalResult[], query: string, ctx: EngineContext, run?: ContextResolveContext) => void | Promise<void>;
+}
+/** Semantic marker for the `ContextProvider` returned by {@link retrieverContext}. */
+export type RetrieverContextProvider = ContextProvider;

package/dist/retrieval/utils.d.ts

@@ -0,0 +1,11 @@
+import type { EngineContext } from "../runtime/types";
+interface EngineContextHolder {
+    readonly engine?: EngineContext;
+}
+/** Throw when retrieval work should stop because the caller aborted the run. */
+export declare function throwIfAborted(ctx?: EngineContext | EngineContextHolder): void;
+/** Assert that an optional numeric option is a positive integer when present. */
+export declare function assertPositiveInteger(name: string, value: number | undefined): void;
+/** Assert that a numeric option is a non-negative integer. */
+export declare function assertNonNegativeInteger(name: string, value: number): void;
+export {};

package/dist/retrieval/vector-store.d.ts

@@ -0,0 +1,13 @@
+import type { EmbeddingVector } from "./types";
+/** Similarity modes supported by the in-memory vector store. */
+export type VectorSimilarity = "cosine" | "dot" | "euclidean";
+/** Validate that a value is a finite, non-empty vector. */
+export declare function assertVector(vector: EmbeddingVector, label?: string): void;
+/** Dot product for equal-length vectors. */
+export declare function dotProduct(a: EmbeddingVector, b: EmbeddingVector): number;
+/** Cosine similarity for equal-length vectors. */
+export declare function cosineSimilarity(a: EmbeddingVector, b: EmbeddingVector): number;
+/** Convert Euclidean distance to a descending similarity score. */
+export declare function euclideanSimilarity(a: EmbeddingVector, b: EmbeddingVector): number;
+/** Score two vectors with the requested similarity mode. */
+export declare function scoreVectors(a: EmbeddingVector, b: EmbeddingVector, similarity: VectorSimilarity): number;

package/dist/runtime/index.js

@@ -3,7 +3,8 @@ import {
   Secret,
   isSecret,
   resolveSecret
-} from "../index-xsv43c5j.js";
+} from "../index-e14g2e48.js";
+import"../index-37x76zdn.js";
 export {
   resolveSecret,
   isSecret,

package/dist/schema/index.js

@@ -1,11 +1,17 @@
+import"../index-c6kgts41.js";
 import {
   asSchema,
   fromJsonSchema,
-  s,
-  toJsonSchema,
+  toJsonSchema
+} from "../index-k0q7x4mz.js";
+import {
+  s
+} from "../index-mnx5mqbs.js";
+import {
   validateJsonSchema
-} from "../index-02s1fjxr.js";
-import"../index-7690reng.js";
+} from "../index-4c15ysa8.js";
+import"../index-ajr3nk10.js";
+import"../index-37x76zdn.js";
 export {
   validateJsonSchema,
   toJsonSchema,

package/dist/session/index.js

@@ -2,6 +2,7 @@ import {
   InMemorySessionStore,
   createSession
 } from "../index-vnby35rm.js";
+import"../index-37x76zdn.js";
 export {
   createSession,
   InMemorySessionStore

package/dist/session-stores/codec.d.ts

@@ -0,0 +1,8 @@
+import type { Message } from "../messages/types";
+import type { SessionStoreCodec } from "./types";
+/** Default JSON codec used by all built-in durable stores. */
+export declare const jsonSessionStoreCodec: SessionStoreCodec;
+export declare function encodeMetadata(codec: SessionStoreCodec, metadata: Readonly<Record<string, unknown>> | undefined): Promise<string | undefined>;
+export declare function decodeMetadata(codec: SessionStoreCodec, payload: string | null | undefined): Promise<Readonly<Record<string, unknown>> | undefined>;
+export declare function encodeMessages(codec: SessionStoreCodec, messages: readonly Message[]): Promise<string[]>;
+export declare function decodeMessages(codec: SessionStoreCodec, payloads: readonly string[]): Promise<Message[]>;

package/dist/session-stores/forge-data.d.ts

@@ -0,0 +1,43 @@
+import type { DatabaseSchema, Db } from "@infinityi/forge/data";
+import type { PostgresClientLike } from "@infinityi/forge/data/dialects/postgres";
+import type { SqliteDriverOptions } from "@infinityi/forge/data/dialects/sqlite";
+import type { Message } from "../messages/types";
+import type { SessionState, SessionStore } from "../session/types";
+import type { SessionStoreCodec } from "./types";
+export interface ForgeDataSessionStoreOptions {
+    readonly db: Db<DatabaseSchema>;
+    readonly tablePrefix?: string;
+    readonly codec?: SessionStoreCodec;
+    readonly closeDbOnClose?: boolean;
+}
+export interface CreateSqliteSessionStoreOptions extends Pick<SqliteDriverOptions, "database" | "filename" | "create"> {
+    readonly tablePrefix?: string;
+    readonly codec?: SessionStoreCodec;
+    readonly migrate?: boolean;
+}
+export interface CreatePostgresSessionStoreOptions {
+    readonly client: PostgresClientLike;
+    readonly closeOnShutdown?: boolean;
+    readonly tablePrefix?: string;
+    readonly codec?: SessionStoreCodec;
+    readonly migrate?: boolean;
+}
+/** SQL-backed {@link SessionStore} implemented over a Forge `Db`. */
+export declare class ForgeDataSessionStore implements SessionStore {
+    private readonly db;
+    private readonly codec;
+    private readonly tables;
+    private readonly closeDbOnClose;
+    private writeQueue;
+    constructor(options: ForgeDataSessionStoreOptions);
+    migrate(): Promise<void>;
+    schemaVersion(): Promise<number | undefined>;
+    load(id: string): Promise<SessionState | undefined>;
+    append(id: string, messages: readonly Message[]): Promise<void>;
+    save(state: SessionState): Promise<void>;
+    delete(id: string): Promise<void>;
+    close(): Promise<void>;
+    private enqueueWrite;
+}
+export declare function createSqliteSessionStore(options?: CreateSqliteSessionStoreOptions): Promise<ForgeDataSessionStore>;
+export declare function createPostgresSessionStore(options: CreatePostgresSessionStoreOptions): Promise<ForgeDataSessionStore>;

package/dist/session-stores/hooks.d.ts

@@ -0,0 +1,4 @@
+import type { SessionStore } from "../session/types";
+import type { CloseableSessionStore, SessionStoreHooks, VersionedSessionStore } from "./types";
+/** Decorate a store with host-owned compaction and archival hooks. */
+export declare function withSessionStoreHooks<T extends SessionStore>(store: T, hooks: SessionStoreHooks): T & Partial<VersionedSessionStore & CloseableSessionStore>;

package/dist/session-stores/ids.d.ts

@@ -0,0 +1,3 @@
+export declare function assertTablePrefix(prefix: string): string;
+export declare function safeSessionKey(id: string): string;
+export declare function sessionFileName(id: string): string;

package/dist/session-stores/index.d.ts

@@ -0,0 +1,20 @@
+/**
+ * `@infinityi/engine-lib/session-stores` — optional durable implementations of
+ * the stable {@link SessionStore} contract.
+ *
+ * These adapters are opt-in so the root package stays lightweight. Built-ins
+ * include Forge SQL/SQLite/PostgreSQL, Redis, and filesystem JSONL stores, plus
+ * common codec, migration, compaction, and archival helpers.
+ *
+ * @module
+ */
+export { jsonSessionStoreCodec } from "./codec";
+export { ForgeDataSessionStore, createPostgresSessionStore, createSqliteSessionStore } from "./forge-data";
+export { withSessionStoreHooks } from "./hooks";
+export { FilesystemJsonlSessionStore } from "./jsonl";
+export { RedisSessionStore } from "./redis";
+export { SESSION_STORE_SCHEMA_VERSION, isCloseableSessionStore, isVersionedSessionStore, migrateSessionStore } from "./versioning";
+export type { CreatePostgresSessionStoreOptions, CreateSqliteSessionStoreOptions, ForgeDataSessionStoreOptions, } from "./forge-data";
+export type { FilesystemJsonlSessionStoreOptions } from "./jsonl";
+export type { RedisSessionStoreClient, RedisSessionStoreOptions, RedisSessionStoreTransaction } from "./redis";
+export type { CloseableSessionStore, SessionArchiveRecord, SessionArchiver, SessionCompactionResult, SessionCompactor, SessionStoreCodec, SessionStoreHookContext, SessionStoreHookOperation, SessionStoreHooks, VersionedSessionStore, } from "./types";