@lov3kaizen/agentsea-embeddings 0.5.1

package/dist/chunking/index.mjs

@@ -0,0 +1,37 @@
+import {
+  BaseChunker,
+  CodeChunker,
+  FixedChunker,
+  MarkdownChunker,
+  RecursiveChunker,
+  SemanticChunker,
+  chunk,
+  createChunker,
+  createCodeChunker,
+  createFixedChunker,
+  createMarkdownChunker,
+  createRecursiveChunker,
+  createSemanticChunker,
+  defaultTokenCounter,
+  mergeSmallChunks,
+  splitLargeChunks
+} from "../chunk-DJAURHAS.mjs";
+import "../chunk-QAITLJ2E.mjs";
+export {
+  BaseChunker,
+  CodeChunker,
+  FixedChunker,
+  MarkdownChunker,
+  RecursiveChunker,
+  SemanticChunker,
+  chunk,
+  createChunker,
+  createCodeChunker,
+  createFixedChunker,
+  createMarkdownChunker,
+  createRecursiveChunker,
+  createSemanticChunker,
+  defaultTokenCounter,
+  mergeSmallChunks,
+  splitLargeChunks
+};

package/dist/embedding.types-CCgPVxt1.d.mts

@@ -0,0 +1,102 @@
+type EmbeddingVector = number[];
+interface EmbeddingResult {
+    vector: EmbeddingVector;
+    text: string;
+    tokenCount: number;
+    cached: boolean;
+    model: string;
+    dimensions: number;
+    latencyMs: number;
+    metadata?: Record<string, unknown>;
+}
+interface BatchEmbeddingResult {
+    results: EmbeddingResult[];
+    totalTokens: number;
+    totalLatencyMs: number;
+    cacheHits: number;
+    cacheMisses: number;
+    failures: number;
+}
+interface EmbeddedChunk {
+    id: string;
+    text: string;
+    vector: EmbeddingVector;
+    index: number;
+    startPosition: number;
+    endPosition: number;
+    tokenCount: number;
+    metadata: ChunkMetadata;
+}
+interface ChunkMetadata {
+    documentId?: string;
+    source?: string;
+    type?: string;
+    page?: number;
+    section?: string;
+    [key: string]: unknown;
+}
+interface EmbeddingModelInfo {
+    name: string;
+    provider: string;
+    dimensions: number;
+    maxTokens: number;
+    maxBatchSize: number;
+    costPer1K?: number;
+    description?: string;
+}
+interface EmbeddingOptions {
+    metadata?: Record<string, unknown>;
+    user?: string;
+    model?: string;
+    skipCache?: boolean;
+    forceCache?: boolean;
+    timeout?: number;
+}
+interface BatchEmbeddingOptions extends EmbeddingOptions {
+    concurrency?: number;
+    onProgress?: (progress: BatchProgress) => void;
+    continueOnError?: boolean;
+    retryFailed?: boolean;
+    maxRetries?: number;
+}
+interface BatchProgress {
+    percent: number;
+    processed: number;
+    total: number;
+    current?: string;
+    elapsedMs: number;
+    estimatedRemainingMs?: number;
+}
+interface DocumentEmbeddingOptions extends BatchEmbeddingOptions {
+    documentId?: string;
+    source?: string;
+    type?: string;
+    chunkMetadata?: Record<string, unknown>;
+}
+interface SearchResult {
+    id: string;
+    text: string;
+    score: number;
+    metadata: ChunkMetadata;
+    distance?: number;
+}
+interface SearchOptions {
+    topK?: number;
+    minScore?: number;
+    filter?: Record<string, unknown>;
+    includeVectors?: boolean;
+    includeMetadata?: boolean;
+    namespace?: string;
+}
+type SimilarityMetric = 'cosine' | 'euclidean' | 'dot_product';
+interface EmbeddingStats {
+    totalEmbeddings: number;
+    totalTokens: number;
+    avgLatencyMs: number;
+    cacheHitRate: number;
+    apiCalls: number;
+    errors: number;
+    estimatedCostUSD: number;
+}
+
+export type { BatchEmbeddingOptions as B, ChunkMetadata as C, DocumentEmbeddingOptions as D, EmbeddingVector as E, SearchOptions as S, EmbeddingResult as a, EmbeddingOptions as b, BatchEmbeddingResult as c, EmbeddedChunk as d, SearchResult as e, EmbeddingStats as f, EmbeddingModelInfo as g, BatchProgress as h, SimilarityMetric as i };

package/dist/embedding.types-CCgPVxt1.d.ts

@@ -0,0 +1,102 @@
+type EmbeddingVector = number[];
+interface EmbeddingResult {
+    vector: EmbeddingVector;
+    text: string;
+    tokenCount: number;
+    cached: boolean;
+    model: string;
+    dimensions: number;
+    latencyMs: number;
+    metadata?: Record<string, unknown>;
+}
+interface BatchEmbeddingResult {
+    results: EmbeddingResult[];
+    totalTokens: number;
+    totalLatencyMs: number;
+    cacheHits: number;
+    cacheMisses: number;
+    failures: number;
+}
+interface EmbeddedChunk {
+    id: string;
+    text: string;
+    vector: EmbeddingVector;
+    index: number;
+    startPosition: number;
+    endPosition: number;
+    tokenCount: number;
+    metadata: ChunkMetadata;
+}
+interface ChunkMetadata {
+    documentId?: string;
+    source?: string;
+    type?: string;
+    page?: number;
+    section?: string;
+    [key: string]: unknown;
+}
+interface EmbeddingModelInfo {
+    name: string;
+    provider: string;
+    dimensions: number;
+    maxTokens: number;
+    maxBatchSize: number;
+    costPer1K?: number;
+    description?: string;
+}
+interface EmbeddingOptions {
+    metadata?: Record<string, unknown>;
+    user?: string;
+    model?: string;
+    skipCache?: boolean;
+    forceCache?: boolean;
+    timeout?: number;
+}
+interface BatchEmbeddingOptions extends EmbeddingOptions {
+    concurrency?: number;
+    onProgress?: (progress: BatchProgress) => void;
+    continueOnError?: boolean;
+    retryFailed?: boolean;
+    maxRetries?: number;
+}
+interface BatchProgress {
+    percent: number;
+    processed: number;
+    total: number;
+    current?: string;
+    elapsedMs: number;
+    estimatedRemainingMs?: number;
+}
+interface DocumentEmbeddingOptions extends BatchEmbeddingOptions {
+    documentId?: string;
+    source?: string;
+    type?: string;
+    chunkMetadata?: Record<string, unknown>;
+}
+interface SearchResult {
+    id: string;
+    text: string;
+    score: number;
+    metadata: ChunkMetadata;
+    distance?: number;
+}
+interface SearchOptions {
+    topK?: number;
+    minScore?: number;
+    filter?: Record<string, unknown>;
+    includeVectors?: boolean;
+    includeMetadata?: boolean;
+    namespace?: string;
+}
+type SimilarityMetric = 'cosine' | 'euclidean' | 'dot_product';
+interface EmbeddingStats {
+    totalEmbeddings: number;
+    totalTokens: number;
+    avgLatencyMs: number;
+    cacheHitRate: number;
+    apiCalls: number;
+    errors: number;
+    estimatedCostUSD: number;
+}
+
+export type { BatchEmbeddingOptions as B, ChunkMetadata as C, DocumentEmbeddingOptions as D, EmbeddingVector as E, SearchOptions as S, EmbeddingResult as a, EmbeddingOptions as b, BatchEmbeddingResult as c, EmbeddedChunk as d, SearchResult as e, EmbeddingStats as f, EmbeddingModelInfo as g, BatchProgress as h, SimilarityMetric as i };

package/dist/index-CeG6God2.d.mts

@@ -0,0 +1,297 @@
+import { E as EmbeddingVector, a as EmbeddingResult, c as BatchEmbeddingResult, g as EmbeddingModelInfo, b as EmbeddingOptions, B as BatchEmbeddingOptions } from './embedding.types-CCgPVxt1.mjs';
+
+type EmbeddingProviderType = 'openai' | 'cohere' | 'voyage' | 'local' | 'huggingface' | 'anthropic' | 'google' | 'custom';
+interface ProviderConfig {
+    type: EmbeddingProviderType;
+    apiKey?: string;
+    baseUrl?: string;
+    model?: string;
+    timeout?: number;
+    maxRetries?: number;
+    retryDelay?: number;
+    headers?: Record<string, string>;
+    options?: Record<string, unknown>;
+}
+interface OpenAIProviderConfig extends ProviderConfig {
+    type: 'openai';
+    organization?: string;
+    apiVersion?: string;
+    model?: string;
+    encodingFormat?: 'float' | 'base64';
+    dimensions?: number;
+}
+interface CohereProviderConfig extends ProviderConfig {
+    type: 'cohere';
+    model?: string;
+    inputType?: 'search_document' | 'search_query' | 'classification' | 'clustering';
+    truncate?: 'NONE' | 'START' | 'END';
+}
+interface VoyageProviderConfig extends ProviderConfig {
+    type: 'voyage';
+    model?: string;
+    inputType?: 'document' | 'query';
+    truncation?: boolean;
+}
+interface LocalProviderConfig extends ProviderConfig {
+    type: 'local';
+    modelPath?: string;
+    modelType?: 'onnx' | 'transformers' | 'sentence-transformers';
+    device?: 'cpu' | 'cuda' | 'mps';
+    batchSize?: number;
+    normalize?: boolean;
+    pooling?: 'mean' | 'cls' | 'max';
+}
+interface HuggingFaceProviderConfig extends ProviderConfig {
+    type: 'huggingface';
+    model?: string;
+    useInferenceApi?: boolean;
+    waitForModel?: boolean;
+}
+interface EmbeddingRequest {
+    text: string;
+    model?: string;
+    user?: string;
+    options?: Record<string, unknown>;
+}
+interface BatchEmbeddingRequest {
+    texts: string[];
+    model?: string;
+    user?: string;
+    options?: Record<string, unknown>;
+}
+interface ProviderResponse {
+    embeddings: EmbeddingVector[];
+    model: string;
+    usage: TokenUsage;
+    metadata?: Record<string, unknown>;
+}
+interface TokenUsage {
+    promptTokens: number;
+    totalTokens: number;
+}
+interface ProviderError {
+    code: string;
+    message: string;
+    status?: number;
+    retryable: boolean;
+    retryAfter?: number;
+    details?: Record<string, unknown>;
+}
+interface ProviderHealth {
+    healthy: boolean;
+    latencyMs: number;
+    lastCheck: number;
+    error?: string;
+    rateLimit?: RateLimitInfo;
+}
+interface RateLimitInfo {
+    remaining: number;
+    limit: number;
+    reset: number;
+    retryAfter?: number;
+}
+interface ProviderMetrics {
+    provider: EmbeddingProviderType;
+    totalRequests: number;
+    successfulRequests: number;
+    failedRequests: number;
+    totalTokens: number;
+    avgLatencyMs: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    p99LatencyMs: number;
+    errorRate: number;
+    rateLimitHits: number;
+    estimatedCostUSD: number;
+}
+interface ModelInfo {
+    id: string;
+    name: string;
+    provider: EmbeddingProviderType;
+    dimensions: number;
+    maxTokens: number;
+    maxBatchSize: number;
+    costPer1K?: number;
+    description?: string;
+    releaseDate?: string;
+    deprecated?: boolean;
+    replacement?: string;
+}
+interface ProviderCapabilities {
+    batch: boolean;
+    maxBatchSize: number;
+    streaming: boolean;
+    customDimensions: boolean;
+    inputTypes: string[];
+    models: string[];
+}
+interface ProviderFactoryOptions {
+    defaultProvider?: EmbeddingProviderType;
+    providers?: Record<EmbeddingProviderType, ProviderConfig>;
+    enableMetrics?: boolean;
+    enableHealthChecks?: boolean;
+    healthCheckInterval?: number;
+}
+interface CustomProviderConfig extends ProviderConfig {
+    type: 'custom';
+    embedFn: (text: string, options?: Record<string, unknown>) => Promise<EmbeddingResult>;
+    batchEmbedFn?: (texts: string[], options?: Record<string, unknown>) => Promise<BatchEmbeddingResult>;
+    modelInfo: ModelInfo;
+}
+
+declare abstract class EmbeddingModel {
+    abstract readonly info: EmbeddingModelInfo;
+    abstract embed(text: string, options?: EmbeddingOptions): Promise<EmbeddingResult>;
+    abstract embedBatch(texts: string[], options?: BatchEmbeddingOptions): Promise<BatchEmbeddingResult>;
+    get dimensions(): number;
+    get maxTokens(): number;
+    get maxBatchSize(): number;
+    get name(): string;
+    get provider(): string;
+    countTokens(text: string): number;
+    exceedsMaxTokens(text: string): boolean;
+    truncateToMaxTokens(text: string): string;
+    static cosineSimilarity(a: EmbeddingVector, b: EmbeddingVector): number;
+    static euclideanDistance(a: EmbeddingVector, b: EmbeddingVector): number;
+    static dotProduct(a: EmbeddingVector, b: EmbeddingVector): number;
+    static normalize(vector: EmbeddingVector): EmbeddingVector;
+    static average(vectors: EmbeddingVector[]): EmbeddingVector;
+    static weightedAverage(vectors: EmbeddingVector[], weights: number[]): EmbeddingVector;
+}
+declare class ModelRegistry {
+    private models;
+    private defaultModel;
+    register(model: EmbeddingModel, isDefault?: boolean): void;
+    get(provider: string, name: string): EmbeddingModel | undefined;
+    getByKey(key: string): EmbeddingModel | undefined;
+    getDefault(): EmbeddingModel | undefined;
+    setDefault(provider: string, name: string): void;
+    list(): EmbeddingModelInfo[];
+    has(provider: string, name: string): boolean;
+    remove(provider: string, name: string): boolean;
+    clear(): void;
+}
+declare const modelRegistry: ModelRegistry;
+
+declare abstract class BaseProvider extends EmbeddingModel {
+    protected config: ProviderConfig;
+    protected metrics: ProviderMetrics;
+    protected health: ProviderHealth;
+    protected latencies: number[];
+    private readonly maxLatencySamples;
+    constructor(config: ProviderConfig);
+    private createInitialMetrics;
+    protected abstract doEmbed(texts: string[], options?: EmbeddingOptions): Promise<{
+        vectors: number[][];
+        tokenCount: number;
+    }>;
+    embed(text: string, options?: EmbeddingOptions): Promise<EmbeddingResult>;
+    embedBatch(texts: string[], options?: BatchEmbeddingOptions): Promise<BatchEmbeddingResult>;
+    protected isRetryable(error: Error): boolean;
+    protected recordLatency(latencyMs: number): void;
+    protected calculatePercentile(p: number): number;
+    protected updateMetrics(): void;
+    getMetrics(): ProviderMetrics;
+    getHealth(): ProviderHealth;
+    checkHealth(): Promise<ProviderHealth>;
+    resetMetrics(): void;
+}
+
+declare class OpenAIProvider extends BaseProvider {
+    private modelInfo;
+    private apiKey;
+    private baseUrl;
+    private organization?;
+    constructor(config: OpenAIProviderConfig);
+    get info(): EmbeddingModelInfo;
+    protected doEmbed(texts: string[], options?: EmbeddingOptions): Promise<{
+        vectors: number[][];
+        tokenCount: number;
+    }>;
+    countTokens(text: string): number;
+}
+declare function createOpenAIProvider(config: OpenAIProviderConfig): OpenAIProvider;
+
+declare class CohereProvider extends BaseProvider {
+    private modelInfo;
+    private apiKey;
+    private baseUrl;
+    private inputType;
+    private truncate;
+    constructor(config: CohereProviderConfig);
+    get info(): EmbeddingModelInfo;
+    protected doEmbed(texts: string[], options?: EmbeddingOptions): Promise<{
+        vectors: number[][];
+        tokenCount: number;
+    }>;
+    setInputType(inputType: CohereProviderConfig['inputType']): this;
+    countTokens(text: string): number;
+}
+declare function createCohereProvider(config: CohereProviderConfig): CohereProvider;
+
+declare class VoyageProvider extends BaseProvider {
+    private modelInfo;
+    private apiKey;
+    private baseUrl;
+    private inputType;
+    private truncation;
+    constructor(config: VoyageProviderConfig);
+    get info(): EmbeddingModelInfo;
+    protected doEmbed(texts: string[], options?: EmbeddingOptions): Promise<{
+        vectors: number[][];
+        tokenCount: number;
+    }>;
+    setInputType(inputType: VoyageProviderConfig['inputType']): this;
+    countTokens(text: string): number;
+}
+declare function createVoyageProvider(config: VoyageProviderConfig): VoyageProvider;
+
+type LocalEmbeddingFn = (texts: string[], options?: Record<string, unknown>) => Promise<number[][]>;
+interface LocalProviderOptions extends LocalProviderConfig {
+    embedFn?: LocalEmbeddingFn;
+    dimensions: number;
+    name?: string;
+    maxTokens?: number;
+    maxBatchSize?: number;
+}
+declare class LocalProvider extends BaseProvider {
+    private modelInfo;
+    private embedFn;
+    private normalize;
+    private batchSize;
+    constructor(config: LocalProviderOptions);
+    get info(): EmbeddingModelInfo;
+    protected doEmbed(texts: string[], options?: EmbeddingOptions): Promise<{
+        vectors: number[][];
+        tokenCount: number;
+    }>;
+    setEmbedFunction(fn: LocalEmbeddingFn): this;
+    countTokens(text: string): number;
+}
+declare function createLocalProvider(config: LocalProviderOptions): LocalProvider;
+declare function createMockProvider(config: {
+    dimensions: number;
+    name?: string;
+    delay?: number;
+}): LocalProvider;
+declare function createRandomProvider(config: {
+    dimensions: number;
+    name?: string;
+}): LocalProvider;
+
+declare class HuggingFaceProvider extends BaseProvider {
+    private modelInfo;
+    private apiKey;
+    private baseUrl;
+    private waitForModel;
+    constructor(config: HuggingFaceProviderConfig);
+    get info(): EmbeddingModelInfo;
+    protected doEmbed(texts: string[], _options?: EmbeddingOptions): Promise<{
+        vectors: number[][];
+        tokenCount: number;
+    }>;
+    countTokens(text: string): number;
+}
+declare function createHuggingFaceProvider(config: HuggingFaceProviderConfig): HuggingFaceProvider;
+
+export { BaseProvider as B, CohereProvider as C, type EmbeddingProviderType as E, HuggingFaceProvider as H, LocalProvider as L, type ModelInfo as M, OpenAIProvider as O, type ProviderConfig as P, type RateLimitInfo as R, type TokenUsage as T, VoyageProvider as V, EmbeddingModel as a, ModelRegistry as b, createOpenAIProvider as c, createCohereProvider as d, createVoyageProvider as e, createLocalProvider as f, createMockProvider as g, createRandomProvider as h, createHuggingFaceProvider as i, type LocalEmbeddingFn as j, type LocalProviderOptions as k, type OpenAIProviderConfig as l, modelRegistry as m, type CohereProviderConfig as n, type VoyageProviderConfig as o, type LocalProviderConfig as p, type HuggingFaceProviderConfig as q, type EmbeddingRequest as r, type BatchEmbeddingRequest as s, type ProviderResponse as t, type ProviderError as u, type ProviderHealth as v, type ProviderMetrics as w, type ProviderCapabilities as x, type ProviderFactoryOptions as y, type CustomProviderConfig as z };