agentdb 3.0.0-alpha.10 → 3.0.0-alpha.12

package/dist/src/benchmark/BenchmarkSuite.d.ts

@@ -0,0 +1,318 @@
+/**
+ * AgentDB Benchmark Suite
+ *
+ * Comprehensive benchmarking framework for measuring:
+ * - Vector insert performance (batch and single)
+ * - Vector search latency (p50, p95, p99)
+ * - Memory consumption
+ * - Concurrent read/write performance
+ * - Quantization performance comparison
+ *
+ * Usage:
+ *   import { BenchmarkSuite, runBenchmarks } from './benchmark/BenchmarkSuite';
+ *
+ *   const suite = new BenchmarkSuite();
+ *   const report = await suite.runAll();
+ *   console.log(JSON.stringify(report, null, 2));
+ *
+ * CLI integration:
+ *   npx tsx packages/agentdb/src/benchmark/BenchmarkSuite.ts
+ */
+/**
+ * Latency statistics with percentiles
+ */
+export interface LatencyStats {
+    /** 50th percentile (median) in milliseconds */
+    p50: number;
+    /** 95th percentile in milliseconds */
+    p95: number;
+    /** 99th percentile in milliseconds */
+    p99: number;
+    /** Mean latency in milliseconds */
+    mean: number;
+    /** Maximum latency in milliseconds */
+    max: number;
+    /** Minimum latency in milliseconds */
+    min: number;
+}
+/**
+ * Result from a single benchmark run
+ */
+export interface BenchmarkResult {
+    /** Benchmark name */
+    name: string;
+    /** Operations per second */
+    opsPerSecond: number;
+    /** Latency statistics */
+    latencyMs: LatencyStats;
+    /** Memory usage in megabytes */
+    memoryMB: number;
+    /** Total benchmark duration in milliseconds */
+    duration: number;
+    /** Total operations performed */
+    operations: number;
+    /** Optional additional metrics */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Complete benchmark report
+ */
+export interface BenchmarkReport {
+    /** Timestamp of the benchmark run */
+    timestamp: string;
+    /** Platform information */
+    platform: {
+        os: string;
+        arch: string;
+        nodeVersion: string;
+    };
+    /** Backend information */
+    backend: {
+        name: string;
+        isNative?: boolean;
+    };
+    /** Configuration used for benchmarks */
+    config: {
+        vectorDimension: number;
+        warmupIterations: number;
+    };
+    /** Individual benchmark results */
+    results: BenchmarkResult[];
+    /** Summary statistics */
+    summary: {
+        totalDuration: number;
+        benchmarksRun: number;
+        benchmarksPassed: number;
+        peakMemoryMB: number;
+    };
+}
+/**
+ * Comparison between two benchmark runs
+ */
+export interface ComparisonReport {
+    /** Baseline report timestamp */
+    baselineTimestamp: string;
+    /** Current report timestamp */
+    currentTimestamp: string;
+    /** Comparison for each benchmark */
+    comparisons: Array<{
+        name: string;
+        baseline: BenchmarkResult | null;
+        current: BenchmarkResult | null;
+        /** Percentage change in ops/sec (positive = improvement) */
+        opsPerSecondChange: number | null;
+        /** Percentage change in p99 latency (negative = improvement) */
+        p99LatencyChange: number | null;
+        /** Percentage change in memory (negative = improvement) */
+        memoryChange: number | null;
+        /** Overall assessment */
+        status: 'improved' | 'regressed' | 'unchanged' | 'new' | 'removed';
+    }>;
+    /** Summary of changes */
+    summary: {
+        improved: number;
+        regressed: number;
+        unchanged: number;
+        new: number;
+        removed: number;
+    };
+}
+/**
+ * Benchmark configuration options
+ */
+export interface BenchmarkConfig {
+    /** Vector dimension for tests (default: 384) */
+    vectorDimension?: number;
+    /** Number of warmup iterations before measuring (default: 100) */
+    warmupIterations?: number;
+    /** Vector counts for insert benchmarks (default: [1000, 10000, 100000]) */
+    insertCounts?: number[];
+    /** Number of search queries to measure (default: 1000) */
+    searchQueries?: number;
+    /** k value for k-NN search (default: 10) */
+    searchK?: number;
+    /** Concurrency levels to test (default: [1, 4, 8, 16]) */
+    concurrencyLevels?: number[];
+    /** Whether to run memory-intensive tests (default: true) */
+    runMemoryTests?: boolean;
+    /** Whether to run quantization tests (default: true) */
+    runQuantizationTests?: boolean;
+}
+/**
+ * Abstract base class for benchmarks
+ */
+export declare abstract class Benchmark {
+    /** Unique benchmark name */
+    abstract readonly name: string;
+    /** Description of what this benchmark measures */
+    abstract readonly description: string;
+    /**
+     * Setup phase - prepare resources before benchmarking
+     */
+    abstract setup(): Promise<void>;
+    /**
+     * Run the benchmark and return results
+     */
+    abstract run(): Promise<BenchmarkResult>;
+    /**
+     * Cleanup phase - release resources after benchmarking
+     */
+    abstract teardown(): Promise<void>;
+    /**
+     * Generate a random normalized vector
+     */
+    protected generateRandomVector(dimension: number): Float32Array;
+    /**
+     * Generate multiple random vectors
+     */
+    protected generateRandomVectors(count: number, dimension: number): Float32Array[];
+    /**
+     * Calculate latency statistics from measurements
+     */
+    protected calculateLatencyStats(measurements: number[]): LatencyStats;
+    /**
+     * Get current memory usage in MB
+     */
+    protected getMemoryUsageMB(): number;
+    /**
+     * Force garbage collection if available
+     */
+    protected forceGC(): void;
+}
+/**
+ * Vector Insert Benchmark
+ *
+ * Tests batch insert performance at different scales:
+ * - 1K vectors (warm cache scenarios)
+ * - 10K vectors (typical use case)
+ * - 100K vectors (stress test)
+ */
+export declare class VectorInsertBenchmark extends Benchmark {
+    readonly name = "VectorInsert";
+    readonly description = "Batch insert performance at different scales";
+    private backend;
+    private config;
+    private vectors1K;
+    private vectors10K;
+    private vectors100K;
+    constructor(config?: BenchmarkConfig);
+    setup(): Promise<void>;
+    run(): Promise<BenchmarkResult>;
+    teardown(): Promise<void>;
+}
+/**
+ * Vector Search Benchmark
+ *
+ * Tests k-NN search latency with percentile measurements
+ */
+export declare class VectorSearchBenchmark extends Benchmark {
+    readonly name = "VectorSearch";
+    readonly description = "k-NN search latency (p50, p95, p99)";
+    private backend;
+    private config;
+    private queryVectors;
+    constructor(config?: BenchmarkConfig);
+    setup(): Promise<void>;
+    run(): Promise<BenchmarkResult>;
+    teardown(): Promise<void>;
+}
+/**
+ * Memory Usage Benchmark
+ *
+ * Tracks memory consumption at different scales
+ */
+export declare class MemoryUsageBenchmark extends Benchmark {
+    readonly name = "MemoryUsage";
+    readonly description = "Memory consumption tracking";
+    private backend;
+    private config;
+    constructor(config?: BenchmarkConfig);
+    setup(): Promise<void>;
+    run(): Promise<BenchmarkResult>;
+    teardown(): Promise<void>;
+}
+/**
+ * Concurrency Benchmark
+ *
+ * Tests concurrent read/write performance
+ */
+export declare class ConcurrencyBenchmark extends Benchmark {
+    readonly name = "Concurrency";
+    readonly description = "Concurrent read/write performance";
+    private backend;
+    private config;
+    constructor(config?: BenchmarkConfig);
+    setup(): Promise<void>;
+    run(): Promise<BenchmarkResult>;
+    teardown(): Promise<void>;
+}
+/**
+ * Quantization Benchmark
+ *
+ * Compares quantized vs unquantized performance
+ */
+export declare class QuantizationBenchmark extends Benchmark {
+    readonly name = "Quantization";
+    readonly description = "Quantized vs unquantized performance comparison";
+    private config;
+    constructor(config?: BenchmarkConfig);
+    setup(): Promise<void>;
+    run(): Promise<BenchmarkResult>;
+    teardown(): Promise<void>;
+}
+/**
+ * BenchmarkSuite - Orchestrates benchmark execution and reporting
+ */
+export declare class BenchmarkSuite {
+    private benchmarks;
+    private config;
+    constructor(config?: BenchmarkConfig);
+    /**
+     * Register a benchmark to the suite
+     */
+    register(benchmark: Benchmark): void;
+    /**
+     * Unregister a benchmark from the suite
+     */
+    unregister(name: string): boolean;
+    /**
+     * Get list of registered benchmark names
+     */
+    listBenchmarks(): string[];
+    /**
+     * Run all registered benchmarks
+     */
+    runAll(): Promise<BenchmarkReport>;
+    /**
+     * Run a specific benchmark by name
+     */
+    runByName(name: string): Promise<BenchmarkResult>;
+    /**
+     * Compare two benchmark reports
+     */
+    compare(baseline: BenchmarkReport, current: BenchmarkReport): ComparisonReport;
+}
+/**
+ * Run benchmarks and return results as JSON
+ *
+ * @param config - Benchmark configuration
+ * @returns Promise<BenchmarkReport>
+ */
+export declare function runBenchmarks(config?: BenchmarkConfig): Promise<BenchmarkReport>;
+/**
+ * Run benchmarks with a specific subset
+ *
+ * @param names - Names of benchmarks to run
+ * @param config - Benchmark configuration
+ * @returns Promise<BenchmarkResult[]>
+ */
+export declare function runSelectedBenchmarks(names: string[], config?: BenchmarkConfig): Promise<BenchmarkResult[]>;
+/**
+ * Format benchmark report as markdown
+ */
+export declare function formatReportAsMarkdown(report: BenchmarkReport): string;
+/**
+ * Format comparison report as markdown
+ */
+export declare function formatComparisonAsMarkdown(comparison: ComparisonReport): string;
+//# sourceMappingURL=BenchmarkSuite.d.ts.map

package/dist/src/benchmark/BenchmarkSuite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BenchmarkSuite.d.ts","sourceRoot":"","sources":["../../../src/benchmark/BenchmarkSuite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAgBH;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+CAA+C;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,yBAAyB;IACzB,SAAS,EAAE,YAAY,CAAC;IACxB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,QAAQ,EAAE;QACR,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,0BAA0B;IAC1B,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF,wCAAwC;IACxC,MAAM,EAAE;QACN,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,mCAAmC;IACnC,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,yBAAyB;IACzB,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,oCAAoC;IACpC,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;QACjC,OAAO,EAAE,eAAe,GAAG,IAAI,CAAC;QAChC,4DAA4D;QAC5D,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,gEAAgE;QAChE,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,2DAA2D;QAC3D,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,yBAAyB;QACzB,MAAM,EAAE,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,SAAS,CAAC;KACpE,CAAC,CAAC;IACH,yBAAyB;IACzB,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,GAAG,EAAE,MAAM,CAAC;QACZ,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,4DAA4D;IAC5D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wDAAwD;IACxD,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAMD;;GAEG;AACH,8BAAsB,SAAS;IAC7B,4BAA4B;IAC5B,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAE/B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAEtC;;OAEG;IACH,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAE/B;;OAEG;IACH,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IAExC;;OAEG;IACH,QAAQ,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAElC;;OAEG;IACH,SAAS,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY;IAmB/D;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,EAAE;IAQjF;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,YAAY;IAyBrE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,MAAM;IAOpC;;OAEG;IACH,SAAS,CAAC,OAAO,IAAI,IAAI;CAK1B;AAMD;;;;;;;GAOG;AACH,qBAAa,qBAAsB,SAAQ,SAAS;IAClD,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,WAAW,kDAAkD;IAEtE,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,WAAW,CAAsB;gBAE7B,MAAM,GAAE,eAAoB;IAclC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBtB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IA8E/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAOhC;AAED;;;;GAIG;AACH,qBAAa,qBAAsB,SAAQ,SAAS;IAClD,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,WAAW,yCAAyC;IAE7D,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAAsB;gBAE9B,MAAM,GAAE,eAAoB;IAclC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBtB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IA+C/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAKhC;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,SAAS;IACjD,QAAQ,CAAC,IAAI,iBAAiB;IAC9B,QAAQ,CAAC,WAAW,iCAAiC;IAErD,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,MAAM,CAA4B;gBAE9B,MAAM,GAAE,eAAoB;IAclC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IAqE/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGhC;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,SAAS;IACjD,QAAQ,CAAC,IAAI,iBAAiB;IAC9B,QAAQ,CAAC,WAAW,uCAAuC;IAE3D,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,MAAM,CAA4B;gBAE9B,MAAM,GAAE,eAAoB;IAclC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBtB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IAsF/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAIhC;AAED;;;;GAIG;AACH,qBAAa,qBAAsB,SAAQ,SAAS;IAClD,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,WAAW,qDAAqD;IAEzE,OAAO,CAAC,MAAM,CAA4B;gBAE9B,MAAM,GAAE,eAAoB;IAclC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IA8J/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGhC;AAMD;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,MAAM,CAA4B;gBAE9B,MAAM,GAAE,eAAoB;IA2BxC;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAIpC;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIjC;;OAEG;IACH,cAAc,IAAI,MAAM,EAAE;IAI1B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,eAAe,CAAC;IAkFxC;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAkBvD;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,gBAAgB;CA4E/E;AAMD;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,MAAM,GAAE,eAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,CAG1F;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,GAAE,eAAoB,GAC3B,OAAO,CAAC,eAAe,EAAE,CAAC,CAc5B;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA+BtE;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,UAAU,EAAE,gBAAgB,GAAG,MAAM,CA6C/E"}