@xdarkicex/openclaw-memory-libravdb 1.3.5

package/sidecar/embed/engine_test.go

@@ -0,0 +1,349 @@
+package embed
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/sugarme/tokenizer"
+)
+
+type fakeMiniLMModel struct {
+	vector []float32
+	err    error
+}
+
+func (m fakeMiniLMModel) Compute(_ string, _ bool) ([]float32, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	return append([]float32(nil), m.vector...), nil
+}
+
+func (m fakeMiniLMModel) TokenCount(_ string, _ bool) (int, error) {
+	if m.err != nil {
+		return 0, m.err
+	}
+	return 4, nil
+}
+
+func (m fakeMiniLMModel) Encode(_ string, _ bool) (*tokenizer.Encoding, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	return &tokenizer.Encoding{
+		Ids:           []int{1, 2, 3, 4},
+		TypeIds:       []int{0, 0, 0, 0},
+		AttentionMask: []int{1, 1, 1, 1},
+		Tokens:        []string{"a", "b", "c", "d"},
+		Offsets:       [][]int{{0, 1}, {1, 2}, {2, 3}, {3, 4}},
+		Words:         []int{0, 1, 2, 3},
+	}, nil
+}
+
+func (m fakeMiniLMModel) ComputeEncoding(_ tokenizer.Encoding) ([]float32, error) {
+	return m.Compute("", true)
+}
+
+func TestNewUnavailableIsNotReady(t *testing.T) {
+	engine := NewUnavailable("missing runtime")
+	if engine.Ready() {
+		t.Fatalf("expected unavailable engine to be not ready")
+	}
+	if engine.Dimensions() != DefaultDimensions {
+		t.Fatalf("expected dimensions %d, got %d", DefaultDimensions, engine.Dimensions())
+	}
+}
+
+func TestNewPrimaryRequiresRuntimePath(t *testing.T) {
+	originalResolver := resolveBundledSpec
+	originalFactory := newONNXLocalBackend
+	t.Cleanup(func() {
+		resolveBundledSpec = originalResolver
+		newONNXLocalBackend = originalFactory
+	})
+
+	resolveBundledSpec = func(cfg Config) (onnxLocalSpec, error) {
+		return onnxLocalSpec{}, errors.New("missing bundled runtime")
+	}
+	engine := NewPrimary("")
+	if !engine.Ready() {
+		t.Fatalf("expected missing runtime path to fall back to a ready local embedder")
+	}
+	if engine.Mode() != "fallback" {
+		t.Fatalf("expected fallback mode, got %q", engine.Mode())
+	}
+
+	resolveBundledSpec = func(cfg Config) (onnxLocalSpec, error) {
+		if cfg.RuntimePath != "/tmp/libonnxruntime.dylib" {
+			t.Fatalf("unexpected runtime path %q", cfg.RuntimePath)
+		}
+		if cfg.Profile != DefaultEmbeddingProfile {
+			t.Fatalf("expected default profile %q, got %q", DefaultEmbeddingProfile, cfg.Profile)
+		}
+		return onnxLocalSpec{RuntimePath: cfg.RuntimePath, Dimensions: 768, Normalize: true, Profile: buildProfile(Profile{
+			Backend:    "bundled",
+			Family:     DefaultEmbeddingProfile,
+			Dimensions: 768,
+			Normalize:  true,
+		})}, nil
+	}
+	newONNXLocalBackend = func(spec onnxLocalSpec) (embeddingBackend, error) {
+		return fakeONNXLocalBackend{vector: make([]float32, 768)}, nil
+	}
+	engine = NewPrimary("/tmp/libonnxruntime.dylib")
+	if !engine.Ready() {
+		t.Fatalf("expected non-empty runtime path to mark engine ready")
+	}
+	if engine.Mode() != "primary" {
+		t.Fatalf("expected primary mode, got %q", engine.Mode())
+	}
+	if engine.Dimensions() != 768 {
+		t.Fatalf("expected dimensions 768, got %d", engine.Dimensions())
+	}
+}
+
+func TestBundledBackendUsesRealModelFactory(t *testing.T) {
+	originalResolver := resolveBundledSpec
+	originalFactory := newONNXLocalBackend
+	t.Cleanup(func() {
+		resolveBundledSpec = originalResolver
+		newONNXLocalBackend = originalFactory
+	})
+
+	resolveBundledSpec = func(cfg Config) (onnxLocalSpec, error) {
+		return onnxLocalSpec{
+			RuntimePath: cfg.RuntimePath,
+			Dimensions:  384,
+			Normalize:   true,
+			Profile: buildProfile(Profile{
+				Backend:    "bundled",
+				Family:     "all-minilm-l6-v2",
+				Dimensions: 384,
+				Normalize:  true,
+			}),
+		}, nil
+	}
+	newONNXLocalBackend = func(spec onnxLocalSpec) (embeddingBackend, error) {
+		if spec.RuntimePath != "/opt/onnx/libonnxruntime.so" {
+			t.Fatalf("unexpected runtime path %q", spec.RuntimePath)
+		}
+		vec := make([]float32, 384)
+		vec[0] = 3
+		vec[1] = 4
+		return miniLMBackend{
+			model:     fakeMiniLMModel{vector: vec},
+			normalize: spec.Normalize,
+		}, nil
+	}
+
+	engine := NewWithConfig(Config{
+		Backend:     "bundled",
+		RuntimePath: "/opt/onnx/libonnxruntime.so",
+		Dimensions:  384,
+		Normalize:   true,
+	})
+	if !engine.Ready() {
+		t.Fatalf("expected bundled engine to be ready")
+	}
+	if engine.Mode() != "primary" {
+		t.Fatalf("expected primary mode, got %q", engine.Mode())
+	}
+
+	vec, err := engine.EmbedDocument(context.Background(), "hello")
+	if err != nil {
+		t.Fatalf("EmbedDocument() error = %v", err)
+	}
+	if len(vec) != 384 {
+		t.Fatalf("expected 384 dimensions, got %d", len(vec))
+	}
+	if vec[0] == 3 || vec[1] == 4 {
+		t.Fatalf("expected normalized output from bundled backend")
+	}
+}
+
+func TestNewWithConfigSupportsPowerUserSeam(t *testing.T) {
+	engine := NewWithConfig(Config{
+		Backend:    "custom-local",
+		ModelPath:  "/models/custom.onnx",
+		Dimensions: 768,
+		Normalize:  true,
+	})
+	if !engine.Ready() {
+		t.Fatalf("expected custom-local engine to be ready")
+	}
+	if engine.Mode() != "custom-local" {
+		t.Fatalf("expected custom-local mode, got %q", engine.Mode())
+	}
+	if engine.Dimensions() != 768 {
+		t.Fatalf("expected dimensions 768, got %d", engine.Dimensions())
+	}
+
+	fallback := NewWithConfig(Config{
+		Backend:    "custom-local",
+		Dimensions: 512,
+	})
+	if fallback.Mode() != "fallback" {
+		t.Fatalf("expected missing model path to fall back, got %q", fallback.Mode())
+	}
+	if fallback.Dimensions() != 512 {
+		t.Fatalf("expected fallback dimensions 512, got %d", fallback.Dimensions())
+	}
+
+	unavailable := NewWithConfig(Config{
+		Backend:    "made-up-backend",
+		Dimensions: 256,
+	})
+	if unavailable.Ready() {
+		t.Fatalf("expected unsupported backend to be unavailable")
+	}
+}
+
+func TestEmbedDeterministic384Dimensions(t *testing.T) {
+	engine := NewFallback("test fallback")
+	first, err := engine.EmbedDocument(context.Background(), "remember the red book")
+	if err != nil {
+		t.Fatalf("EmbedDocument(first) error = %v", err)
+	}
+	second, err := engine.EmbedDocument(context.Background(), "remember the red book")
+	if err != nil {
+		t.Fatalf("EmbedDocument(second) error = %v", err)
+	}
+	other, err := engine.EmbedDocument(context.Background(), "different text entirely")
+	if err != nil {
+		t.Fatalf("EmbedDocument(other) error = %v", err)
+	}
+
+	if len(first) != DefaultDimensions {
+		t.Fatalf("expected %d dimensions, got %d", DefaultDimensions, len(first))
+	}
+	if len(second) != DefaultDimensions || len(other) != DefaultDimensions {
+		t.Fatalf("expected stable %d-dimensional output", DefaultDimensions)
+	}
+	for i := range first {
+		if first[i] != second[i] {
+			t.Fatalf("expected deterministic embedding at index %d", i)
+		}
+	}
+
+	var differs bool
+	for i := range first {
+		if first[i] != other[i] {
+			differs = true
+			break
+		}
+	}
+	if !differs {
+		t.Fatalf("expected different text to change the embedding")
+	}
+}
+
+func TestBundledFallsBackToConfiguredFallbackProfile(t *testing.T) {
+	originalResolver := resolveBundledSpec
+	originalFactory := newONNXLocalBackend
+	t.Cleanup(func() {
+		resolveBundledSpec = originalResolver
+		newONNXLocalBackend = originalFactory
+	})
+
+	resolveCalls := make([]string, 0, 2)
+	resolveBundledSpec = func(cfg Config) (onnxLocalSpec, error) {
+		resolveCalls = append(resolveCalls, cfg.Profile)
+		if cfg.Profile == DefaultEmbeddingProfile {
+			return onnxLocalSpec{}, errors.New("primary profile unavailable")
+		}
+		return onnxLocalSpec{
+			RuntimePath: cfg.RuntimePath,
+			Dimensions:  384,
+			Normalize:   true,
+			Profile: buildProfile(Profile{
+				Backend:    "bundled",
+				Family:     FallbackEmbeddingProfile,
+				Dimensions: 384,
+				Normalize:  true,
+			}),
+		}, nil
+	}
+	newONNXLocalBackend = func(spec onnxLocalSpec) (embeddingBackend, error) {
+		return fakeONNXLocalBackend{vector: make([]float32, spec.Dimensions)}, nil
+	}
+
+	engine := NewWithConfig(Config{
+		Backend:         "bundled",
+		Profile:         DefaultEmbeddingProfile,
+		FallbackProfile: FallbackEmbeddingProfile,
+		RuntimePath:     "/opt/onnx/libonnxruntime.so",
+	})
+	if !engine.Ready() {
+		t.Fatalf("expected bundled engine to be ready via fallback profile")
+	}
+	if engine.Profile().Family != FallbackEmbeddingProfile {
+		t.Fatalf("expected fallback family %q, got %q", FallbackEmbeddingProfile, engine.Profile().Family)
+	}
+	if len(resolveCalls) != 2 || resolveCalls[0] != DefaultEmbeddingProfile || resolveCalls[1] != FallbackEmbeddingProfile {
+		t.Fatalf("unexpected resolve order %v", resolveCalls)
+	}
+}
+
+func TestNomicUsesAsymmetricDocumentAndQueryPrefixes(t *testing.T) {
+	originalResolver := resolveBundledSpec
+	originalFactory := newONNXLocalBackend
+	t.Cleanup(func() {
+		resolveBundledSpec = originalResolver
+		newONNXLocalBackend = originalFactory
+	})
+
+	resolveBundledSpec = func(cfg Config) (onnxLocalSpec, error) {
+		return onnxLocalSpec{
+			Family:     "nomic-embed-text-v1.5",
+			Dimensions: 768,
+			Normalize:  true,
+			Profile: buildProfile(Profile{
+				Backend:    "onnx-local",
+				Family:     "nomic-embed-text-v1.5",
+				Dimensions: 768,
+				Normalize:  true,
+			}),
+		}, nil
+	}
+
+	var calls []string
+	newONNXLocalBackend = func(spec onnxLocalSpec) (embeddingBackend, error) {
+		return deterministicRecorderBackend{calls: &calls}, nil
+	}
+
+	engine := NewWithConfig(Config{
+		Backend:     "bundled",
+		Profile:     "nomic-embed-text-v1.5",
+		RuntimePath: "/opt/onnx/libonnxruntime.so",
+	})
+	if !engine.Ready() {
+		t.Fatalf("expected nomic engine to be ready")
+	}
+
+	if _, err := engine.EmbedDocument(context.Background(), "project note"); err != nil {
+		t.Fatalf("EmbedDocument() error = %v", err)
+	}
+	if _, err := engine.EmbedQuery(context.Background(), "project note"); err != nil {
+		t.Fatalf("EmbedQuery() error = %v", err)
+	}
+
+	if len(calls) != 3 {
+		t.Fatalf("expected 3 backend calls including dimension probe, got %d", len(calls))
+	}
+	if calls[1] != "search_document: project note" {
+		t.Fatalf("unexpected document prefix call %q", calls[1])
+	}
+	if calls[2] != "search_query: project note" {
+		t.Fatalf("unexpected query prefix call %q", calls[2])
+	}
+}
+
+type deterministicRecorderBackend struct {
+	calls *[]string
+}
+
+func (b deterministicRecorderBackend) Embed(text string, dimensions int) ([]float32, error) {
+	*b.calls = append(*b.calls, text)
+	return make([]float32, dimensions), nil
+}

package/sidecar/embed/matryoshka.go

@@ -0,0 +1,93 @@
+package embed
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"strings"
+)
+
+const (
+	DimsL1 = 64
+	DimsL2 = 256
+	DimsL3 = 768
+)
+
+type MatryoshkaVec struct {
+	L1 []float32
+	L2 []float32
+	L3 []float32
+}
+
+type MatryoshkaEmbedder interface {
+	Embedder
+	EmbedDocumentM(ctx context.Context, text string) (MatryoshkaVec, error)
+	EmbedQueryM(ctx context.Context, text string) (MatryoshkaVec, error)
+}
+
+func SupportsMatryoshka(e Embedder) bool {
+	if e == nil {
+		return false
+	}
+	return strings.EqualFold(e.Profile().Family, "nomic-embed-text-v1.5")
+}
+
+func NewMatryoshkaVec(full []float32) (MatryoshkaVec, error) {
+	if len(full) < DimsL3 {
+		return MatryoshkaVec{}, fmt.Errorf("matryoshka requires %d-dim vector, got %d", DimsL3, len(full))
+	}
+	return MatryoshkaVec{
+		L1: truncateAndNormalize(full, DimsL1),
+		L2: truncateAndNormalize(full, DimsL2),
+		L3: truncateAndNormalize(full, DimsL3),
+	}, nil
+}
+
+func truncateAndNormalize(v []float32, dims int) []float32 {
+	if dims <= 0 {
+		return nil
+	}
+	if dims > len(v) {
+		dims = len(v)
+	}
+
+	t := make([]float32, dims)
+	copy(t, v[:dims])
+
+	var norm float64
+	for _, x := range t {
+		norm += float64(x) * float64(x)
+	}
+	norm = math.Sqrt(norm)
+	if norm < 1e-9 {
+		return t
+	}
+
+	scale := float32(norm)
+	for i := range t {
+		t[i] /= scale
+	}
+	return t
+}
+
+func (e *Engine) EmbedDocumentM(ctx context.Context, text string) (MatryoshkaVec, error) {
+	if !SupportsMatryoshka(e) {
+		return MatryoshkaVec{}, fmt.Errorf("matryoshka unavailable for profile family %q", e.Profile().Family)
+	}
+	full, err := e.EmbedDocument(ctx, text)
+	if err != nil {
+		return MatryoshkaVec{}, err
+	}
+	return NewMatryoshkaVec(full)
+}
+
+func (e *Engine) EmbedQueryM(ctx context.Context, text string) (MatryoshkaVec, error) {
+	if !SupportsMatryoshka(e) {
+		return MatryoshkaVec{}, fmt.Errorf("matryoshka unavailable for profile family %q", e.Profile().Family)
+	}
+	full, err := e.EmbedQuery(ctx, text)
+	if err != nil {
+		return MatryoshkaVec{}, err
+	}
+	return NewMatryoshkaVec(full)
+}

package/sidecar/embed/matryoshka_test.go

@@ -0,0 +1,150 @@
+package embed
+
+import (
+	"context"
+	"math"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestTruncateAndNormalize(t *testing.T) {
+	full := randomUnitVec(DimsL3)
+	mv, err := NewMatryoshkaVec(full)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct {
+		name string
+		v    []float32
+	}{
+		{"L1", mv.L1},
+		{"L2", mv.L2},
+		{"L3", mv.L3},
+	} {
+		var norm float64
+		for _, x := range tc.v {
+			norm += float64(x) * float64(x)
+		}
+		norm = math.Sqrt(norm)
+		if math.Abs(norm-1.0) > 1e-5 {
+			t.Errorf("%s: L2 norm = %.8f, want 1.0", tc.name, norm)
+		}
+	}
+}
+
+func TestMatryoshkaSimilarityPreservation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("requires real Nomic model")
+	}
+
+	engine := loadNomicEngine(t)
+	ctx := context.Background()
+
+	pairs := []struct {
+		doc   string
+		query string
+	}{
+		{
+			doc:   "The eviction formula uses logarithmic frequency damping.",
+			query: "how does the system decide which model to remove",
+		},
+		{
+			doc:   "Compaction clusters session turns into summarized records.",
+			query: "memory cleanup after a conversation ends",
+		},
+		{
+			doc:   "Nomic embeddings support Matryoshka representation learning.",
+			query: "truncated vectors for fast approximate search",
+		},
+	}
+
+	tiers := []struct {
+		label     string
+		threshold float64
+		selectVec func(MatryoshkaVec) []float32
+	}{
+		{
+			label:     "L2 (256d)",
+			threshold: 0.90,
+			selectVec: func(v MatryoshkaVec) []float32 { return v.L2 },
+		},
+		{
+			label:     "L1 (64d)",
+			threshold: 0.70,
+			selectVec: func(v MatryoshkaVec) []float32 { return v.L1 },
+		},
+	}
+
+	for _, pair := range pairs {
+		dFull, err := engine.EmbedDocumentM(ctx, pair.doc)
+		if err != nil {
+			t.Fatalf("EmbedDocumentM() error = %v", err)
+		}
+		qFull, err := engine.EmbedQueryM(ctx, pair.query)
+		if err != nil {
+			t.Fatalf("EmbedQueryM() error = %v", err)
+		}
+
+		simFull := cosineEval(dFull.L3, qFull.L3)
+		for _, tier := range tiers {
+			simTier := cosineEval(tier.selectVec(dFull), tier.selectVec(qFull))
+			if math.Abs(simFull) < 0.05 {
+				continue
+			}
+
+			ratio := simTier / simFull
+			if ratio < tier.threshold {
+				label := pair.doc
+				if len(label) > 40 {
+					label = label[:40]
+				}
+				t.Errorf("%s preservation below threshold for pair %q: full=%.4f tier=%.4f ratio=%.4f threshold=%.2f",
+					tier.label, label, simFull, simTier, ratio, tier.threshold)
+			}
+		}
+	}
+}
+
+func randomUnitVec(dims int) []float32 {
+	rng := rand.New(rand.NewSource(42))
+	vec := make([]float32, dims)
+	var norm float64
+	for i := range vec {
+		value := rng.NormFloat64()
+		vec[i] = float32(value)
+		norm += value * value
+	}
+	norm = math.Sqrt(norm)
+	scale := float32(norm)
+	for i := range vec {
+		vec[i] /= scale
+	}
+	return vec
+}
+
+func loadNomicEngine(t *testing.T) *Engine {
+	t.Helper()
+
+	runtimePath := strings.TrimSpace(os.Getenv("LIBRAVDB_EVAL_ONNX_RUNTIME"))
+	modelDir := strings.TrimSpace(os.Getenv("LIBRAVDB_EVAL_NOMIC_DIR"))
+	if runtimePath == "" || modelDir == "" {
+		t.Skip("set LIBRAVDB_EVAL_ONNX_RUNTIME and LIBRAVDB_EVAL_NOMIC_DIR to run real Nomic Matryoshka tests")
+	}
+
+	engine := NewWithConfig(Config{
+		Backend:     "onnx-local",
+		Profile:     "nomic-embed-text-v1.5",
+		RuntimePath: runtimePath,
+		ModelPath:   modelDir,
+	})
+	if !engine.Ready() {
+		t.Fatalf("engine not ready: %s", engine.Reason())
+	}
+	if !SupportsMatryoshka(engine) {
+		t.Fatalf("expected Nomic engine to support Matryoshka")
+	}
+	return engine
+}