@xdarkicex/openclaw-memory-libravdb 1.3.5

package/sidecar/embed/profiles.go

@@ -0,0 +1,39 @@
+package embed
+
+import "strings"
+
+const (
+	DefaultEmbeddingProfile  = "nomic-embed-text-v1.5"
+	FallbackEmbeddingProfile = "all-minilm-l6-v2"
+)
+
+type modelProfile struct {
+	Name             string
+	Family           string
+	Dimensions       int
+	Normalize        bool
+	MaxContextTokens int
+}
+
+var shippedProfiles = map[string]modelProfile{
+	"all-minilm-l6-v2": {
+		Name:             "all-minilm-l6-v2",
+		Family:           "all-minilm-l6-v2",
+		Dimensions:       384,
+		Normalize:        true,
+		MaxContextTokens: 128,
+	},
+	"nomic-embed-text-v1.5": {
+		Name:             "nomic-embed-text-v1.5",
+		Family:           "nomic-embed-text-v1.5",
+		Dimensions:       768,
+		Normalize:        true,
+		MaxContextTokens: 8192,
+	},
+}
+
+func lookupProfile(name string) (modelProfile, bool) {
+	name = strings.TrimSpace(strings.ToLower(name))
+	profile, ok := shippedProfiles[name]
+	return profile, ok
+}

package/sidecar/go.mod

@@ -0,0 +1,21 @@
+module github.com/xDarkicex/openclaw-memory-libravdb/sidecar
+
+go 1.25.1
+
+require (
+	github.com/sugarme/tokenizer v0.3.0
+	github.com/yalue/onnxruntime_go v1.21.0
+)
+
+require (
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v2 v2.15.0 // indirect
+	github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
+	golang.org/x/text v0.25.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/sugarme/tokenizer => github.com/clems4ever/tokenizer v0.0.0-20250926133620-9ddc80533c43

package/sidecar/go.sum

@@ -0,0 +1,30 @@
+github.com/clems4ever/tokenizer v0.0.0-20250926133620-9ddc80533c43 h1:j8YQypEqa5OjqbGciCNb9hOcYbo1oTVuEjd/iu9U2SY=
+github.com/clems4ever/tokenizer v0.0.0-20250926133620-9ddc80533c43/go.mod h1:VJ+DLK5ZEZwzvODOWwY0cw+B1dabTd3nCB5HuFCItCc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8=
+github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89ymCaN+5lPH/MwnfSVg4rzGDh4d+iy4=
+github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw=
+github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI=
+github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

package/sidecar/health/check.go

@@ -0,0 +1,33 @@
+package health
+
+import (
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/embed"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/store"
+)
+
+type Status struct {
+	OK      bool   `json:"ok"`
+	Message string `json:"message"`
+}
+
+func Check(embedder embed.Embedder, st *store.Store) Status {
+	if embedder == nil {
+		return Status{OK: false, Message: "embedder unavailable"}
+	}
+	if !embedder.Ready() {
+		if embedder.Reason() != "" {
+			return Status{OK: false, Message: embedder.Reason()}
+		}
+		return Status{OK: false, Message: "embedder not ready"}
+	}
+	if embedder.Mode() == "fallback" {
+		if embedder.Reason() != "" {
+			return Status{OK: false, Message: embedder.Reason()}
+		}
+		return Status{OK: false, Message: "embedder running in deterministic fallback mode"}
+	}
+	if st == nil {
+		return Status{OK: false, Message: "store unavailable"}
+	}
+	return Status{OK: true, Message: "ok"}
+}

package/sidecar/health/check_test.go

@@ -0,0 +1,55 @@
+package health
+
+import (
+	"context"
+	"testing"
+
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/embed"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/store"
+)
+
+type fakeEmbedder struct {
+	ready   bool
+	reason  string
+	mode    string
+	profile embed.Profile
+}
+
+func (f fakeEmbedder) EmbedDocument(context.Context, string) ([]float32, error) {
+	return make([]float32, 1), nil
+}
+func (f fakeEmbedder) EmbedQuery(context.Context, string) ([]float32, error) {
+	return make([]float32, 1), nil
+}
+func (f fakeEmbedder) Dimensions() int        { return 1 }
+func (f fakeEmbedder) Profile() embed.Profile { return f.profile }
+func (f fakeEmbedder) Ready() bool            { return f.ready }
+func (f fakeEmbedder) Reason() string         { return f.reason }
+func (f fakeEmbedder) Mode() string           { return f.mode }
+
+func TestCheckRejectsFallbackEmbedder(t *testing.T) {
+	status := Check(fakeEmbedder{ready: true, mode: "fallback", reason: "bundled embedder unavailable"}, &store.Store{})
+	if status.OK {
+		t.Fatalf("expected fallback embedder to fail health")
+	}
+	if status.Message != "bundled embedder unavailable" {
+		t.Fatalf("unexpected message %q", status.Message)
+	}
+}
+
+func TestCheckRejectsNotReadyEmbedderWithReason(t *testing.T) {
+	status := Check(fakeEmbedder{ready: false, reason: "missing onnx runtime"}, &store.Store{})
+	if status.OK {
+		t.Fatalf("expected not-ready embedder to fail health")
+	}
+	if status.Message != "missing onnx runtime" {
+		t.Fatalf("unexpected message %q", status.Message)
+	}
+}
+
+func TestCheckAcceptsReadyPrimaryEmbedder(t *testing.T) {
+	status := Check(fakeEmbedder{ready: true, mode: "primary"}, &store.Store{})
+	if !status.OK {
+		t.Fatalf("expected primary embedder to pass health, got %+v", status)
+	}
+}

package/sidecar/main.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/compact"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/config"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/embed"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/health"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/model"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/server"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/store"
+	"github.com/xDarkicex/openclaw-memory-libravdb/sidecar/summarize"
+)
+
+func main() {
+	cfg := config.FromEnv()
+	if err := preflightONNXRuntime(cfg); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	embedder := embed.NewWithConfig(embed.Config{
+		Backend:         cfg.EmbeddingBackend,
+		Profile:         cfg.EmbeddingProfile,
+		FallbackProfile: cfg.FallbackProfile,
+		RuntimePath:     cfg.ONNXRuntimePath,
+		ModelPath:       cfg.EmbeddingModelPath,
+		TokenizerPath:   cfg.EmbeddingTokenizerPath,
+		Dimensions:      cfg.EmbeddingDimensions,
+		Normalize:       cfg.EmbeddingNormalize,
+	})
+	summarizerRuntimePath := cfg.SummarizerRuntimePath
+	if summarizerRuntimePath == "" {
+		summarizerRuntimePath = cfg.ONNXRuntimePath
+	}
+	extractive := summarize.NewExtractive(embedder, "extractive")
+	configuredSummarizer := summarize.NewWithDeps(summarize.Config{
+		Backend:       cfg.SummarizerBackend,
+		Profile:       cfg.SummarizerProfile,
+		RuntimePath:   summarizerRuntimePath,
+		ModelPath:     cfg.SummarizerModelPath,
+		TokenizerPath: cfg.SummarizerTokenizerPath,
+		Model:         cfg.SummarizerModel,
+		Endpoint:      cfg.SummarizerEndpoint,
+	}, summarize.Dependencies{
+		Embedder: embedder,
+		Registry: model.DefaultRegistry(),
+	})
+	var abstractive summarize.Summarizer
+	if configuredSummarizer != nil && configuredSummarizer.Ready() && configuredSummarizer.Mode() != "extractive" {
+		abstractive = configuredSummarizer
+	}
+	st, err := store.Open(cfg.DBPath, embedder)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	if err := st.BackfillDirtyTiers(context.Background()); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	status := health.Check(embedder, st)
+	if !status.OK {
+		fmt.Fprintln(os.Stderr, status.Message)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	srv := server.New(embedder, extractive, abstractive, st, compact.GatingConfig{
+		W1c: cfg.GatingW1c,
+		W2c: cfg.GatingW2c,
+		W3c: cfg.GatingW3c,
+		W1t: cfg.GatingW1t,
+		W2t: cfg.GatingW2t,
+		W3t: cfg.GatingW3t,
+		TechNorm:  cfg.GatingTechNorm,
+		Threshold: cfg.GatingThreshold,
+	})
+	listener, endpoint, cleanup, err := server.Listen()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	defer cleanup()
+
+	fmt.Println(endpoint)
+	if err := server.Serve(ctx, listener, srv); err != nil && !errors.Is(err, server.ErrServerClosed) {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}
+
+func preflightONNXRuntime(cfg config.Config) error {
+	paths := make([]string, 0, 2)
+
+	embeddingPath, err := resolvedRuntimePath(cfg.EmbeddingBackend, cfg.ONNXRuntimePath)
+	if err != nil {
+		return err
+	}
+	if embeddingPath != "" {
+		paths = append(paths, embeddingPath)
+	}
+
+	summarizerRuntimePath := cfg.SummarizerRuntimePath
+	if summarizerRuntimePath == "" {
+		summarizerRuntimePath = cfg.ONNXRuntimePath
+	}
+	summarizerPath, err := resolvedRuntimePath(cfg.SummarizerBackend, summarizerRuntimePath)
+	if err != nil {
+		return err
+	}
+	if summarizerPath != "" {
+		paths = append(paths, summarizerPath)
+	}
+
+	seen := map[string]struct{}{}
+	for _, runtimePath := range paths {
+		if _, ok := seen[runtimePath]; ok {
+			continue
+		}
+		seen[runtimePath] = struct{}{}
+		if _, err := os.Stat(runtimePath); err != nil {
+			if os.IsNotExist(err) {
+				return fmt.Errorf("ONNX Runtime library not found at %s\nRun scripts/setup.sh to unpack it.", runtimePath)
+			}
+			return fmt.Errorf("failed to stat ONNX Runtime library %s: %w", runtimePath, err)
+		}
+	}
+
+	return nil
+}
+
+func resolvedRuntimePath(backend, explicit string) (string, error) {
+	switch backend {
+	case "", "bundled", "onnx-local":
+		return embed.ResolveRuntimePath(embed.Config{
+			Backend:     backend,
+			RuntimePath: explicit,
+		})
+	default:
+		return "", nil
+	}
+}

package/sidecar/model/encoder.go

@@ -0,0 +1,222 @@
+package model
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/sugarme/tokenizer"
+	"github.com/sugarme/tokenizer/pretrained"
+	ort "github.com/yalue/onnxruntime_go"
+)
+
+type EncoderSpec struct {
+	Key              string
+	Profile          Profile
+	RuntimePath      string
+	ModelPath        string
+	TokenizerPath    string
+	InputNames       []string
+	OutputName       string
+	Dimensions       int
+	AddSpecialTokens bool
+	Pooling          string
+}
+
+type EncoderModel struct {
+	key              string
+	registry         *Registry
+	tokenizer        tokenizer.Tokenizer
+	session          *ort.DynamicAdvancedSession
+	inputNames       []string
+	dimensions       int
+	addSpecialTokens bool
+	pooling          string
+}
+
+func (r *Registry) LoadEncoder(spec EncoderSpec) (*EncoderModel, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if err := r.ensureRuntimeLocked(strings.TrimSpace(spec.RuntimePath)); err != nil {
+		return nil, fmt.Errorf("failed to initialize onnx runtime: %w", err)
+	}
+	if spec.Key == "" {
+		spec.Key = spec.Profile.Name
+	}
+	if loaded, ok := r.loaded[spec.Key]; ok && loaded.encoder != nil {
+		loaded.lastAccess = timeNow()
+		return loaded.encoder, nil
+	}
+
+	tk, err := pretrained.FromFile(spec.TokenizerPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load tokenizer: %w", err)
+	}
+	session, err := ort.NewDynamicAdvancedSession(spec.ModelPath, spec.InputNames, []string{spec.OutputName}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create onnx session: %w", err)
+	}
+
+	encoder := &EncoderModel{
+		key:              spec.Key,
+		registry:         r,
+		tokenizer:        *tk,
+		session:          session,
+		inputNames:       append([]string(nil), spec.InputNames...),
+		dimensions:       spec.Dimensions,
+		addSpecialTokens: spec.AddSpecialTokens,
+		pooling:          spec.Pooling,
+	}
+	r.loaded[spec.Key] = &loadedModel{
+		key:           spec.Key,
+		profile:       spec.Profile,
+		lastAccess:    timeNow(),
+		useCount:      0,
+		reservedBytes: fileSize(spec.ModelPath) + fileSize(spec.TokenizerPath),
+		closeFn:       session.Destroy,
+		encoder:       encoder,
+	}
+	if err := r.maybeEvictLocked(timeNow()); err != nil {
+		return nil, err
+	}
+	return encoder, nil
+}
+
+func (m *EncoderModel) EmbedText(text string) ([]float32, error) {
+	encoding, err := m.EncodeText(text, m.addSpecialTokens)
+	if err != nil {
+		return nil, err
+	}
+	return m.EmbedEncoding(*encoding)
+}
+
+func (m *EncoderModel) TokenCount(text string) (int, error) {
+	encoding, err := m.EncodeText(text, m.addSpecialTokens)
+	if err != nil {
+		return 0, err
+	}
+	return len(encoding.Ids), nil
+}
+
+func (m *EncoderModel) EncodeText(text string, addSpecialTokens bool) (*tokenizer.Encoding, error) {
+	m.registry.mu.Lock()
+	m.registry.touchLocked(m.key)
+	m.registry.mu.Unlock()
+
+	input := tokenizer.NewSingleEncodeInput(tokenizer.NewRawInputSequence(text))
+	encoding, err := m.tokenizer.Encode(input, addSpecialTokens)
+	if err != nil {
+		return nil, fmt.Errorf("failed to tokenize sentence: %w", err)
+	}
+	if encoding == nil || len(encoding.Ids) == 0 {
+		return nil, fmt.Errorf("tokenizer returned no encodings")
+	}
+	return encoding, nil
+}
+
+func (m *EncoderModel) EmbedEncoding(encoding tokenizer.Encoding) ([]float32, error) {
+	batchSize := 1
+	seqLength := len(encoding.Ids)
+	inputShape := ort.NewShape(int64(batchSize), int64(seqLength))
+	inputs := make([]ort.Value, 0, len(m.inputNames))
+	encodings := []tokenizer.Encoding{encoding}
+
+	for _, name := range m.inputNames {
+		data, err := inputTensorData(name, encodings, seqLength)
+		if err != nil {
+			return nil, err
+		}
+		tensor, err := ort.NewTensor(inputShape, data)
+		if err != nil {
+			return nil, fmt.Errorf("failed creating %s tensor: %w", name, err)
+		}
+		defer tensor.Destroy()
+		inputs = append(inputs, tensor)
+	}
+
+	outputShape := ort.NewShape(int64(batchSize), int64(m.dimensions))
+	useMeanPooling := m.pooling == "mean"
+	if useMeanPooling {
+		outputShape = ort.NewShape(int64(batchSize), int64(seqLength), int64(m.dimensions))
+	}
+	outputTensor, err := ort.NewEmptyTensor[float32](outputShape)
+	if err != nil {
+		return nil, fmt.Errorf("failed creating output tensor: %w", err)
+	}
+	defer outputTensor.Destroy()
+
+	if err := m.session.Run(inputs, []ort.Value{outputTensor}); err != nil {
+		return nil, fmt.Errorf("failed to run onnx session: %w", err)
+	}
+
+	flat := outputTensor.GetData()
+	expected := batchSize * m.dimensions
+	if useMeanPooling {
+		expected = batchSize * seqLength * m.dimensions
+	}
+	if len(flat) != expected {
+		return nil, fmt.Errorf("unexpected output tensor size: got %d elements, expected %d", len(flat), expected)
+	}
+
+	if useMeanPooling {
+		return meanPoolLastHiddenState(flat, encoding.AttentionMask, seqLength, m.dimensions), nil
+	}
+
+	vec := make([]float32, m.dimensions)
+	copy(vec, flat[:m.dimensions])
+	return vec, nil
+}
+
+func inputTensorData(name string, encodings []tokenizer.Encoding, seqLength int) ([]int64, error) {
+	data := make([]int64, len(encodings)*seqLength)
+	for batch := range encodings {
+		switch name {
+		case "input_ids":
+			for i, id := range encodings[batch].Ids {
+				data[batch*seqLength+i] = int64(id)
+			}
+		case "attention_mask":
+			for i, mask := range encodings[batch].AttentionMask {
+				data[batch*seqLength+i] = int64(mask)
+			}
+		case "token_type_ids":
+			for i, typeID := range encodings[batch].TypeIds {
+				data[batch*seqLength+i] = int64(typeID)
+			}
+		default:
+			return nil, fmt.Errorf("unsupported input tensor name %q", name)
+		}
+	}
+	return data, nil
+}
+
+func meanPoolLastHiddenState(flat []float32, attentionMask []int, seqLength int, dimensions int) []float32 {
+	vec := make([]float32, dimensions)
+	var denom float32
+	for tokenIdx := 0; tokenIdx < seqLength; tokenIdx++ {
+		if tokenIdx >= len(attentionMask) || attentionMask[tokenIdx] == 0 {
+			continue
+		}
+		denom++
+		base := tokenIdx * dimensions
+		for dim := 0; dim < dimensions; dim++ {
+			vec[dim] += flat[base+dim]
+		}
+	}
+	if denom == 0 {
+		return vec
+	}
+	for dim := range vec {
+		vec[dim] /= denom
+	}
+	return vec
+}
+
+func fileSize(path string) int64 {
+	info, err := os.Stat(path)
+	if err != nil {
+		return 0
+	}
+	return info.Size()
+}