@xdarkicex/openclaw-memory-libravdb 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/docs/README.md +14 -0
- package/docs/architecture-decisions/README.md +6 -0
- package/docs/architecture-decisions/adr-001-onnx-over-ollama.md +21 -0
- package/docs/architecture-decisions/adr-002-libravdb-over-lancedb.md +19 -0
- package/docs/architecture-decisions/adr-003-convex-gating-over-threshold.md +27 -0
- package/docs/architecture-decisions/adr-004-sidecar-over-native-ts.md +21 -0
- package/docs/architecture.md +188 -0
- package/docs/contributing.md +76 -0
- package/docs/dependencies.md +38 -0
- package/docs/embedding-profiles.md +42 -0
- package/docs/gating.md +329 -0
- package/docs/implementation.md +381 -0
- package/docs/installation.md +272 -0
- package/docs/mathematics.md +695 -0
- package/docs/models.md +63 -0
- package/docs/problem.md +64 -0
- package/docs/security.md +86 -0
- package/openclaw.plugin.json +84 -0
- package/package.json +41 -0
- package/scripts/build-sidecar.sh +30 -0
- package/scripts/postinstall.js +169 -0
- package/scripts/setup.sh +20 -0
- package/scripts/setup.ts +505 -0
- package/scripts/sidecar-release.d.ts +4 -0
- package/scripts/sidecar-release.js +17 -0
- package/sidecar/cmd/inspect_onnx/main.go +105 -0
- package/sidecar/compact/gate.go +273 -0
- package/sidecar/compact/gate_test.go +85 -0
- package/sidecar/compact/summarize.go +345 -0
- package/sidecar/compact/summarize_test.go +319 -0
- package/sidecar/compact/tokens.go +11 -0
- package/sidecar/config/config.go +119 -0
- package/sidecar/config/config_test.go +75 -0
- package/sidecar/embed/engine.go +696 -0
- package/sidecar/embed/engine_test.go +349 -0
- package/sidecar/embed/matryoshka.go +93 -0
- package/sidecar/embed/matryoshka_test.go +150 -0
- package/sidecar/embed/onnx_local.go +319 -0
- package/sidecar/embed/onnx_local_test.go +159 -0
- package/sidecar/embed/profile_contract_test.go +71 -0
- package/sidecar/embed/profile_eval_test.go +923 -0
- package/sidecar/embed/profiles.go +39 -0
- package/sidecar/go.mod +21 -0
- package/sidecar/go.sum +30 -0
- package/sidecar/health/check.go +33 -0
- package/sidecar/health/check_test.go +55 -0
- package/sidecar/main.go +151 -0
- package/sidecar/model/encoder.go +222 -0
- package/sidecar/model/registry.go +262 -0
- package/sidecar/model/registry_test.go +102 -0
- package/sidecar/model/seq2seq.go +133 -0
- package/sidecar/server/rpc.go +343 -0
- package/sidecar/server/rpc_test.go +350 -0
- package/sidecar/server/transport.go +160 -0
- package/sidecar/store/libravdb.go +676 -0
- package/sidecar/store/libravdb_test.go +472 -0
- package/sidecar/summarize/engine.go +360 -0
- package/sidecar/summarize/engine_test.go +148 -0
- package/sidecar/summarize/onnx_local.go +494 -0
- package/sidecar/summarize/onnx_local_test.go +48 -0
- package/sidecar/summarize/profiles.go +52 -0
- package/sidecar/summarize/tokenizer.go +13 -0
- package/sidecar/summarize/tokenizer_hf.go +76 -0
- package/sidecar/summarize/util.go +13 -0
- package/src/cli.ts +205 -0
- package/src/context-engine.ts +195 -0
- package/src/index.ts +27 -0
- package/src/memory-provider.ts +24 -0
- package/src/openclaw-plugin-sdk.d.ts +53 -0
- package/src/plugin-runtime.ts +67 -0
- package/src/recall-cache.ts +34 -0
- package/src/recall-utils.ts +22 -0
- package/src/rpc.ts +84 -0
- package/src/scoring.ts +58 -0
- package/src/sidecar.ts +506 -0
- package/src/tokens.ts +36 -0
- package/src/types.ts +146 -0
- package/tsconfig.json +20 -0
- package/tsconfig.tests.json +12 -0
package/scripts/setup.ts
ADDED
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { createWriteStream, cpSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { mkdtempSync } from "node:fs";
|
|
4
|
+
import net from "node:net";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { Readable } from "node:stream";
|
|
8
|
+
import { pipeline } from "node:stream/promises";
|
|
9
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
10
|
+
import { fileURLToPath } from "node:url";
|
|
11
|
+
|
|
12
|
+
type AssetSpec = {
|
|
13
|
+
name: string;
|
|
14
|
+
dest: string;
|
|
15
|
+
sha256?: string;
|
|
16
|
+
url?: string;
|
|
17
|
+
optional?: boolean;
|
|
18
|
+
localSource?: string;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
type RuntimeSpec = {
|
|
22
|
+
archiveName: string;
|
|
23
|
+
url: string;
|
|
24
|
+
archiveSha256?: string;
|
|
25
|
+
archiveChecksumURL?: string;
|
|
26
|
+
extractedLib: string;
|
|
27
|
+
format: "tgz" | "zip";
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
31
|
+
const sidecarDir = path.join(rootDir, "sidecar");
|
|
32
|
+
const sidecarBinDir = path.join(rootDir, ".sidecar-bin");
|
|
33
|
+
const modelsDir = path.join(sidecarBinDir, "models");
|
|
34
|
+
const runtimeDir = path.join(sidecarBinDir, "onnxruntime");
|
|
35
|
+
const binaryName = process.platform === "win32" ? "libravdb-sidecar.exe" : "libravdb-sidecar";
|
|
36
|
+
const sidecarBinary = path.join(sidecarBinDir, binaryName);
|
|
37
|
+
|
|
38
|
+
const nomicAssets: AssetSpec[] = [
|
|
39
|
+
{
|
|
40
|
+
name: "nomic-embed-text-v1.5 model",
|
|
41
|
+
dest: path.join(modelsDir, "nomic-embed-text-v1.5", "model.onnx"),
|
|
42
|
+
localSource: path.join(rootDir, ".models", "nomic-embed-text-v1.5", "model.onnx"),
|
|
43
|
+
url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx",
|
|
44
|
+
sha256: "147d5aa88c2101237358e17796cf3a227cead1ec304ec34b465bb08e9d952965",
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
name: "nomic-embed-text-v1.5 tokenizer",
|
|
48
|
+
dest: path.join(modelsDir, "nomic-embed-text-v1.5", "tokenizer.json"),
|
|
49
|
+
localSource: path.join(rootDir, ".models", "nomic-embed-text-v1.5", "tokenizer.json"),
|
|
50
|
+
url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json",
|
|
51
|
+
sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
const miniLMAssets: AssetSpec[] = [
|
|
56
|
+
{
|
|
57
|
+
name: "all-minilm-l6-v2 model",
|
|
58
|
+
dest: path.join(modelsDir, "all-minilm-l6-v2", "model.onnx"),
|
|
59
|
+
localSource: path.join(rootDir, ".models", "all-minilm-l6-v2", "model.onnx"),
|
|
60
|
+
url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx",
|
|
61
|
+
sha256: "759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e",
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
name: "all-minilm-l6-v2 tokenizer",
|
|
65
|
+
dest: path.join(modelsDir, "all-minilm-l6-v2", "tokenizer.json"),
|
|
66
|
+
localSource: path.join(rootDir, ".models", "all-minilm-l6-v2", "tokenizer.json"),
|
|
67
|
+
url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json",
|
|
68
|
+
sha256: "da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0",
|
|
69
|
+
},
|
|
70
|
+
];
|
|
71
|
+
|
|
72
|
+
const t5Assets: AssetSpec[] = [
|
|
73
|
+
{
|
|
74
|
+
name: "t5-small encoder",
|
|
75
|
+
dest: path.join(modelsDir, "t5-small", "encoder_model.onnx"),
|
|
76
|
+
localSource: path.join(rootDir, ".models", "t5-small", "encoder_model.onnx"),
|
|
77
|
+
url: "https://huggingface.co/optimum/t5-small/resolve/main/encoder_model.onnx",
|
|
78
|
+
sha256: "41d326633f1b85f526508cc0db78a5d40877c292c1b6dccae2eacd7d2a53480d",
|
|
79
|
+
optional: true,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
name: "t5-small decoder",
|
|
83
|
+
dest: path.join(modelsDir, "t5-small", "decoder_model.onnx"),
|
|
84
|
+
localSource: path.join(rootDir, ".models", "t5-small", "decoder_model.onnx"),
|
|
85
|
+
url: "https://huggingface.co/optimum/t5-small/resolve/main/decoder_model.onnx",
|
|
86
|
+
sha256: "0a1451011d61bcc796a87b7306c503562e910f110f884d0cc08532972c2cc584",
|
|
87
|
+
optional: true,
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
name: "t5-small tokenizer",
|
|
91
|
+
dest: path.join(modelsDir, "t5-small", "tokenizer.json"),
|
|
92
|
+
localSource: path.join(rootDir, ".models", "t5-small", "tokenizer.json"),
|
|
93
|
+
url: "https://huggingface.co/optimum/t5-small/resolve/main/tokenizer.json",
|
|
94
|
+
sha256: "5f0ed8ab5b8cfa9812bb73752f1d80c292e52bcf5a87a144dc9ab2d251056cbb",
|
|
95
|
+
optional: true,
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
name: "t5-small tokenizer config",
|
|
99
|
+
dest: path.join(modelsDir, "t5-small", "tokenizer_config.json"),
|
|
100
|
+
localSource: path.join(rootDir, ".models", "t5-small", "tokenizer_config.json"),
|
|
101
|
+
url: "https://huggingface.co/optimum/t5-small/resolve/main/tokenizer_config.json",
|
|
102
|
+
sha256: "4969f8d76ef05a16553bd2b07b3501673ae8d36972aea88a0f78ad31a3ff2de9",
|
|
103
|
+
optional: true,
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
name: "t5-small config",
|
|
107
|
+
dest: path.join(modelsDir, "t5-small", "config.json"),
|
|
108
|
+
localSource: path.join(rootDir, ".models", "t5-small", "config.json"),
|
|
109
|
+
url: "https://huggingface.co/optimum/t5-small/resolve/main/config.json",
|
|
110
|
+
sha256: "d112428e703aa7ea0d6b17a77e9739fcc15b87653779d9b7942d5ecbc61c00ed",
|
|
111
|
+
optional: true,
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
|
|
115
|
+
const runtimeSpecs: Record<string, RuntimeSpec> = {
|
|
116
|
+
"darwin-arm64": {
|
|
117
|
+
archiveName: "onnxruntime-osx-arm64-1.23.0.tgz",
|
|
118
|
+
url: "https://github.com/microsoft/onnxruntime/releases/download/v1.23.0/onnxruntime-osx-arm64-1.23.0.tgz",
|
|
119
|
+
archiveSha256: "8182db0ebb5caa21036a3c78178f17fabb98a7916bdab454467c8f4cf34bcfdf",
|
|
120
|
+
extractedLib: path.join(runtimeDir, "onnxruntime-osx-arm64-1.23.0", "lib", "libonnxruntime.dylib"),
|
|
121
|
+
format: "tgz",
|
|
122
|
+
},
|
|
123
|
+
"linux-x64": {
|
|
124
|
+
archiveName: "onnxruntime-linux-x64-1.23.0.tgz",
|
|
125
|
+
url: "https://github.com/microsoft/onnxruntime/releases/download/v1.23.0/onnxruntime-linux-x64-1.23.0.tgz",
|
|
126
|
+
archiveChecksumURL: "https://github.com/microsoft/onnxruntime/releases/download/v1.23.0/onnxruntime-linux-x64-1.23.0.tgz.sha256",
|
|
127
|
+
extractedLib: path.join(runtimeDir, "onnxruntime-linux-x64-1.23.0", "lib", "libonnxruntime.so"),
|
|
128
|
+
format: "tgz",
|
|
129
|
+
},
|
|
130
|
+
"win32-x64": {
|
|
131
|
+
archiveName: "onnxruntime-win-x64-1.23.0.zip",
|
|
132
|
+
url: "https://github.com/microsoft/onnxruntime/releases/download/v1.23.0/onnxruntime-win-x64-1.23.0.zip",
|
|
133
|
+
archiveChecksumURL: "https://github.com/microsoft/onnxruntime/releases/download/v1.23.0/onnxruntime-win-x64-1.23.0.zip.sha256",
|
|
134
|
+
extractedLib: path.join(runtimeDir, "onnxruntime-win-x64-1.23.0", "lib", "onnxruntime.dll"),
|
|
135
|
+
format: "zip",
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
async function main(): Promise<void> {
|
|
140
|
+
console.log("[openclaw-memory-libravdb] Building Go sidecar...");
|
|
141
|
+
buildSidecar();
|
|
142
|
+
|
|
143
|
+
mkdirSync(sidecarBinDir, { recursive: true });
|
|
144
|
+
mkdirSync(modelsDir, { recursive: true });
|
|
145
|
+
mkdirSync(runtimeDir, { recursive: true });
|
|
146
|
+
|
|
147
|
+
console.log("[openclaw-memory-libravdb] Provisioning embedding model...");
|
|
148
|
+
for (const asset of nomicAssets) {
|
|
149
|
+
await ensureAsset(asset);
|
|
150
|
+
}
|
|
151
|
+
writeEmbeddingManifest();
|
|
152
|
+
for (const asset of miniLMAssets) {
|
|
153
|
+
await ensureAsset(asset);
|
|
154
|
+
}
|
|
155
|
+
writeMiniLMManifest();
|
|
156
|
+
|
|
157
|
+
console.log("[openclaw-memory-libravdb] Provisioning ONNX runtime...");
|
|
158
|
+
await ensureRuntime();
|
|
159
|
+
|
|
160
|
+
console.log("[openclaw-memory-libravdb] Provisioning summarizer model...");
|
|
161
|
+
await ensureOptionalAssets(t5Assets, writeSummarizerManifest);
|
|
162
|
+
|
|
163
|
+
console.log("[openclaw-memory-libravdb] Verifying sidecar health...");
|
|
164
|
+
await verifySidecarHealth();
|
|
165
|
+
|
|
166
|
+
console.log("[openclaw-memory-libravdb] Setup complete.");
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function buildSidecar(): void {
|
|
170
|
+
const result = spawnSync(process.execPath, [path.join(rootDir, "scripts", "postinstall.js")], {
|
|
171
|
+
cwd: rootDir,
|
|
172
|
+
stdio: "inherit",
|
|
173
|
+
env: process.env,
|
|
174
|
+
});
|
|
175
|
+
if (result.status !== 0) {
|
|
176
|
+
process.exit(result.status ?? 1);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function ensureAsset(spec: AssetSpec): Promise<void> {
|
|
181
|
+
mkdirSync(path.dirname(spec.dest), { recursive: true });
|
|
182
|
+
|
|
183
|
+
if (existsSync(spec.dest) && await verifyFile(spec.dest, spec.sha256)) {
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (existsSync(spec.dest)) {
|
|
188
|
+
rmSync(spec.dest, { force: true });
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (spec.localSource && existsSync(spec.localSource) && await verifyFile(spec.localSource, spec.sha256)) {
|
|
192
|
+
cpSync(spec.localSource, spec.dest);
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (!spec.url) {
|
|
197
|
+
throw new Error(`No download URL available for required asset: ${spec.name}`);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
await downloadToFile(spec.url, spec.dest);
|
|
201
|
+
if (!await verifyFile(spec.dest, spec.sha256)) {
|
|
202
|
+
rmSync(spec.dest, { force: true });
|
|
203
|
+
throw new Error(`SHA-256 verification failed for ${spec.name}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async function ensureOptionalAssets(assets: AssetSpec[], onSuccess: () => void): Promise<void> {
|
|
208
|
+
try {
|
|
209
|
+
for (const asset of assets) {
|
|
210
|
+
await ensureAsset(asset);
|
|
211
|
+
}
|
|
212
|
+
onSuccess();
|
|
213
|
+
} catch (error) {
|
|
214
|
+
console.warn(`[openclaw-memory-libravdb] Optional summarizer provisioning skipped: ${(error as Error).message}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function writeEmbeddingManifest(): void {
|
|
219
|
+
const dir = path.join(modelsDir, "nomic-embed-text-v1.5");
|
|
220
|
+
mkdirSync(dir, { recursive: true });
|
|
221
|
+
writeFileSync(path.join(dir, "embedding.json"), `${JSON.stringify({
|
|
222
|
+
backend: "onnx-local",
|
|
223
|
+
profile: "nomic-embed-text-v1.5",
|
|
224
|
+
family: "nomic-embed-text-v1.5",
|
|
225
|
+
model: "model.onnx",
|
|
226
|
+
tokenizer: "tokenizer.json",
|
|
227
|
+
dimensions: 768,
|
|
228
|
+
normalize: true,
|
|
229
|
+
inputNames: ["input_ids", "attention_mask", "token_type_ids"],
|
|
230
|
+
outputName: "last_hidden_state",
|
|
231
|
+
pooling: "mean",
|
|
232
|
+
addSpecialTokens: true,
|
|
233
|
+
}, null, 2)}\n`);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function writeMiniLMManifest(): void {
|
|
237
|
+
const dir = path.join(modelsDir, "all-minilm-l6-v2");
|
|
238
|
+
mkdirSync(dir, { recursive: true });
|
|
239
|
+
writeFileSync(path.join(dir, "embedding.json"), `${JSON.stringify({
|
|
240
|
+
backend: "onnx-local",
|
|
241
|
+
profile: "all-minilm-l6-v2",
|
|
242
|
+
family: "all-minilm-l6-v2",
|
|
243
|
+
model: "model.onnx",
|
|
244
|
+
tokenizer: "tokenizer.json",
|
|
245
|
+
dimensions: 384,
|
|
246
|
+
normalize: true,
|
|
247
|
+
pooling: "mean",
|
|
248
|
+
addSpecialTokens: true,
|
|
249
|
+
}, null, 2)}\n`);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function writeSummarizerManifest(): void {
|
|
253
|
+
const dir = path.join(modelsDir, "t5-small");
|
|
254
|
+
mkdirSync(dir, { recursive: true });
|
|
255
|
+
writeFileSync(path.join(dir, "summarizer.json"), `${JSON.stringify({
|
|
256
|
+
backend: "onnx-local",
|
|
257
|
+
profile: "t5-small",
|
|
258
|
+
family: "t5-small",
|
|
259
|
+
encoder: "encoder_model.onnx",
|
|
260
|
+
decoder: "decoder_model.onnx",
|
|
261
|
+
tokenizer: "tokenizer.json",
|
|
262
|
+
maxContextTokens: 512,
|
|
263
|
+
}, null, 2)}\n`);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
async function ensureRuntime(): Promise<void> {
|
|
267
|
+
const spec = runtimeSpecs[`${process.platform}-${process.arch}`];
|
|
268
|
+
if (!spec) {
|
|
269
|
+
throw new Error(`Unsupported runtime platform: ${process.platform}/${process.arch}`);
|
|
270
|
+
}
|
|
271
|
+
if (existsSync(spec.extractedLib)) {
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
mkdirSync(runtimeDir, { recursive: true });
|
|
276
|
+
const archivePath = path.join(runtimeDir, spec.archiveName);
|
|
277
|
+
const localArchive = path.join(rootDir, ".models", "onnxruntime", spec.archiveName);
|
|
278
|
+
|
|
279
|
+
if (!(existsSync(archivePath) && await verifyArchive(archivePath, spec))) {
|
|
280
|
+
rmSync(archivePath, { force: true });
|
|
281
|
+
|
|
282
|
+
if (existsSync(localArchive) && await verifyArchive(localArchive, spec)) {
|
|
283
|
+
cpSync(localArchive, archivePath);
|
|
284
|
+
} else {
|
|
285
|
+
await downloadToFile(spec.url, archivePath);
|
|
286
|
+
if (!await verifyArchive(archivePath, spec)) {
|
|
287
|
+
rmSync(archivePath, { force: true });
|
|
288
|
+
throw new Error(`SHA-256 verification failed for runtime archive ${spec.archiveName}`);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
await extractRuntimeArchive(spec, archivePath);
|
|
294
|
+
if (!existsSync(spec.extractedLib)) {
|
|
295
|
+
throw new Error(`Runtime archive extracted but library missing: ${spec.extractedLib}`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
async function verifyArchive(archivePath: string, spec: RuntimeSpec): Promise<boolean> {
|
|
300
|
+
if (spec.archiveSha256) {
|
|
301
|
+
return await verifyFile(archivePath, spec.archiveSha256);
|
|
302
|
+
}
|
|
303
|
+
if (!spec.archiveChecksumURL) {
|
|
304
|
+
return false;
|
|
305
|
+
}
|
|
306
|
+
const expected = await fetchChecksum(spec.archiveChecksumURL);
|
|
307
|
+
if (!expected) {
|
|
308
|
+
return false;
|
|
309
|
+
}
|
|
310
|
+
return await verifyFile(archivePath, expected);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
async function fetchChecksum(url: string): Promise<string | null> {
|
|
314
|
+
try {
|
|
315
|
+
const response = await fetch(url);
|
|
316
|
+
if (!response.ok) {
|
|
317
|
+
return null;
|
|
318
|
+
}
|
|
319
|
+
const text = (await response.text()).trim();
|
|
320
|
+
const match = text.match(/[a-f0-9]{64}/i);
|
|
321
|
+
return match ? match[0].toLowerCase() : null;
|
|
322
|
+
} catch {
|
|
323
|
+
return null;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function extractRuntimeArchive(spec: RuntimeSpec, archivePath: string): Promise<void> {
|
|
328
|
+
if (spec.format === "tgz") {
|
|
329
|
+
const result = spawnSync("tar", ["-xzf", archivePath, "-C", runtimeDir], {
|
|
330
|
+
stdio: "inherit",
|
|
331
|
+
});
|
|
332
|
+
if (result.status !== 0) {
|
|
333
|
+
throw new Error(`Failed to extract runtime archive ${spec.archiveName}`);
|
|
334
|
+
}
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
if (process.platform !== "win32") {
|
|
339
|
+
throw new Error(`ZIP runtime extraction is only supported on Windows for ${spec.archiveName}`);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const result = spawnSync("powershell", [
|
|
343
|
+
"-NoProfile",
|
|
344
|
+
"-Command",
|
|
345
|
+
`Expand-Archive -Path "${archivePath}" -DestinationPath "${runtimeDir}" -Force`,
|
|
346
|
+
], { stdio: "inherit" });
|
|
347
|
+
if (result.status !== 0) {
|
|
348
|
+
throw new Error(`Failed to extract runtime archive ${spec.archiveName}`);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
async function verifySidecarHealth(): Promise<void> {
|
|
353
|
+
if (!existsSync(sidecarBinary)) {
|
|
354
|
+
throw new Error(`Sidecar binary not found after setup: ${sidecarBinary}`);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const tempDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-libravdb-setup-"));
|
|
358
|
+
const child = spawn(sidecarBinary, [], {
|
|
359
|
+
cwd: rootDir,
|
|
360
|
+
env: {
|
|
361
|
+
...process.env,
|
|
362
|
+
LIBRAVDB_DB_PATH: tempDir,
|
|
363
|
+
LIBRAVDB_SUMMARIZER_BACKEND: "bundled",
|
|
364
|
+
LIBRAVDB_SUMMARIZER_PROFILE: "",
|
|
365
|
+
},
|
|
366
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
try {
|
|
370
|
+
const endpoint = await waitForEndpoint(child);
|
|
371
|
+
const health = await callHealth(endpoint);
|
|
372
|
+
if (!health.ok) {
|
|
373
|
+
throw new Error(`Sidecar health check failed: ${health.message ?? "unknown error"}`);
|
|
374
|
+
}
|
|
375
|
+
} finally {
|
|
376
|
+
child.kill();
|
|
377
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function waitForEndpoint(child: ReturnType<typeof spawn>): Promise<string> {
|
|
382
|
+
return new Promise((resolve, reject) => {
|
|
383
|
+
let settled = false;
|
|
384
|
+
let stderr = "";
|
|
385
|
+
const timeout = setTimeout(() => {
|
|
386
|
+
if (settled) {
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
settled = true;
|
|
390
|
+
reject(new Error(`Timed out waiting for sidecar endpoint${stderr ? `: ${stderr.trim()}` : ""}`));
|
|
391
|
+
}, 10000);
|
|
392
|
+
|
|
393
|
+
const finishResolve = (endpoint: string) => {
|
|
394
|
+
if (settled) {
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
settled = true;
|
|
398
|
+
clearTimeout(timeout);
|
|
399
|
+
resolve(endpoint);
|
|
400
|
+
};
|
|
401
|
+
|
|
402
|
+
const finishReject = (error: Error) => {
|
|
403
|
+
if (settled) {
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
settled = true;
|
|
407
|
+
clearTimeout(timeout);
|
|
408
|
+
reject(error);
|
|
409
|
+
};
|
|
410
|
+
|
|
411
|
+
child.stderr?.setEncoding("utf8");
|
|
412
|
+
child.stderr?.on("data", (chunk: string) => {
|
|
413
|
+
stderr += chunk;
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
child.once("error", (error) => finishReject(error));
|
|
417
|
+
child.once("exit", (code) => {
|
|
418
|
+
finishReject(new Error(
|
|
419
|
+
`Sidecar exited before advertising endpoint (code ${code ?? "unknown"})${stderr ? `: ${stderr.trim()}` : ""}`,
|
|
420
|
+
));
|
|
421
|
+
});
|
|
422
|
+
|
|
423
|
+
child.stdout?.setEncoding("utf8");
|
|
424
|
+
child.stdout?.on("data", (chunk: string) => {
|
|
425
|
+
const line = chunk.split("\n").map((v) => v.trim()).find(Boolean);
|
|
426
|
+
if (!line) {
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
finishResolve(line);
|
|
430
|
+
});
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function callHealth(endpoint: string): Promise<{ ok?: boolean; message?: string }> {
|
|
435
|
+
return new Promise((resolve, reject) => {
|
|
436
|
+
const socket = endpoint.startsWith("tcp:")
|
|
437
|
+
? connectTcp(endpoint)
|
|
438
|
+
: net.createConnection(endpoint);
|
|
439
|
+
|
|
440
|
+
let buf = "";
|
|
441
|
+
const cleanup = () => socket.destroy();
|
|
442
|
+
|
|
443
|
+
socket.setEncoding("utf8");
|
|
444
|
+
socket.once("error", (error) => {
|
|
445
|
+
cleanup();
|
|
446
|
+
reject(error);
|
|
447
|
+
});
|
|
448
|
+
socket.on("data", (chunk) => {
|
|
449
|
+
buf += chunk;
|
|
450
|
+
const line = buf.split("\n").find((value) => value.trim());
|
|
451
|
+
if (!line) {
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
cleanup();
|
|
455
|
+
try {
|
|
456
|
+
const msg = JSON.parse(line) as { result?: { ok?: boolean; message?: string }; error?: { message?: string } };
|
|
457
|
+
if (msg.error?.message) {
|
|
458
|
+
reject(new Error(msg.error.message));
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
resolve(msg.result ?? {});
|
|
462
|
+
} catch (error) {
|
|
463
|
+
reject(error);
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
socket.once("connect", () => {
|
|
467
|
+
socket.write(`${JSON.stringify({ jsonrpc: "2.0", id: 1, method: "health", params: {} })}\n`);
|
|
468
|
+
});
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
function connectTcp(endpoint: string): net.Socket {
|
|
473
|
+
const raw = endpoint.slice("tcp:".length);
|
|
474
|
+
const [host, portText] = raw.split(":");
|
|
475
|
+
const port = Number(portText);
|
|
476
|
+
if (!host || !Number.isFinite(port)) {
|
|
477
|
+
throw new Error(`Invalid TCP sidecar endpoint: ${endpoint}`);
|
|
478
|
+
}
|
|
479
|
+
return net.createConnection({ host, port });
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
async function verifyFile(filePath: string, expectedSha256?: string): Promise<boolean> {
|
|
483
|
+
if (!expectedSha256) {
|
|
484
|
+
return existsSync(filePath);
|
|
485
|
+
}
|
|
486
|
+
if (!existsSync(filePath) || !statSync(filePath).isFile()) {
|
|
487
|
+
return false;
|
|
488
|
+
}
|
|
489
|
+
const actual = createHash("sha256").update(readFileSync(filePath)).digest("hex");
|
|
490
|
+
return actual === expectedSha256.toLowerCase();
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
async function downloadToFile(url: string, dest: string): Promise<void> {
|
|
494
|
+
const response = await fetch(url);
|
|
495
|
+
if (!response.ok || !response.body) {
|
|
496
|
+
throw new Error(`Download failed: ${url} (${response.status})`);
|
|
497
|
+
}
|
|
498
|
+
mkdirSync(path.dirname(dest), { recursive: true });
|
|
499
|
+
const tempPath = `${dest}.tmp`;
|
|
500
|
+
await pipeline(Readable.fromWeb(response.body as any), createWriteStream(tempPath));
|
|
501
|
+
rmSync(dest, { force: true });
|
|
502
|
+
renameSync(tempPath, dest);
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
await main();
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export const SIDECAR_RELEASE_REPO: string;
|
|
2
|
+
export const SIDECAR_RELEASE_TARGETS: Record<string, string>;
|
|
3
|
+
export function detectSidecarReleaseTarget(platform?: string, arch?: string): string | null;
|
|
4
|
+
export function buildSidecarReleaseAssetURL(version: string, target: string): string;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const SIDECAR_RELEASE_REPO = "https://github.com/xDarkicex/openclaw-memory-libravdb/releases/download";
|
|
2
|
+
|
|
3
|
+
export const SIDECAR_RELEASE_TARGETS = {
|
|
4
|
+
"darwin-arm64": "clawdb-sidecar-darwin-arm64",
|
|
5
|
+
"darwin-x64": "clawdb-sidecar-darwin-amd64",
|
|
6
|
+
"linux-x64": "clawdb-sidecar-linux-amd64",
|
|
7
|
+
"linux-arm64": "clawdb-sidecar-linux-arm64",
|
|
8
|
+
"win32-x64": "clawdb-sidecar-windows-amd64.exe",
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export function detectSidecarReleaseTarget(platform = process.platform, arch = process.arch) {
|
|
12
|
+
return SIDECAR_RELEASE_TARGETS[`${platform}-${arch}`] ?? null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function buildSidecarReleaseAssetURL(version, target) {
|
|
16
|
+
return `${SIDECAR_RELEASE_REPO}/v${version}/${target}`;
|
|
17
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
package main
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"flag"
|
|
5
|
+
"fmt"
|
|
6
|
+
"os"
|
|
7
|
+
"path/filepath"
|
|
8
|
+
"strings"
|
|
9
|
+
"time"
|
|
10
|
+
|
|
11
|
+
ort "github.com/yalue/onnxruntime_go"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
func main() {
|
|
15
|
+
var (
|
|
16
|
+
asComment bool
|
|
17
|
+
ortLib string
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
flag.BoolVar(&asComment, "comment", false, "wrap output as a Go comment block")
|
|
21
|
+
flag.StringVar(&ortLib, "ort-lib", strings.TrimSpace(os.Getenv("ORT_LIB_PATH")), "path to libonnxruntime shared library; defaults to ORT_LIB_PATH")
|
|
22
|
+
flag.Parse()
|
|
23
|
+
|
|
24
|
+
if flag.NArg() == 0 {
|
|
25
|
+
fmt.Fprintln(os.Stderr, "usage: inspect_onnx [-comment] [-ort-lib /path/to/libonnxruntime] <model.onnx> [more-models.onnx]")
|
|
26
|
+
os.Exit(2)
|
|
27
|
+
}
|
|
28
|
+
if strings.TrimSpace(ortLib) == "" {
|
|
29
|
+
fmt.Fprintln(os.Stderr, "inspect_onnx: ORT library path is required via -ort-lib or ORT_LIB_PATH")
|
|
30
|
+
os.Exit(2)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
ort.SetSharedLibraryPath(ortLib)
|
|
34
|
+
if err := ort.InitializeEnvironment(); err != nil {
|
|
35
|
+
fmt.Fprintf(os.Stderr, "inspect_onnx: ort init: %v\n", err)
|
|
36
|
+
os.Exit(1)
|
|
37
|
+
}
|
|
38
|
+
defer func() {
|
|
39
|
+
if err := ort.DestroyEnvironment(); err != nil {
|
|
40
|
+
fmt.Fprintf(os.Stderr, "inspect_onnx: ort shutdown: %v\n", err)
|
|
41
|
+
os.Exit(1)
|
|
42
|
+
}
|
|
43
|
+
}()
|
|
44
|
+
|
|
45
|
+
for i, path := range flag.Args() {
|
|
46
|
+
inputs, outputs, err := ort.GetInputOutputInfo(path)
|
|
47
|
+
if err != nil {
|
|
48
|
+
fmt.Fprintf(os.Stderr, "inspect_onnx: inspect %s: %v\n", path, err)
|
|
49
|
+
os.Exit(1)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
record := formatRecord(filepath.Base(path), inputs, outputs, asComment)
|
|
53
|
+
if i > 0 {
|
|
54
|
+
fmt.Println()
|
|
55
|
+
}
|
|
56
|
+
fmt.Print(record)
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
func formatRecord(model string, inputs, outputs []ort.InputOutputInfo, asComment bool) string {
|
|
61
|
+
lines := []string{
|
|
62
|
+
"--- ONNX Graph Record ---",
|
|
63
|
+
fmt.Sprintf("Model: %s", model),
|
|
64
|
+
fmt.Sprintf("Inspected: %s", time.Now().Format("2006-01-02 15:04:05 MST")),
|
|
65
|
+
"",
|
|
66
|
+
"INPUTS:",
|
|
67
|
+
}
|
|
68
|
+
lines = append(lines, formatInfos(inputs)...)
|
|
69
|
+
lines = append(lines, "")
|
|
70
|
+
lines = append(lines, "OUTPUTS:")
|
|
71
|
+
lines = append(lines, formatInfos(outputs)...)
|
|
72
|
+
lines = append(lines, "-------------------------")
|
|
73
|
+
|
|
74
|
+
if !asComment {
|
|
75
|
+
return strings.Join(lines, "\n") + "\n"
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
for i, line := range lines {
|
|
79
|
+
lines[i] = "// " + line
|
|
80
|
+
}
|
|
81
|
+
return strings.Join(lines, "\n") + "\n"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
func formatInfos(items []ort.InputOutputInfo) []string {
|
|
85
|
+
if len(items) == 0 {
|
|
86
|
+
return []string{" <none>"}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
lines := make([]string, 0, len(items))
|
|
90
|
+
for _, item := range items {
|
|
91
|
+
lines = append(lines, fmt.Sprintf(" %-40s %-10s %s", item.Name, item.DataType, formatDims(item.Dimensions)))
|
|
92
|
+
}
|
|
93
|
+
return lines
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
func formatDims(shape ort.Shape) string {
|
|
97
|
+
if len(shape) == 0 {
|
|
98
|
+
return "[]"
|
|
99
|
+
}
|
|
100
|
+
parts := make([]string, len(shape))
|
|
101
|
+
for i, dim := range shape {
|
|
102
|
+
parts[i] = fmt.Sprintf("%d", dim)
|
|
103
|
+
}
|
|
104
|
+
return "[" + strings.Join(parts, " ") + "]"
|
|
105
|
+
}
|