memorix 1.7.1 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -1
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/assets/star-history-dark.svg +4 -4
- package/assets/star-history-light.svg +4 -4
- package/assets/star-history.json +47 -3
- package/dist/cli/index.js +423 -74
- package/dist/index.js +396 -73
- package/dist/maintenance-runner.js +393 -69
- package/dist/media-audio-runner.js +393 -69
- package/dist/media-video-runner.js +393 -69
- package/dist/memcode-runtime/CHANGELOG.md +43 -1
- package/dist/sdk.js +395 -72
- package/dist/vector-backfill-runner.js +393 -69
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CONFIGURATION.md +57 -1
- package/docs/DEVELOPMENT.md +6 -2
- package/docs/PERFORMANCE.md +13 -4
- package/docs/README.md +3 -1
- package/docs/SETUP.md +2 -1
- package/package.json +1 -1
- package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
- package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
- package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/copilot/memorix/plugin.json +1 -1
- package/plugins/gemini/memorix/gemini-extension.json +1 -1
- package/plugins/omp/memorix/package.json +1 -1
- package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/pi/memorix/package.json +1 -1
- package/server.json +2 -2
- package/src/cli/commands/config-migrate.ts +7 -0
- package/src/cli/commands/doctor.ts +5 -1
- package/src/cli/commands/init.ts +17 -0
- package/src/cli/commands/status.ts +2 -0
- package/src/config/resolved-config.ts +110 -12
- package/src/config/toml-loader.ts +8 -0
- package/src/config/yaml-loader.ts +9 -0
- package/src/llm/provider.ts +33 -7
- package/src/multimodal/image-loader.ts +6 -3
- package/src/rerank/http-provider.ts +125 -0
- package/src/rerank/index.ts +81 -0
- package/src/rerank/policy.ts +88 -0
- package/src/store/orama-store.ts +111 -65
package/dist/sdk.js
CHANGED
|
@@ -2781,6 +2781,7 @@ function loadYamlConfig(projectRoot) {
|
|
|
2781
2781
|
llm: { ...userConfig.llm, ...projectConfig.llm },
|
|
2782
2782
|
agent: { ...userConfig.agent, ...projectConfig.agent },
|
|
2783
2783
|
embedding: { ...userConfig.embedding, ...projectConfig.embedding },
|
|
2784
|
+
rerank: { ...userConfig.rerank, ...projectConfig.rerank },
|
|
2784
2785
|
git: { ...userConfig.git, ...projectConfig.git },
|
|
2785
2786
|
codegraph: { ...userConfig.codegraph, ...projectConfig.codegraph },
|
|
2786
2787
|
behavior: { ...userConfig.behavior, ...projectConfig.behavior },
|
|
@@ -3056,6 +3057,7 @@ function mergeTomlConfig(base, override) {
|
|
|
3056
3057
|
llm: { ...base.memory?.llm, ...override.memory?.llm }
|
|
3057
3058
|
},
|
|
3058
3059
|
embedding: { ...base.embedding, ...override.embedding },
|
|
3060
|
+
rerank: { ...base.rerank, ...override.rerank },
|
|
3059
3061
|
hooks: { ...base.hooks, ...override.hooks },
|
|
3060
3062
|
git: { ...base.git, ...override.git },
|
|
3061
3063
|
codegraph: { ...base.codegraph, ...override.codegraph },
|
|
@@ -3211,6 +3213,7 @@ __export(resolved_config_exports, {
|
|
|
3211
3213
|
getResolvedConfigForCwd: () => getResolvedConfigForCwd,
|
|
3212
3214
|
getResolvedEmbeddingLane: () => getResolvedEmbeddingLane,
|
|
3213
3215
|
getResolvedMemoryLane: () => getResolvedMemoryLane,
|
|
3216
|
+
getResolvedRerankLane: () => getResolvedRerankLane,
|
|
3214
3217
|
resetResolvedConfigCache: () => resetResolvedConfigCache
|
|
3215
3218
|
});
|
|
3216
3219
|
import { homedir as homedir5 } from "os";
|
|
@@ -3243,6 +3246,16 @@ function getResolvedConfig(options = {}) {
|
|
|
3243
3246
|
legacy.llm?.baseUrl
|
|
3244
3247
|
);
|
|
3245
3248
|
const openRouterMemoryLlmApiKey = isOpenRouterMemoryLane(memoryLlmProvider, memoryLlmBaseUrl) ? process.env.OPENROUTER_API_KEY : void 0;
|
|
3249
|
+
const memoryLlmApiKey = first(
|
|
3250
|
+
process.env.MEMORIX_LLM_API_KEY,
|
|
3251
|
+
process.env.MEMORIX_API_KEY,
|
|
3252
|
+
toml.memory?.llm?.api_key,
|
|
3253
|
+
yaml.llm?.apiKey,
|
|
3254
|
+
legacy.llm?.apiKey,
|
|
3255
|
+
process.env.OPENAI_API_KEY,
|
|
3256
|
+
process.env.ANTHROPIC_API_KEY,
|
|
3257
|
+
openRouterMemoryLlmApiKey
|
|
3258
|
+
);
|
|
3246
3259
|
const resolved = {
|
|
3247
3260
|
agent: {
|
|
3248
3261
|
provider: first(
|
|
@@ -3283,16 +3296,7 @@ function getResolvedConfig(options = {}) {
|
|
|
3283
3296
|
provider: memoryLlmProvider,
|
|
3284
3297
|
model: memoryLlmModel,
|
|
3285
3298
|
baseUrl: memoryLlmBaseUrl,
|
|
3286
|
-
apiKey:
|
|
3287
|
-
process.env.MEMORIX_LLM_API_KEY,
|
|
3288
|
-
process.env.MEMORIX_API_KEY,
|
|
3289
|
-
toml.memory?.llm?.api_key,
|
|
3290
|
-
yaml.llm?.apiKey,
|
|
3291
|
-
legacy.llm?.apiKey,
|
|
3292
|
-
process.env.OPENAI_API_KEY,
|
|
3293
|
-
process.env.ANTHROPIC_API_KEY,
|
|
3294
|
-
openRouterMemoryLlmApiKey
|
|
3295
|
-
)
|
|
3299
|
+
apiKey: memoryLlmApiKey
|
|
3296
3300
|
}
|
|
3297
3301
|
},
|
|
3298
3302
|
embedding: {
|
|
@@ -3302,6 +3306,12 @@ function getResolvedConfig(options = {}) {
|
|
|
3302
3306
|
apiKey: first(process.env.MEMORIX_EMBEDDING_API_KEY, toml.embedding?.api_key, yaml.embedding?.apiKey, legacy.embeddingApi?.apiKey, openRouterEmbeddingApiKey),
|
|
3303
3307
|
dimensions: firstNumber(parseNumber(process.env.MEMORIX_EMBEDDING_DIMENSIONS), toml.embedding?.dimensions, yaml.embedding?.dimensions, legacy.embeddingApi?.dimensions)
|
|
3304
3308
|
},
|
|
3309
|
+
rerank: resolveRerankLane({
|
|
3310
|
+
toml,
|
|
3311
|
+
yaml,
|
|
3312
|
+
memoryLlmApiKey,
|
|
3313
|
+
memoryLlmBaseUrl
|
|
3314
|
+
}),
|
|
3305
3315
|
git: {
|
|
3306
3316
|
autoHook: firstBool(toml.git?.auto_hook, yaml.git?.autoHook),
|
|
3307
3317
|
ingestOnCommit: firstBool(toml.git?.ingest_on_commit, yaml.git?.ingestOnCommit),
|
|
@@ -3371,6 +3381,9 @@ function getResolvedMemoryLane(options = {}) {
|
|
|
3371
3381
|
function getResolvedEmbeddingLane(options = {}) {
|
|
3372
3382
|
return getResolvedConfig(options).embedding;
|
|
3373
3383
|
}
|
|
3384
|
+
function getResolvedRerankLane(options = {}) {
|
|
3385
|
+
return getResolvedConfig(options).rerank;
|
|
3386
|
+
}
|
|
3374
3387
|
function resetResolvedConfigCache() {
|
|
3375
3388
|
}
|
|
3376
3389
|
function first(...values) {
|
|
@@ -3413,6 +3426,10 @@ function getEnvSourceNames() {
|
|
|
3413
3426
|
"MEMORIX_EMBEDDING_BASE_URL",
|
|
3414
3427
|
"MEMORIX_EMBEDDING_MODEL",
|
|
3415
3428
|
"MEMORIX_EMBEDDING_DIMENSIONS",
|
|
3429
|
+
"MEMORIX_RERANK_PROVIDER",
|
|
3430
|
+
"MEMORIX_RERANK_MODEL",
|
|
3431
|
+
"MEMORIX_RERANK_BASE_URL",
|
|
3432
|
+
"MEMORIX_RERANK_API_KEY",
|
|
3416
3433
|
"MEMORIX_CODEGRAPH_EXTERNAL_CONTEXT",
|
|
3417
3434
|
"MEMORIX_CODEGRAPH_EXTERNAL_COMMAND",
|
|
3418
3435
|
"MEMORIX_CODEGRAPH_EXTERNAL_TIMEOUT_MS",
|
|
@@ -3436,6 +3453,63 @@ function normalizeExternalContext(value) {
|
|
|
3436
3453
|
if (normalized === "auto" || normalized === "off") return normalized;
|
|
3437
3454
|
return void 0;
|
|
3438
3455
|
}
|
|
3456
|
+
function normalizeRerankProvider(value) {
|
|
3457
|
+
const normalized = value?.trim().toLowerCase();
|
|
3458
|
+
if (normalized === "http") return "http";
|
|
3459
|
+
return "off";
|
|
3460
|
+
}
|
|
3461
|
+
function resolveRerankLane(args) {
|
|
3462
|
+
const provider2 = normalizeRerankProvider(first(
|
|
3463
|
+
process.env.MEMORIX_RERANK_PROVIDER,
|
|
3464
|
+
args.toml.rerank?.provider,
|
|
3465
|
+
args.yaml.rerank?.provider,
|
|
3466
|
+
"off"
|
|
3467
|
+
));
|
|
3468
|
+
const model = first(
|
|
3469
|
+
process.env.MEMORIX_RERANK_MODEL,
|
|
3470
|
+
args.toml.rerank?.model,
|
|
3471
|
+
args.yaml.rerank?.model
|
|
3472
|
+
);
|
|
3473
|
+
const explicitBaseUrl = first(
|
|
3474
|
+
process.env.MEMORIX_RERANK_BASE_URL,
|
|
3475
|
+
args.toml.rerank?.base_url,
|
|
3476
|
+
args.yaml.rerank?.baseUrl
|
|
3477
|
+
);
|
|
3478
|
+
const baseUrl = provider2 === "http" ? first(explicitBaseUrl, args.memoryLlmBaseUrl) : explicitBaseUrl;
|
|
3479
|
+
const dedicatedApiKey = first(
|
|
3480
|
+
process.env.MEMORIX_RERANK_API_KEY,
|
|
3481
|
+
args.toml.rerank?.api_key,
|
|
3482
|
+
args.yaml.rerank?.apiKey
|
|
3483
|
+
);
|
|
3484
|
+
const sameEndpoint = provider2 === "http" && sameTrustedEndpoint(baseUrl, args.memoryLlmBaseUrl);
|
|
3485
|
+
const inheritedApiKey = sameEndpoint ? args.memoryLlmApiKey : void 0;
|
|
3486
|
+
const apiKey = first(dedicatedApiKey, inheritedApiKey);
|
|
3487
|
+
const canSendRequest = provider2 === "http" && Boolean(baseUrl) && (Boolean(apiKey) || !args.memoryLlmApiKey || sameEndpoint);
|
|
3488
|
+
return { provider: provider2, model, baseUrl, apiKey, canSendRequest };
|
|
3489
|
+
}
|
|
3490
|
+
function sameTrustedEndpoint(left, right) {
|
|
3491
|
+
const a = normalizeTrustedEndpoint(left);
|
|
3492
|
+
const b = normalizeTrustedEndpoint(right);
|
|
3493
|
+
return Boolean(a && b && a === b);
|
|
3494
|
+
}
|
|
3495
|
+
function normalizeTrustedEndpoint(value) {
|
|
3496
|
+
const trimmed = value?.trim();
|
|
3497
|
+
if (!trimmed) return void 0;
|
|
3498
|
+
try {
|
|
3499
|
+
const url = new URL(trimmed);
|
|
3500
|
+
const protocol = url.protocol.toLowerCase();
|
|
3501
|
+
const hostname = url.hostname.toLowerCase();
|
|
3502
|
+
const port = url.port;
|
|
3503
|
+
let pathname = url.pathname.replace(/\/+$/, "");
|
|
3504
|
+
if (pathname.toLowerCase().endsWith("/rerank")) {
|
|
3505
|
+
pathname = pathname.slice(0, -"/rerank".length).replace(/\/+$/, "");
|
|
3506
|
+
}
|
|
3507
|
+
return `${protocol}//${hostname}${port ? `:${port}` : ""}${pathname}`;
|
|
3508
|
+
} catch {
|
|
3509
|
+
const fallback = trimmed.replace(/\/+$/, "").replace(/\/rerank$/i, "").replace(/\/+$/, "");
|
|
3510
|
+
return fallback.toLowerCase() || void 0;
|
|
3511
|
+
}
|
|
3512
|
+
}
|
|
3439
3513
|
var init_resolved_config = __esm({
|
|
3440
3514
|
"src/config/resolved-config.ts"() {
|
|
3441
3515
|
"use strict";
|
|
@@ -5671,9 +5745,27 @@ __export(provider_exports2, {
|
|
|
5671
5745
|
getLLMConfig: () => getLLMConfig,
|
|
5672
5746
|
initLLM: () => initLLM,
|
|
5673
5747
|
isLLMEnabled: () => isLLMEnabled,
|
|
5748
|
+
normalizeOpenAICompatibleBaseUrl: () => normalizeOpenAICompatibleBaseUrl,
|
|
5674
5749
|
parseLLMTimeoutMs: () => parseLLMTimeoutMs,
|
|
5675
5750
|
setLLMConfig: () => setLLMConfig
|
|
5676
5751
|
});
|
|
5752
|
+
function normalizeOpenAICompatibleBaseUrl(baseUrl) {
|
|
5753
|
+
const base = baseUrl.trim().replace(/\/+$/, "");
|
|
5754
|
+
if (!base) return base;
|
|
5755
|
+
let pathname = "";
|
|
5756
|
+
try {
|
|
5757
|
+
pathname = new URL(base).pathname.replace(/\/+$/, "");
|
|
5758
|
+
} catch {
|
|
5759
|
+
pathname = base.split(/[?#]/, 1)[0].replace(/\/+$/, "");
|
|
5760
|
+
}
|
|
5761
|
+
const lastSegment = pathname.split("/").filter(Boolean).at(-1);
|
|
5762
|
+
if (lastSegment && OPENAI_API_VERSION_SEGMENT.test(lastSegment)) {
|
|
5763
|
+
return base;
|
|
5764
|
+
}
|
|
5765
|
+
const queryOrHash = base.match(/[?#].*$/)?.[0] ?? "";
|
|
5766
|
+
const pathWithoutQuery = base.slice(0, base.length - queryOrHash.length).replace(/\/+$/, "");
|
|
5767
|
+
return `${pathWithoutQuery}/v1${queryOrHash}`;
|
|
5768
|
+
}
|
|
5677
5769
|
function parseLLMTimeoutMs(raw) {
|
|
5678
5770
|
if (raw === void 0 || raw.trim() === "") return LLM_TIMEOUT_DEFAULT_MS;
|
|
5679
5771
|
const parsed = Number(raw);
|
|
@@ -5777,8 +5869,7 @@ async function callLLM(systemPrompt, userMessage, signal) {
|
|
|
5777
5869
|
}
|
|
5778
5870
|
async function callOpenAICompatible(systemPrompt, userMessage, signal) {
|
|
5779
5871
|
const config = currentConfig;
|
|
5780
|
-
|
|
5781
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
5872
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
5782
5873
|
const url = `${base}/chat/completions`;
|
|
5783
5874
|
const fetchSignal = signal ? AbortSignal.any([signal, AbortSignal.timeout(LLM_CALL_TIMEOUT_MS)]) : AbortSignal.timeout(LLM_CALL_TIMEOUT_MS);
|
|
5784
5875
|
const response = await fetch(url, {
|
|
@@ -5857,8 +5948,7 @@ async function callLLMWithTools(messages, tools, signal) {
|
|
|
5857
5948
|
}
|
|
5858
5949
|
async function callOpenAIWithTools(messages, tools, signal) {
|
|
5859
5950
|
const config = currentConfig;
|
|
5860
|
-
|
|
5861
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
5951
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
5862
5952
|
const url = `${base}/chat/completions`;
|
|
5863
5953
|
const openaiMessages = messages.map((msg) => {
|
|
5864
5954
|
if (msg.role === "tool") {
|
|
@@ -6008,8 +6098,7 @@ async function callAnthropicWithTools(messages, tools, signal) {
|
|
|
6008
6098
|
}
|
|
6009
6099
|
async function* callOpenAIWithToolsStream(messages, tools) {
|
|
6010
6100
|
const config = currentConfig;
|
|
6011
|
-
|
|
6012
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
6101
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
6013
6102
|
const url = `${base}/chat/completions`;
|
|
6014
6103
|
const openaiMessages = messages.map((msg) => {
|
|
6015
6104
|
if (msg.role === "tool") {
|
|
@@ -6257,7 +6346,7 @@ async function* callAnthropicWithToolsStream(messages, tools) {
|
|
|
6257
6346
|
}
|
|
6258
6347
|
};
|
|
6259
6348
|
}
|
|
6260
|
-
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
6349
|
+
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, OPENAI_API_VERSION_SEGMENT, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
6261
6350
|
var init_provider2 = __esm({
|
|
6262
6351
|
"src/llm/provider.ts"() {
|
|
6263
6352
|
"use strict";
|
|
@@ -6267,6 +6356,7 @@ var init_provider2 = __esm({
|
|
|
6267
6356
|
LLM_TIMEOUT_MIN_MS = 1e3;
|
|
6268
6357
|
LLM_TIMEOUT_MAX_MS = 3e5;
|
|
6269
6358
|
MAX_LLM_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
6359
|
+
OPENAI_API_VERSION_SEGMENT = /^v\d+(?:[a-z][a-z0-9-]*)?$/i;
|
|
6270
6360
|
LLM_CALL_TIMEOUT_MS = parseLLMTimeoutMs(process.env.MEMORIX_LLM_TIMEOUT_MS);
|
|
6271
6361
|
PROVIDER_DEFAULTS = {
|
|
6272
6362
|
openai: { baseUrl: "https://api.openai.com/v1", model: "gpt-4.1-nano" },
|
|
@@ -6326,6 +6416,48 @@ Rules:
|
|
|
6326
6416
|
}
|
|
6327
6417
|
});
|
|
6328
6418
|
|
|
6419
|
+
// src/rerank/policy.ts
|
|
6420
|
+
function parseRerankTimeoutMs(raw) {
|
|
6421
|
+
const value = raw?.trim();
|
|
6422
|
+
if (!value) return DEFAULT_RERANK_TIMEOUT_MS;
|
|
6423
|
+
const parsed = Number(value);
|
|
6424
|
+
if (!Number.isInteger(parsed) || Number.isNaN(parsed)) {
|
|
6425
|
+
console.warn(
|
|
6426
|
+
`[memorix] MEMORIX_RERANK_TIMEOUT_MS="${raw}" is invalid (must be a positive integer between ${RERANK_TIMEOUT_MIN_MS}-${RERANK_TIMEOUT_MAX_MS}ms). Using default ${DEFAULT_RERANK_TIMEOUT_MS}ms.`
|
|
6427
|
+
);
|
|
6428
|
+
return DEFAULT_RERANK_TIMEOUT_MS;
|
|
6429
|
+
}
|
|
6430
|
+
if (parsed < RERANK_TIMEOUT_MIN_MS) return RERANK_TIMEOUT_MIN_MS;
|
|
6431
|
+
if (parsed > RERANK_TIMEOUT_MAX_MS) return RERANK_TIMEOUT_MAX_MS;
|
|
6432
|
+
return parsed;
|
|
6433
|
+
}
|
|
6434
|
+
function resolveRerankTimeoutMs(explicitTimeoutMs) {
|
|
6435
|
+
if (typeof explicitTimeoutMs === "number" && Number.isFinite(explicitTimeoutMs) && explicitTimeoutMs > 0) {
|
|
6436
|
+
return explicitTimeoutMs;
|
|
6437
|
+
}
|
|
6438
|
+
return parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
6439
|
+
}
|
|
6440
|
+
function shouldAttemptHttpRerank(input) {
|
|
6441
|
+
return input.quality === "thorough" && input.hasQuery && input.candidateCount >= MIN_RERANK_CANDIDATES;
|
|
6442
|
+
}
|
|
6443
|
+
function shouldAttemptLlmRerank(input) {
|
|
6444
|
+
if (input.httpRerankConfigured) return false;
|
|
6445
|
+
if (input.quality !== "thorough" || input.tier !== "heavy" || !input.hasQuery) return false;
|
|
6446
|
+
if (input.candidateCount < MIN_RERANK_CANDIDATES) return false;
|
|
6447
|
+
return input.topScore > 0 && input.secondScore / input.topScore > 0.7;
|
|
6448
|
+
}
|
|
6449
|
+
var DEFAULT_RERANK_TIMEOUT_MS, RERANK_TIMEOUT_MIN_MS, RERANK_TIMEOUT_MAX_MS, MIN_RERANK_CANDIDATES;
|
|
6450
|
+
var init_policy = __esm({
|
|
6451
|
+
"src/rerank/policy.ts"() {
|
|
6452
|
+
"use strict";
|
|
6453
|
+
init_esm_shims();
|
|
6454
|
+
DEFAULT_RERANK_TIMEOUT_MS = 3e4;
|
|
6455
|
+
RERANK_TIMEOUT_MIN_MS = 1e3;
|
|
6456
|
+
RERANK_TIMEOUT_MAX_MS = 3e5;
|
|
6457
|
+
MIN_RERANK_CANDIDATES = 3;
|
|
6458
|
+
}
|
|
6459
|
+
});
|
|
6460
|
+
|
|
6329
6461
|
// src/timeout.ts
|
|
6330
6462
|
function withTimeout(promise, ms, label) {
|
|
6331
6463
|
return new Promise((resolve2, reject) => {
|
|
@@ -6611,6 +6743,148 @@ var init_aliases = __esm({
|
|
|
6611
6743
|
}
|
|
6612
6744
|
});
|
|
6613
6745
|
|
|
6746
|
+
// src/rerank/http-provider.ts
|
|
6747
|
+
function buildRerankUrl(baseUrl) {
|
|
6748
|
+
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
|
6749
|
+
if (trimmed.endsWith("/rerank")) return trimmed;
|
|
6750
|
+
return `${trimmed}/rerank`;
|
|
6751
|
+
}
|
|
6752
|
+
function parseRerankResponse(payload) {
|
|
6753
|
+
const rows = extractRows(payload);
|
|
6754
|
+
if (rows.length === 0) {
|
|
6755
|
+
throw new Error("Neural rerank response contained no ranked results");
|
|
6756
|
+
}
|
|
6757
|
+
return rows.sort((a, b) => b.score - a.score);
|
|
6758
|
+
}
|
|
6759
|
+
function extractRows(payload) {
|
|
6760
|
+
if (Array.isArray(payload)) {
|
|
6761
|
+
return payload.map(rowFromUnknown).filter((row) => row !== null);
|
|
6762
|
+
}
|
|
6763
|
+
if (payload && typeof payload === "object" && "results" in payload) {
|
|
6764
|
+
const results = payload.results;
|
|
6765
|
+
if (Array.isArray(results)) {
|
|
6766
|
+
return results.map(rowFromUnknown).filter((row) => row !== null);
|
|
6767
|
+
}
|
|
6768
|
+
}
|
|
6769
|
+
throw new Error("Neural rerank response was not Cohere or TEI shaped");
|
|
6770
|
+
}
|
|
6771
|
+
function rowFromUnknown(value) {
|
|
6772
|
+
if (!value || typeof value !== "object") return null;
|
|
6773
|
+
const row = value;
|
|
6774
|
+
if (typeof row.index !== "number" || !Number.isFinite(row.index)) return null;
|
|
6775
|
+
const score = typeof row.relevance_score === "number" ? row.relevance_score : typeof row.score === "number" ? row.score : null;
|
|
6776
|
+
if (score === null || !Number.isFinite(score)) return null;
|
|
6777
|
+
return { index: row.index, score };
|
|
6778
|
+
}
|
|
6779
|
+
function buildRerankBody(query, documents, model) {
|
|
6780
|
+
const body = {
|
|
6781
|
+
query,
|
|
6782
|
+
documents,
|
|
6783
|
+
top_n: documents.length,
|
|
6784
|
+
return_documents: false
|
|
6785
|
+
};
|
|
6786
|
+
const trimmed = model?.trim();
|
|
6787
|
+
if (trimmed) body.model = trimmed;
|
|
6788
|
+
return body;
|
|
6789
|
+
}
|
|
6790
|
+
async function rerankViaHttp(query, documents, config) {
|
|
6791
|
+
if (documents.length === 0) return [];
|
|
6792
|
+
const url = buildRerankUrl(config.baseUrl);
|
|
6793
|
+
const headers = {
|
|
6794
|
+
"content-type": "application/json"
|
|
6795
|
+
};
|
|
6796
|
+
if (config.apiKey) {
|
|
6797
|
+
headers.authorization = `Bearer ${config.apiKey}`;
|
|
6798
|
+
}
|
|
6799
|
+
const controller = new AbortController();
|
|
6800
|
+
const timeoutMs = resolveRerankTimeoutMs(config.timeoutMs);
|
|
6801
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
6802
|
+
try {
|
|
6803
|
+
const fetchImpl = config.fetchImpl ?? fetch;
|
|
6804
|
+
const response = await fetchImpl(url, {
|
|
6805
|
+
method: "POST",
|
|
6806
|
+
headers,
|
|
6807
|
+
signal: controller.signal,
|
|
6808
|
+
body: JSON.stringify(buildRerankBody(query, documents, config.model))
|
|
6809
|
+
});
|
|
6810
|
+
if (!response.ok) {
|
|
6811
|
+
throw new Error(`Neural rerank HTTP ${response.status} from ${url}`);
|
|
6812
|
+
}
|
|
6813
|
+
return parseRerankResponse(await response.json());
|
|
6814
|
+
} finally {
|
|
6815
|
+
clearTimeout(timer);
|
|
6816
|
+
}
|
|
6817
|
+
}
|
|
6818
|
+
var init_http_provider = __esm({
|
|
6819
|
+
"src/rerank/http-provider.ts"() {
|
|
6820
|
+
"use strict";
|
|
6821
|
+
init_esm_shims();
|
|
6822
|
+
init_policy();
|
|
6823
|
+
}
|
|
6824
|
+
});
|
|
6825
|
+
|
|
6826
|
+
// src/rerank/index.ts
|
|
6827
|
+
var rerank_exports = {};
|
|
6828
|
+
__export(rerank_exports, {
|
|
6829
|
+
DEFAULT_RERANK_TIMEOUT_MS: () => DEFAULT_RERANK_TIMEOUT_MS,
|
|
6830
|
+
MIN_RERANK_CANDIDATES: () => MIN_RERANK_CANDIDATES,
|
|
6831
|
+
isNeuralRerankEnabled: () => isNeuralRerankEnabled,
|
|
6832
|
+
neuralRerankCandidates: () => neuralRerankCandidates,
|
|
6833
|
+
parseRerankTimeoutMs: () => parseRerankTimeoutMs,
|
|
6834
|
+
resolveRerankTimeoutMs: () => resolveRerankTimeoutMs,
|
|
6835
|
+
shouldAttemptHttpRerank: () => shouldAttemptHttpRerank,
|
|
6836
|
+
shouldAttemptLlmRerank: () => shouldAttemptLlmRerank
|
|
6837
|
+
});
|
|
6838
|
+
function isNeuralRerankEnabled(options = {}) {
|
|
6839
|
+
const lane = getResolvedRerankLane(options);
|
|
6840
|
+
return lane.provider === "http" && Boolean(lane.baseUrl);
|
|
6841
|
+
}
|
|
6842
|
+
async function neuralRerankCandidates(query, candidates, options = {}) {
|
|
6843
|
+
if (!isNeuralRerankEnabled(options) || candidates.length === 0) return null;
|
|
6844
|
+
const lane = getResolvedRerankLane(options);
|
|
6845
|
+
if (!lane.canSendRequest || !lane.baseUrl) return null;
|
|
6846
|
+
const toRerank = candidates.slice(0, MAX_RERANK);
|
|
6847
|
+
const rest = candidates.slice(MAX_RERANK);
|
|
6848
|
+
const documents = toRerank.map((candidate) => candidateText(candidate));
|
|
6849
|
+
const ranked = await rerankViaHttp(query, documents, {
|
|
6850
|
+
provider: "http",
|
|
6851
|
+
model: lane.model,
|
|
6852
|
+
baseUrl: lane.baseUrl,
|
|
6853
|
+
apiKey: lane.apiKey,
|
|
6854
|
+
timeoutMs: resolveRerankTimeoutMs()
|
|
6855
|
+
});
|
|
6856
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6857
|
+
const reranked = [];
|
|
6858
|
+
for (const row of ranked) {
|
|
6859
|
+
const candidate = toRerank[row.index];
|
|
6860
|
+
if (!candidate || seen.has(row.index)) continue;
|
|
6861
|
+
reranked.push(candidate);
|
|
6862
|
+
seen.add(row.index);
|
|
6863
|
+
}
|
|
6864
|
+
for (const [index, candidate] of toRerank.entries()) {
|
|
6865
|
+
if (!seen.has(index)) reranked.push(candidate);
|
|
6866
|
+
}
|
|
6867
|
+
reranked.push(...rest);
|
|
6868
|
+
return reranked;
|
|
6869
|
+
}
|
|
6870
|
+
function candidateText(candidate) {
|
|
6871
|
+
const narrative = candidate.narrative?.trim();
|
|
6872
|
+
if (narrative) return `${candidate.title} \u2014 ${narrative.slice(0, 200)}`;
|
|
6873
|
+
return candidate.title;
|
|
6874
|
+
}
|
|
6875
|
+
var MAX_RERANK;
|
|
6876
|
+
var init_rerank = __esm({
|
|
6877
|
+
"src/rerank/index.ts"() {
|
|
6878
|
+
"use strict";
|
|
6879
|
+
init_esm_shims();
|
|
6880
|
+
init_resolved_config();
|
|
6881
|
+
init_http_provider();
|
|
6882
|
+
init_policy();
|
|
6883
|
+
init_policy();
|
|
6884
|
+
MAX_RERANK = 10;
|
|
6885
|
+
}
|
|
6886
|
+
});
|
|
6887
|
+
|
|
6614
6888
|
// src/llm/quality.ts
|
|
6615
6889
|
var quality_exports = {};
|
|
6616
6890
|
__export(quality_exports, {
|
|
@@ -6650,9 +6924,9 @@ async function rerankResults(query, candidates) {
|
|
|
6650
6924
|
if (!isLLMEnabled() || candidates.length <= 2) {
|
|
6651
6925
|
return { reranked: candidates, usedLLM: false };
|
|
6652
6926
|
}
|
|
6653
|
-
const
|
|
6654
|
-
const toRerank = candidates.slice(0,
|
|
6655
|
-
const rest = candidates.slice(
|
|
6927
|
+
const MAX_RERANK2 = 10;
|
|
6928
|
+
const toRerank = candidates.slice(0, MAX_RERANK2);
|
|
6929
|
+
const rest = candidates.slice(MAX_RERANK2);
|
|
6656
6930
|
try {
|
|
6657
6931
|
const candidateList = toRerank.map(
|
|
6658
6932
|
(c) => `[ID: ${c.id}] (${c.type}) ${c.title}${c.narrative ? ` \u2014 ${c.narrative.substring(0, 100)}` : ""}`
|
|
@@ -6876,7 +7150,7 @@ __export(orama_store_exports, {
|
|
|
6876
7150
|
searchObservations: () => searchObservations,
|
|
6877
7151
|
updateObservationMetadata: () => updateObservationMetadata
|
|
6878
7152
|
});
|
|
6879
|
-
import { create, insert as insert2, search, remove as remove2, update, count, getByID } from "@orama/orama";
|
|
7153
|
+
import { create, insert as insert2, insertMultiple, search, remove as remove2, update, count, getByID } from "@orama/orama";
|
|
6880
7154
|
function getLastSearchMode(projectId) {
|
|
6881
7155
|
return lastSearchModeByProject.get(projectId ?? SEARCH_MODE_DEFAULT_KEY) ?? "fulltext";
|
|
6882
7156
|
}
|
|
@@ -7088,7 +7362,7 @@ async function hydrateIndex(observations2, options = {}) {
|
|
|
7088
7362
|
}
|
|
7089
7363
|
const deferCachedVectors = options.deferCachedVectors === true && Boolean(indexEmbeddingProvider?.getCachedEmbeddings);
|
|
7090
7364
|
const cachedVectors = deferCachedVectors ? candidates.map(() => null) : await getCachedEmbeddings(candidates.map(({ observation }) => observationEmbeddingText(observation)));
|
|
7091
|
-
|
|
7365
|
+
const documents = [];
|
|
7092
7366
|
for (let index = 0; index < candidates.length; index++) {
|
|
7093
7367
|
const { observation: obs, id } = candidates[index];
|
|
7094
7368
|
try {
|
|
@@ -7122,18 +7396,35 @@ async function hydrateIndex(observations2, options = {}) {
|
|
|
7122
7396
|
knowledgeLayer: resolveKnowledgeLayer("observation", obs.sourceDetail, obs.source),
|
|
7123
7397
|
...compatibleVector ? { embedding: compatibleVector } : {}
|
|
7124
7398
|
};
|
|
7125
|
-
|
|
7126
|
-
rememberObservationDoc(doc);
|
|
7127
|
-
inserted++;
|
|
7128
|
-
if (inserted % 50 === 0) {
|
|
7129
|
-
await new Promise((resolve2) => setImmediate(resolve2));
|
|
7130
|
-
}
|
|
7131
|
-
if (inserted % 200 === 0) {
|
|
7132
|
-
await new Promise((resolve2) => setTimeout(resolve2, 0));
|
|
7133
|
-
}
|
|
7399
|
+
documents.push(doc);
|
|
7134
7400
|
} catch {
|
|
7135
7401
|
}
|
|
7136
7402
|
}
|
|
7403
|
+
let inserted = 0;
|
|
7404
|
+
const hydrationBatchSize = 200;
|
|
7405
|
+
for (let start = 0; start < documents.length; start += hydrationBatchSize) {
|
|
7406
|
+
const batch = documents.slice(start, start + hydrationBatchSize);
|
|
7407
|
+
try {
|
|
7408
|
+
await insertMultiple(database, batch, batch.length);
|
|
7409
|
+
for (const doc of batch) rememberObservationDoc(doc);
|
|
7410
|
+
inserted += batch.length;
|
|
7411
|
+
} catch {
|
|
7412
|
+
for (const doc of batch) {
|
|
7413
|
+
if (getByID(database, doc.id)) {
|
|
7414
|
+
rememberObservationDoc(doc);
|
|
7415
|
+
inserted++;
|
|
7416
|
+
continue;
|
|
7417
|
+
}
|
|
7418
|
+
try {
|
|
7419
|
+
await insert2(database, doc);
|
|
7420
|
+
rememberObservationDoc(doc);
|
|
7421
|
+
inserted++;
|
|
7422
|
+
} catch {
|
|
7423
|
+
}
|
|
7424
|
+
}
|
|
7425
|
+
}
|
|
7426
|
+
await new Promise((resolve2) => setImmediate(resolve2));
|
|
7427
|
+
}
|
|
7137
7428
|
deferredCachedVectorHydration = options.skipCachedVectors ? null : deferCachedVectors ? attachCachedVectors(database, candidates).catch(() => {
|
|
7138
7429
|
}) : null;
|
|
7139
7430
|
return inserted;
|
|
@@ -7528,46 +7819,78 @@ async function searchObservations(options) {
|
|
|
7528
7819
|
}
|
|
7529
7820
|
}
|
|
7530
7821
|
}
|
|
7531
|
-
const
|
|
7532
|
-
|
|
7533
|
-
|
|
7534
|
-
|
|
7535
|
-
})
|
|
7536
|
-
|
|
7537
|
-
|
|
7538
|
-
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
7542
|
-
|
|
7543
|
-
|
|
7544
|
-
|
|
7545
|
-
|
|
7546
|
-
|
|
7547
|
-
|
|
7548
|
-
|
|
7549
|
-
|
|
7550
|
-
|
|
7551
|
-
|
|
7552
|
-
|
|
7553
|
-
|
|
7554
|
-
|
|
7555
|
-
|
|
7556
|
-
|
|
7557
|
-
|
|
7558
|
-
|
|
7559
|
-
|
|
7560
|
-
|
|
7561
|
-
|
|
7562
|
-
|
|
7563
|
-
|
|
7564
|
-
|
|
7565
|
-
|
|
7566
|
-
|
|
7822
|
+
const httpGate = shouldAttemptHttpRerank({
|
|
7823
|
+
quality,
|
|
7824
|
+
hasQuery: Boolean(hasQuery),
|
|
7825
|
+
candidateCount: intermediate.length
|
|
7826
|
+
});
|
|
7827
|
+
const llmGateWithoutHttp = shouldAttemptLlmRerank({
|
|
7828
|
+
quality,
|
|
7829
|
+
tier,
|
|
7830
|
+
hasQuery: Boolean(hasQuery),
|
|
7831
|
+
candidateCount: intermediate.length,
|
|
7832
|
+
topScore: intermediate[0]?.score ?? 0,
|
|
7833
|
+
secondScore: intermediate[1]?.score ?? 0,
|
|
7834
|
+
httpRerankConfigured: false
|
|
7835
|
+
});
|
|
7836
|
+
if (httpGate || llmGateWithoutHttp) {
|
|
7837
|
+
const narrativeMap = /* @__PURE__ */ new Map();
|
|
7838
|
+
for (const hit of results.hits) {
|
|
7839
|
+
const doc = hit.document;
|
|
7840
|
+
narrativeMap.set(makeEntryKey(doc.projectId, doc.observationId), doc.narrative);
|
|
7841
|
+
}
|
|
7842
|
+
const RERANK_TOP_K = 5;
|
|
7843
|
+
const toRerank = intermediate.slice(0, RERANK_TOP_K);
|
|
7844
|
+
const candidates = toRerank.map((e2, index) => ({
|
|
7845
|
+
id: `r${index + 1}`,
|
|
7846
|
+
title: e2.title,
|
|
7847
|
+
type: e2.type,
|
|
7848
|
+
score: e2.score,
|
|
7849
|
+
narrative: narrativeMap.get(makeEntryKey(e2.projectId, e2.id))
|
|
7850
|
+
}));
|
|
7851
|
+
const RERANK_TIMEOUT_MS = parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
7852
|
+
const candidateMap = new Map(candidates.map((candidate, index) => [candidate.id, toRerank[index]]));
|
|
7853
|
+
const { isNeuralRerankEnabled: isNeuralRerankEnabled2, neuralRerankCandidates: neuralRerankCandidates2 } = await Promise.resolve().then(() => (init_rerank(), rerank_exports));
|
|
7854
|
+
const httpConfigured = isNeuralRerankEnabled2();
|
|
7855
|
+
if (httpConfigured && httpGate) {
|
|
7856
|
+
try {
|
|
7857
|
+
const neural = await withTimeout(
|
|
7858
|
+
neuralRerankCandidates2(originalQuery, candidates),
|
|
7859
|
+
RERANK_TIMEOUT_MS,
|
|
7860
|
+
"neural rerank"
|
|
7861
|
+
);
|
|
7862
|
+
if (neural && neural.length > 0) {
|
|
7863
|
+
const rerankedTop = neural.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7864
|
+
if (rerankedTop.length > 0) {
|
|
7865
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7866
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + neural rerank");
|
|
7867
|
+
mark("rerank(usedNeural=true)");
|
|
7868
|
+
}
|
|
7567
7869
|
}
|
|
7870
|
+
} catch {
|
|
7871
|
+
console.error("[memorix] neural rerank failed or timed out; keeping original order");
|
|
7568
7872
|
}
|
|
7569
|
-
}
|
|
7570
|
-
|
|
7873
|
+
} else if (!httpConfigured && llmGateWithoutHttp) {
|
|
7874
|
+
try {
|
|
7875
|
+
const { rerankResults: rerankResults2 } = await Promise.resolve().then(() => (init_quality(), quality_exports));
|
|
7876
|
+
const { reranked, usedLLM } = await withTimeout(
|
|
7877
|
+
rerankResults2(originalQuery, candidates),
|
|
7878
|
+
RERANK_TIMEOUT_MS,
|
|
7879
|
+
"LLM rerank"
|
|
7880
|
+
);
|
|
7881
|
+
mark(`rerank(usedLLM=${usedLLM})`);
|
|
7882
|
+
if (usedLLM) {
|
|
7883
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + LLM rerank");
|
|
7884
|
+
const rerankedTop = reranked.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7885
|
+
if (rerankedTop.length > 0) {
|
|
7886
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7887
|
+
}
|
|
7888
|
+
}
|
|
7889
|
+
} catch {
|
|
7890
|
+
console.error("[memorix] LLM rerank failed or timed out, using original order");
|
|
7891
|
+
}
|
|
7892
|
+
} else {
|
|
7893
|
+
mark(`rerank(skipped,tier=${tier})`);
|
|
7571
7894
|
}
|
|
7572
7895
|
} else {
|
|
7573
7896
|
mark(`rerank(skipped,tier=${tier})`);
|
|
@@ -7748,6 +8071,7 @@ var init_orama_store = __esm({
|
|
|
7748
8071
|
init_project_affinity();
|
|
7749
8072
|
init_intent_detector();
|
|
7750
8073
|
init_query_expansion();
|
|
8074
|
+
init_policy();
|
|
7751
8075
|
init_timeout();
|
|
7752
8076
|
db = null;
|
|
7753
8077
|
dbInitPromise = null;
|
|
@@ -161203,8 +161527,7 @@ async function callVisionLLM(systemPrompt, imageBase64, mimeType) {
|
|
|
161203
161527
|
if (!apiKey) {
|
|
161204
161528
|
throw new Error("No LLM API key configured for image analysis.");
|
|
161205
161529
|
}
|
|
161206
|
-
|
|
161207
|
-
if (!baseUrl.endsWith("/v1")) baseUrl += "/v1";
|
|
161530
|
+
const baseUrl = normalizeOpenAICompatibleBaseUrl(getLLMBaseUrl("https://api.openai.com/v1"));
|
|
161208
161531
|
const model = getLLMModel("gpt-4o");
|
|
161209
161532
|
const response = await fetch(`${baseUrl}/chat/completions`, {
|
|
161210
161533
|
method: "POST",
|
|
@@ -167080,7 +167403,7 @@ The path should point to a directory containing a .git folder.`
|
|
|
167080
167403
|
};
|
|
167081
167404
|
const server = existingServer ?? new McpServer({
|
|
167082
167405
|
name: "memorix",
|
|
167083
|
-
version: true ? "1.7.
|
|
167406
|
+
version: true ? "1.7.3" : "1.0.1"
|
|
167084
167407
|
});
|
|
167085
167408
|
const originalRegisterTool = server.registerTool.bind(server);
|
|
167086
167409
|
server.registerTool = ((name, ...args) => {
|