memorix 1.7.2 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -1
- package/assets/star-history-dark.svg +4 -4
- package/assets/star-history-light.svg +4 -4
- package/assets/star-history.json +47 -3
- package/dist/cli/index.js +396 -64
- package/dist/index.js +368 -62
- package/dist/maintenance-runner.js +365 -58
- package/dist/media-audio-runner.js +365 -58
- package/dist/media-video-runner.js +365 -58
- package/dist/memcode-runtime/CHANGELOG.md +27 -1
- package/dist/sdk.js +367 -61
- package/dist/vector-backfill-runner.js +365 -58
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CONFIGURATION.md +57 -1
- package/docs/PERFORMANCE.md +3 -1
- package/docs/SETUP.md +2 -1
- package/package.json +1 -1
- package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
- package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
- package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/copilot/memorix/plugin.json +1 -1
- package/plugins/gemini/memorix/gemini-extension.json +1 -1
- package/plugins/omp/memorix/package.json +1 -1
- package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/pi/memorix/package.json +1 -1
- package/server.json +2 -2
- package/src/cli/commands/config-migrate.ts +7 -0
- package/src/cli/commands/doctor.ts +5 -1
- package/src/cli/commands/init.ts +17 -0
- package/src/cli/commands/status.ts +2 -0
- package/src/config/resolved-config.ts +110 -12
- package/src/config/toml-loader.ts +8 -0
- package/src/config/yaml-loader.ts +9 -0
- package/src/llm/provider.ts +33 -7
- package/src/multimodal/image-loader.ts +6 -3
- package/src/rerank/http-provider.ts +125 -0
- package/src/rerank/index.ts +81 -0
- package/src/rerank/policy.ts +88 -0
- package/src/store/orama-store.ts +79 -53
|
@@ -176,6 +176,7 @@ function loadYamlConfig(projectRoot) {
|
|
|
176
176
|
llm: { ...userConfig.llm, ...projectConfig.llm },
|
|
177
177
|
agent: { ...userConfig.agent, ...projectConfig.agent },
|
|
178
178
|
embedding: { ...userConfig.embedding, ...projectConfig.embedding },
|
|
179
|
+
rerank: { ...userConfig.rerank, ...projectConfig.rerank },
|
|
179
180
|
git: { ...userConfig.git, ...projectConfig.git },
|
|
180
181
|
codegraph: { ...userConfig.codegraph, ...projectConfig.codegraph },
|
|
181
182
|
behavior: { ...userConfig.behavior, ...projectConfig.behavior },
|
|
@@ -2819,6 +2820,7 @@ function mergeTomlConfig(base, override) {
|
|
|
2819
2820
|
llm: { ...base.memory?.llm, ...override.memory?.llm }
|
|
2820
2821
|
},
|
|
2821
2822
|
embedding: { ...base.embedding, ...override.embedding },
|
|
2823
|
+
rerank: { ...base.rerank, ...override.rerank },
|
|
2822
2824
|
hooks: { ...base.hooks, ...override.hooks },
|
|
2823
2825
|
git: { ...base.git, ...override.git },
|
|
2824
2826
|
codegraph: { ...base.codegraph, ...override.codegraph },
|
|
@@ -2915,6 +2917,7 @@ __export(resolved_config_exports, {
|
|
|
2915
2917
|
getResolvedConfigForCwd: () => getResolvedConfigForCwd,
|
|
2916
2918
|
getResolvedEmbeddingLane: () => getResolvedEmbeddingLane,
|
|
2917
2919
|
getResolvedMemoryLane: () => getResolvedMemoryLane,
|
|
2920
|
+
getResolvedRerankLane: () => getResolvedRerankLane,
|
|
2918
2921
|
resetResolvedConfigCache: () => resetResolvedConfigCache
|
|
2919
2922
|
});
|
|
2920
2923
|
import { homedir as homedir5 } from "os";
|
|
@@ -2947,6 +2950,16 @@ function getResolvedConfig(options = {}) {
|
|
|
2947
2950
|
legacy.llm?.baseUrl
|
|
2948
2951
|
);
|
|
2949
2952
|
const openRouterMemoryLlmApiKey = isOpenRouterMemoryLane(memoryLlmProvider, memoryLlmBaseUrl) ? process.env.OPENROUTER_API_KEY : void 0;
|
|
2953
|
+
const memoryLlmApiKey = first(
|
|
2954
|
+
process.env.MEMORIX_LLM_API_KEY,
|
|
2955
|
+
process.env.MEMORIX_API_KEY,
|
|
2956
|
+
toml.memory?.llm?.api_key,
|
|
2957
|
+
yaml.llm?.apiKey,
|
|
2958
|
+
legacy.llm?.apiKey,
|
|
2959
|
+
process.env.OPENAI_API_KEY,
|
|
2960
|
+
process.env.ANTHROPIC_API_KEY,
|
|
2961
|
+
openRouterMemoryLlmApiKey
|
|
2962
|
+
);
|
|
2950
2963
|
const resolved = {
|
|
2951
2964
|
agent: {
|
|
2952
2965
|
provider: first(
|
|
@@ -2987,16 +3000,7 @@ function getResolvedConfig(options = {}) {
|
|
|
2987
3000
|
provider: memoryLlmProvider,
|
|
2988
3001
|
model: memoryLlmModel,
|
|
2989
3002
|
baseUrl: memoryLlmBaseUrl,
|
|
2990
|
-
apiKey:
|
|
2991
|
-
process.env.MEMORIX_LLM_API_KEY,
|
|
2992
|
-
process.env.MEMORIX_API_KEY,
|
|
2993
|
-
toml.memory?.llm?.api_key,
|
|
2994
|
-
yaml.llm?.apiKey,
|
|
2995
|
-
legacy.llm?.apiKey,
|
|
2996
|
-
process.env.OPENAI_API_KEY,
|
|
2997
|
-
process.env.ANTHROPIC_API_KEY,
|
|
2998
|
-
openRouterMemoryLlmApiKey
|
|
2999
|
-
)
|
|
3003
|
+
apiKey: memoryLlmApiKey
|
|
3000
3004
|
}
|
|
3001
3005
|
},
|
|
3002
3006
|
embedding: {
|
|
@@ -3006,6 +3010,12 @@ function getResolvedConfig(options = {}) {
|
|
|
3006
3010
|
apiKey: first(process.env.MEMORIX_EMBEDDING_API_KEY, toml.embedding?.api_key, yaml.embedding?.apiKey, legacy.embeddingApi?.apiKey, openRouterEmbeddingApiKey),
|
|
3007
3011
|
dimensions: firstNumber(parseNumber(process.env.MEMORIX_EMBEDDING_DIMENSIONS), toml.embedding?.dimensions, yaml.embedding?.dimensions, legacy.embeddingApi?.dimensions)
|
|
3008
3012
|
},
|
|
3013
|
+
rerank: resolveRerankLane({
|
|
3014
|
+
toml,
|
|
3015
|
+
yaml,
|
|
3016
|
+
memoryLlmApiKey,
|
|
3017
|
+
memoryLlmBaseUrl
|
|
3018
|
+
}),
|
|
3009
3019
|
git: {
|
|
3010
3020
|
autoHook: firstBool(toml.git?.auto_hook, yaml.git?.autoHook),
|
|
3011
3021
|
ingestOnCommit: firstBool(toml.git?.ingest_on_commit, yaml.git?.ingestOnCommit),
|
|
@@ -3075,6 +3085,9 @@ function getResolvedMemoryLane(options = {}) {
|
|
|
3075
3085
|
function getResolvedEmbeddingLane(options = {}) {
|
|
3076
3086
|
return getResolvedConfig(options).embedding;
|
|
3077
3087
|
}
|
|
3088
|
+
function getResolvedRerankLane(options = {}) {
|
|
3089
|
+
return getResolvedConfig(options).rerank;
|
|
3090
|
+
}
|
|
3078
3091
|
function resetResolvedConfigCache() {
|
|
3079
3092
|
}
|
|
3080
3093
|
function first(...values) {
|
|
@@ -3117,6 +3130,10 @@ function getEnvSourceNames() {
|
|
|
3117
3130
|
"MEMORIX_EMBEDDING_BASE_URL",
|
|
3118
3131
|
"MEMORIX_EMBEDDING_MODEL",
|
|
3119
3132
|
"MEMORIX_EMBEDDING_DIMENSIONS",
|
|
3133
|
+
"MEMORIX_RERANK_PROVIDER",
|
|
3134
|
+
"MEMORIX_RERANK_MODEL",
|
|
3135
|
+
"MEMORIX_RERANK_BASE_URL",
|
|
3136
|
+
"MEMORIX_RERANK_API_KEY",
|
|
3120
3137
|
"MEMORIX_CODEGRAPH_EXTERNAL_CONTEXT",
|
|
3121
3138
|
"MEMORIX_CODEGRAPH_EXTERNAL_COMMAND",
|
|
3122
3139
|
"MEMORIX_CODEGRAPH_EXTERNAL_TIMEOUT_MS",
|
|
@@ -3140,6 +3157,63 @@ function normalizeExternalContext(value) {
|
|
|
3140
3157
|
if (normalized === "auto" || normalized === "off") return normalized;
|
|
3141
3158
|
return void 0;
|
|
3142
3159
|
}
|
|
3160
|
+
function normalizeRerankProvider(value) {
|
|
3161
|
+
const normalized = value?.trim().toLowerCase();
|
|
3162
|
+
if (normalized === "http") return "http";
|
|
3163
|
+
return "off";
|
|
3164
|
+
}
|
|
3165
|
+
function resolveRerankLane(args) {
|
|
3166
|
+
const provider2 = normalizeRerankProvider(first(
|
|
3167
|
+
process.env.MEMORIX_RERANK_PROVIDER,
|
|
3168
|
+
args.toml.rerank?.provider,
|
|
3169
|
+
args.yaml.rerank?.provider,
|
|
3170
|
+
"off"
|
|
3171
|
+
));
|
|
3172
|
+
const model = first(
|
|
3173
|
+
process.env.MEMORIX_RERANK_MODEL,
|
|
3174
|
+
args.toml.rerank?.model,
|
|
3175
|
+
args.yaml.rerank?.model
|
|
3176
|
+
);
|
|
3177
|
+
const explicitBaseUrl = first(
|
|
3178
|
+
process.env.MEMORIX_RERANK_BASE_URL,
|
|
3179
|
+
args.toml.rerank?.base_url,
|
|
3180
|
+
args.yaml.rerank?.baseUrl
|
|
3181
|
+
);
|
|
3182
|
+
const baseUrl = provider2 === "http" ? first(explicitBaseUrl, args.memoryLlmBaseUrl) : explicitBaseUrl;
|
|
3183
|
+
const dedicatedApiKey = first(
|
|
3184
|
+
process.env.MEMORIX_RERANK_API_KEY,
|
|
3185
|
+
args.toml.rerank?.api_key,
|
|
3186
|
+
args.yaml.rerank?.apiKey
|
|
3187
|
+
);
|
|
3188
|
+
const sameEndpoint = provider2 === "http" && sameTrustedEndpoint(baseUrl, args.memoryLlmBaseUrl);
|
|
3189
|
+
const inheritedApiKey = sameEndpoint ? args.memoryLlmApiKey : void 0;
|
|
3190
|
+
const apiKey = first(dedicatedApiKey, inheritedApiKey);
|
|
3191
|
+
const canSendRequest = provider2 === "http" && Boolean(baseUrl) && (Boolean(apiKey) || !args.memoryLlmApiKey || sameEndpoint);
|
|
3192
|
+
return { provider: provider2, model, baseUrl, apiKey, canSendRequest };
|
|
3193
|
+
}
|
|
3194
|
+
function sameTrustedEndpoint(left, right) {
|
|
3195
|
+
const a = normalizeTrustedEndpoint(left);
|
|
3196
|
+
const b = normalizeTrustedEndpoint(right);
|
|
3197
|
+
return Boolean(a && b && a === b);
|
|
3198
|
+
}
|
|
3199
|
+
function normalizeTrustedEndpoint(value) {
|
|
3200
|
+
const trimmed = value?.trim();
|
|
3201
|
+
if (!trimmed) return void 0;
|
|
3202
|
+
try {
|
|
3203
|
+
const url = new URL(trimmed);
|
|
3204
|
+
const protocol = url.protocol.toLowerCase();
|
|
3205
|
+
const hostname = url.hostname.toLowerCase();
|
|
3206
|
+
const port = url.port;
|
|
3207
|
+
let pathname = url.pathname.replace(/\/+$/, "");
|
|
3208
|
+
if (pathname.toLowerCase().endsWith("/rerank")) {
|
|
3209
|
+
pathname = pathname.slice(0, -"/rerank".length).replace(/\/+$/, "");
|
|
3210
|
+
}
|
|
3211
|
+
return `${protocol}//${hostname}${port ? `:${port}` : ""}${pathname}`;
|
|
3212
|
+
} catch {
|
|
3213
|
+
const fallback = trimmed.replace(/\/+$/, "").replace(/\/rerank$/i, "").replace(/\/+$/, "");
|
|
3214
|
+
return fallback.toLowerCase() || void 0;
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
3143
3217
|
var init_resolved_config = __esm({
|
|
3144
3218
|
"src/config/resolved-config.ts"() {
|
|
3145
3219
|
"use strict";
|
|
@@ -5375,9 +5449,27 @@ __export(provider_exports2, {
|
|
|
5375
5449
|
getLLMConfig: () => getLLMConfig,
|
|
5376
5450
|
initLLM: () => initLLM,
|
|
5377
5451
|
isLLMEnabled: () => isLLMEnabled,
|
|
5452
|
+
normalizeOpenAICompatibleBaseUrl: () => normalizeOpenAICompatibleBaseUrl,
|
|
5378
5453
|
parseLLMTimeoutMs: () => parseLLMTimeoutMs,
|
|
5379
5454
|
setLLMConfig: () => setLLMConfig
|
|
5380
5455
|
});
|
|
5456
|
+
function normalizeOpenAICompatibleBaseUrl(baseUrl) {
|
|
5457
|
+
const base = baseUrl.trim().replace(/\/+$/, "");
|
|
5458
|
+
if (!base) return base;
|
|
5459
|
+
let pathname = "";
|
|
5460
|
+
try {
|
|
5461
|
+
pathname = new URL(base).pathname.replace(/\/+$/, "");
|
|
5462
|
+
} catch {
|
|
5463
|
+
pathname = base.split(/[?#]/, 1)[0].replace(/\/+$/, "");
|
|
5464
|
+
}
|
|
5465
|
+
const lastSegment = pathname.split("/").filter(Boolean).at(-1);
|
|
5466
|
+
if (lastSegment && OPENAI_API_VERSION_SEGMENT.test(lastSegment)) {
|
|
5467
|
+
return base;
|
|
5468
|
+
}
|
|
5469
|
+
const queryOrHash = base.match(/[?#].*$/)?.[0] ?? "";
|
|
5470
|
+
const pathWithoutQuery = base.slice(0, base.length - queryOrHash.length).replace(/\/+$/, "");
|
|
5471
|
+
return `${pathWithoutQuery}/v1${queryOrHash}`;
|
|
5472
|
+
}
|
|
5381
5473
|
function parseLLMTimeoutMs(raw) {
|
|
5382
5474
|
if (raw === void 0 || raw.trim() === "") return LLM_TIMEOUT_DEFAULT_MS;
|
|
5383
5475
|
const parsed = Number(raw);
|
|
@@ -5481,8 +5573,7 @@ async function callLLM(systemPrompt, userMessage, signal) {
|
|
|
5481
5573
|
}
|
|
5482
5574
|
async function callOpenAICompatible(systemPrompt, userMessage, signal) {
|
|
5483
5575
|
const config = currentConfig;
|
|
5484
|
-
|
|
5485
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
5576
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
5486
5577
|
const url = `${base}/chat/completions`;
|
|
5487
5578
|
const fetchSignal = signal ? AbortSignal.any([signal, AbortSignal.timeout(LLM_CALL_TIMEOUT_MS)]) : AbortSignal.timeout(LLM_CALL_TIMEOUT_MS);
|
|
5488
5579
|
const response = await fetch(url, {
|
|
@@ -5561,8 +5652,7 @@ async function callLLMWithTools(messages, tools, signal) {
|
|
|
5561
5652
|
}
|
|
5562
5653
|
async function callOpenAIWithTools(messages, tools, signal) {
|
|
5563
5654
|
const config = currentConfig;
|
|
5564
|
-
|
|
5565
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
5655
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
5566
5656
|
const url = `${base}/chat/completions`;
|
|
5567
5657
|
const openaiMessages = messages.map((msg) => {
|
|
5568
5658
|
if (msg.role === "tool") {
|
|
@@ -5712,8 +5802,7 @@ async function callAnthropicWithTools(messages, tools, signal) {
|
|
|
5712
5802
|
}
|
|
5713
5803
|
async function* callOpenAIWithToolsStream(messages, tools) {
|
|
5714
5804
|
const config = currentConfig;
|
|
5715
|
-
|
|
5716
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
5805
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
5717
5806
|
const url = `${base}/chat/completions`;
|
|
5718
5807
|
const openaiMessages = messages.map((msg) => {
|
|
5719
5808
|
if (msg.role === "tool") {
|
|
@@ -5961,7 +6050,7 @@ async function* callAnthropicWithToolsStream(messages, tools) {
|
|
|
5961
6050
|
}
|
|
5962
6051
|
};
|
|
5963
6052
|
}
|
|
5964
|
-
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
6053
|
+
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, OPENAI_API_VERSION_SEGMENT, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
5965
6054
|
var init_provider2 = __esm({
|
|
5966
6055
|
"src/llm/provider.ts"() {
|
|
5967
6056
|
"use strict";
|
|
@@ -5971,6 +6060,7 @@ var init_provider2 = __esm({
|
|
|
5971
6060
|
LLM_TIMEOUT_MIN_MS = 1e3;
|
|
5972
6061
|
LLM_TIMEOUT_MAX_MS = 3e5;
|
|
5973
6062
|
MAX_LLM_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
6063
|
+
OPENAI_API_VERSION_SEGMENT = /^v\d+(?:[a-z][a-z0-9-]*)?$/i;
|
|
5974
6064
|
LLM_CALL_TIMEOUT_MS = parseLLMTimeoutMs(process.env.MEMORIX_LLM_TIMEOUT_MS);
|
|
5975
6065
|
PROVIDER_DEFAULTS = {
|
|
5976
6066
|
openai: { baseUrl: "https://api.openai.com/v1", model: "gpt-4.1-nano" },
|
|
@@ -6030,6 +6120,48 @@ Rules:
|
|
|
6030
6120
|
}
|
|
6031
6121
|
});
|
|
6032
6122
|
|
|
6123
|
+
// src/rerank/policy.ts
|
|
6124
|
+
function parseRerankTimeoutMs(raw) {
|
|
6125
|
+
const value = raw?.trim();
|
|
6126
|
+
if (!value) return DEFAULT_RERANK_TIMEOUT_MS;
|
|
6127
|
+
const parsed = Number(value);
|
|
6128
|
+
if (!Number.isInteger(parsed) || Number.isNaN(parsed)) {
|
|
6129
|
+
console.warn(
|
|
6130
|
+
`[memorix] MEMORIX_RERANK_TIMEOUT_MS="${raw}" is invalid (must be a positive integer between ${RERANK_TIMEOUT_MIN_MS}-${RERANK_TIMEOUT_MAX_MS}ms). Using default ${DEFAULT_RERANK_TIMEOUT_MS}ms.`
|
|
6131
|
+
);
|
|
6132
|
+
return DEFAULT_RERANK_TIMEOUT_MS;
|
|
6133
|
+
}
|
|
6134
|
+
if (parsed < RERANK_TIMEOUT_MIN_MS) return RERANK_TIMEOUT_MIN_MS;
|
|
6135
|
+
if (parsed > RERANK_TIMEOUT_MAX_MS) return RERANK_TIMEOUT_MAX_MS;
|
|
6136
|
+
return parsed;
|
|
6137
|
+
}
|
|
6138
|
+
function resolveRerankTimeoutMs(explicitTimeoutMs) {
|
|
6139
|
+
if (typeof explicitTimeoutMs === "number" && Number.isFinite(explicitTimeoutMs) && explicitTimeoutMs > 0) {
|
|
6140
|
+
return explicitTimeoutMs;
|
|
6141
|
+
}
|
|
6142
|
+
return parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
6143
|
+
}
|
|
6144
|
+
function shouldAttemptHttpRerank(input) {
|
|
6145
|
+
return input.quality === "thorough" && input.hasQuery && input.candidateCount >= MIN_RERANK_CANDIDATES;
|
|
6146
|
+
}
|
|
6147
|
+
function shouldAttemptLlmRerank(input) {
|
|
6148
|
+
if (input.httpRerankConfigured) return false;
|
|
6149
|
+
if (input.quality !== "thorough" || input.tier !== "heavy" || !input.hasQuery) return false;
|
|
6150
|
+
if (input.candidateCount < MIN_RERANK_CANDIDATES) return false;
|
|
6151
|
+
return input.topScore > 0 && input.secondScore / input.topScore > 0.7;
|
|
6152
|
+
}
|
|
6153
|
+
var DEFAULT_RERANK_TIMEOUT_MS, RERANK_TIMEOUT_MIN_MS, RERANK_TIMEOUT_MAX_MS, MIN_RERANK_CANDIDATES;
|
|
6154
|
+
var init_policy = __esm({
|
|
6155
|
+
"src/rerank/policy.ts"() {
|
|
6156
|
+
"use strict";
|
|
6157
|
+
init_esm_shims();
|
|
6158
|
+
DEFAULT_RERANK_TIMEOUT_MS = 3e4;
|
|
6159
|
+
RERANK_TIMEOUT_MIN_MS = 1e3;
|
|
6160
|
+
RERANK_TIMEOUT_MAX_MS = 3e5;
|
|
6161
|
+
MIN_RERANK_CANDIDATES = 3;
|
|
6162
|
+
}
|
|
6163
|
+
});
|
|
6164
|
+
|
|
6033
6165
|
// src/timeout.ts
|
|
6034
6166
|
function withTimeout(promise, ms, label) {
|
|
6035
6167
|
return new Promise((resolve, reject) => {
|
|
@@ -6315,6 +6447,148 @@ var init_aliases = __esm({
|
|
|
6315
6447
|
}
|
|
6316
6448
|
});
|
|
6317
6449
|
|
|
6450
|
+
// src/rerank/http-provider.ts
|
|
6451
|
+
function buildRerankUrl(baseUrl) {
|
|
6452
|
+
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
|
6453
|
+
if (trimmed.endsWith("/rerank")) return trimmed;
|
|
6454
|
+
return `${trimmed}/rerank`;
|
|
6455
|
+
}
|
|
6456
|
+
function parseRerankResponse(payload) {
|
|
6457
|
+
const rows = extractRows(payload);
|
|
6458
|
+
if (rows.length === 0) {
|
|
6459
|
+
throw new Error("Neural rerank response contained no ranked results");
|
|
6460
|
+
}
|
|
6461
|
+
return rows.sort((a, b) => b.score - a.score);
|
|
6462
|
+
}
|
|
6463
|
+
function extractRows(payload) {
|
|
6464
|
+
if (Array.isArray(payload)) {
|
|
6465
|
+
return payload.map(rowFromUnknown).filter((row) => row !== null);
|
|
6466
|
+
}
|
|
6467
|
+
if (payload && typeof payload === "object" && "results" in payload) {
|
|
6468
|
+
const results = payload.results;
|
|
6469
|
+
if (Array.isArray(results)) {
|
|
6470
|
+
return results.map(rowFromUnknown).filter((row) => row !== null);
|
|
6471
|
+
}
|
|
6472
|
+
}
|
|
6473
|
+
throw new Error("Neural rerank response was not Cohere or TEI shaped");
|
|
6474
|
+
}
|
|
6475
|
+
function rowFromUnknown(value) {
|
|
6476
|
+
if (!value || typeof value !== "object") return null;
|
|
6477
|
+
const row = value;
|
|
6478
|
+
if (typeof row.index !== "number" || !Number.isFinite(row.index)) return null;
|
|
6479
|
+
const score = typeof row.relevance_score === "number" ? row.relevance_score : typeof row.score === "number" ? row.score : null;
|
|
6480
|
+
if (score === null || !Number.isFinite(score)) return null;
|
|
6481
|
+
return { index: row.index, score };
|
|
6482
|
+
}
|
|
6483
|
+
function buildRerankBody(query, documents, model) {
|
|
6484
|
+
const body = {
|
|
6485
|
+
query,
|
|
6486
|
+
documents,
|
|
6487
|
+
top_n: documents.length,
|
|
6488
|
+
return_documents: false
|
|
6489
|
+
};
|
|
6490
|
+
const trimmed = model?.trim();
|
|
6491
|
+
if (trimmed) body.model = trimmed;
|
|
6492
|
+
return body;
|
|
6493
|
+
}
|
|
6494
|
+
async function rerankViaHttp(query, documents, config) {
|
|
6495
|
+
if (documents.length === 0) return [];
|
|
6496
|
+
const url = buildRerankUrl(config.baseUrl);
|
|
6497
|
+
const headers = {
|
|
6498
|
+
"content-type": "application/json"
|
|
6499
|
+
};
|
|
6500
|
+
if (config.apiKey) {
|
|
6501
|
+
headers.authorization = `Bearer ${config.apiKey}`;
|
|
6502
|
+
}
|
|
6503
|
+
const controller = new AbortController();
|
|
6504
|
+
const timeoutMs = resolveRerankTimeoutMs(config.timeoutMs);
|
|
6505
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
6506
|
+
try {
|
|
6507
|
+
const fetchImpl = config.fetchImpl ?? fetch;
|
|
6508
|
+
const response = await fetchImpl(url, {
|
|
6509
|
+
method: "POST",
|
|
6510
|
+
headers,
|
|
6511
|
+
signal: controller.signal,
|
|
6512
|
+
body: JSON.stringify(buildRerankBody(query, documents, config.model))
|
|
6513
|
+
});
|
|
6514
|
+
if (!response.ok) {
|
|
6515
|
+
throw new Error(`Neural rerank HTTP ${response.status} from ${url}`);
|
|
6516
|
+
}
|
|
6517
|
+
return parseRerankResponse(await response.json());
|
|
6518
|
+
} finally {
|
|
6519
|
+
clearTimeout(timer);
|
|
6520
|
+
}
|
|
6521
|
+
}
|
|
6522
|
+
var init_http_provider = __esm({
|
|
6523
|
+
"src/rerank/http-provider.ts"() {
|
|
6524
|
+
"use strict";
|
|
6525
|
+
init_esm_shims();
|
|
6526
|
+
init_policy();
|
|
6527
|
+
}
|
|
6528
|
+
});
|
|
6529
|
+
|
|
6530
|
+
// src/rerank/index.ts
|
|
6531
|
+
var rerank_exports = {};
|
|
6532
|
+
__export(rerank_exports, {
|
|
6533
|
+
DEFAULT_RERANK_TIMEOUT_MS: () => DEFAULT_RERANK_TIMEOUT_MS,
|
|
6534
|
+
MIN_RERANK_CANDIDATES: () => MIN_RERANK_CANDIDATES,
|
|
6535
|
+
isNeuralRerankEnabled: () => isNeuralRerankEnabled,
|
|
6536
|
+
neuralRerankCandidates: () => neuralRerankCandidates,
|
|
6537
|
+
parseRerankTimeoutMs: () => parseRerankTimeoutMs,
|
|
6538
|
+
resolveRerankTimeoutMs: () => resolveRerankTimeoutMs,
|
|
6539
|
+
shouldAttemptHttpRerank: () => shouldAttemptHttpRerank,
|
|
6540
|
+
shouldAttemptLlmRerank: () => shouldAttemptLlmRerank
|
|
6541
|
+
});
|
|
6542
|
+
function isNeuralRerankEnabled(options = {}) {
|
|
6543
|
+
const lane = getResolvedRerankLane(options);
|
|
6544
|
+
return lane.provider === "http" && Boolean(lane.baseUrl);
|
|
6545
|
+
}
|
|
6546
|
+
async function neuralRerankCandidates(query, candidates, options = {}) {
|
|
6547
|
+
if (!isNeuralRerankEnabled(options) || candidates.length === 0) return null;
|
|
6548
|
+
const lane = getResolvedRerankLane(options);
|
|
6549
|
+
if (!lane.canSendRequest || !lane.baseUrl) return null;
|
|
6550
|
+
const toRerank = candidates.slice(0, MAX_RERANK);
|
|
6551
|
+
const rest = candidates.slice(MAX_RERANK);
|
|
6552
|
+
const documents = toRerank.map((candidate) => candidateText(candidate));
|
|
6553
|
+
const ranked = await rerankViaHttp(query, documents, {
|
|
6554
|
+
provider: "http",
|
|
6555
|
+
model: lane.model,
|
|
6556
|
+
baseUrl: lane.baseUrl,
|
|
6557
|
+
apiKey: lane.apiKey,
|
|
6558
|
+
timeoutMs: resolveRerankTimeoutMs()
|
|
6559
|
+
});
|
|
6560
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6561
|
+
const reranked = [];
|
|
6562
|
+
for (const row of ranked) {
|
|
6563
|
+
const candidate = toRerank[row.index];
|
|
6564
|
+
if (!candidate || seen.has(row.index)) continue;
|
|
6565
|
+
reranked.push(candidate);
|
|
6566
|
+
seen.add(row.index);
|
|
6567
|
+
}
|
|
6568
|
+
for (const [index, candidate] of toRerank.entries()) {
|
|
6569
|
+
if (!seen.has(index)) reranked.push(candidate);
|
|
6570
|
+
}
|
|
6571
|
+
reranked.push(...rest);
|
|
6572
|
+
return reranked;
|
|
6573
|
+
}
|
|
6574
|
+
function candidateText(candidate) {
|
|
6575
|
+
const narrative = candidate.narrative?.trim();
|
|
6576
|
+
if (narrative) return `${candidate.title} \u2014 ${narrative.slice(0, 200)}`;
|
|
6577
|
+
return candidate.title;
|
|
6578
|
+
}
|
|
6579
|
+
var MAX_RERANK;
|
|
6580
|
+
var init_rerank = __esm({
|
|
6581
|
+
"src/rerank/index.ts"() {
|
|
6582
|
+
"use strict";
|
|
6583
|
+
init_esm_shims();
|
|
6584
|
+
init_resolved_config();
|
|
6585
|
+
init_http_provider();
|
|
6586
|
+
init_policy();
|
|
6587
|
+
init_policy();
|
|
6588
|
+
MAX_RERANK = 10;
|
|
6589
|
+
}
|
|
6590
|
+
});
|
|
6591
|
+
|
|
6318
6592
|
// src/llm/quality.ts
|
|
6319
6593
|
var quality_exports = {};
|
|
6320
6594
|
__export(quality_exports, {
|
|
@@ -6354,9 +6628,9 @@ async function rerankResults(query, candidates) {
|
|
|
6354
6628
|
if (!isLLMEnabled() || candidates.length <= 2) {
|
|
6355
6629
|
return { reranked: candidates, usedLLM: false };
|
|
6356
6630
|
}
|
|
6357
|
-
const
|
|
6358
|
-
const toRerank = candidates.slice(0,
|
|
6359
|
-
const rest = candidates.slice(
|
|
6631
|
+
const MAX_RERANK2 = 10;
|
|
6632
|
+
const toRerank = candidates.slice(0, MAX_RERANK2);
|
|
6633
|
+
const rest = candidates.slice(MAX_RERANK2);
|
|
6360
6634
|
try {
|
|
6361
6635
|
const candidateList = toRerank.map(
|
|
6362
6636
|
(c) => `[ID: ${c.id}] (${c.type}) ${c.title}${c.narrative ? ` \u2014 ${c.narrative.substring(0, 100)}` : ""}`
|
|
@@ -7249,46 +7523,78 @@ async function searchObservations(options) {
|
|
|
7249
7523
|
}
|
|
7250
7524
|
}
|
|
7251
7525
|
}
|
|
7252
|
-
const
|
|
7253
|
-
|
|
7254
|
-
|
|
7255
|
-
|
|
7256
|
-
})
|
|
7257
|
-
|
|
7258
|
-
|
|
7259
|
-
|
|
7260
|
-
|
|
7261
|
-
|
|
7262
|
-
|
|
7263
|
-
|
|
7264
|
-
|
|
7265
|
-
|
|
7266
|
-
|
|
7267
|
-
|
|
7268
|
-
|
|
7269
|
-
|
|
7270
|
-
|
|
7271
|
-
|
|
7272
|
-
|
|
7273
|
-
|
|
7274
|
-
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7278
|
-
|
|
7279
|
-
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7283
|
-
|
|
7284
|
-
|
|
7285
|
-
|
|
7286
|
-
|
|
7287
|
-
|
|
7526
|
+
const httpGate = shouldAttemptHttpRerank({
|
|
7527
|
+
quality,
|
|
7528
|
+
hasQuery: Boolean(hasQuery),
|
|
7529
|
+
candidateCount: intermediate.length
|
|
7530
|
+
});
|
|
7531
|
+
const llmGateWithoutHttp = shouldAttemptLlmRerank({
|
|
7532
|
+
quality,
|
|
7533
|
+
tier,
|
|
7534
|
+
hasQuery: Boolean(hasQuery),
|
|
7535
|
+
candidateCount: intermediate.length,
|
|
7536
|
+
topScore: intermediate[0]?.score ?? 0,
|
|
7537
|
+
secondScore: intermediate[1]?.score ?? 0,
|
|
7538
|
+
httpRerankConfigured: false
|
|
7539
|
+
});
|
|
7540
|
+
if (httpGate || llmGateWithoutHttp) {
|
|
7541
|
+
const narrativeMap = /* @__PURE__ */ new Map();
|
|
7542
|
+
for (const hit of results.hits) {
|
|
7543
|
+
const doc = hit.document;
|
|
7544
|
+
narrativeMap.set(makeEntryKey(doc.projectId, doc.observationId), doc.narrative);
|
|
7545
|
+
}
|
|
7546
|
+
const RERANK_TOP_K = 5;
|
|
7547
|
+
const toRerank = intermediate.slice(0, RERANK_TOP_K);
|
|
7548
|
+
const candidates = toRerank.map((e2, index) => ({
|
|
7549
|
+
id: `r${index + 1}`,
|
|
7550
|
+
title: e2.title,
|
|
7551
|
+
type: e2.type,
|
|
7552
|
+
score: e2.score,
|
|
7553
|
+
narrative: narrativeMap.get(makeEntryKey(e2.projectId, e2.id))
|
|
7554
|
+
}));
|
|
7555
|
+
const RERANK_TIMEOUT_MS = parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
7556
|
+
const candidateMap = new Map(candidates.map((candidate, index) => [candidate.id, toRerank[index]]));
|
|
7557
|
+
const { isNeuralRerankEnabled: isNeuralRerankEnabled2, neuralRerankCandidates: neuralRerankCandidates2 } = await Promise.resolve().then(() => (init_rerank(), rerank_exports));
|
|
7558
|
+
const httpConfigured = isNeuralRerankEnabled2();
|
|
7559
|
+
if (httpConfigured && httpGate) {
|
|
7560
|
+
try {
|
|
7561
|
+
const neural = await withTimeout(
|
|
7562
|
+
neuralRerankCandidates2(originalQuery, candidates),
|
|
7563
|
+
RERANK_TIMEOUT_MS,
|
|
7564
|
+
"neural rerank"
|
|
7565
|
+
);
|
|
7566
|
+
if (neural && neural.length > 0) {
|
|
7567
|
+
const rerankedTop = neural.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7568
|
+
if (rerankedTop.length > 0) {
|
|
7569
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7570
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + neural rerank");
|
|
7571
|
+
mark("rerank(usedNeural=true)");
|
|
7572
|
+
}
|
|
7288
7573
|
}
|
|
7574
|
+
} catch {
|
|
7575
|
+
console.error("[memorix] neural rerank failed or timed out; keeping original order");
|
|
7289
7576
|
}
|
|
7290
|
-
}
|
|
7291
|
-
|
|
7577
|
+
} else if (!httpConfigured && llmGateWithoutHttp) {
|
|
7578
|
+
try {
|
|
7579
|
+
const { rerankResults: rerankResults2 } = await Promise.resolve().then(() => (init_quality(), quality_exports));
|
|
7580
|
+
const { reranked, usedLLM } = await withTimeout(
|
|
7581
|
+
rerankResults2(originalQuery, candidates),
|
|
7582
|
+
RERANK_TIMEOUT_MS,
|
|
7583
|
+
"LLM rerank"
|
|
7584
|
+
);
|
|
7585
|
+
mark(`rerank(usedLLM=${usedLLM})`);
|
|
7586
|
+
if (usedLLM) {
|
|
7587
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + LLM rerank");
|
|
7588
|
+
const rerankedTop = reranked.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7589
|
+
if (rerankedTop.length > 0) {
|
|
7590
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7591
|
+
}
|
|
7592
|
+
}
|
|
7593
|
+
} catch {
|
|
7594
|
+
console.error("[memorix] LLM rerank failed or timed out, using original order");
|
|
7595
|
+
}
|
|
7596
|
+
} else {
|
|
7597
|
+
mark(`rerank(skipped,tier=${tier})`);
|
|
7292
7598
|
}
|
|
7293
7599
|
} else {
|
|
7294
7600
|
mark(`rerank(skipped,tier=${tier})`);
|
|
@@ -7469,6 +7775,7 @@ var init_orama_store = __esm({
|
|
|
7469
7775
|
init_project_affinity();
|
|
7470
7776
|
init_intent_detector();
|
|
7471
7777
|
init_query_expansion();
|
|
7778
|
+
init_policy();
|
|
7472
7779
|
init_timeout();
|
|
7473
7780
|
db = null;
|
|
7474
7781
|
dbInitPromise = null;
|