memorix 1.7.2 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -1
- package/assets/star-history-dark.svg +4 -4
- package/assets/star-history-light.svg +4 -4
- package/assets/star-history.json +47 -3
- package/dist/cli/index.js +396 -64
- package/dist/index.js +368 -62
- package/dist/maintenance-runner.js +365 -58
- package/dist/media-audio-runner.js +365 -58
- package/dist/media-video-runner.js +365 -58
- package/dist/memcode-runtime/CHANGELOG.md +27 -1
- package/dist/sdk.js +367 -61
- package/dist/vector-backfill-runner.js +365 -58
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CONFIGURATION.md +57 -1
- package/docs/PERFORMANCE.md +3 -1
- package/docs/SETUP.md +2 -1
- package/package.json +1 -1
- package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
- package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
- package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/copilot/memorix/plugin.json +1 -1
- package/plugins/gemini/memorix/gemini-extension.json +1 -1
- package/plugins/omp/memorix/package.json +1 -1
- package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/pi/memorix/package.json +1 -1
- package/server.json +2 -2
- package/src/cli/commands/config-migrate.ts +7 -0
- package/src/cli/commands/doctor.ts +5 -1
- package/src/cli/commands/init.ts +17 -0
- package/src/cli/commands/status.ts +2 -0
- package/src/config/resolved-config.ts +110 -12
- package/src/config/toml-loader.ts +8 -0
- package/src/config/yaml-loader.ts +9 -0
- package/src/llm/provider.ts +33 -7
- package/src/multimodal/image-loader.ts +6 -3
- package/src/rerank/http-provider.ts +125 -0
- package/src/rerank/index.ts +81 -0
- package/src/rerank/policy.ts +88 -0
- package/src/store/orama-store.ts +79 -53
|
@@ -1327,6 +1327,7 @@ function loadYamlConfig(projectRoot) {
|
|
|
1327
1327
|
llm: { ...userConfig.llm, ...projectConfig.llm },
|
|
1328
1328
|
agent: { ...userConfig.agent, ...projectConfig.agent },
|
|
1329
1329
|
embedding: { ...userConfig.embedding, ...projectConfig.embedding },
|
|
1330
|
+
rerank: { ...userConfig.rerank, ...projectConfig.rerank },
|
|
1330
1331
|
git: { ...userConfig.git, ...projectConfig.git },
|
|
1331
1332
|
codegraph: { ...userConfig.codegraph, ...projectConfig.codegraph },
|
|
1332
1333
|
behavior: { ...userConfig.behavior, ...projectConfig.behavior },
|
|
@@ -1563,6 +1564,7 @@ function mergeTomlConfig(base, override) {
|
|
|
1563
1564
|
llm: { ...base.memory?.llm, ...override.memory?.llm }
|
|
1564
1565
|
},
|
|
1565
1566
|
embedding: { ...base.embedding, ...override.embedding },
|
|
1567
|
+
rerank: { ...base.rerank, ...override.rerank },
|
|
1566
1568
|
hooks: { ...base.hooks, ...override.hooks },
|
|
1567
1569
|
git: { ...base.git, ...override.git },
|
|
1568
1570
|
codegraph: { ...base.codegraph, ...override.codegraph },
|
|
@@ -1659,6 +1661,7 @@ __export(resolved_config_exports, {
|
|
|
1659
1661
|
getResolvedConfigForCwd: () => getResolvedConfigForCwd,
|
|
1660
1662
|
getResolvedEmbeddingLane: () => getResolvedEmbeddingLane,
|
|
1661
1663
|
getResolvedMemoryLane: () => getResolvedMemoryLane,
|
|
1664
|
+
getResolvedRerankLane: () => getResolvedRerankLane,
|
|
1662
1665
|
resetResolvedConfigCache: () => resetResolvedConfigCache
|
|
1663
1666
|
});
|
|
1664
1667
|
import { homedir as homedir5 } from "os";
|
|
@@ -1691,6 +1694,16 @@ function getResolvedConfig(options = {}) {
|
|
|
1691
1694
|
legacy.llm?.baseUrl
|
|
1692
1695
|
);
|
|
1693
1696
|
const openRouterMemoryLlmApiKey = isOpenRouterMemoryLane(memoryLlmProvider, memoryLlmBaseUrl) ? process.env.OPENROUTER_API_KEY : void 0;
|
|
1697
|
+
const memoryLlmApiKey = first(
|
|
1698
|
+
process.env.MEMORIX_LLM_API_KEY,
|
|
1699
|
+
process.env.MEMORIX_API_KEY,
|
|
1700
|
+
toml.memory?.llm?.api_key,
|
|
1701
|
+
yaml.llm?.apiKey,
|
|
1702
|
+
legacy.llm?.apiKey,
|
|
1703
|
+
process.env.OPENAI_API_KEY,
|
|
1704
|
+
process.env.ANTHROPIC_API_KEY,
|
|
1705
|
+
openRouterMemoryLlmApiKey
|
|
1706
|
+
);
|
|
1694
1707
|
const resolved = {
|
|
1695
1708
|
agent: {
|
|
1696
1709
|
provider: first(
|
|
@@ -1731,16 +1744,7 @@ function getResolvedConfig(options = {}) {
|
|
|
1731
1744
|
provider: memoryLlmProvider,
|
|
1732
1745
|
model: memoryLlmModel,
|
|
1733
1746
|
baseUrl: memoryLlmBaseUrl,
|
|
1734
|
-
apiKey:
|
|
1735
|
-
process.env.MEMORIX_LLM_API_KEY,
|
|
1736
|
-
process.env.MEMORIX_API_KEY,
|
|
1737
|
-
toml.memory?.llm?.api_key,
|
|
1738
|
-
yaml.llm?.apiKey,
|
|
1739
|
-
legacy.llm?.apiKey,
|
|
1740
|
-
process.env.OPENAI_API_KEY,
|
|
1741
|
-
process.env.ANTHROPIC_API_KEY,
|
|
1742
|
-
openRouterMemoryLlmApiKey
|
|
1743
|
-
)
|
|
1747
|
+
apiKey: memoryLlmApiKey
|
|
1744
1748
|
}
|
|
1745
1749
|
},
|
|
1746
1750
|
embedding: {
|
|
@@ -1750,6 +1754,12 @@ function getResolvedConfig(options = {}) {
|
|
|
1750
1754
|
apiKey: first(process.env.MEMORIX_EMBEDDING_API_KEY, toml.embedding?.api_key, yaml.embedding?.apiKey, legacy.embeddingApi?.apiKey, openRouterEmbeddingApiKey),
|
|
1751
1755
|
dimensions: firstNumber(parseNumber(process.env.MEMORIX_EMBEDDING_DIMENSIONS), toml.embedding?.dimensions, yaml.embedding?.dimensions, legacy.embeddingApi?.dimensions)
|
|
1752
1756
|
},
|
|
1757
|
+
rerank: resolveRerankLane({
|
|
1758
|
+
toml,
|
|
1759
|
+
yaml,
|
|
1760
|
+
memoryLlmApiKey,
|
|
1761
|
+
memoryLlmBaseUrl
|
|
1762
|
+
}),
|
|
1753
1763
|
git: {
|
|
1754
1764
|
autoHook: firstBool(toml.git?.auto_hook, yaml.git?.autoHook),
|
|
1755
1765
|
ingestOnCommit: firstBool(toml.git?.ingest_on_commit, yaml.git?.ingestOnCommit),
|
|
@@ -1819,6 +1829,9 @@ function getResolvedMemoryLane(options = {}) {
|
|
|
1819
1829
|
function getResolvedEmbeddingLane(options = {}) {
|
|
1820
1830
|
return getResolvedConfig(options).embedding;
|
|
1821
1831
|
}
|
|
1832
|
+
function getResolvedRerankLane(options = {}) {
|
|
1833
|
+
return getResolvedConfig(options).rerank;
|
|
1834
|
+
}
|
|
1822
1835
|
function resetResolvedConfigCache() {
|
|
1823
1836
|
}
|
|
1824
1837
|
function first(...values) {
|
|
@@ -1861,6 +1874,10 @@ function getEnvSourceNames() {
|
|
|
1861
1874
|
"MEMORIX_EMBEDDING_BASE_URL",
|
|
1862
1875
|
"MEMORIX_EMBEDDING_MODEL",
|
|
1863
1876
|
"MEMORIX_EMBEDDING_DIMENSIONS",
|
|
1877
|
+
"MEMORIX_RERANK_PROVIDER",
|
|
1878
|
+
"MEMORIX_RERANK_MODEL",
|
|
1879
|
+
"MEMORIX_RERANK_BASE_URL",
|
|
1880
|
+
"MEMORIX_RERANK_API_KEY",
|
|
1864
1881
|
"MEMORIX_CODEGRAPH_EXTERNAL_CONTEXT",
|
|
1865
1882
|
"MEMORIX_CODEGRAPH_EXTERNAL_COMMAND",
|
|
1866
1883
|
"MEMORIX_CODEGRAPH_EXTERNAL_TIMEOUT_MS",
|
|
@@ -1884,6 +1901,63 @@ function normalizeExternalContext(value) {
|
|
|
1884
1901
|
if (normalized === "auto" || normalized === "off") return normalized;
|
|
1885
1902
|
return void 0;
|
|
1886
1903
|
}
|
|
1904
|
+
function normalizeRerankProvider(value) {
|
|
1905
|
+
const normalized = value?.trim().toLowerCase();
|
|
1906
|
+
if (normalized === "http") return "http";
|
|
1907
|
+
return "off";
|
|
1908
|
+
}
|
|
1909
|
+
function resolveRerankLane(args) {
|
|
1910
|
+
const provider2 = normalizeRerankProvider(first(
|
|
1911
|
+
process.env.MEMORIX_RERANK_PROVIDER,
|
|
1912
|
+
args.toml.rerank?.provider,
|
|
1913
|
+
args.yaml.rerank?.provider,
|
|
1914
|
+
"off"
|
|
1915
|
+
));
|
|
1916
|
+
const model = first(
|
|
1917
|
+
process.env.MEMORIX_RERANK_MODEL,
|
|
1918
|
+
args.toml.rerank?.model,
|
|
1919
|
+
args.yaml.rerank?.model
|
|
1920
|
+
);
|
|
1921
|
+
const explicitBaseUrl = first(
|
|
1922
|
+
process.env.MEMORIX_RERANK_BASE_URL,
|
|
1923
|
+
args.toml.rerank?.base_url,
|
|
1924
|
+
args.yaml.rerank?.baseUrl
|
|
1925
|
+
);
|
|
1926
|
+
const baseUrl = provider2 === "http" ? first(explicitBaseUrl, args.memoryLlmBaseUrl) : explicitBaseUrl;
|
|
1927
|
+
const dedicatedApiKey = first(
|
|
1928
|
+
process.env.MEMORIX_RERANK_API_KEY,
|
|
1929
|
+
args.toml.rerank?.api_key,
|
|
1930
|
+
args.yaml.rerank?.apiKey
|
|
1931
|
+
);
|
|
1932
|
+
const sameEndpoint = provider2 === "http" && sameTrustedEndpoint(baseUrl, args.memoryLlmBaseUrl);
|
|
1933
|
+
const inheritedApiKey = sameEndpoint ? args.memoryLlmApiKey : void 0;
|
|
1934
|
+
const apiKey = first(dedicatedApiKey, inheritedApiKey);
|
|
1935
|
+
const canSendRequest = provider2 === "http" && Boolean(baseUrl) && (Boolean(apiKey) || !args.memoryLlmApiKey || sameEndpoint);
|
|
1936
|
+
return { provider: provider2, model, baseUrl, apiKey, canSendRequest };
|
|
1937
|
+
}
|
|
1938
|
+
function sameTrustedEndpoint(left, right) {
|
|
1939
|
+
const a = normalizeTrustedEndpoint(left);
|
|
1940
|
+
const b = normalizeTrustedEndpoint(right);
|
|
1941
|
+
return Boolean(a && b && a === b);
|
|
1942
|
+
}
|
|
1943
|
+
function normalizeTrustedEndpoint(value) {
|
|
1944
|
+
const trimmed = value?.trim();
|
|
1945
|
+
if (!trimmed) return void 0;
|
|
1946
|
+
try {
|
|
1947
|
+
const url = new URL(trimmed);
|
|
1948
|
+
const protocol = url.protocol.toLowerCase();
|
|
1949
|
+
const hostname = url.hostname.toLowerCase();
|
|
1950
|
+
const port = url.port;
|
|
1951
|
+
let pathname = url.pathname.replace(/\/+$/, "");
|
|
1952
|
+
if (pathname.toLowerCase().endsWith("/rerank")) {
|
|
1953
|
+
pathname = pathname.slice(0, -"/rerank".length).replace(/\/+$/, "");
|
|
1954
|
+
}
|
|
1955
|
+
return `${protocol}//${hostname}${port ? `:${port}` : ""}${pathname}`;
|
|
1956
|
+
} catch {
|
|
1957
|
+
const fallback = trimmed.replace(/\/+$/, "").replace(/\/rerank$/i, "").replace(/\/+$/, "");
|
|
1958
|
+
return fallback.toLowerCase() || void 0;
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1887
1961
|
var init_resolved_config = __esm({
|
|
1888
1962
|
"src/config/resolved-config.ts"() {
|
|
1889
1963
|
"use strict";
|
|
@@ -2020,9 +2094,27 @@ __export(provider_exports, {
|
|
|
2020
2094
|
getLLMConfig: () => getLLMConfig,
|
|
2021
2095
|
initLLM: () => initLLM,
|
|
2022
2096
|
isLLMEnabled: () => isLLMEnabled,
|
|
2097
|
+
normalizeOpenAICompatibleBaseUrl: () => normalizeOpenAICompatibleBaseUrl,
|
|
2023
2098
|
parseLLMTimeoutMs: () => parseLLMTimeoutMs,
|
|
2024
2099
|
setLLMConfig: () => setLLMConfig
|
|
2025
2100
|
});
|
|
2101
|
+
function normalizeOpenAICompatibleBaseUrl(baseUrl) {
|
|
2102
|
+
const base = baseUrl.trim().replace(/\/+$/, "");
|
|
2103
|
+
if (!base) return base;
|
|
2104
|
+
let pathname = "";
|
|
2105
|
+
try {
|
|
2106
|
+
pathname = new URL(base).pathname.replace(/\/+$/, "");
|
|
2107
|
+
} catch {
|
|
2108
|
+
pathname = base.split(/[?#]/, 1)[0].replace(/\/+$/, "");
|
|
2109
|
+
}
|
|
2110
|
+
const lastSegment = pathname.split("/").filter(Boolean).at(-1);
|
|
2111
|
+
if (lastSegment && OPENAI_API_VERSION_SEGMENT.test(lastSegment)) {
|
|
2112
|
+
return base;
|
|
2113
|
+
}
|
|
2114
|
+
const queryOrHash = base.match(/[?#].*$/)?.[0] ?? "";
|
|
2115
|
+
const pathWithoutQuery = base.slice(0, base.length - queryOrHash.length).replace(/\/+$/, "");
|
|
2116
|
+
return `${pathWithoutQuery}/v1${queryOrHash}`;
|
|
2117
|
+
}
|
|
2026
2118
|
function parseLLMTimeoutMs(raw) {
|
|
2027
2119
|
if (raw === void 0 || raw.trim() === "") return LLM_TIMEOUT_DEFAULT_MS;
|
|
2028
2120
|
const parsed = Number(raw);
|
|
@@ -2126,8 +2218,7 @@ async function callLLM(systemPrompt, userMessage, signal) {
|
|
|
2126
2218
|
}
|
|
2127
2219
|
async function callOpenAICompatible(systemPrompt, userMessage, signal) {
|
|
2128
2220
|
const config = currentConfig;
|
|
2129
|
-
|
|
2130
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
2221
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
2131
2222
|
const url = `${base}/chat/completions`;
|
|
2132
2223
|
const fetchSignal = signal ? AbortSignal.any([signal, AbortSignal.timeout(LLM_CALL_TIMEOUT_MS)]) : AbortSignal.timeout(LLM_CALL_TIMEOUT_MS);
|
|
2133
2224
|
const response = await fetch(url, {
|
|
@@ -2206,8 +2297,7 @@ async function callLLMWithTools(messages, tools, signal) {
|
|
|
2206
2297
|
}
|
|
2207
2298
|
async function callOpenAIWithTools(messages, tools, signal) {
|
|
2208
2299
|
const config = currentConfig;
|
|
2209
|
-
|
|
2210
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
2300
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
2211
2301
|
const url = `${base}/chat/completions`;
|
|
2212
2302
|
const openaiMessages = messages.map((msg) => {
|
|
2213
2303
|
if (msg.role === "tool") {
|
|
@@ -2357,8 +2447,7 @@ async function callAnthropicWithTools(messages, tools, signal) {
|
|
|
2357
2447
|
}
|
|
2358
2448
|
async function* callOpenAIWithToolsStream(messages, tools) {
|
|
2359
2449
|
const config = currentConfig;
|
|
2360
|
-
|
|
2361
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
2450
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
2362
2451
|
const url = `${base}/chat/completions`;
|
|
2363
2452
|
const openaiMessages = messages.map((msg) => {
|
|
2364
2453
|
if (msg.role === "tool") {
|
|
@@ -2606,7 +2695,7 @@ async function* callAnthropicWithToolsStream(messages, tools) {
|
|
|
2606
2695
|
}
|
|
2607
2696
|
};
|
|
2608
2697
|
}
|
|
2609
|
-
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
2698
|
+
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, OPENAI_API_VERSION_SEGMENT, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
2610
2699
|
var init_provider = __esm({
|
|
2611
2700
|
"src/llm/provider.ts"() {
|
|
2612
2701
|
"use strict";
|
|
@@ -2616,6 +2705,7 @@ var init_provider = __esm({
|
|
|
2616
2705
|
LLM_TIMEOUT_MIN_MS = 1e3;
|
|
2617
2706
|
LLM_TIMEOUT_MAX_MS = 3e5;
|
|
2618
2707
|
MAX_LLM_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
2708
|
+
OPENAI_API_VERSION_SEGMENT = /^v\d+(?:[a-z][a-z0-9-]*)?$/i;
|
|
2619
2709
|
LLM_CALL_TIMEOUT_MS = parseLLMTimeoutMs(process.env.MEMORIX_LLM_TIMEOUT_MS);
|
|
2620
2710
|
PROVIDER_DEFAULTS = {
|
|
2621
2711
|
openai: { baseUrl: "https://api.openai.com/v1", model: "gpt-4.1-nano" },
|
|
@@ -5544,6 +5634,48 @@ Rules:
|
|
|
5544
5634
|
}
|
|
5545
5635
|
});
|
|
5546
5636
|
|
|
5637
|
+
// src/rerank/policy.ts
|
|
5638
|
+
function parseRerankTimeoutMs(raw) {
|
|
5639
|
+
const value = raw?.trim();
|
|
5640
|
+
if (!value) return DEFAULT_RERANK_TIMEOUT_MS;
|
|
5641
|
+
const parsed = Number(value);
|
|
5642
|
+
if (!Number.isInteger(parsed) || Number.isNaN(parsed)) {
|
|
5643
|
+
console.warn(
|
|
5644
|
+
`[memorix] MEMORIX_RERANK_TIMEOUT_MS="${raw}" is invalid (must be a positive integer between ${RERANK_TIMEOUT_MIN_MS}-${RERANK_TIMEOUT_MAX_MS}ms). Using default ${DEFAULT_RERANK_TIMEOUT_MS}ms.`
|
|
5645
|
+
);
|
|
5646
|
+
return DEFAULT_RERANK_TIMEOUT_MS;
|
|
5647
|
+
}
|
|
5648
|
+
if (parsed < RERANK_TIMEOUT_MIN_MS) return RERANK_TIMEOUT_MIN_MS;
|
|
5649
|
+
if (parsed > RERANK_TIMEOUT_MAX_MS) return RERANK_TIMEOUT_MAX_MS;
|
|
5650
|
+
return parsed;
|
|
5651
|
+
}
|
|
5652
|
+
function resolveRerankTimeoutMs(explicitTimeoutMs) {
|
|
5653
|
+
if (typeof explicitTimeoutMs === "number" && Number.isFinite(explicitTimeoutMs) && explicitTimeoutMs > 0) {
|
|
5654
|
+
return explicitTimeoutMs;
|
|
5655
|
+
}
|
|
5656
|
+
return parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
5657
|
+
}
|
|
5658
|
+
function shouldAttemptHttpRerank(input) {
|
|
5659
|
+
return input.quality === "thorough" && input.hasQuery && input.candidateCount >= MIN_RERANK_CANDIDATES;
|
|
5660
|
+
}
|
|
5661
|
+
function shouldAttemptLlmRerank(input) {
|
|
5662
|
+
if (input.httpRerankConfigured) return false;
|
|
5663
|
+
if (input.quality !== "thorough" || input.tier !== "heavy" || !input.hasQuery) return false;
|
|
5664
|
+
if (input.candidateCount < MIN_RERANK_CANDIDATES) return false;
|
|
5665
|
+
return input.topScore > 0 && input.secondScore / input.topScore > 0.7;
|
|
5666
|
+
}
|
|
5667
|
+
var DEFAULT_RERANK_TIMEOUT_MS, RERANK_TIMEOUT_MIN_MS, RERANK_TIMEOUT_MAX_MS, MIN_RERANK_CANDIDATES;
|
|
5668
|
+
var init_policy = __esm({
|
|
5669
|
+
"src/rerank/policy.ts"() {
|
|
5670
|
+
"use strict";
|
|
5671
|
+
init_esm_shims();
|
|
5672
|
+
DEFAULT_RERANK_TIMEOUT_MS = 3e4;
|
|
5673
|
+
RERANK_TIMEOUT_MIN_MS = 1e3;
|
|
5674
|
+
RERANK_TIMEOUT_MAX_MS = 3e5;
|
|
5675
|
+
MIN_RERANK_CANDIDATES = 3;
|
|
5676
|
+
}
|
|
5677
|
+
});
|
|
5678
|
+
|
|
5547
5679
|
// src/timeout.ts
|
|
5548
5680
|
function withTimeout(promise, ms, label) {
|
|
5549
5681
|
return new Promise((resolve, reject) => {
|
|
@@ -5829,6 +5961,148 @@ var init_aliases = __esm({
|
|
|
5829
5961
|
}
|
|
5830
5962
|
});
|
|
5831
5963
|
|
|
5964
|
+
// src/rerank/http-provider.ts
|
|
5965
|
+
function buildRerankUrl(baseUrl) {
|
|
5966
|
+
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
|
5967
|
+
if (trimmed.endsWith("/rerank")) return trimmed;
|
|
5968
|
+
return `${trimmed}/rerank`;
|
|
5969
|
+
}
|
|
5970
|
+
function parseRerankResponse(payload) {
|
|
5971
|
+
const rows = extractRows(payload);
|
|
5972
|
+
if (rows.length === 0) {
|
|
5973
|
+
throw new Error("Neural rerank response contained no ranked results");
|
|
5974
|
+
}
|
|
5975
|
+
return rows.sort((a, b) => b.score - a.score);
|
|
5976
|
+
}
|
|
5977
|
+
function extractRows(payload) {
|
|
5978
|
+
if (Array.isArray(payload)) {
|
|
5979
|
+
return payload.map(rowFromUnknown).filter((row) => row !== null);
|
|
5980
|
+
}
|
|
5981
|
+
if (payload && typeof payload === "object" && "results" in payload) {
|
|
5982
|
+
const results = payload.results;
|
|
5983
|
+
if (Array.isArray(results)) {
|
|
5984
|
+
return results.map(rowFromUnknown).filter((row) => row !== null);
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
5987
|
+
throw new Error("Neural rerank response was not Cohere or TEI shaped");
|
|
5988
|
+
}
|
|
5989
|
+
function rowFromUnknown(value) {
|
|
5990
|
+
if (!value || typeof value !== "object") return null;
|
|
5991
|
+
const row = value;
|
|
5992
|
+
if (typeof row.index !== "number" || !Number.isFinite(row.index)) return null;
|
|
5993
|
+
const score = typeof row.relevance_score === "number" ? row.relevance_score : typeof row.score === "number" ? row.score : null;
|
|
5994
|
+
if (score === null || !Number.isFinite(score)) return null;
|
|
5995
|
+
return { index: row.index, score };
|
|
5996
|
+
}
|
|
5997
|
+
function buildRerankBody(query, documents, model) {
|
|
5998
|
+
const body = {
|
|
5999
|
+
query,
|
|
6000
|
+
documents,
|
|
6001
|
+
top_n: documents.length,
|
|
6002
|
+
return_documents: false
|
|
6003
|
+
};
|
|
6004
|
+
const trimmed = model?.trim();
|
|
6005
|
+
if (trimmed) body.model = trimmed;
|
|
6006
|
+
return body;
|
|
6007
|
+
}
|
|
6008
|
+
async function rerankViaHttp(query, documents, config) {
|
|
6009
|
+
if (documents.length === 0) return [];
|
|
6010
|
+
const url = buildRerankUrl(config.baseUrl);
|
|
6011
|
+
const headers = {
|
|
6012
|
+
"content-type": "application/json"
|
|
6013
|
+
};
|
|
6014
|
+
if (config.apiKey) {
|
|
6015
|
+
headers.authorization = `Bearer ${config.apiKey}`;
|
|
6016
|
+
}
|
|
6017
|
+
const controller = new AbortController();
|
|
6018
|
+
const timeoutMs = resolveRerankTimeoutMs(config.timeoutMs);
|
|
6019
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
6020
|
+
try {
|
|
6021
|
+
const fetchImpl = config.fetchImpl ?? fetch;
|
|
6022
|
+
const response = await fetchImpl(url, {
|
|
6023
|
+
method: "POST",
|
|
6024
|
+
headers,
|
|
6025
|
+
signal: controller.signal,
|
|
6026
|
+
body: JSON.stringify(buildRerankBody(query, documents, config.model))
|
|
6027
|
+
});
|
|
6028
|
+
if (!response.ok) {
|
|
6029
|
+
throw new Error(`Neural rerank HTTP ${response.status} from ${url}`);
|
|
6030
|
+
}
|
|
6031
|
+
return parseRerankResponse(await response.json());
|
|
6032
|
+
} finally {
|
|
6033
|
+
clearTimeout(timer);
|
|
6034
|
+
}
|
|
6035
|
+
}
|
|
6036
|
+
var init_http_provider = __esm({
|
|
6037
|
+
"src/rerank/http-provider.ts"() {
|
|
6038
|
+
"use strict";
|
|
6039
|
+
init_esm_shims();
|
|
6040
|
+
init_policy();
|
|
6041
|
+
}
|
|
6042
|
+
});
|
|
6043
|
+
|
|
6044
|
+
// src/rerank/index.ts
|
|
6045
|
+
var rerank_exports = {};
|
|
6046
|
+
__export(rerank_exports, {
|
|
6047
|
+
DEFAULT_RERANK_TIMEOUT_MS: () => DEFAULT_RERANK_TIMEOUT_MS,
|
|
6048
|
+
MIN_RERANK_CANDIDATES: () => MIN_RERANK_CANDIDATES,
|
|
6049
|
+
isNeuralRerankEnabled: () => isNeuralRerankEnabled,
|
|
6050
|
+
neuralRerankCandidates: () => neuralRerankCandidates,
|
|
6051
|
+
parseRerankTimeoutMs: () => parseRerankTimeoutMs,
|
|
6052
|
+
resolveRerankTimeoutMs: () => resolveRerankTimeoutMs,
|
|
6053
|
+
shouldAttemptHttpRerank: () => shouldAttemptHttpRerank,
|
|
6054
|
+
shouldAttemptLlmRerank: () => shouldAttemptLlmRerank
|
|
6055
|
+
});
|
|
6056
|
+
function isNeuralRerankEnabled(options = {}) {
|
|
6057
|
+
const lane = getResolvedRerankLane(options);
|
|
6058
|
+
return lane.provider === "http" && Boolean(lane.baseUrl);
|
|
6059
|
+
}
|
|
6060
|
+
async function neuralRerankCandidates(query, candidates, options = {}) {
|
|
6061
|
+
if (!isNeuralRerankEnabled(options) || candidates.length === 0) return null;
|
|
6062
|
+
const lane = getResolvedRerankLane(options);
|
|
6063
|
+
if (!lane.canSendRequest || !lane.baseUrl) return null;
|
|
6064
|
+
const toRerank = candidates.slice(0, MAX_RERANK);
|
|
6065
|
+
const rest = candidates.slice(MAX_RERANK);
|
|
6066
|
+
const documents = toRerank.map((candidate) => candidateText(candidate));
|
|
6067
|
+
const ranked = await rerankViaHttp(query, documents, {
|
|
6068
|
+
provider: "http",
|
|
6069
|
+
model: lane.model,
|
|
6070
|
+
baseUrl: lane.baseUrl,
|
|
6071
|
+
apiKey: lane.apiKey,
|
|
6072
|
+
timeoutMs: resolveRerankTimeoutMs()
|
|
6073
|
+
});
|
|
6074
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6075
|
+
const reranked = [];
|
|
6076
|
+
for (const row of ranked) {
|
|
6077
|
+
const candidate = toRerank[row.index];
|
|
6078
|
+
if (!candidate || seen.has(row.index)) continue;
|
|
6079
|
+
reranked.push(candidate);
|
|
6080
|
+
seen.add(row.index);
|
|
6081
|
+
}
|
|
6082
|
+
for (const [index, candidate] of toRerank.entries()) {
|
|
6083
|
+
if (!seen.has(index)) reranked.push(candidate);
|
|
6084
|
+
}
|
|
6085
|
+
reranked.push(...rest);
|
|
6086
|
+
return reranked;
|
|
6087
|
+
}
|
|
6088
|
+
function candidateText(candidate) {
|
|
6089
|
+
const narrative = candidate.narrative?.trim();
|
|
6090
|
+
if (narrative) return `${candidate.title} \u2014 ${narrative.slice(0, 200)}`;
|
|
6091
|
+
return candidate.title;
|
|
6092
|
+
}
|
|
6093
|
+
var MAX_RERANK;
|
|
6094
|
+
var init_rerank = __esm({
|
|
6095
|
+
"src/rerank/index.ts"() {
|
|
6096
|
+
"use strict";
|
|
6097
|
+
init_esm_shims();
|
|
6098
|
+
init_resolved_config();
|
|
6099
|
+
init_http_provider();
|
|
6100
|
+
init_policy();
|
|
6101
|
+
init_policy();
|
|
6102
|
+
MAX_RERANK = 10;
|
|
6103
|
+
}
|
|
6104
|
+
});
|
|
6105
|
+
|
|
5832
6106
|
// src/llm/quality.ts
|
|
5833
6107
|
var quality_exports = {};
|
|
5834
6108
|
__export(quality_exports, {
|
|
@@ -5868,9 +6142,9 @@ async function rerankResults(query, candidates) {
|
|
|
5868
6142
|
if (!isLLMEnabled() || candidates.length <= 2) {
|
|
5869
6143
|
return { reranked: candidates, usedLLM: false };
|
|
5870
6144
|
}
|
|
5871
|
-
const
|
|
5872
|
-
const toRerank = candidates.slice(0,
|
|
5873
|
-
const rest = candidates.slice(
|
|
6145
|
+
const MAX_RERANK2 = 10;
|
|
6146
|
+
const toRerank = candidates.slice(0, MAX_RERANK2);
|
|
6147
|
+
const rest = candidates.slice(MAX_RERANK2);
|
|
5874
6148
|
try {
|
|
5875
6149
|
const candidateList = toRerank.map(
|
|
5876
6150
|
(c) => `[ID: ${c.id}] (${c.type}) ${c.title}${c.narrative ? ` \u2014 ${c.narrative.substring(0, 100)}` : ""}`
|
|
@@ -6763,46 +7037,78 @@ async function searchObservations(options) {
|
|
|
6763
7037
|
}
|
|
6764
7038
|
}
|
|
6765
7039
|
}
|
|
6766
|
-
const
|
|
6767
|
-
|
|
6768
|
-
|
|
6769
|
-
|
|
6770
|
-
})
|
|
6771
|
-
|
|
6772
|
-
|
|
6773
|
-
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
|
|
6783
|
-
|
|
6784
|
-
|
|
6785
|
-
|
|
6786
|
-
|
|
6787
|
-
|
|
6788
|
-
|
|
6789
|
-
|
|
6790
|
-
|
|
6791
|
-
|
|
6792
|
-
|
|
6793
|
-
|
|
6794
|
-
|
|
6795
|
-
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6799
|
-
|
|
6800
|
-
|
|
6801
|
-
|
|
7040
|
+
const httpGate = shouldAttemptHttpRerank({
|
|
7041
|
+
quality,
|
|
7042
|
+
hasQuery: Boolean(hasQuery),
|
|
7043
|
+
candidateCount: intermediate.length
|
|
7044
|
+
});
|
|
7045
|
+
const llmGateWithoutHttp = shouldAttemptLlmRerank({
|
|
7046
|
+
quality,
|
|
7047
|
+
tier,
|
|
7048
|
+
hasQuery: Boolean(hasQuery),
|
|
7049
|
+
candidateCount: intermediate.length,
|
|
7050
|
+
topScore: intermediate[0]?.score ?? 0,
|
|
7051
|
+
secondScore: intermediate[1]?.score ?? 0,
|
|
7052
|
+
httpRerankConfigured: false
|
|
7053
|
+
});
|
|
7054
|
+
if (httpGate || llmGateWithoutHttp) {
|
|
7055
|
+
const narrativeMap = /* @__PURE__ */ new Map();
|
|
7056
|
+
for (const hit of results.hits) {
|
|
7057
|
+
const doc = hit.document;
|
|
7058
|
+
narrativeMap.set(makeEntryKey(doc.projectId, doc.observationId), doc.narrative);
|
|
7059
|
+
}
|
|
7060
|
+
const RERANK_TOP_K = 5;
|
|
7061
|
+
const toRerank = intermediate.slice(0, RERANK_TOP_K);
|
|
7062
|
+
const candidates = toRerank.map((e2, index) => ({
|
|
7063
|
+
id: `r${index + 1}`,
|
|
7064
|
+
title: e2.title,
|
|
7065
|
+
type: e2.type,
|
|
7066
|
+
score: e2.score,
|
|
7067
|
+
narrative: narrativeMap.get(makeEntryKey(e2.projectId, e2.id))
|
|
7068
|
+
}));
|
|
7069
|
+
const RERANK_TIMEOUT_MS = parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
7070
|
+
const candidateMap = new Map(candidates.map((candidate, index) => [candidate.id, toRerank[index]]));
|
|
7071
|
+
const { isNeuralRerankEnabled: isNeuralRerankEnabled2, neuralRerankCandidates: neuralRerankCandidates2 } = await Promise.resolve().then(() => (init_rerank(), rerank_exports));
|
|
7072
|
+
const httpConfigured = isNeuralRerankEnabled2();
|
|
7073
|
+
if (httpConfigured && httpGate) {
|
|
7074
|
+
try {
|
|
7075
|
+
const neural = await withTimeout(
|
|
7076
|
+
neuralRerankCandidates2(originalQuery, candidates),
|
|
7077
|
+
RERANK_TIMEOUT_MS,
|
|
7078
|
+
"neural rerank"
|
|
7079
|
+
);
|
|
7080
|
+
if (neural && neural.length > 0) {
|
|
7081
|
+
const rerankedTop = neural.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7082
|
+
if (rerankedTop.length > 0) {
|
|
7083
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7084
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + neural rerank");
|
|
7085
|
+
mark("rerank(usedNeural=true)");
|
|
7086
|
+
}
|
|
6802
7087
|
}
|
|
7088
|
+
} catch {
|
|
7089
|
+
console.error("[memorix] neural rerank failed or timed out; keeping original order");
|
|
6803
7090
|
}
|
|
6804
|
-
}
|
|
6805
|
-
|
|
7091
|
+
} else if (!httpConfigured && llmGateWithoutHttp) {
|
|
7092
|
+
try {
|
|
7093
|
+
const { rerankResults: rerankResults2 } = await Promise.resolve().then(() => (init_quality(), quality_exports));
|
|
7094
|
+
const { reranked, usedLLM } = await withTimeout(
|
|
7095
|
+
rerankResults2(originalQuery, candidates),
|
|
7096
|
+
RERANK_TIMEOUT_MS,
|
|
7097
|
+
"LLM rerank"
|
|
7098
|
+
);
|
|
7099
|
+
mark(`rerank(usedLLM=${usedLLM})`);
|
|
7100
|
+
if (usedLLM) {
|
|
7101
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + LLM rerank");
|
|
7102
|
+
const rerankedTop = reranked.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7103
|
+
if (rerankedTop.length > 0) {
|
|
7104
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7105
|
+
}
|
|
7106
|
+
}
|
|
7107
|
+
} catch {
|
|
7108
|
+
console.error("[memorix] LLM rerank failed or timed out, using original order");
|
|
7109
|
+
}
|
|
7110
|
+
} else {
|
|
7111
|
+
mark(`rerank(skipped,tier=${tier})`);
|
|
6806
7112
|
}
|
|
6807
7113
|
} else {
|
|
6808
7114
|
mark(`rerank(skipped,tier=${tier})`);
|
|
@@ -6983,6 +7289,7 @@ var init_orama_store = __esm({
|
|
|
6983
7289
|
init_project_affinity();
|
|
6984
7290
|
init_intent_detector();
|
|
6985
7291
|
init_query_expansion();
|
|
7292
|
+
init_policy();
|
|
6986
7293
|
init_timeout();
|
|
6987
7294
|
db = null;
|
|
6988
7295
|
dbInitPromise = null;
|