memorix 1.7.1 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -1
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/assets/star-history-dark.svg +4 -4
- package/assets/star-history-light.svg +4 -4
- package/assets/star-history.json +47 -3
- package/dist/cli/index.js +423 -74
- package/dist/index.js +396 -73
- package/dist/maintenance-runner.js +393 -69
- package/dist/media-audio-runner.js +393 -69
- package/dist/media-video-runner.js +393 -69
- package/dist/memcode-runtime/CHANGELOG.md +43 -1
- package/dist/sdk.js +395 -72
- package/dist/vector-backfill-runner.js +393 -69
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CONFIGURATION.md +57 -1
- package/docs/DEVELOPMENT.md +6 -2
- package/docs/PERFORMANCE.md +13 -4
- package/docs/README.md +3 -1
- package/docs/SETUP.md +2 -1
- package/package.json +1 -1
- package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
- package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
- package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/copilot/memorix/plugin.json +1 -1
- package/plugins/gemini/memorix/gemini-extension.json +1 -1
- package/plugins/omp/memorix/package.json +1 -1
- package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/pi/memorix/package.json +1 -1
- package/server.json +2 -2
- package/src/cli/commands/config-migrate.ts +7 -0
- package/src/cli/commands/doctor.ts +5 -1
- package/src/cli/commands/init.ts +17 -0
- package/src/cli/commands/status.ts +2 -0
- package/src/config/resolved-config.ts +110 -12
- package/src/config/toml-loader.ts +8 -0
- package/src/config/yaml-loader.ts +9 -0
- package/src/llm/provider.ts +33 -7
- package/src/multimodal/image-loader.ts +6 -3
- package/src/rerank/http-provider.ts +125 -0
- package/src/rerank/index.ts +81 -0
- package/src/rerank/policy.ts +88 -0
- package/src/store/orama-store.ts +111 -65
|
@@ -1327,6 +1327,7 @@ function loadYamlConfig(projectRoot) {
|
|
|
1327
1327
|
llm: { ...userConfig.llm, ...projectConfig.llm },
|
|
1328
1328
|
agent: { ...userConfig.agent, ...projectConfig.agent },
|
|
1329
1329
|
embedding: { ...userConfig.embedding, ...projectConfig.embedding },
|
|
1330
|
+
rerank: { ...userConfig.rerank, ...projectConfig.rerank },
|
|
1330
1331
|
git: { ...userConfig.git, ...projectConfig.git },
|
|
1331
1332
|
codegraph: { ...userConfig.codegraph, ...projectConfig.codegraph },
|
|
1332
1333
|
behavior: { ...userConfig.behavior, ...projectConfig.behavior },
|
|
@@ -1563,6 +1564,7 @@ function mergeTomlConfig(base, override) {
|
|
|
1563
1564
|
llm: { ...base.memory?.llm, ...override.memory?.llm }
|
|
1564
1565
|
},
|
|
1565
1566
|
embedding: { ...base.embedding, ...override.embedding },
|
|
1567
|
+
rerank: { ...base.rerank, ...override.rerank },
|
|
1566
1568
|
hooks: { ...base.hooks, ...override.hooks },
|
|
1567
1569
|
git: { ...base.git, ...override.git },
|
|
1568
1570
|
codegraph: { ...base.codegraph, ...override.codegraph },
|
|
@@ -1659,6 +1661,7 @@ __export(resolved_config_exports, {
|
|
|
1659
1661
|
getResolvedConfigForCwd: () => getResolvedConfigForCwd,
|
|
1660
1662
|
getResolvedEmbeddingLane: () => getResolvedEmbeddingLane,
|
|
1661
1663
|
getResolvedMemoryLane: () => getResolvedMemoryLane,
|
|
1664
|
+
getResolvedRerankLane: () => getResolvedRerankLane,
|
|
1662
1665
|
resetResolvedConfigCache: () => resetResolvedConfigCache
|
|
1663
1666
|
});
|
|
1664
1667
|
import { homedir as homedir5 } from "os";
|
|
@@ -1691,6 +1694,16 @@ function getResolvedConfig(options = {}) {
|
|
|
1691
1694
|
legacy.llm?.baseUrl
|
|
1692
1695
|
);
|
|
1693
1696
|
const openRouterMemoryLlmApiKey = isOpenRouterMemoryLane(memoryLlmProvider, memoryLlmBaseUrl) ? process.env.OPENROUTER_API_KEY : void 0;
|
|
1697
|
+
const memoryLlmApiKey = first(
|
|
1698
|
+
process.env.MEMORIX_LLM_API_KEY,
|
|
1699
|
+
process.env.MEMORIX_API_KEY,
|
|
1700
|
+
toml.memory?.llm?.api_key,
|
|
1701
|
+
yaml.llm?.apiKey,
|
|
1702
|
+
legacy.llm?.apiKey,
|
|
1703
|
+
process.env.OPENAI_API_KEY,
|
|
1704
|
+
process.env.ANTHROPIC_API_KEY,
|
|
1705
|
+
openRouterMemoryLlmApiKey
|
|
1706
|
+
);
|
|
1694
1707
|
const resolved = {
|
|
1695
1708
|
agent: {
|
|
1696
1709
|
provider: first(
|
|
@@ -1731,16 +1744,7 @@ function getResolvedConfig(options = {}) {
|
|
|
1731
1744
|
provider: memoryLlmProvider,
|
|
1732
1745
|
model: memoryLlmModel,
|
|
1733
1746
|
baseUrl: memoryLlmBaseUrl,
|
|
1734
|
-
apiKey:
|
|
1735
|
-
process.env.MEMORIX_LLM_API_KEY,
|
|
1736
|
-
process.env.MEMORIX_API_KEY,
|
|
1737
|
-
toml.memory?.llm?.api_key,
|
|
1738
|
-
yaml.llm?.apiKey,
|
|
1739
|
-
legacy.llm?.apiKey,
|
|
1740
|
-
process.env.OPENAI_API_KEY,
|
|
1741
|
-
process.env.ANTHROPIC_API_KEY,
|
|
1742
|
-
openRouterMemoryLlmApiKey
|
|
1743
|
-
)
|
|
1747
|
+
apiKey: memoryLlmApiKey
|
|
1744
1748
|
}
|
|
1745
1749
|
},
|
|
1746
1750
|
embedding: {
|
|
@@ -1750,6 +1754,12 @@ function getResolvedConfig(options = {}) {
|
|
|
1750
1754
|
apiKey: first(process.env.MEMORIX_EMBEDDING_API_KEY, toml.embedding?.api_key, yaml.embedding?.apiKey, legacy.embeddingApi?.apiKey, openRouterEmbeddingApiKey),
|
|
1751
1755
|
dimensions: firstNumber(parseNumber(process.env.MEMORIX_EMBEDDING_DIMENSIONS), toml.embedding?.dimensions, yaml.embedding?.dimensions, legacy.embeddingApi?.dimensions)
|
|
1752
1756
|
},
|
|
1757
|
+
rerank: resolveRerankLane({
|
|
1758
|
+
toml,
|
|
1759
|
+
yaml,
|
|
1760
|
+
memoryLlmApiKey,
|
|
1761
|
+
memoryLlmBaseUrl
|
|
1762
|
+
}),
|
|
1753
1763
|
git: {
|
|
1754
1764
|
autoHook: firstBool(toml.git?.auto_hook, yaml.git?.autoHook),
|
|
1755
1765
|
ingestOnCommit: firstBool(toml.git?.ingest_on_commit, yaml.git?.ingestOnCommit),
|
|
@@ -1819,6 +1829,9 @@ function getResolvedMemoryLane(options = {}) {
|
|
|
1819
1829
|
function getResolvedEmbeddingLane(options = {}) {
|
|
1820
1830
|
return getResolvedConfig(options).embedding;
|
|
1821
1831
|
}
|
|
1832
|
+
function getResolvedRerankLane(options = {}) {
|
|
1833
|
+
return getResolvedConfig(options).rerank;
|
|
1834
|
+
}
|
|
1822
1835
|
function resetResolvedConfigCache() {
|
|
1823
1836
|
}
|
|
1824
1837
|
function first(...values) {
|
|
@@ -1861,6 +1874,10 @@ function getEnvSourceNames() {
|
|
|
1861
1874
|
"MEMORIX_EMBEDDING_BASE_URL",
|
|
1862
1875
|
"MEMORIX_EMBEDDING_MODEL",
|
|
1863
1876
|
"MEMORIX_EMBEDDING_DIMENSIONS",
|
|
1877
|
+
"MEMORIX_RERANK_PROVIDER",
|
|
1878
|
+
"MEMORIX_RERANK_MODEL",
|
|
1879
|
+
"MEMORIX_RERANK_BASE_URL",
|
|
1880
|
+
"MEMORIX_RERANK_API_KEY",
|
|
1864
1881
|
"MEMORIX_CODEGRAPH_EXTERNAL_CONTEXT",
|
|
1865
1882
|
"MEMORIX_CODEGRAPH_EXTERNAL_COMMAND",
|
|
1866
1883
|
"MEMORIX_CODEGRAPH_EXTERNAL_TIMEOUT_MS",
|
|
@@ -1884,6 +1901,63 @@ function normalizeExternalContext(value) {
|
|
|
1884
1901
|
if (normalized === "auto" || normalized === "off") return normalized;
|
|
1885
1902
|
return void 0;
|
|
1886
1903
|
}
|
|
1904
|
+
function normalizeRerankProvider(value) {
|
|
1905
|
+
const normalized = value?.trim().toLowerCase();
|
|
1906
|
+
if (normalized === "http") return "http";
|
|
1907
|
+
return "off";
|
|
1908
|
+
}
|
|
1909
|
+
function resolveRerankLane(args) {
|
|
1910
|
+
const provider2 = normalizeRerankProvider(first(
|
|
1911
|
+
process.env.MEMORIX_RERANK_PROVIDER,
|
|
1912
|
+
args.toml.rerank?.provider,
|
|
1913
|
+
args.yaml.rerank?.provider,
|
|
1914
|
+
"off"
|
|
1915
|
+
));
|
|
1916
|
+
const model = first(
|
|
1917
|
+
process.env.MEMORIX_RERANK_MODEL,
|
|
1918
|
+
args.toml.rerank?.model,
|
|
1919
|
+
args.yaml.rerank?.model
|
|
1920
|
+
);
|
|
1921
|
+
const explicitBaseUrl = first(
|
|
1922
|
+
process.env.MEMORIX_RERANK_BASE_URL,
|
|
1923
|
+
args.toml.rerank?.base_url,
|
|
1924
|
+
args.yaml.rerank?.baseUrl
|
|
1925
|
+
);
|
|
1926
|
+
const baseUrl = provider2 === "http" ? first(explicitBaseUrl, args.memoryLlmBaseUrl) : explicitBaseUrl;
|
|
1927
|
+
const dedicatedApiKey = first(
|
|
1928
|
+
process.env.MEMORIX_RERANK_API_KEY,
|
|
1929
|
+
args.toml.rerank?.api_key,
|
|
1930
|
+
args.yaml.rerank?.apiKey
|
|
1931
|
+
);
|
|
1932
|
+
const sameEndpoint = provider2 === "http" && sameTrustedEndpoint(baseUrl, args.memoryLlmBaseUrl);
|
|
1933
|
+
const inheritedApiKey = sameEndpoint ? args.memoryLlmApiKey : void 0;
|
|
1934
|
+
const apiKey = first(dedicatedApiKey, inheritedApiKey);
|
|
1935
|
+
const canSendRequest = provider2 === "http" && Boolean(baseUrl) && (Boolean(apiKey) || !args.memoryLlmApiKey || sameEndpoint);
|
|
1936
|
+
return { provider: provider2, model, baseUrl, apiKey, canSendRequest };
|
|
1937
|
+
}
|
|
1938
|
+
function sameTrustedEndpoint(left, right) {
|
|
1939
|
+
const a = normalizeTrustedEndpoint(left);
|
|
1940
|
+
const b = normalizeTrustedEndpoint(right);
|
|
1941
|
+
return Boolean(a && b && a === b);
|
|
1942
|
+
}
|
|
1943
|
+
function normalizeTrustedEndpoint(value) {
|
|
1944
|
+
const trimmed = value?.trim();
|
|
1945
|
+
if (!trimmed) return void 0;
|
|
1946
|
+
try {
|
|
1947
|
+
const url = new URL(trimmed);
|
|
1948
|
+
const protocol = url.protocol.toLowerCase();
|
|
1949
|
+
const hostname = url.hostname.toLowerCase();
|
|
1950
|
+
const port = url.port;
|
|
1951
|
+
let pathname = url.pathname.replace(/\/+$/, "");
|
|
1952
|
+
if (pathname.toLowerCase().endsWith("/rerank")) {
|
|
1953
|
+
pathname = pathname.slice(0, -"/rerank".length).replace(/\/+$/, "");
|
|
1954
|
+
}
|
|
1955
|
+
return `${protocol}//${hostname}${port ? `:${port}` : ""}${pathname}`;
|
|
1956
|
+
} catch {
|
|
1957
|
+
const fallback = trimmed.replace(/\/+$/, "").replace(/\/rerank$/i, "").replace(/\/+$/, "");
|
|
1958
|
+
return fallback.toLowerCase() || void 0;
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1887
1961
|
var init_resolved_config = __esm({
|
|
1888
1962
|
"src/config/resolved-config.ts"() {
|
|
1889
1963
|
"use strict";
|
|
@@ -2020,9 +2094,27 @@ __export(provider_exports, {
|
|
|
2020
2094
|
getLLMConfig: () => getLLMConfig,
|
|
2021
2095
|
initLLM: () => initLLM,
|
|
2022
2096
|
isLLMEnabled: () => isLLMEnabled,
|
|
2097
|
+
normalizeOpenAICompatibleBaseUrl: () => normalizeOpenAICompatibleBaseUrl,
|
|
2023
2098
|
parseLLMTimeoutMs: () => parseLLMTimeoutMs,
|
|
2024
2099
|
setLLMConfig: () => setLLMConfig
|
|
2025
2100
|
});
|
|
2101
|
+
function normalizeOpenAICompatibleBaseUrl(baseUrl) {
|
|
2102
|
+
const base = baseUrl.trim().replace(/\/+$/, "");
|
|
2103
|
+
if (!base) return base;
|
|
2104
|
+
let pathname = "";
|
|
2105
|
+
try {
|
|
2106
|
+
pathname = new URL(base).pathname.replace(/\/+$/, "");
|
|
2107
|
+
} catch {
|
|
2108
|
+
pathname = base.split(/[?#]/, 1)[0].replace(/\/+$/, "");
|
|
2109
|
+
}
|
|
2110
|
+
const lastSegment = pathname.split("/").filter(Boolean).at(-1);
|
|
2111
|
+
if (lastSegment && OPENAI_API_VERSION_SEGMENT.test(lastSegment)) {
|
|
2112
|
+
return base;
|
|
2113
|
+
}
|
|
2114
|
+
const queryOrHash = base.match(/[?#].*$/)?.[0] ?? "";
|
|
2115
|
+
const pathWithoutQuery = base.slice(0, base.length - queryOrHash.length).replace(/\/+$/, "");
|
|
2116
|
+
return `${pathWithoutQuery}/v1${queryOrHash}`;
|
|
2117
|
+
}
|
|
2026
2118
|
function parseLLMTimeoutMs(raw) {
|
|
2027
2119
|
if (raw === void 0 || raw.trim() === "") return LLM_TIMEOUT_DEFAULT_MS;
|
|
2028
2120
|
const parsed = Number(raw);
|
|
@@ -2126,8 +2218,7 @@ async function callLLM(systemPrompt, userMessage, signal) {
|
|
|
2126
2218
|
}
|
|
2127
2219
|
async function callOpenAICompatible(systemPrompt, userMessage, signal) {
|
|
2128
2220
|
const config = currentConfig;
|
|
2129
|
-
|
|
2130
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
2221
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
2131
2222
|
const url = `${base}/chat/completions`;
|
|
2132
2223
|
const fetchSignal = signal ? AbortSignal.any([signal, AbortSignal.timeout(LLM_CALL_TIMEOUT_MS)]) : AbortSignal.timeout(LLM_CALL_TIMEOUT_MS);
|
|
2133
2224
|
const response = await fetch(url, {
|
|
@@ -2206,8 +2297,7 @@ async function callLLMWithTools(messages, tools, signal) {
|
|
|
2206
2297
|
}
|
|
2207
2298
|
async function callOpenAIWithTools(messages, tools, signal) {
|
|
2208
2299
|
const config = currentConfig;
|
|
2209
|
-
|
|
2210
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
2300
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
2211
2301
|
const url = `${base}/chat/completions`;
|
|
2212
2302
|
const openaiMessages = messages.map((msg) => {
|
|
2213
2303
|
if (msg.role === "tool") {
|
|
@@ -2357,8 +2447,7 @@ async function callAnthropicWithTools(messages, tools, signal) {
|
|
|
2357
2447
|
}
|
|
2358
2448
|
async function* callOpenAIWithToolsStream(messages, tools) {
|
|
2359
2449
|
const config = currentConfig;
|
|
2360
|
-
|
|
2361
|
-
if (!base.endsWith("/v1")) base += "/v1";
|
|
2450
|
+
const base = normalizeOpenAICompatibleBaseUrl(config.baseUrl);
|
|
2362
2451
|
const url = `${base}/chat/completions`;
|
|
2363
2452
|
const openaiMessages = messages.map((msg) => {
|
|
2364
2453
|
if (msg.role === "tool") {
|
|
@@ -2606,7 +2695,7 @@ async function* callAnthropicWithToolsStream(messages, tools) {
|
|
|
2606
2695
|
}
|
|
2607
2696
|
};
|
|
2608
2697
|
}
|
|
2609
|
-
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
2698
|
+
var LLM_TIMEOUT_DEFAULT_MS, LLM_TIMEOUT_MIN_MS, LLM_TIMEOUT_MAX_MS, MAX_LLM_RESPONSE_BYTES, OPENAI_API_VERSION_SEGMENT, LLM_CALL_TIMEOUT_MS, PROVIDER_DEFAULTS, currentConfig;
|
|
2610
2699
|
var init_provider = __esm({
|
|
2611
2700
|
"src/llm/provider.ts"() {
|
|
2612
2701
|
"use strict";
|
|
@@ -2616,6 +2705,7 @@ var init_provider = __esm({
|
|
|
2616
2705
|
LLM_TIMEOUT_MIN_MS = 1e3;
|
|
2617
2706
|
LLM_TIMEOUT_MAX_MS = 3e5;
|
|
2618
2707
|
MAX_LLM_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
2708
|
+
OPENAI_API_VERSION_SEGMENT = /^v\d+(?:[a-z][a-z0-9-]*)?$/i;
|
|
2619
2709
|
LLM_CALL_TIMEOUT_MS = parseLLMTimeoutMs(process.env.MEMORIX_LLM_TIMEOUT_MS);
|
|
2620
2710
|
PROVIDER_DEFAULTS = {
|
|
2621
2711
|
openai: { baseUrl: "https://api.openai.com/v1", model: "gpt-4.1-nano" },
|
|
@@ -5544,6 +5634,48 @@ Rules:
|
|
|
5544
5634
|
}
|
|
5545
5635
|
});
|
|
5546
5636
|
|
|
5637
|
+
// src/rerank/policy.ts
|
|
5638
|
+
function parseRerankTimeoutMs(raw) {
|
|
5639
|
+
const value = raw?.trim();
|
|
5640
|
+
if (!value) return DEFAULT_RERANK_TIMEOUT_MS;
|
|
5641
|
+
const parsed = Number(value);
|
|
5642
|
+
if (!Number.isInteger(parsed) || Number.isNaN(parsed)) {
|
|
5643
|
+
console.warn(
|
|
5644
|
+
`[memorix] MEMORIX_RERANK_TIMEOUT_MS="${raw}" is invalid (must be a positive integer between ${RERANK_TIMEOUT_MIN_MS}-${RERANK_TIMEOUT_MAX_MS}ms). Using default ${DEFAULT_RERANK_TIMEOUT_MS}ms.`
|
|
5645
|
+
);
|
|
5646
|
+
return DEFAULT_RERANK_TIMEOUT_MS;
|
|
5647
|
+
}
|
|
5648
|
+
if (parsed < RERANK_TIMEOUT_MIN_MS) return RERANK_TIMEOUT_MIN_MS;
|
|
5649
|
+
if (parsed > RERANK_TIMEOUT_MAX_MS) return RERANK_TIMEOUT_MAX_MS;
|
|
5650
|
+
return parsed;
|
|
5651
|
+
}
|
|
5652
|
+
function resolveRerankTimeoutMs(explicitTimeoutMs) {
|
|
5653
|
+
if (typeof explicitTimeoutMs === "number" && Number.isFinite(explicitTimeoutMs) && explicitTimeoutMs > 0) {
|
|
5654
|
+
return explicitTimeoutMs;
|
|
5655
|
+
}
|
|
5656
|
+
return parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
5657
|
+
}
|
|
5658
|
+
function shouldAttemptHttpRerank(input) {
|
|
5659
|
+
return input.quality === "thorough" && input.hasQuery && input.candidateCount >= MIN_RERANK_CANDIDATES;
|
|
5660
|
+
}
|
|
5661
|
+
function shouldAttemptLlmRerank(input) {
|
|
5662
|
+
if (input.httpRerankConfigured) return false;
|
|
5663
|
+
if (input.quality !== "thorough" || input.tier !== "heavy" || !input.hasQuery) return false;
|
|
5664
|
+
if (input.candidateCount < MIN_RERANK_CANDIDATES) return false;
|
|
5665
|
+
return input.topScore > 0 && input.secondScore / input.topScore > 0.7;
|
|
5666
|
+
}
|
|
5667
|
+
var DEFAULT_RERANK_TIMEOUT_MS, RERANK_TIMEOUT_MIN_MS, RERANK_TIMEOUT_MAX_MS, MIN_RERANK_CANDIDATES;
|
|
5668
|
+
var init_policy = __esm({
|
|
5669
|
+
"src/rerank/policy.ts"() {
|
|
5670
|
+
"use strict";
|
|
5671
|
+
init_esm_shims();
|
|
5672
|
+
DEFAULT_RERANK_TIMEOUT_MS = 3e4;
|
|
5673
|
+
RERANK_TIMEOUT_MIN_MS = 1e3;
|
|
5674
|
+
RERANK_TIMEOUT_MAX_MS = 3e5;
|
|
5675
|
+
MIN_RERANK_CANDIDATES = 3;
|
|
5676
|
+
}
|
|
5677
|
+
});
|
|
5678
|
+
|
|
5547
5679
|
// src/timeout.ts
|
|
5548
5680
|
function withTimeout(promise, ms, label) {
|
|
5549
5681
|
return new Promise((resolve, reject) => {
|
|
@@ -5829,6 +5961,148 @@ var init_aliases = __esm({
|
|
|
5829
5961
|
}
|
|
5830
5962
|
});
|
|
5831
5963
|
|
|
5964
|
+
// src/rerank/http-provider.ts
|
|
5965
|
+
function buildRerankUrl(baseUrl) {
|
|
5966
|
+
const trimmed = baseUrl.trim().replace(/\/+$/, "");
|
|
5967
|
+
if (trimmed.endsWith("/rerank")) return trimmed;
|
|
5968
|
+
return `${trimmed}/rerank`;
|
|
5969
|
+
}
|
|
5970
|
+
function parseRerankResponse(payload) {
|
|
5971
|
+
const rows = extractRows(payload);
|
|
5972
|
+
if (rows.length === 0) {
|
|
5973
|
+
throw new Error("Neural rerank response contained no ranked results");
|
|
5974
|
+
}
|
|
5975
|
+
return rows.sort((a, b) => b.score - a.score);
|
|
5976
|
+
}
|
|
5977
|
+
function extractRows(payload) {
|
|
5978
|
+
if (Array.isArray(payload)) {
|
|
5979
|
+
return payload.map(rowFromUnknown).filter((row) => row !== null);
|
|
5980
|
+
}
|
|
5981
|
+
if (payload && typeof payload === "object" && "results" in payload) {
|
|
5982
|
+
const results = payload.results;
|
|
5983
|
+
if (Array.isArray(results)) {
|
|
5984
|
+
return results.map(rowFromUnknown).filter((row) => row !== null);
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
5987
|
+
throw new Error("Neural rerank response was not Cohere or TEI shaped");
|
|
5988
|
+
}
|
|
5989
|
+
function rowFromUnknown(value) {
|
|
5990
|
+
if (!value || typeof value !== "object") return null;
|
|
5991
|
+
const row = value;
|
|
5992
|
+
if (typeof row.index !== "number" || !Number.isFinite(row.index)) return null;
|
|
5993
|
+
const score = typeof row.relevance_score === "number" ? row.relevance_score : typeof row.score === "number" ? row.score : null;
|
|
5994
|
+
if (score === null || !Number.isFinite(score)) return null;
|
|
5995
|
+
return { index: row.index, score };
|
|
5996
|
+
}
|
|
5997
|
+
function buildRerankBody(query, documents, model) {
|
|
5998
|
+
const body = {
|
|
5999
|
+
query,
|
|
6000
|
+
documents,
|
|
6001
|
+
top_n: documents.length,
|
|
6002
|
+
return_documents: false
|
|
6003
|
+
};
|
|
6004
|
+
const trimmed = model?.trim();
|
|
6005
|
+
if (trimmed) body.model = trimmed;
|
|
6006
|
+
return body;
|
|
6007
|
+
}
|
|
6008
|
+
async function rerankViaHttp(query, documents, config) {
|
|
6009
|
+
if (documents.length === 0) return [];
|
|
6010
|
+
const url = buildRerankUrl(config.baseUrl);
|
|
6011
|
+
const headers = {
|
|
6012
|
+
"content-type": "application/json"
|
|
6013
|
+
};
|
|
6014
|
+
if (config.apiKey) {
|
|
6015
|
+
headers.authorization = `Bearer ${config.apiKey}`;
|
|
6016
|
+
}
|
|
6017
|
+
const controller = new AbortController();
|
|
6018
|
+
const timeoutMs = resolveRerankTimeoutMs(config.timeoutMs);
|
|
6019
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
6020
|
+
try {
|
|
6021
|
+
const fetchImpl = config.fetchImpl ?? fetch;
|
|
6022
|
+
const response = await fetchImpl(url, {
|
|
6023
|
+
method: "POST",
|
|
6024
|
+
headers,
|
|
6025
|
+
signal: controller.signal,
|
|
6026
|
+
body: JSON.stringify(buildRerankBody(query, documents, config.model))
|
|
6027
|
+
});
|
|
6028
|
+
if (!response.ok) {
|
|
6029
|
+
throw new Error(`Neural rerank HTTP ${response.status} from ${url}`);
|
|
6030
|
+
}
|
|
6031
|
+
return parseRerankResponse(await response.json());
|
|
6032
|
+
} finally {
|
|
6033
|
+
clearTimeout(timer);
|
|
6034
|
+
}
|
|
6035
|
+
}
|
|
6036
|
+
var init_http_provider = __esm({
|
|
6037
|
+
"src/rerank/http-provider.ts"() {
|
|
6038
|
+
"use strict";
|
|
6039
|
+
init_esm_shims();
|
|
6040
|
+
init_policy();
|
|
6041
|
+
}
|
|
6042
|
+
});
|
|
6043
|
+
|
|
6044
|
+
// src/rerank/index.ts
|
|
6045
|
+
var rerank_exports = {};
|
|
6046
|
+
__export(rerank_exports, {
|
|
6047
|
+
DEFAULT_RERANK_TIMEOUT_MS: () => DEFAULT_RERANK_TIMEOUT_MS,
|
|
6048
|
+
MIN_RERANK_CANDIDATES: () => MIN_RERANK_CANDIDATES,
|
|
6049
|
+
isNeuralRerankEnabled: () => isNeuralRerankEnabled,
|
|
6050
|
+
neuralRerankCandidates: () => neuralRerankCandidates,
|
|
6051
|
+
parseRerankTimeoutMs: () => parseRerankTimeoutMs,
|
|
6052
|
+
resolveRerankTimeoutMs: () => resolveRerankTimeoutMs,
|
|
6053
|
+
shouldAttemptHttpRerank: () => shouldAttemptHttpRerank,
|
|
6054
|
+
shouldAttemptLlmRerank: () => shouldAttemptLlmRerank
|
|
6055
|
+
});
|
|
6056
|
+
function isNeuralRerankEnabled(options = {}) {
|
|
6057
|
+
const lane = getResolvedRerankLane(options);
|
|
6058
|
+
return lane.provider === "http" && Boolean(lane.baseUrl);
|
|
6059
|
+
}
|
|
6060
|
+
async function neuralRerankCandidates(query, candidates, options = {}) {
|
|
6061
|
+
if (!isNeuralRerankEnabled(options) || candidates.length === 0) return null;
|
|
6062
|
+
const lane = getResolvedRerankLane(options);
|
|
6063
|
+
if (!lane.canSendRequest || !lane.baseUrl) return null;
|
|
6064
|
+
const toRerank = candidates.slice(0, MAX_RERANK);
|
|
6065
|
+
const rest = candidates.slice(MAX_RERANK);
|
|
6066
|
+
const documents = toRerank.map((candidate) => candidateText(candidate));
|
|
6067
|
+
const ranked = await rerankViaHttp(query, documents, {
|
|
6068
|
+
provider: "http",
|
|
6069
|
+
model: lane.model,
|
|
6070
|
+
baseUrl: lane.baseUrl,
|
|
6071
|
+
apiKey: lane.apiKey,
|
|
6072
|
+
timeoutMs: resolveRerankTimeoutMs()
|
|
6073
|
+
});
|
|
6074
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6075
|
+
const reranked = [];
|
|
6076
|
+
for (const row of ranked) {
|
|
6077
|
+
const candidate = toRerank[row.index];
|
|
6078
|
+
if (!candidate || seen.has(row.index)) continue;
|
|
6079
|
+
reranked.push(candidate);
|
|
6080
|
+
seen.add(row.index);
|
|
6081
|
+
}
|
|
6082
|
+
for (const [index, candidate] of toRerank.entries()) {
|
|
6083
|
+
if (!seen.has(index)) reranked.push(candidate);
|
|
6084
|
+
}
|
|
6085
|
+
reranked.push(...rest);
|
|
6086
|
+
return reranked;
|
|
6087
|
+
}
|
|
6088
|
+
function candidateText(candidate) {
|
|
6089
|
+
const narrative = candidate.narrative?.trim();
|
|
6090
|
+
if (narrative) return `${candidate.title} \u2014 ${narrative.slice(0, 200)}`;
|
|
6091
|
+
return candidate.title;
|
|
6092
|
+
}
|
|
6093
|
+
var MAX_RERANK;
|
|
6094
|
+
var init_rerank = __esm({
|
|
6095
|
+
"src/rerank/index.ts"() {
|
|
6096
|
+
"use strict";
|
|
6097
|
+
init_esm_shims();
|
|
6098
|
+
init_resolved_config();
|
|
6099
|
+
init_http_provider();
|
|
6100
|
+
init_policy();
|
|
6101
|
+
init_policy();
|
|
6102
|
+
MAX_RERANK = 10;
|
|
6103
|
+
}
|
|
6104
|
+
});
|
|
6105
|
+
|
|
5832
6106
|
// src/llm/quality.ts
|
|
5833
6107
|
var quality_exports = {};
|
|
5834
6108
|
__export(quality_exports, {
|
|
@@ -5868,9 +6142,9 @@ async function rerankResults(query, candidates) {
|
|
|
5868
6142
|
if (!isLLMEnabled() || candidates.length <= 2) {
|
|
5869
6143
|
return { reranked: candidates, usedLLM: false };
|
|
5870
6144
|
}
|
|
5871
|
-
const
|
|
5872
|
-
const toRerank = candidates.slice(0,
|
|
5873
|
-
const rest = candidates.slice(
|
|
6145
|
+
const MAX_RERANK2 = 10;
|
|
6146
|
+
const toRerank = candidates.slice(0, MAX_RERANK2);
|
|
6147
|
+
const rest = candidates.slice(MAX_RERANK2);
|
|
5874
6148
|
try {
|
|
5875
6149
|
const candidateList = toRerank.map(
|
|
5876
6150
|
(c) => `[ID: ${c.id}] (${c.type}) ${c.title}${c.narrative ? ` \u2014 ${c.narrative.substring(0, 100)}` : ""}`
|
|
@@ -6094,7 +6368,7 @@ __export(orama_store_exports, {
|
|
|
6094
6368
|
searchObservations: () => searchObservations,
|
|
6095
6369
|
updateObservationMetadata: () => updateObservationMetadata
|
|
6096
6370
|
});
|
|
6097
|
-
import { create, insert as insert2, search, remove as remove2, update, count, getByID } from "@orama/orama";
|
|
6371
|
+
import { create, insert as insert2, insertMultiple, search, remove as remove2, update, count, getByID } from "@orama/orama";
|
|
6098
6372
|
function getLastSearchMode(projectId) {
|
|
6099
6373
|
return lastSearchModeByProject.get(projectId ?? SEARCH_MODE_DEFAULT_KEY) ?? "fulltext";
|
|
6100
6374
|
}
|
|
@@ -6306,7 +6580,7 @@ async function hydrateIndex(observations2, options = {}) {
|
|
|
6306
6580
|
}
|
|
6307
6581
|
const deferCachedVectors = options.deferCachedVectors === true && Boolean(indexEmbeddingProvider?.getCachedEmbeddings);
|
|
6308
6582
|
const cachedVectors = deferCachedVectors ? candidates.map(() => null) : await getCachedEmbeddings(candidates.map(({ observation }) => observationEmbeddingText(observation)));
|
|
6309
|
-
|
|
6583
|
+
const documents = [];
|
|
6310
6584
|
for (let index = 0; index < candidates.length; index++) {
|
|
6311
6585
|
const { observation: obs, id } = candidates[index];
|
|
6312
6586
|
try {
|
|
@@ -6340,18 +6614,35 @@ async function hydrateIndex(observations2, options = {}) {
|
|
|
6340
6614
|
knowledgeLayer: resolveKnowledgeLayer("observation", obs.sourceDetail, obs.source),
|
|
6341
6615
|
...compatibleVector ? { embedding: compatibleVector } : {}
|
|
6342
6616
|
};
|
|
6343
|
-
|
|
6344
|
-
rememberObservationDoc(doc);
|
|
6345
|
-
inserted++;
|
|
6346
|
-
if (inserted % 50 === 0) {
|
|
6347
|
-
await new Promise((resolve) => setImmediate(resolve));
|
|
6348
|
-
}
|
|
6349
|
-
if (inserted % 200 === 0) {
|
|
6350
|
-
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
6351
|
-
}
|
|
6617
|
+
documents.push(doc);
|
|
6352
6618
|
} catch {
|
|
6353
6619
|
}
|
|
6354
6620
|
}
|
|
6621
|
+
let inserted = 0;
|
|
6622
|
+
const hydrationBatchSize = 200;
|
|
6623
|
+
for (let start = 0; start < documents.length; start += hydrationBatchSize) {
|
|
6624
|
+
const batch = documents.slice(start, start + hydrationBatchSize);
|
|
6625
|
+
try {
|
|
6626
|
+
await insertMultiple(database, batch, batch.length);
|
|
6627
|
+
for (const doc of batch) rememberObservationDoc(doc);
|
|
6628
|
+
inserted += batch.length;
|
|
6629
|
+
} catch {
|
|
6630
|
+
for (const doc of batch) {
|
|
6631
|
+
if (getByID(database, doc.id)) {
|
|
6632
|
+
rememberObservationDoc(doc);
|
|
6633
|
+
inserted++;
|
|
6634
|
+
continue;
|
|
6635
|
+
}
|
|
6636
|
+
try {
|
|
6637
|
+
await insert2(database, doc);
|
|
6638
|
+
rememberObservationDoc(doc);
|
|
6639
|
+
inserted++;
|
|
6640
|
+
} catch {
|
|
6641
|
+
}
|
|
6642
|
+
}
|
|
6643
|
+
}
|
|
6644
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
6645
|
+
}
|
|
6355
6646
|
deferredCachedVectorHydration = options.skipCachedVectors ? null : deferCachedVectors ? attachCachedVectors(database, candidates).catch(() => {
|
|
6356
6647
|
}) : null;
|
|
6357
6648
|
return inserted;
|
|
@@ -6746,46 +7037,78 @@ async function searchObservations(options) {
|
|
|
6746
7037
|
}
|
|
6747
7038
|
}
|
|
6748
7039
|
}
|
|
6749
|
-
const
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
})
|
|
6754
|
-
|
|
6755
|
-
|
|
6756
|
-
|
|
6757
|
-
|
|
6758
|
-
|
|
6759
|
-
|
|
6760
|
-
|
|
6761
|
-
|
|
6762
|
-
|
|
6763
|
-
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
|
|
6767
|
-
|
|
6768
|
-
|
|
6769
|
-
|
|
6770
|
-
|
|
6771
|
-
|
|
6772
|
-
|
|
6773
|
-
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
|
|
6783
|
-
|
|
6784
|
-
|
|
7040
|
+
const httpGate = shouldAttemptHttpRerank({
|
|
7041
|
+
quality,
|
|
7042
|
+
hasQuery: Boolean(hasQuery),
|
|
7043
|
+
candidateCount: intermediate.length
|
|
7044
|
+
});
|
|
7045
|
+
const llmGateWithoutHttp = shouldAttemptLlmRerank({
|
|
7046
|
+
quality,
|
|
7047
|
+
tier,
|
|
7048
|
+
hasQuery: Boolean(hasQuery),
|
|
7049
|
+
candidateCount: intermediate.length,
|
|
7050
|
+
topScore: intermediate[0]?.score ?? 0,
|
|
7051
|
+
secondScore: intermediate[1]?.score ?? 0,
|
|
7052
|
+
httpRerankConfigured: false
|
|
7053
|
+
});
|
|
7054
|
+
if (httpGate || llmGateWithoutHttp) {
|
|
7055
|
+
const narrativeMap = /* @__PURE__ */ new Map();
|
|
7056
|
+
for (const hit of results.hits) {
|
|
7057
|
+
const doc = hit.document;
|
|
7058
|
+
narrativeMap.set(makeEntryKey(doc.projectId, doc.observationId), doc.narrative);
|
|
7059
|
+
}
|
|
7060
|
+
const RERANK_TOP_K = 5;
|
|
7061
|
+
const toRerank = intermediate.slice(0, RERANK_TOP_K);
|
|
7062
|
+
const candidates = toRerank.map((e2, index) => ({
|
|
7063
|
+
id: `r${index + 1}`,
|
|
7064
|
+
title: e2.title,
|
|
7065
|
+
type: e2.type,
|
|
7066
|
+
score: e2.score,
|
|
7067
|
+
narrative: narrativeMap.get(makeEntryKey(e2.projectId, e2.id))
|
|
7068
|
+
}));
|
|
7069
|
+
const RERANK_TIMEOUT_MS = parseRerankTimeoutMs(process.env.MEMORIX_RERANK_TIMEOUT_MS);
|
|
7070
|
+
const candidateMap = new Map(candidates.map((candidate, index) => [candidate.id, toRerank[index]]));
|
|
7071
|
+
const { isNeuralRerankEnabled: isNeuralRerankEnabled2, neuralRerankCandidates: neuralRerankCandidates2 } = await Promise.resolve().then(() => (init_rerank(), rerank_exports));
|
|
7072
|
+
const httpConfigured = isNeuralRerankEnabled2();
|
|
7073
|
+
if (httpConfigured && httpGate) {
|
|
7074
|
+
try {
|
|
7075
|
+
const neural = await withTimeout(
|
|
7076
|
+
neuralRerankCandidates2(originalQuery, candidates),
|
|
7077
|
+
RERANK_TIMEOUT_MS,
|
|
7078
|
+
"neural rerank"
|
|
7079
|
+
);
|
|
7080
|
+
if (neural && neural.length > 0) {
|
|
7081
|
+
const rerankedTop = neural.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7082
|
+
if (rerankedTop.length > 0) {
|
|
7083
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7084
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + neural rerank");
|
|
7085
|
+
mark("rerank(usedNeural=true)");
|
|
7086
|
+
}
|
|
6785
7087
|
}
|
|
7088
|
+
} catch {
|
|
7089
|
+
console.error("[memorix] neural rerank failed or timed out; keeping original order");
|
|
6786
7090
|
}
|
|
6787
|
-
}
|
|
6788
|
-
|
|
7091
|
+
} else if (!httpConfigured && llmGateWithoutHttp) {
|
|
7092
|
+
try {
|
|
7093
|
+
const { rerankResults: rerankResults2 } = await Promise.resolve().then(() => (init_quality(), quality_exports));
|
|
7094
|
+
const { reranked, usedLLM } = await withTimeout(
|
|
7095
|
+
rerankResults2(originalQuery, candidates),
|
|
7096
|
+
RERANK_TIMEOUT_MS,
|
|
7097
|
+
"LLM rerank"
|
|
7098
|
+
);
|
|
7099
|
+
mark(`rerank(usedLLM=${usedLLM})`);
|
|
7100
|
+
if (usedLLM) {
|
|
7101
|
+
lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? "fulltext") + " + LLM rerank");
|
|
7102
|
+
const rerankedTop = reranked.map((row) => candidateMap.get(row.id)).filter((entry) => entry != null);
|
|
7103
|
+
if (rerankedTop.length > 0) {
|
|
7104
|
+
intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
|
|
7105
|
+
}
|
|
7106
|
+
}
|
|
7107
|
+
} catch {
|
|
7108
|
+
console.error("[memorix] LLM rerank failed or timed out, using original order");
|
|
7109
|
+
}
|
|
7110
|
+
} else {
|
|
7111
|
+
mark(`rerank(skipped,tier=${tier})`);
|
|
6789
7112
|
}
|
|
6790
7113
|
} else {
|
|
6791
7114
|
mark(`rerank(skipped,tier=${tier})`);
|
|
@@ -6966,6 +7289,7 @@ var init_orama_store = __esm({
|
|
|
6966
7289
|
init_project_affinity();
|
|
6967
7290
|
init_intent_detector();
|
|
6968
7291
|
init_query_expansion();
|
|
7292
|
+
init_policy();
|
|
6969
7293
|
init_timeout();
|
|
6970
7294
|
db = null;
|
|
6971
7295
|
dbInitPromise = null;
|