@memory-river/core 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +222 -0
- package/README.zh-TW.md +186 -0
- package/dist/api.d.ts +100 -0
- package/dist/api.js +156 -0
- package/dist/cognition/causal-attribution.d.ts +36 -0
- package/dist/cognition/causal-attribution.js +239 -0
- package/dist/cognition/causal-engine.d.ts +105 -0
- package/dist/cognition/causal-engine.js +150 -0
- package/dist/cognition/conflict-detector.d.ts +39 -0
- package/dist/cognition/conflict-detector.js +193 -0
- package/dist/cognition/global-working-memory.d.ts +53 -0
- package/dist/cognition/global-working-memory.js +211 -0
- package/dist/cognition/hooks-engine.d.ts +99 -0
- package/dist/cognition/hooks-engine.js +672 -0
- package/dist/cognition/ralph-core.d.ts +28 -0
- package/dist/cognition/ralph-core.js +104 -0
- package/dist/distill/concentrator-adapter.d.ts +167 -0
- package/dist/distill/concentrator-adapter.js +1876 -0
- package/dist/engine.d.ts +402 -0
- package/dist/engine.js +2254 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +3 -0
- package/dist/lifecycle/cleanup-engine.d.ts +80 -0
- package/dist/lifecycle/cleanup-engine.js +162 -0
- package/dist/lifecycle/cleanup-state.d.ts +34 -0
- package/dist/lifecycle/cleanup-state.js +50 -0
- package/dist/lifecycle/night-consolidation.d.ts +102 -0
- package/dist/lifecycle/night-consolidation.js +640 -0
- package/dist/lifecycle/night-recovery.d.ts +40 -0
- package/dist/lifecycle/night-recovery.js +107 -0
- package/dist/paths.d.ts +17 -0
- package/dist/paths.js +16 -0
- package/dist/pipeline/capsule-bridge.d.ts +35 -0
- package/dist/pipeline/capsule-bridge.js +86 -0
- package/dist/pipeline/compact-request.d.ts +30 -0
- package/dist/pipeline/compact-request.js +66 -0
- package/dist/pipeline/inbox-watcher.d.ts +112 -0
- package/dist/pipeline/inbox-watcher.js +1039 -0
- package/dist/ports.d.ts +29 -0
- package/dist/ports.js +1 -0
- package/dist/providers/embedder-v5.d.ts +46 -0
- package/dist/providers/embedder-v5.js +155 -0
- package/dist/providers/ollama-embedding.d.ts +25 -0
- package/dist/providers/ollama-embedding.js +166 -0
- package/dist/retrieval/abstractness-judge.d.ts +14 -0
- package/dist/retrieval/abstractness-judge.js +87 -0
- package/dist/retrieval/coverage-selection.d.ts +3 -0
- package/dist/retrieval/coverage-selection.js +53 -0
- package/dist/retrieval/cross-encoder-gate.d.ts +40 -0
- package/dist/retrieval/cross-encoder-gate.js +239 -0
- package/dist/retrieval/retriever-v4.d.ts +78 -0
- package/dist/retrieval/retriever-v4.js +1200 -0
- package/dist/skills/validate.d.ts +6 -0
- package/dist/skills/validate.js +69 -0
- package/dist/storage.d.ts +19 -0
- package/dist/storage.js +54 -0
- package/dist/store/aux-table-maintenance.d.ts +5 -0
- package/dist/store/aux-table-maintenance.js +64 -0
- package/dist/store/graph-enumerator.d.ts +21 -0
- package/dist/store/graph-enumerator.js +185 -0
- package/dist/store/graph-store.d.ts +107 -0
- package/dist/store/graph-store.js +478 -0
- package/dist/store/status-manager.d.ts +44 -0
- package/dist/store/status-manager.js +235 -0
- package/dist/store/store-v4.d.ts +339 -0
- package/dist/store/store-v4.js +2871 -0
- package/dist/transcript/keyword-search.d.ts +9 -0
- package/dist/transcript/keyword-search.js +67 -0
- package/dist/transcript/rehydrate-keyword.d.ts +6 -0
- package/dist/transcript/rehydrate-keyword.js +29 -0
- package/dist/transcript/rehydrate.d.ts +33 -0
- package/dist/transcript/rehydrate.js +285 -0
- package/dist/transcript/transcript-archive.d.ts +46 -0
- package/dist/transcript/transcript-archive.js +516 -0
- package/dist/types.d.ts +409 -0
- package/dist/types.js +104 -0
- package/dist/util/bounded-map.d.ts +1 -0
- package/dist/util/bounded-map.js +8 -0
- package/dist/util/rate-limiter.d.ts +12 -0
- package/dist/util/rate-limiter.js +54 -0
- package/dist/util/session-identity.d.ts +65 -0
- package/dist/util/session-identity.js +227 -0
- package/dist/util/util-hash.d.ts +1 -0
- package/dist/util/util-hash.js +4 -0
- package/package.json +59 -0
package/dist/ports.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface LlmClient {
|
|
2
|
+
/** 回傳純文字。實作端自行負責 fallback / rate limit / circuit breaker。 */
|
|
3
|
+
generate(prompt: string, opts?: {
|
|
4
|
+
purpose?: string;
|
|
5
|
+
maxTokens?: number;
|
|
6
|
+
}): Promise<string>;
|
|
7
|
+
}
|
|
8
|
+
export interface EmbeddingProvider {
|
|
9
|
+
embed(text: string, mode?: 'store' | 'query'): Promise<number[]>;
|
|
10
|
+
embedBatch(texts: string[]): Promise<number[][]>;
|
|
11
|
+
getDimensions(): number;
|
|
12
|
+
healthCheck?(): Promise<boolean>;
|
|
13
|
+
}
|
|
14
|
+
export interface Logger {
|
|
15
|
+
info(msg: string, meta?: unknown): void;
|
|
16
|
+
warn(msg: string, meta?: unknown): void;
|
|
17
|
+
error(msg: string, meta?: unknown): void;
|
|
18
|
+
}
|
|
19
|
+
export interface Notifier {
|
|
20
|
+
/** 夜間日報、降級告警。沒提供就靜默跳過。 */
|
|
21
|
+
notify(message: string): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
export interface SessionFileAccess {
|
|
24
|
+
/** 宿主若有「session 對話檔」概念才需要實作;沒有就回 null,compact 功能自動停用。 */
|
|
25
|
+
resolveSessionFile(identity: {
|
|
26
|
+
sessionKey?: string;
|
|
27
|
+
sessionId?: string;
|
|
28
|
+
}): string | null;
|
|
29
|
+
}
|
package/dist/ports.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder v5 - Qwen3-Embedding via Ollama (Instruction-Aware)
|
|
3
|
+
* memory-river v5
|
|
4
|
+
*
|
|
5
|
+
* [鐵律] 使用 Ollama 本地部署 Qwen3-Embedding-0.6B
|
|
6
|
+
* 向量維度:1024(與 Gemini 3072維 不相容,強制使用獨立的 lancedb-v5-qwen)
|
|
7
|
+
*
|
|
8
|
+
* Instruction-Aware 前輟綁定:
|
|
9
|
+
* - 寫入:瑪「Summarize this memory concisely:」
|
|
10
|
+
* - 讀取:「Retrieve similar memory records from Memory River knowledge base:」
|
|
11
|
+
*/
|
|
12
|
+
import type { PluginConfig } from "../types.js";
|
|
13
|
+
export declare class Embedder {
|
|
14
|
+
private ollamaUrl;
|
|
15
|
+
private model;
|
|
16
|
+
apiKey: string;
|
|
17
|
+
dimensions: number;
|
|
18
|
+
constructor(config: PluginConfig["embedding"] & {
|
|
19
|
+
ollamaUrl: string;
|
|
20
|
+
embeddingModel?: string;
|
|
21
|
+
});
|
|
22
|
+
/**
|
|
23
|
+
* 單筆查詢向量化(Instruction-Aware)
|
|
24
|
+
* @param text 原始查詢文字
|
|
25
|
+
* @param mode 'store' → "Summarize this memory concisely:" | 'query' → "Retrieve similar..."
|
|
26
|
+
* @param retries 重試次數
|
|
27
|
+
*/
|
|
28
|
+
embed(text: string, mode?: 'store' | 'query', retries?: number): Promise<number[]>;
|
|
29
|
+
/**
|
|
30
|
+
* 批次向量化(寫入時使用,無 Instruction-Aware 前輟)
|
|
31
|
+
* @param texts 字串陣列
|
|
32
|
+
*/
|
|
33
|
+
embedBatch(texts: string[], retries?: number): Promise<number[][]>;
|
|
34
|
+
/** 回報維度 */
|
|
35
|
+
getDimensions(): number;
|
|
36
|
+
/** 回報模型名 */
|
|
37
|
+
getModel(): string;
|
|
38
|
+
/** Ping 測試 */
|
|
39
|
+
healthCheck(): Promise<boolean>;
|
|
40
|
+
/** embed() 的 Float32Array 版本(v4 相容) */
|
|
41
|
+
embedText(text: string): Promise<Float32Array>;
|
|
42
|
+
/** embedBatch() 的 Float32Array 版本(v4 相容) */
|
|
43
|
+
embedTextBatch(texts: string[], _concurrency?: number): Promise<Float32Array[]>;
|
|
44
|
+
/** Ollama 不支援 generate(僅相容性 stub) */
|
|
45
|
+
generate(prompt: string): Promise<string>;
|
|
46
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder v5 - Qwen3-Embedding via Ollama (Instruction-Aware)
|
|
3
|
+
* memory-river v5
|
|
4
|
+
*
|
|
5
|
+
* [鐵律] 使用 Ollama 本地部署 Qwen3-Embedding-0.6B
|
|
6
|
+
* 向量維度:1024(與 Gemini 3072維 不相容,強制使用獨立的 lancedb-v5-qwen)
|
|
7
|
+
*
|
|
8
|
+
* Instruction-Aware 前輟綁定:
|
|
9
|
+
* - 寫入:瑪「Summarize this memory concisely:」
|
|
10
|
+
* - 讀取:「Retrieve similar memory records from Memory River knowledge base:」
|
|
11
|
+
*/
|
|
12
|
+
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
13
|
+
// ── Ollama 全域並發限制(防止大量並發打垮 Ollama)───────────────────────────
|
|
14
|
+
class Semaphore {
|
|
15
|
+
count;
|
|
16
|
+
queue = [];
|
|
17
|
+
constructor(limit) { this.count = limit; }
|
|
18
|
+
async acquire() {
|
|
19
|
+
if (this.count > 0) {
|
|
20
|
+
this.count--;
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
return new Promise(resolve => this.queue.push(resolve));
|
|
24
|
+
}
|
|
25
|
+
release() {
|
|
26
|
+
if (this.queue.length > 0) {
|
|
27
|
+
this.queue.shift()();
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
this.count++;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const ollamaSemaphore = new Semaphore(4); // 最多 4 路並發
|
|
35
|
+
function chunkArray(arr, size) {
|
|
36
|
+
const chunks = [];
|
|
37
|
+
for (let i = 0; i < arr.length; i += size)
|
|
38
|
+
chunks.push(arr.slice(i, i + size));
|
|
39
|
+
return chunks;
|
|
40
|
+
}
|
|
41
|
+
export class Embedder {
|
|
42
|
+
ollamaUrl;
|
|
43
|
+
model;
|
|
44
|
+
// v4 相容性欄位(v5 不需要 apiKey,但其他模組有型別預期)
|
|
45
|
+
apiKey = "";
|
|
46
|
+
dimensions = 1024;
|
|
47
|
+
constructor(config) {
|
|
48
|
+
this.ollamaUrl = config.ollamaUrl;
|
|
49
|
+
// 綁死 Qwen3 Embedding 模型
|
|
50
|
+
this.model = config.embeddingModel || "hf.co/Qwen/Qwen3-Embedding-0.6B-GGUF";
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* 單筆查詢向量化(Instruction-Aware)
|
|
54
|
+
* @param text 原始查詢文字
|
|
55
|
+
* @param mode 'store' → "Summarize this memory concisely:" | 'query' → "Retrieve similar..."
|
|
56
|
+
* @param retries 重試次數
|
|
57
|
+
*/
|
|
58
|
+
async embed(text, mode = 'query', retries = 3) {
|
|
59
|
+
const prefix = mode === 'store'
|
|
60
|
+
? "Summarize this memory concisely:"
|
|
61
|
+
: "Retrieve similar memory records from Memory River knowledge base:";
|
|
62
|
+
await ollamaSemaphore.acquire();
|
|
63
|
+
try {
|
|
64
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
65
|
+
try {
|
|
66
|
+
const response = await fetch(`${this.ollamaUrl}/api/embeddings`, {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers: { "Content-Type": "application/json" },
|
|
69
|
+
body: JSON.stringify({
|
|
70
|
+
model: this.model,
|
|
71
|
+
prompt: `${prefix} ${text}`,
|
|
72
|
+
}),
|
|
73
|
+
});
|
|
74
|
+
if (!response.ok) {
|
|
75
|
+
const errText = await response.text();
|
|
76
|
+
if (response.status === 503 || response.status === 429) {
|
|
77
|
+
// 模型尚未載入或被限流,等待後重試
|
|
78
|
+
if (attempt < retries) {
|
|
79
|
+
const backoff = attempt * 2000;
|
|
80
|
+
console.warn(`[Embedder-v5] Ollama returned ${response.status}; retrying in ${backoff}ms... (${attempt}/${retries})`);
|
|
81
|
+
await sleep(backoff);
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
throw new Error(`Ollama embedding failed (${response.status}): ${errText}`);
|
|
86
|
+
}
|
|
87
|
+
const data = await response.json();
|
|
88
|
+
if (!data.embedding || !Array.isArray(data.embedding)) {
|
|
89
|
+
throw new Error(`Invalid Ollama response: missing embedding field`);
|
|
90
|
+
}
|
|
91
|
+
return data.embedding;
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
if (attempt >= retries)
|
|
95
|
+
throw err;
|
|
96
|
+
console.warn(`[Embedder-v5] embed() failed (${attempt}/${retries}): ${err.message}`);
|
|
97
|
+
await sleep(attempt * 1000);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
throw new Error("Should not reach here");
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
ollamaSemaphore.release();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* 批次向量化(寫入時使用,無 Instruction-Aware 前輟)
|
|
108
|
+
* @param texts 字串陣列
|
|
109
|
+
*/
|
|
110
|
+
async embedBatch(texts, retries = 3) {
|
|
111
|
+
// Ollama /api/embeddings 只接受單一 prompt 字串,不支援陣列
|
|
112
|
+
// 改用並發多支 embed() 呼叫,並加 concurrency 控制
|
|
113
|
+
const BATCH_CONCURRENCY = 4;
|
|
114
|
+
const results = [];
|
|
115
|
+
for (let i = 0; i < texts.length; i += BATCH_CONCURRENCY) {
|
|
116
|
+
const chunk = texts.slice(i, i + BATCH_CONCURRENCY);
|
|
117
|
+
const embeddings = await Promise.all(chunk.map(text => this.embed(text, 'store', retries)));
|
|
118
|
+
results.push(...embeddings);
|
|
119
|
+
}
|
|
120
|
+
return results;
|
|
121
|
+
}
|
|
122
|
+
/** 回報維度 */
|
|
123
|
+
getDimensions() {
|
|
124
|
+
return 1024;
|
|
125
|
+
}
|
|
126
|
+
/** 回報模型名 */
|
|
127
|
+
getModel() {
|
|
128
|
+
return this.model;
|
|
129
|
+
}
|
|
130
|
+
/** Ping 測試 */
|
|
131
|
+
async healthCheck() {
|
|
132
|
+
try {
|
|
133
|
+
const response = await fetch(`${this.ollamaUrl}/api/tags`, { method: "GET" });
|
|
134
|
+
return response.ok;
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// ── v4 相容性包裝 ──────────────────────────────────────
|
|
141
|
+
/** embed() 的 Float32Array 版本(v4 相容) */
|
|
142
|
+
async embedText(text) {
|
|
143
|
+
const vec = await this.embed(text);
|
|
144
|
+
return new Float32Array(vec);
|
|
145
|
+
}
|
|
146
|
+
/** embedBatch() 的 Float32Array 版本(v4 相容) */
|
|
147
|
+
async embedTextBatch(texts, _concurrency = 5) {
|
|
148
|
+
const vecs = await this.embedBatch(texts);
|
|
149
|
+
return vecs.map(v => new Float32Array(v));
|
|
150
|
+
}
|
|
151
|
+
/** Ollama 不支援 generate(僅相容性 stub) */
|
|
152
|
+
async generate(prompt) {
|
|
153
|
+
throw new Error("[Embedder-v5] generate() 不支援,請使用 Ollama /api/generate 接口");
|
|
154
|
+
}
|
|
155
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OllamaEmbeddingFunction - LanceDB 0.14 compatible embedding function
|
|
3
|
+
* for Qwen3-Embedding via Ollama
|
|
4
|
+
*/
|
|
5
|
+
import { TextEmbeddingFunction } from "@lancedb/lancedb/embedding";
|
|
6
|
+
import { Float32 } from "apache-arrow";
|
|
7
|
+
export declare class OllamaEmbeddingFunction extends TextEmbeddingFunction {
|
|
8
|
+
private url;
|
|
9
|
+
private model;
|
|
10
|
+
private _initialized;
|
|
11
|
+
constructor(options?: {
|
|
12
|
+
url?: string;
|
|
13
|
+
model?: string;
|
|
14
|
+
});
|
|
15
|
+
init(): Promise<void>;
|
|
16
|
+
ndims(): number;
|
|
17
|
+
toJSON(): object;
|
|
18
|
+
embeddingDataType(): Float32;
|
|
19
|
+
/**
|
|
20
|
+
* generateEmbeddings — LanceDB calls this for source column (store mode)
|
|
21
|
+
* Uses "Summarize this memory concisely:" prefix
|
|
22
|
+
*/
|
|
23
|
+
generateEmbeddings(data: string[]): Promise<number[][]>;
|
|
24
|
+
private _embedOne;
|
|
25
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
|
|
2
|
+
function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
|
|
3
|
+
var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
|
|
4
|
+
var target = !descriptorIn && ctor ? contextIn["static"] ? ctor : ctor.prototype : null;
|
|
5
|
+
var descriptor = descriptorIn || (target ? Object.getOwnPropertyDescriptor(target, contextIn.name) : {});
|
|
6
|
+
var _, done = false;
|
|
7
|
+
for (var i = decorators.length - 1; i >= 0; i--) {
|
|
8
|
+
var context = {};
|
|
9
|
+
for (var p in contextIn) context[p] = p === "access" ? {} : contextIn[p];
|
|
10
|
+
for (var p in contextIn.access) context.access[p] = contextIn.access[p];
|
|
11
|
+
context.addInitializer = function (f) { if (done) throw new TypeError("Cannot add initializers after decoration has completed"); extraInitializers.push(accept(f || null)); };
|
|
12
|
+
var result = (0, decorators[i])(kind === "accessor" ? { get: descriptor.get, set: descriptor.set } : descriptor[key], context);
|
|
13
|
+
if (kind === "accessor") {
|
|
14
|
+
if (result === void 0) continue;
|
|
15
|
+
if (result === null || typeof result !== "object") throw new TypeError("Object expected");
|
|
16
|
+
if (_ = accept(result.get)) descriptor.get = _;
|
|
17
|
+
if (_ = accept(result.set)) descriptor.set = _;
|
|
18
|
+
if (_ = accept(result.init)) initializers.unshift(_);
|
|
19
|
+
}
|
|
20
|
+
else if (_ = accept(result)) {
|
|
21
|
+
if (kind === "field") initializers.unshift(_);
|
|
22
|
+
else descriptor[key] = _;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
if (target) Object.defineProperty(target, contextIn.name, descriptor);
|
|
26
|
+
done = true;
|
|
27
|
+
};
|
|
28
|
+
var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
|
|
29
|
+
var useValue = arguments.length > 2;
|
|
30
|
+
for (var i = 0; i < initializers.length; i++) {
|
|
31
|
+
value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
|
|
32
|
+
}
|
|
33
|
+
return useValue ? value : void 0;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* OllamaEmbeddingFunction - LanceDB 0.14 compatible embedding function
|
|
37
|
+
* for Qwen3-Embedding via Ollama
|
|
38
|
+
*/
|
|
39
|
+
import { TextEmbeddingFunction, register } from "@lancedb/lancedb/embedding";
|
|
40
|
+
import { Float32 } from "apache-arrow";
|
|
41
|
+
const SEMAPHORE_LIMIT = 4;
|
|
42
|
+
// ── Semaphore for Ollama concurrency control ────────────────────────────────
|
|
43
|
+
class Semaphore {
|
|
44
|
+
count;
|
|
45
|
+
queue = [];
|
|
46
|
+
constructor(limit) { this.count = limit; }
|
|
47
|
+
async acquire() {
|
|
48
|
+
if (this.count > 0) {
|
|
49
|
+
this.count--;
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
return new Promise(resolve => this.queue.push(resolve));
|
|
53
|
+
}
|
|
54
|
+
release() {
|
|
55
|
+
if (this.queue.length > 0) {
|
|
56
|
+
this.queue.shift()();
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.count++;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const ollamaSemaphore = new Semaphore(SEMAPHORE_LIMIT);
|
|
64
|
+
function sleep(ms) {
|
|
65
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
66
|
+
}
|
|
67
|
+
// ── Ollama Embedding Function ────────────────────────────────────────────────
|
|
68
|
+
let OllamaEmbeddingFunction = (() => {
|
|
69
|
+
let _classDecorators = [register("ollama")];
|
|
70
|
+
let _classDescriptor;
|
|
71
|
+
let _classExtraInitializers = [];
|
|
72
|
+
let _classThis;
|
|
73
|
+
let _classSuper = TextEmbeddingFunction;
|
|
74
|
+
var OllamaEmbeddingFunction = class extends _classSuper {
|
|
75
|
+
static { _classThis = this; }
|
|
76
|
+
static {
|
|
77
|
+
const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
|
|
78
|
+
__esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
|
|
79
|
+
OllamaEmbeddingFunction = _classThis = _classDescriptor.value;
|
|
80
|
+
if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
|
|
81
|
+
__runInitializers(_classThis, _classExtraInitializers);
|
|
82
|
+
}
|
|
83
|
+
url;
|
|
84
|
+
model;
|
|
85
|
+
_initialized = false;
|
|
86
|
+
constructor(options) {
|
|
87
|
+
super();
|
|
88
|
+
this.url = options?.url ?? "http://localhost:11434";
|
|
89
|
+
this.model = options?.model ?? "hf.co/Qwen/Qwen3-Embedding-0.6B-GGUF";
|
|
90
|
+
}
|
|
91
|
+
async init() {
|
|
92
|
+
this._initialized = true;
|
|
93
|
+
}
|
|
94
|
+
ndims() {
|
|
95
|
+
return 1024;
|
|
96
|
+
}
|
|
97
|
+
toJSON() {
|
|
98
|
+
return {
|
|
99
|
+
url: this.url,
|
|
100
|
+
model: this.model,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
embeddingDataType() {
|
|
104
|
+
// arrow Float32
|
|
105
|
+
return new Float32();
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* generateEmbeddings — LanceDB calls this for source column (store mode)
|
|
109
|
+
* Uses "Summarize this memory concisely:" prefix
|
|
110
|
+
*/
|
|
111
|
+
async generateEmbeddings(data) {
|
|
112
|
+
await ollamaSemaphore.acquire();
|
|
113
|
+
try {
|
|
114
|
+
const results = [];
|
|
115
|
+
for (const text of data) {
|
|
116
|
+
const embedding = await this._embedOne(text, "store");
|
|
117
|
+
results.push(embedding);
|
|
118
|
+
}
|
|
119
|
+
return results;
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
ollamaSemaphore.release();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
async _embedOne(text, mode) {
|
|
126
|
+
const prefix = mode === "store"
|
|
127
|
+
? "Summarize this memory concisely:"
|
|
128
|
+
: "Retrieve similar memory records from Memory River knowledge base:";
|
|
129
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
130
|
+
try {
|
|
131
|
+
const response = await fetch(`${this.url}/api/embeddings`, {
|
|
132
|
+
method: "POST",
|
|
133
|
+
headers: { "Content-Type": "application/json" },
|
|
134
|
+
body: JSON.stringify({
|
|
135
|
+
model: this.model,
|
|
136
|
+
prompt: `${prefix} ${text}`,
|
|
137
|
+
}),
|
|
138
|
+
});
|
|
139
|
+
if (!response.ok) {
|
|
140
|
+
const errText = await response.text();
|
|
141
|
+
if (response.status === 503 || response.status === 429) {
|
|
142
|
+
if (attempt < 3) {
|
|
143
|
+
await sleep(attempt * 2000);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
throw new Error(`Ollama ${response.status}: ${errText}`);
|
|
148
|
+
}
|
|
149
|
+
const json = await response.json();
|
|
150
|
+
if (!json.embedding || !Array.isArray(json.embedding)) {
|
|
151
|
+
throw new Error("Missing embedding in response");
|
|
152
|
+
}
|
|
153
|
+
return json.embedding;
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
if (attempt >= 3)
|
|
157
|
+
throw err;
|
|
158
|
+
await sleep(attempt * 1000);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
throw new Error("unreachable");
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
return OllamaEmbeddingFunction = _classThis;
|
|
165
|
+
})();
|
|
166
|
+
export { OllamaEmbeddingFunction };
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface AbstractnessJudgement {
|
|
2
|
+
isAbstract: boolean;
|
|
3
|
+
abstractness: number;
|
|
4
|
+
reasons: string[];
|
|
5
|
+
ruleHits: {
|
|
6
|
+
entityCount: number;
|
|
7
|
+
hasMetaNarration: boolean;
|
|
8
|
+
abstractRatio: number;
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
export declare function countEntities(text: string): number;
|
|
12
|
+
export declare function hasMetaNarration(text: string): boolean;
|
|
13
|
+
export declare function abstractNounRatio(text: string): number;
|
|
14
|
+
export declare function judgeAbstractness(text: string): AbstractnessJudgement;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
const ABSTRACT_STOPWORDS = new Set([
|
|
2
|
+
'用戶', '使用者', '系統', '工具', '方式', '策略', '場景', '方法', '建議',
|
|
3
|
+
'錯誤', '問題', '資訊', '資料', '內容', '結果', '過程', '操作', '執行',
|
|
4
|
+
'功能', '設定', '配置', '流程', '機制', '邏輯', '概念', '應用', '處理',
|
|
5
|
+
'根據', '調整', '傳遞', '解決', '理解', '指令', '完成', '任務', '類型',
|
|
6
|
+
'選用', '需要', '描述', '進行', '推理', '成功', '所有',
|
|
7
|
+
]);
|
|
8
|
+
const META_NARRATION_PATTERNS = [
|
|
9
|
+
/^(用戶|使用者|User)\s*(正在|已|將|想要|開始)/,
|
|
10
|
+
/^AI\s*(已|將|提供|選用|理解|執行|完成|建議)/,
|
|
11
|
+
/^系統\s*(已|將|正在|提供|執行)/,
|
|
12
|
+
/^(模型|Model|LLM)\s*(已|將|提供|選用)/,
|
|
13
|
+
/(已成功|已完成).{0,15}(任務|流程|操作|執行)$/,
|
|
14
|
+
];
|
|
15
|
+
function isAbstractChineseChunk(word) {
|
|
16
|
+
for (const stopword of ABSTRACT_STOPWORDS) {
|
|
17
|
+
if (word.includes(stopword))
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
export function countEntities(text) {
|
|
23
|
+
let count = 0;
|
|
24
|
+
count += (text.match(/\d+/g) || []).length;
|
|
25
|
+
count += (text.match(/(\/[a-zA-Z0-9_\-./]+|[a-zA-Z]:\\[^\s]+)/g) || []).length;
|
|
26
|
+
count += (text.match(/\b[a-z][a-zA-Z0-9_]*[A-Z][a-zA-Z0-9_]*\b/g) || []).length;
|
|
27
|
+
count += (text.match(/[a-zA-Z_][a-zA-Z0-9_]*\(\)/g) || []).length;
|
|
28
|
+
count += (text.match(/[「『"'][^」』"']+[」』"']/g) || []).length;
|
|
29
|
+
count += (text.match(/\b[A-Z][a-zA-Z0-9]+\b/g) || []).length;
|
|
30
|
+
const chineseProperNouns = text.match(/[\u4e00-\u9fa5]{2,4}/g) || [];
|
|
31
|
+
for (const word of chineseProperNouns) {
|
|
32
|
+
if (!isAbstractChineseChunk(word))
|
|
33
|
+
count += 0.3;
|
|
34
|
+
}
|
|
35
|
+
return Math.floor(count);
|
|
36
|
+
}
|
|
37
|
+
export function hasMetaNarration(text) {
|
|
38
|
+
return META_NARRATION_PATTERNS.some((pattern) => pattern.test(text));
|
|
39
|
+
}
|
|
40
|
+
export function abstractNounRatio(text) {
|
|
41
|
+
const tokens = text.match(/[\u4e00-\u9fa5]{2,4}/g) || [];
|
|
42
|
+
if (tokens.length === 0)
|
|
43
|
+
return 0;
|
|
44
|
+
const abstractCount = tokens.filter((token) => isAbstractChineseChunk(token)).length;
|
|
45
|
+
return abstractCount / tokens.length;
|
|
46
|
+
}
|
|
47
|
+
export function judgeAbstractness(text) {
|
|
48
|
+
const cleanText = text.replace(/\[#[^\]]+\]/g, '').trim();
|
|
49
|
+
if (text.trim().length < 15) {
|
|
50
|
+
return {
|
|
51
|
+
isAbstract: false,
|
|
52
|
+
abstractness: 0,
|
|
53
|
+
reasons: [],
|
|
54
|
+
ruleHits: { entityCount: 0, hasMetaNarration: false, abstractRatio: 0 },
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
const entityCount = countEntities(cleanText);
|
|
58
|
+
const hasMeta = hasMetaNarration(cleanText);
|
|
59
|
+
const abstractRatio = abstractNounRatio(cleanText);
|
|
60
|
+
const reasons = [];
|
|
61
|
+
if (entityCount < 2)
|
|
62
|
+
reasons.push('low_entity');
|
|
63
|
+
if (hasMeta)
|
|
64
|
+
reasons.push('meta_narration');
|
|
65
|
+
if (abstractRatio > 0.3)
|
|
66
|
+
reasons.push('high_abstract_ratio');
|
|
67
|
+
const isAbstract = entityCount < 2 && (hasMeta || abstractRatio > 0.3);
|
|
68
|
+
let abstractness = 0;
|
|
69
|
+
if (entityCount === 0)
|
|
70
|
+
abstractness += 0.4;
|
|
71
|
+
else if (entityCount === 1)
|
|
72
|
+
abstractness += 0.2;
|
|
73
|
+
if (hasMeta)
|
|
74
|
+
abstractness += 0.3;
|
|
75
|
+
abstractness += Math.min(abstractRatio, 0.5) * 0.6;
|
|
76
|
+
abstractness = Math.min(abstractness, 1.0);
|
|
77
|
+
return {
|
|
78
|
+
isAbstract,
|
|
79
|
+
abstractness,
|
|
80
|
+
reasons,
|
|
81
|
+
ruleHits: {
|
|
82
|
+
entityCount,
|
|
83
|
+
hasMetaNarration: hasMeta,
|
|
84
|
+
abstractRatio,
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export declare function mmrOrder(relevance: number[], vectors: (Float32Array | number[] | null | undefined)[], lambda: number): number[];
|
|
2
|
+
export declare function isCoverageSelectionEnabled(env?: NodeJS.ProcessEnv): boolean;
|
|
3
|
+
export declare function coverageLambda(env?: NodeJS.ProcessEnv): number;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
function cosineNumber(a, b) {
|
|
2
|
+
if (!a || !b || a.length === 0 || b.length === 0)
|
|
3
|
+
return 0;
|
|
4
|
+
let dot = 0;
|
|
5
|
+
let na = 0;
|
|
6
|
+
let nb = 0;
|
|
7
|
+
const length = Math.min(a.length, b.length);
|
|
8
|
+
for (let i = 0; i < length; i++) {
|
|
9
|
+
const av = Number(a[i]);
|
|
10
|
+
const bv = Number(b[i]);
|
|
11
|
+
if (!Number.isFinite(av) || !Number.isFinite(bv))
|
|
12
|
+
continue;
|
|
13
|
+
dot += av * bv;
|
|
14
|
+
na += av * av;
|
|
15
|
+
nb += bv * bv;
|
|
16
|
+
}
|
|
17
|
+
const denom = Math.sqrt(na) * Math.sqrt(nb);
|
|
18
|
+
return denom > 0 ? dot / denom : 0;
|
|
19
|
+
}
|
|
20
|
+
export function mmrOrder(relevance, vectors, lambda) {
|
|
21
|
+
const lo = Math.min(...relevance);
|
|
22
|
+
const hi = Math.max(...relevance);
|
|
23
|
+
const rel = relevance.map(value => (hi > lo ? (value - lo) / (hi - lo) : 0));
|
|
24
|
+
const selected = [];
|
|
25
|
+
const remaining = new Set(relevance.map((_, index) => index));
|
|
26
|
+
while (remaining.size > 0) {
|
|
27
|
+
let best = -1;
|
|
28
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
29
|
+
for (const index of remaining) {
|
|
30
|
+
let maxSim = 0;
|
|
31
|
+
for (const chosen of selected) {
|
|
32
|
+
const sim = cosineNumber(vectors[index], vectors[chosen]);
|
|
33
|
+
if (sim > maxSim)
|
|
34
|
+
maxSim = sim;
|
|
35
|
+
}
|
|
36
|
+
const score = lambda * rel[index] - (1 - lambda) * maxSim;
|
|
37
|
+
if (score > bestScore || (score === bestScore && index < best)) {
|
|
38
|
+
bestScore = score;
|
|
39
|
+
best = index;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
selected.push(best);
|
|
43
|
+
remaining.delete(best);
|
|
44
|
+
}
|
|
45
|
+
return selected;
|
|
46
|
+
}
|
|
47
|
+
export function isCoverageSelectionEnabled(env = process.env) {
|
|
48
|
+
return env.MR_COVERAGE_SELECTION === "1";
|
|
49
|
+
}
|
|
50
|
+
export function coverageLambda(env = process.env) {
|
|
51
|
+
const parsed = Number(env.MR_COVERAGE_LAMBDA);
|
|
52
|
+
return Number.isFinite(parsed) ? parsed : 0.5;
|
|
53
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export interface CrossEncoderGateCandidate {
|
|
2
|
+
entry: {
|
|
3
|
+
text: string;
|
|
4
|
+
};
|
|
5
|
+
fusedScore?: number;
|
|
6
|
+
finalScore?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface RerankPair {
|
|
9
|
+
query: string;
|
|
10
|
+
passage: string;
|
|
11
|
+
}
|
|
12
|
+
export interface CrossEncoderScorer {
|
|
13
|
+
scorePairs(pairs: RerankPair[]): Promise<{
|
|
14
|
+
logits: number[];
|
|
15
|
+
}>;
|
|
16
|
+
}
|
|
17
|
+
export interface CrossEncoderGateOptions {
|
|
18
|
+
env?: NodeJS.ProcessEnv;
|
|
19
|
+
cacheDir?: string;
|
|
20
|
+
modelDir?: string;
|
|
21
|
+
topK?: number;
|
|
22
|
+
scorer?: CrossEncoderScorer;
|
|
23
|
+
logger?: Pick<Console, "warn" | "log">;
|
|
24
|
+
}
|
|
25
|
+
export interface ScoredCandidate<T extends CrossEncoderGateCandidate> {
|
|
26
|
+
candidate: T;
|
|
27
|
+
logit: number;
|
|
28
|
+
}
|
|
29
|
+
export interface ScoreCandidatesResult<T extends CrossEncoderGateCandidate> {
|
|
30
|
+
scored: ScoredCandidate<T>[];
|
|
31
|
+
timingMs: number;
|
|
32
|
+
}
|
|
33
|
+
export declare function isCragCrossEncoderGateEnabled(env?: NodeJS.ProcessEnv): boolean;
|
|
34
|
+
export declare function containsCjk(text: string): boolean;
|
|
35
|
+
export declare function cragGateThresholdForText(query: string, passage: string, env?: NodeJS.ProcessEnv): number;
|
|
36
|
+
export declare function cragGateTopK(env?: NodeJS.ProcessEnv): number;
|
|
37
|
+
export declare function scoreCandidates<T extends CrossEncoderGateCandidate>(query: string, candidates: T[], options?: CrossEncoderGateOptions): Promise<ScoreCandidatesResult<T> | null>;
|
|
38
|
+
export declare function applyCragCrossEncoderGate<T extends CrossEncoderGateCandidate>(query: string, candidates: T[], options?: CrossEncoderGateOptions): Promise<T[]>;
|
|
39
|
+
export declare function __setCragCrossEncoderScorerForTests(scorer: CrossEncoderScorer | null): void;
|
|
40
|
+
export declare function __resetCragCrossEncoderForTests(): void;
|