ahok-skill 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc +8 -0
- package/Dockerfile +59 -0
- package/RAW_SKILL.md +219 -0
- package/README.md +277 -0
- package/SKILL.md +58 -0
- package/bin/opm.js +268 -0
- package/data/openmemory.sqlite +0 -0
- package/data/openmemory.sqlite-shm +0 -0
- package/data/openmemory.sqlite-wal +0 -0
- package/dist/ai/graph.js +293 -0
- package/dist/ai/mcp.js +397 -0
- package/dist/cli.js +78 -0
- package/dist/core/cfg.js +87 -0
- package/dist/core/db.js +636 -0
- package/dist/core/memory.js +116 -0
- package/dist/core/migrate.js +227 -0
- package/dist/core/models.js +105 -0
- package/dist/core/telemetry.js +57 -0
- package/dist/core/types.js +2 -0
- package/dist/core/vector/postgres.js +52 -0
- package/dist/core/vector/valkey.js +246 -0
- package/dist/core/vector_store.js +2 -0
- package/dist/index.js +44 -0
- package/dist/memory/decay.js +301 -0
- package/dist/memory/embed.js +675 -0
- package/dist/memory/hsg.js +959 -0
- package/dist/memory/reflect.js +131 -0
- package/dist/memory/user_summary.js +99 -0
- package/dist/migrate.js +9 -0
- package/dist/ops/compress.js +255 -0
- package/dist/ops/dynamics.js +189 -0
- package/dist/ops/extract.js +333 -0
- package/dist/ops/ingest.js +214 -0
- package/dist/server/index.js +109 -0
- package/dist/server/middleware/auth.js +137 -0
- package/dist/server/routes/auth.js +186 -0
- package/dist/server/routes/compression.js +108 -0
- package/dist/server/routes/dashboard.js +399 -0
- package/dist/server/routes/docs.js +241 -0
- package/dist/server/routes/dynamics.js +312 -0
- package/dist/server/routes/ide.js +280 -0
- package/dist/server/routes/index.js +33 -0
- package/dist/server/routes/keys.js +132 -0
- package/dist/server/routes/langgraph.js +61 -0
- package/dist/server/routes/memory.js +213 -0
- package/dist/server/routes/sources.js +140 -0
- package/dist/server/routes/system.js +63 -0
- package/dist/server/routes/temporal.js +293 -0
- package/dist/server/routes/users.js +101 -0
- package/dist/server/routes/vercel.js +57 -0
- package/dist/server/server.js +211 -0
- package/dist/server.js +3 -0
- package/dist/sources/base.js +223 -0
- package/dist/sources/github.js +171 -0
- package/dist/sources/google_drive.js +166 -0
- package/dist/sources/google_sheets.js +112 -0
- package/dist/sources/google_slides.js +139 -0
- package/dist/sources/index.js +34 -0
- package/dist/sources/notion.js +165 -0
- package/dist/sources/onedrive.js +143 -0
- package/dist/sources/web_crawler.js +166 -0
- package/dist/temporal_graph/index.js +20 -0
- package/dist/temporal_graph/query.js +240 -0
- package/dist/temporal_graph/store.js +116 -0
- package/dist/temporal_graph/timeline.js +241 -0
- package/dist/temporal_graph/types.js +2 -0
- package/dist/utils/chunking.js +60 -0
- package/dist/utils/index.js +31 -0
- package/dist/utils/keyword.js +94 -0
- package/dist/utils/text.js +120 -0
- package/nodemon.json +7 -0
- package/package.json +50 -0
- package/references/api_reference.md +66 -0
- package/references/examples.md +45 -0
- package/src/ai/graph.ts +363 -0
- package/src/ai/mcp.ts +494 -0
- package/src/cli.ts +94 -0
- package/src/core/cfg.ts +110 -0
- package/src/core/db.ts +1052 -0
- package/src/core/memory.ts +99 -0
- package/src/core/migrate.ts +302 -0
- package/src/core/models.ts +107 -0
- package/src/core/telemetry.ts +47 -0
- package/src/core/types.ts +130 -0
- package/src/core/vector/postgres.ts +61 -0
- package/src/core/vector/valkey.ts +261 -0
- package/src/core/vector_store.ts +9 -0
- package/src/index.ts +5 -0
- package/src/memory/decay.ts +427 -0
- package/src/memory/embed.ts +707 -0
- package/src/memory/hsg.ts +1245 -0
- package/src/memory/reflect.ts +158 -0
- package/src/memory/user_summary.ts +110 -0
- package/src/migrate.ts +8 -0
- package/src/ops/compress.ts +296 -0
- package/src/ops/dynamics.ts +272 -0
- package/src/ops/extract.ts +360 -0
- package/src/ops/ingest.ts +286 -0
- package/src/server/index.ts +159 -0
- package/src/server/middleware/auth.ts +156 -0
- package/src/server/routes/auth.ts +223 -0
- package/src/server/routes/compression.ts +106 -0
- package/src/server/routes/dashboard.ts +420 -0
- package/src/server/routes/docs.ts +380 -0
- package/src/server/routes/dynamics.ts +516 -0
- package/src/server/routes/ide.ts +283 -0
- package/src/server/routes/index.ts +32 -0
- package/src/server/routes/keys.ts +131 -0
- package/src/server/routes/langgraph.ts +71 -0
- package/src/server/routes/memory.ts +440 -0
- package/src/server/routes/sources.ts +111 -0
- package/src/server/routes/system.ts +68 -0
- package/src/server/routes/temporal.ts +335 -0
- package/src/server/routes/users.ts +111 -0
- package/src/server/routes/vercel.ts +55 -0
- package/src/server/server.js +215 -0
- package/src/server.ts +1 -0
- package/src/sources/base.ts +257 -0
- package/src/sources/github.ts +156 -0
- package/src/sources/google_drive.ts +144 -0
- package/src/sources/google_sheets.ts +85 -0
- package/src/sources/google_slides.ts +115 -0
- package/src/sources/index.ts +19 -0
- package/src/sources/notion.ts +148 -0
- package/src/sources/onedrive.ts +131 -0
- package/src/sources/web_crawler.ts +161 -0
- package/src/temporal_graph/index.ts +4 -0
- package/src/temporal_graph/query.ts +299 -0
- package/src/temporal_graph/store.ts +156 -0
- package/src/temporal_graph/timeline.ts +319 -0
- package/src/temporal_graph/types.ts +41 -0
- package/src/utils/chunking.ts +66 -0
- package/src/utils/index.ts +25 -0
- package/src/utils/keyword.ts +137 -0
- package/src/utils/text.ts +115 -0
- package/tests/test_api_workspace_management.ts +413 -0
- package/tests/test_bulk_delete.ts +267 -0
- package/tests/test_omnibus.ts +166 -0
- package/tests/test_workspace_management.ts +278 -0
- package/tests/verify.ts +104 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ValkeyVectorStore = void 0;
|
|
7
|
+
const ioredis_1 = __importDefault(require("ioredis"));
|
|
8
|
+
const cfg_1 = require("../cfg");
|
|
9
|
+
const embed_1 = require("../../memory/embed");
|
|
10
|
+
class ValkeyVectorStore {
|
|
11
|
+
client;
|
|
12
|
+
constructor() {
|
|
13
|
+
this.client = new ioredis_1.default({
|
|
14
|
+
host: cfg_1.env.valkey_host || "localhost",
|
|
15
|
+
port: cfg_1.env.valkey_port || 6379,
|
|
16
|
+
password: cfg_1.env.valkey_password,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
getKey(id, sector) {
|
|
20
|
+
return `vec:${sector}:${id}`;
|
|
21
|
+
}
|
|
22
|
+
async storeVector(id, sector, vector, dim, user_id) {
|
|
23
|
+
const key = this.getKey(id, sector);
|
|
24
|
+
const buf = (0, embed_1.vectorToBuffer)(vector);
|
|
25
|
+
// Store as Hash: v (blob), dim (int), user_id (string)
|
|
26
|
+
await this.client.hset(key, {
|
|
27
|
+
v: buf,
|
|
28
|
+
dim: dim,
|
|
29
|
+
user_id: user_id || "anonymous",
|
|
30
|
+
id: id,
|
|
31
|
+
sector: sector
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
async deleteVector(id, sector) {
|
|
35
|
+
const key = this.getKey(id, sector);
|
|
36
|
+
await this.client.del(key);
|
|
37
|
+
}
|
|
38
|
+
async deleteVectors(id) {
|
|
39
|
+
// This is inefficient in Redis without an index on ID across sectors.
|
|
40
|
+
// We might need to track which sectors an ID has.
|
|
41
|
+
// For now, we can scan or just assume we know the sectors?
|
|
42
|
+
// The interface implies we delete ALL vectors for this ID.
|
|
43
|
+
// In Postgres we did `delete from vectors where id=$1`.
|
|
44
|
+
// In Redis, we might need to know the sectors.
|
|
45
|
+
// Or we can use `keys vec:*:${id}` which is slow.
|
|
46
|
+
// Better approach: maintain a set of sectors for each ID?
|
|
47
|
+
// Or just iterate over known sectors (from hsg.ts sectors list).
|
|
48
|
+
// I'll import `sectors` from `../../memory/hsg`? No, circular dependency risk.
|
|
49
|
+
// I'll use a scan for now as it's safer, or just accept it might be slow.
|
|
50
|
+
// Actually, `keys` is blocking. `scan` is better.
|
|
51
|
+
// But `id` is at the end of the key `vec:{sector}:{id}`.
|
|
52
|
+
// Pattern: `vec:*:${id}`.
|
|
53
|
+
let cursor = "0";
|
|
54
|
+
do {
|
|
55
|
+
const res = await this.client.scan(cursor, "MATCH", `vec:*:${id}`, "COUNT", 100);
|
|
56
|
+
cursor = res[0];
|
|
57
|
+
const keys = res[1];
|
|
58
|
+
if (keys.length)
|
|
59
|
+
await this.client.del(...keys);
|
|
60
|
+
} while (cursor !== "0");
|
|
61
|
+
}
|
|
62
|
+
async searchSimilar(sector, queryVec, topK) {
|
|
63
|
+
// Try to use FT.SEARCH if index exists.
|
|
64
|
+
// Index name assumption: `idx:{sector}`
|
|
65
|
+
// Query: `*=>[KNN {k} @v $blob AS score]`
|
|
66
|
+
const indexName = `idx:${sector}`;
|
|
67
|
+
const blob = (0, embed_1.vectorToBuffer)(queryVec);
|
|
68
|
+
try {
|
|
69
|
+
// Check if index exists (optional, or just try search)
|
|
70
|
+
// We'll try the search.
|
|
71
|
+
// FT.SEARCH idx:sector "*=>[KNN 10 @v $blob AS score]" PARAMS 2 blob "\x..." DIALECT 2
|
|
72
|
+
const res = await this.client.call("FT.SEARCH", indexName, `*=>[KNN ${topK} @v $blob AS score]`, "PARAMS", "2", "blob", blob, "DIALECT", "2");
|
|
73
|
+
// Parse result
|
|
74
|
+
// [total_results, key1, [field1, val1, ...], key2, ...]
|
|
75
|
+
// We need to parse the array.
|
|
76
|
+
const count = res[0];
|
|
77
|
+
const results = [];
|
|
78
|
+
for (let i = 1; i < res.length; i += 2) {
|
|
79
|
+
const key = res[i]; // e.g. vec:semantic:123
|
|
80
|
+
const fields = res[i + 1];
|
|
81
|
+
let id = "";
|
|
82
|
+
let score = 0;
|
|
83
|
+
// fields is array [k, v, k, v...]
|
|
84
|
+
for (let j = 0; j < fields.length; j += 2) {
|
|
85
|
+
const f = fields[j];
|
|
86
|
+
const v = fields[j + 1];
|
|
87
|
+
if (f === "id")
|
|
88
|
+
id = v;
|
|
89
|
+
if (f === "score")
|
|
90
|
+
score = 1 - (parseFloat(v) / 2); // Cosine distance to similarity?
|
|
91
|
+
// Valkey HNSW usually returns distance.
|
|
92
|
+
// If metric is COSINE, distance is 1 - cosine_sim.
|
|
93
|
+
// So sim = 1 - dist.
|
|
94
|
+
// Wait, standard RediSearch KNN distance depends on metric.
|
|
95
|
+
// Assuming COSINE metric.
|
|
96
|
+
}
|
|
97
|
+
// If id not in fields, extract from key
|
|
98
|
+
if (!id) {
|
|
99
|
+
const parts = key.split(":");
|
|
100
|
+
id = parts[parts.length - 1];
|
|
101
|
+
}
|
|
102
|
+
// If score not found (should be there as 'score'), default 0
|
|
103
|
+
// Actually, 'score' is the alias we gave.
|
|
104
|
+
// Wait, in FT.SEARCH response, the score is returned if we ask for it?
|
|
105
|
+
// Yes, "AS score" puts it in the fields.
|
|
106
|
+
// Correction: RediSearch returns distance by default for KNN?
|
|
107
|
+
// Yes, but we aliased it "AS score".
|
|
108
|
+
// So it should be in the fields as "score".
|
|
109
|
+
// Distance conversion:
|
|
110
|
+
// If distance is d, and we want cosine similarity.
|
|
111
|
+
// Cosine distance = 1 - cosine similarity.
|
|
112
|
+
// So similarity = 1 - distance.
|
|
113
|
+
results.push({ id, score: 1 - parseFloat(fields.find((f, idx) => f === "score" && idx % 2 === 0 ? false : fields[idx - 1] === "score") || "0") });
|
|
114
|
+
// The find logic above is tricky.
|
|
115
|
+
// Let's just loop.
|
|
116
|
+
}
|
|
117
|
+
// Fix score parsing loop
|
|
118
|
+
results.length = 0; // clear
|
|
119
|
+
for (let i = 1; i < res.length; i += 2) {
|
|
120
|
+
const key = res[i];
|
|
121
|
+
const fields = res[i + 1];
|
|
122
|
+
let id = "";
|
|
123
|
+
let dist = 0;
|
|
124
|
+
for (let j = 0; j < fields.length; j += 2) {
|
|
125
|
+
if (fields[j] === "id")
|
|
126
|
+
id = fields[j + 1];
|
|
127
|
+
if (fields[j] === "score")
|
|
128
|
+
dist = parseFloat(fields[j + 1]);
|
|
129
|
+
}
|
|
130
|
+
if (!id)
|
|
131
|
+
id = key.split(":").pop();
|
|
132
|
+
results.push({ id, score: 1 - dist });
|
|
133
|
+
}
|
|
134
|
+
return results;
|
|
135
|
+
}
|
|
136
|
+
catch (e) {
|
|
137
|
+
console.warn(`[Valkey] FT.SEARCH failed for ${sector}, falling back to scan (slow):`, e);
|
|
138
|
+
// Fallback: Scan all vectors in sector and compute cosine sim
|
|
139
|
+
// This is very slow but ensures correctness if index is missing.
|
|
140
|
+
let cursor = "0";
|
|
141
|
+
const allVecs = [];
|
|
142
|
+
do {
|
|
143
|
+
const res = await this.client.scan(cursor, "MATCH", `vec:${sector}:*`, "COUNT", 100);
|
|
144
|
+
cursor = res[0];
|
|
145
|
+
const keys = res[1];
|
|
146
|
+
if (keys.length) {
|
|
147
|
+
// Pipeline get all
|
|
148
|
+
const pipe = this.client.pipeline();
|
|
149
|
+
keys.forEach(k => pipe.hget(k, "v"));
|
|
150
|
+
const buffers = await pipe.exec();
|
|
151
|
+
buffers?.forEach((b, idx) => {
|
|
152
|
+
if (b && b[1]) {
|
|
153
|
+
const buf = b[1];
|
|
154
|
+
const id = keys[idx].split(":").pop();
|
|
155
|
+
allVecs.push({ id, vector: (0, embed_1.bufferToVector)(buf) });
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
} while (cursor !== "0");
|
|
160
|
+
const sims = allVecs.map(v => ({
|
|
161
|
+
id: v.id,
|
|
162
|
+
score: this.cosineSimilarity(queryVec, v.vector)
|
|
163
|
+
}));
|
|
164
|
+
sims.sort((a, b) => b.score - a.score);
|
|
165
|
+
return sims.slice(0, topK);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
cosineSimilarity(a, b) {
|
|
169
|
+
if (a.length !== b.length)
|
|
170
|
+
return 0;
|
|
171
|
+
let dot = 0, na = 0, nb = 0;
|
|
172
|
+
for (let i = 0; i < a.length; i++) {
|
|
173
|
+
dot += a[i] * b[i];
|
|
174
|
+
na += a[i] * a[i];
|
|
175
|
+
nb += b[i] * b[i];
|
|
176
|
+
}
|
|
177
|
+
return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
|
|
178
|
+
}
|
|
179
|
+
async getVector(id, sector) {
|
|
180
|
+
const key = this.getKey(id, sector);
|
|
181
|
+
const res = await this.client.hmget(key, "v", "dim");
|
|
182
|
+
if (!res[0])
|
|
183
|
+
return null;
|
|
184
|
+
return {
|
|
185
|
+
vector: (0, embed_1.bufferToVector)(res[0]),
|
|
186
|
+
dim: parseInt(res[1])
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
async getVectorsById(id) {
|
|
190
|
+
// Scan for vec:*:{id}
|
|
191
|
+
const results = [];
|
|
192
|
+
let cursor = "0";
|
|
193
|
+
do {
|
|
194
|
+
const res = await this.client.scan(cursor, "MATCH", `vec:*:${id}`, "COUNT", 100);
|
|
195
|
+
cursor = res[0];
|
|
196
|
+
const keys = res[1];
|
|
197
|
+
if (keys.length) {
|
|
198
|
+
const pipe = this.client.pipeline();
|
|
199
|
+
keys.forEach(k => pipe.hmget(k, "v", "dim"));
|
|
200
|
+
const res = await pipe.exec();
|
|
201
|
+
res?.forEach((r, idx) => {
|
|
202
|
+
if (r && r[1]) {
|
|
203
|
+
const [v, dim] = r[1];
|
|
204
|
+
const key = keys[idx];
|
|
205
|
+
const parts = key.split(":");
|
|
206
|
+
const sector = parts[1];
|
|
207
|
+
results.push({
|
|
208
|
+
sector,
|
|
209
|
+
vector: (0, embed_1.bufferToVector)(v),
|
|
210
|
+
dim: parseInt(dim)
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
} while (cursor !== "0");
|
|
216
|
+
return results;
|
|
217
|
+
}
|
|
218
|
+
async getVectorsBySector(sector) {
|
|
219
|
+
const results = [];
|
|
220
|
+
let cursor = "0";
|
|
221
|
+
do {
|
|
222
|
+
const res = await this.client.scan(cursor, "MATCH", `vec:${sector}:*`, "COUNT", 100);
|
|
223
|
+
cursor = res[0];
|
|
224
|
+
const keys = res[1];
|
|
225
|
+
if (keys.length) {
|
|
226
|
+
const pipe = this.client.pipeline();
|
|
227
|
+
keys.forEach(k => pipe.hmget(k, "v", "dim"));
|
|
228
|
+
const res = await pipe.exec();
|
|
229
|
+
res?.forEach((r, idx) => {
|
|
230
|
+
if (r && r[1]) {
|
|
231
|
+
const [v, dim] = r[1];
|
|
232
|
+
const key = keys[idx];
|
|
233
|
+
const id = key.split(":").pop();
|
|
234
|
+
results.push({
|
|
235
|
+
id,
|
|
236
|
+
vector: (0, embed_1.bufferToVector)(v),
|
|
237
|
+
dim: parseInt(dim)
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
} while (cursor !== "0");
|
|
243
|
+
return results;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
exports.ValkeyVectorStore = ValkeyVectorStore;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
19
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
20
|
+
};
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.sources = void 0;
|
|
40
|
+
// Export core functionality for use as a package
|
|
41
|
+
__exportStar(require("./core/memory"), exports);
|
|
42
|
+
__exportStar(require("./server/index"), exports);
|
|
43
|
+
__exportStar(require("./ops/ingest"), exports);
|
|
44
|
+
exports.sources = __importStar(require("./sources"));
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.on_query_hit = exports.apply_decay = exports.dec_q = exports.inc_q = void 0;
|
|
4
|
+
const db_1 = require("../core/db");
|
|
5
|
+
const utils_1 = require("../utils");
|
|
6
|
+
const cfg_1 = require("../core/cfg");
|
|
7
|
+
const parse_int = (x, d) => Number.isFinite(+x) ? Math.floor(+x) : d;
|
|
8
|
+
const parse_f = (x, d) => (Number.isFinite(+x) ? +x : d);
|
|
9
|
+
const parse_bool = (x, d) => x === "true" ? true : x === "false" ? false : d;
|
|
10
|
+
const clamp_f = (v, a, b) => Math.min(b, Math.max(a, v));
|
|
11
|
+
const clamp_i = (v, a, b) => Math.min(b, Math.max(a, Math.floor(v)));
|
|
12
|
+
const tick = () => new Promise((r) => setImmediate(r));
|
|
13
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
14
|
+
const mean = (arr) => arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : 0;
|
|
15
|
+
const l2 = (v) => Math.sqrt(v.reduce((s, x) => s + x * x, 0));
|
|
16
|
+
const normalize = (v) => {
|
|
17
|
+
const n = l2(v) || 1;
|
|
18
|
+
for (let i = 0; i < v.length; i++)
|
|
19
|
+
v[i] /= n;
|
|
20
|
+
return v;
|
|
21
|
+
};
|
|
22
|
+
const chunkz = (arr, k) => {
|
|
23
|
+
const n = Math.max(1, k | 0), out = Array.from({ length: n }, () => []);
|
|
24
|
+
for (let i = 0; i < arr.length; i++)
|
|
25
|
+
out[i % n].push(arr[i]);
|
|
26
|
+
return out;
|
|
27
|
+
};
|
|
28
|
+
const make_decay_cfg = () => ({
|
|
29
|
+
threads: parse_int(process.env.OM_DECAY_THREADS, 3),
|
|
30
|
+
cold_threshold: parse_f(process.env.OM_DECAY_COLD_THRESHOLD, 0.25),
|
|
31
|
+
reinforce_on_query: parse_bool(process.env.OM_DECAY_REINFORCE_ON_QUERY, true),
|
|
32
|
+
regeneration_enabled: parse_bool(process.env.OM_REGENERATION_ENABLED, true),
|
|
33
|
+
max_vec_dim: parse_int(process.env.OM_MAX_VECTOR_DIM, cfg_1.env.vec_dim || 1536),
|
|
34
|
+
min_vec_dim: parse_int(process.env.OM_MIN_VECTOR_DIM, 64),
|
|
35
|
+
summary_layers: clamp_i(parse_int(process.env.OM_SUMMARY_LAYERS, 3), 1, 3),
|
|
36
|
+
lambda_hot: 0.005,
|
|
37
|
+
lambda_warm: 0.02,
|
|
38
|
+
lambda_cold: 0.05,
|
|
39
|
+
time_unit_ms: 86_400_000,
|
|
40
|
+
});
|
|
41
|
+
const cfg = make_decay_cfg();
|
|
42
|
+
let active_q = 0;
|
|
43
|
+
let last_decay = 0;
|
|
44
|
+
const cooldown = 60000;
|
|
45
|
+
const inc_q = () => active_q++;
|
|
46
|
+
exports.inc_q = inc_q;
|
|
47
|
+
const dec_q = () => active_q--;
|
|
48
|
+
exports.dec_q = dec_q;
|
|
49
|
+
const pick_tier = (m, now_ts) => {
|
|
50
|
+
const dt = Math.max(0, now_ts - (m.last_seen_at || m.updated_at || now_ts));
|
|
51
|
+
const recent = dt < 6 * 86_400_000;
|
|
52
|
+
const high = (m.coactivations || 0) > 5 || (m.salience || 0) > 0.7;
|
|
53
|
+
if (recent && high)
|
|
54
|
+
return "hot";
|
|
55
|
+
if (recent || (m.salience || 0) > 0.4)
|
|
56
|
+
return "warm";
|
|
57
|
+
return "cold";
|
|
58
|
+
};
|
|
59
|
+
const compress_vector = (vec, f, min_dim = 64, max_dim = 1536) => {
|
|
60
|
+
const src = vec.length ? vec : [1];
|
|
61
|
+
const tgt_dim = Math.max(min_dim, Math.min(max_dim, Math.floor(src.length * clamp_f(f, 0.0, 1.0))));
|
|
62
|
+
const dim = Math.max(min_dim, Math.min(src.length, tgt_dim));
|
|
63
|
+
if (dim >= src.length)
|
|
64
|
+
return src.slice(0);
|
|
65
|
+
const pooled = [];
|
|
66
|
+
const bucket = Math.ceil(src.length / dim);
|
|
67
|
+
for (let i = 0; i < src.length; i += bucket)
|
|
68
|
+
pooled.push(mean(src.slice(i, i + bucket)));
|
|
69
|
+
normalize(pooled);
|
|
70
|
+
return pooled;
|
|
71
|
+
};
|
|
72
|
+
const compress_summary = (txt, f, layers = 3) => {
|
|
73
|
+
const t = (txt || "").trim();
|
|
74
|
+
if (!t)
|
|
75
|
+
return "";
|
|
76
|
+
const lay = clamp_i(layers, 1, 3);
|
|
77
|
+
const trunc = (s, n) => s.length <= n ? s : s.slice(0, n - 1).trimEnd() + "…";
|
|
78
|
+
const sumz = (s) => summarize_quick(s);
|
|
79
|
+
const keys = (s, k = 5) => top_keywords(s, k).join(" ");
|
|
80
|
+
if (f > 0.8)
|
|
81
|
+
return trunc(t, 200);
|
|
82
|
+
if (f > 0.4)
|
|
83
|
+
return trunc(sumz(t), lay >= 2 ? 80 : 200);
|
|
84
|
+
return keys(t, lay >= 3 ? 5 : 3);
|
|
85
|
+
};
|
|
86
|
+
const fingerprint_mem = (m) => {
|
|
87
|
+
const base = (m.id + "|" + (m.summary || m.content || "")).trim();
|
|
88
|
+
const vec = hash_to_vec(base, 32);
|
|
89
|
+
normalize(vec);
|
|
90
|
+
const summary = top_keywords(m.summary || m.content || "", 3).join(" ");
|
|
91
|
+
return { vector: vec, summary };
|
|
92
|
+
};
|
|
93
|
+
const hash_to_vec = (s, d = 32) => {
|
|
94
|
+
let h = 2166136261 >>> 0;
|
|
95
|
+
for (let i = 0; i < s.length; i++) {
|
|
96
|
+
h ^= s.charCodeAt(i);
|
|
97
|
+
h = Math.imul(h, 16777619) >>> 0;
|
|
98
|
+
}
|
|
99
|
+
const out = new Array(Math.max(2, d | 0)).fill(0);
|
|
100
|
+
let x = h || 1;
|
|
101
|
+
for (let i = 0; i < out.length; i++) {
|
|
102
|
+
x ^= x << 13;
|
|
103
|
+
x ^= x >>> 17;
|
|
104
|
+
x ^= x << 5;
|
|
105
|
+
out[i] = ((x >>> 0) / 0xffffffff) * 2 - 1;
|
|
106
|
+
}
|
|
107
|
+
normalize(out);
|
|
108
|
+
return out;
|
|
109
|
+
};
|
|
110
|
+
const summarize_quick = (t) => {
|
|
111
|
+
const sents = t.split(/(?<=[.!?])\s+/).filter(Boolean);
|
|
112
|
+
if (!sents.length)
|
|
113
|
+
return t;
|
|
114
|
+
const score = (s) => top_keywords(s, 6).length + Math.min(3, s.match(/[,;:]/g)?.length || 0);
|
|
115
|
+
const top = sents
|
|
116
|
+
.map((s, i) => ({ s, i, sc: score(s) }))
|
|
117
|
+
.sort((a, b) => b.sc - a.sc || a.i - b.i)
|
|
118
|
+
.slice(0, Math.min(3, Math.ceil(sents.length / 3)))
|
|
119
|
+
.sort((a, b) => a.i - b.i)
|
|
120
|
+
.map((x) => x.s)
|
|
121
|
+
.join(" ");
|
|
122
|
+
return top || sents[0];
|
|
123
|
+
};
|
|
124
|
+
const stop = new Set([
|
|
125
|
+
"the",
|
|
126
|
+
"a",
|
|
127
|
+
"an",
|
|
128
|
+
"to",
|
|
129
|
+
"of",
|
|
130
|
+
"and",
|
|
131
|
+
"or",
|
|
132
|
+
"in",
|
|
133
|
+
"on",
|
|
134
|
+
"for",
|
|
135
|
+
"with",
|
|
136
|
+
"at",
|
|
137
|
+
"by",
|
|
138
|
+
"is",
|
|
139
|
+
"it",
|
|
140
|
+
"be",
|
|
141
|
+
"as",
|
|
142
|
+
"are",
|
|
143
|
+
"was",
|
|
144
|
+
"were",
|
|
145
|
+
"from",
|
|
146
|
+
"that",
|
|
147
|
+
"this",
|
|
148
|
+
"these",
|
|
149
|
+
"those",
|
|
150
|
+
"but",
|
|
151
|
+
"if",
|
|
152
|
+
"then",
|
|
153
|
+
"so",
|
|
154
|
+
"than",
|
|
155
|
+
"into",
|
|
156
|
+
"over",
|
|
157
|
+
"under",
|
|
158
|
+
"about",
|
|
159
|
+
"via",
|
|
160
|
+
"vs",
|
|
161
|
+
"not",
|
|
162
|
+
]);
|
|
163
|
+
const top_keywords = (t, k = 5) => {
|
|
164
|
+
const words = (t.toLowerCase().match(/[a-z0-9]+/g) || []).filter((w) => !stop.has(w));
|
|
165
|
+
if (!words.length)
|
|
166
|
+
return [];
|
|
167
|
+
const freq = new Map();
|
|
168
|
+
for (const w of words)
|
|
169
|
+
freq.set(w, (freq.get(w) || 0) + 1);
|
|
170
|
+
return Array.from(freq.entries())
|
|
171
|
+
.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))
|
|
172
|
+
.slice(0, k)
|
|
173
|
+
.map(([w]) => w);
|
|
174
|
+
};
|
|
175
|
+
const apply_decay = async () => {
|
|
176
|
+
if (active_q > 0) {
|
|
177
|
+
console.log(`[decay] skipped - ${active_q} active queries`);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
const now_ts = Date.now();
|
|
181
|
+
if (now_ts - last_decay < cooldown) {
|
|
182
|
+
console.log(`[decay] skipped - cooldown active (${((cooldown - (now_ts - last_decay)) / 1000).toFixed(0)}s remaining)`);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
last_decay = now_ts;
|
|
186
|
+
const t0 = performance.now();
|
|
187
|
+
const segments = await db_1.q.get_segments.all();
|
|
188
|
+
let tot_proc = 0, tot_chg = 0, tot_comp = 0, tot_fp = 0;
|
|
189
|
+
const tier_counts = { hot: 0, warm: 0, cold: 0 };
|
|
190
|
+
for (const seg of segments) {
|
|
191
|
+
const segment = seg.segment;
|
|
192
|
+
const rows = await (0, db_1.all_async)("select id,content,summary,salience,decay_lambda,last_seen_at,updated_at,primary_sector,coactivations from memories where segment=?", [segment]);
|
|
193
|
+
const decay_ratio = cfg_1.env.decay_ratio;
|
|
194
|
+
const batch_sz = Math.max(1, Math.floor(rows.length * decay_ratio));
|
|
195
|
+
const start_idx = Math.floor(Math.random() * Math.max(1, rows.length - batch_sz + 1));
|
|
196
|
+
const batch = rows.slice(start_idx, start_idx + batch_sz);
|
|
197
|
+
const parts = chunkz(batch, cfg.threads);
|
|
198
|
+
await Promise.all(parts.map(async (part) => {
|
|
199
|
+
for (const m of part) {
|
|
200
|
+
const tier = pick_tier(m, now_ts);
|
|
201
|
+
tier_counts[tier]++;
|
|
202
|
+
const lam = tier === "hot"
|
|
203
|
+
? cfg.lambda_hot
|
|
204
|
+
: tier === "warm"
|
|
205
|
+
? cfg.lambda_warm
|
|
206
|
+
: cfg.lambda_cold;
|
|
207
|
+
const dt = Math.max(0, (now_ts - (m.last_seen_at || m.updated_at)) /
|
|
208
|
+
cfg.time_unit_ms);
|
|
209
|
+
const act = Math.max(0, m.coactivations || 0);
|
|
210
|
+
const sal = clamp_f((m.salience || 0.5) * (1 + Math.log1p(act)), 0, 1);
|
|
211
|
+
const f = Math.exp(-lam * (dt / (sal + 0.1)));
|
|
212
|
+
let new_sal = clamp_f(sal * f, 0, 1);
|
|
213
|
+
let changed = Math.abs(new_sal - m.salience) > 0.001;
|
|
214
|
+
let compressed = false;
|
|
215
|
+
let fingerprinted = false;
|
|
216
|
+
if (f < 0.7) {
|
|
217
|
+
const sector = m.primary_sector || "semantic";
|
|
218
|
+
const vec_row = await db_1.vector_store.getVector(m.id, sector);
|
|
219
|
+
if (vec_row && vec_row.vector) {
|
|
220
|
+
const vec = typeof vec_row.vector === "string"
|
|
221
|
+
? JSON.parse(vec_row.vector)
|
|
222
|
+
: vec_row.vector;
|
|
223
|
+
const before_len = Array.isArray(vec)
|
|
224
|
+
? vec.length
|
|
225
|
+
: 0;
|
|
226
|
+
if (before_len > 0) {
|
|
227
|
+
const new_vec = compress_vector(vec, f, cfg.min_vec_dim, cfg.max_vec_dim);
|
|
228
|
+
const new_summary = compress_summary(m.summary || m.content || "", f, cfg.summary_layers);
|
|
229
|
+
if (new_vec.length < before_len) {
|
|
230
|
+
await db_1.vector_store.storeVector(m.id, sector, new_vec, new_vec.length);
|
|
231
|
+
compressed = true;
|
|
232
|
+
tot_comp++;
|
|
233
|
+
}
|
|
234
|
+
if (new_summary !== (m.summary || "")) {
|
|
235
|
+
await (0, db_1.run_async)("update memories set summary=? where id=?", [new_summary, m.id]);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
changed = true;
|
|
240
|
+
}
|
|
241
|
+
if (f < Math.max(0.3, cfg.cold_threshold)) {
|
|
242
|
+
const sector = m.primary_sector || "semantic";
|
|
243
|
+
const fp = fingerprint_mem(m);
|
|
244
|
+
await db_1.vector_store.storeVector(m.id, sector, fp.vector, fp.vector.length);
|
|
245
|
+
await (0, db_1.run_async)("update memories set summary=? where id=?", [fp.summary, m.id]);
|
|
246
|
+
fingerprinted = true;
|
|
247
|
+
tot_fp++;
|
|
248
|
+
changed = true;
|
|
249
|
+
}
|
|
250
|
+
if (changed) {
|
|
251
|
+
await (0, db_1.run_async)(`update ${db_1.memories_table} set salience=?,updated_at=? where id=?`, [new_sal, (0, utils_1.now)(), m.id]);
|
|
252
|
+
tot_chg++;
|
|
253
|
+
}
|
|
254
|
+
tot_proc++;
|
|
255
|
+
await tick();
|
|
256
|
+
}
|
|
257
|
+
}));
|
|
258
|
+
if (seg !== segments[segments.length - 1]) {
|
|
259
|
+
await sleep(cfg_1.env.decay_sleep_ms);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
const tot = performance.now() - t0;
|
|
263
|
+
// Use stderr for debug output to avoid breaking MCP JSON-RPC protocol
|
|
264
|
+
console.error(`[decay-2.0] ${tot_chg}/${tot_proc} | tiers: hot=${tier_counts.hot} warm=${tier_counts.warm} cold=${tier_counts.cold} | compressed=${tot_comp} fingerprinted=${tot_fp} | ${tot.toFixed(1)}ms across ${segments.length} segments`);
|
|
265
|
+
};
|
|
266
|
+
exports.apply_decay = apply_decay;
|
|
267
|
+
const on_query_hit = async (mem_id, sector, reembed) => {
|
|
268
|
+
if (!cfg.regeneration_enabled && !cfg.reinforce_on_query)
|
|
269
|
+
return;
|
|
270
|
+
const m = await db_1.q.get_mem.get(mem_id);
|
|
271
|
+
if (!m)
|
|
272
|
+
return;
|
|
273
|
+
let updated = false;
|
|
274
|
+
if (cfg.regeneration_enabled && reembed) {
|
|
275
|
+
const vec_row = await db_1.vector_store.getVector(mem_id, sector);
|
|
276
|
+
if (vec_row && vec_row.vector) {
|
|
277
|
+
const vec = typeof vec_row.vector === "string"
|
|
278
|
+
? JSON.parse(vec_row.vector)
|
|
279
|
+
: vec_row.vector;
|
|
280
|
+
if (Array.isArray(vec) && vec.length <= 64) {
|
|
281
|
+
try {
|
|
282
|
+
const base = m.summary || m.content || "";
|
|
283
|
+
const new_vec = await reembed(base);
|
|
284
|
+
await db_1.vector_store.storeVector(mem_id, sector, new_vec, new_vec.length);
|
|
285
|
+
updated = true;
|
|
286
|
+
}
|
|
287
|
+
catch (e) { }
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
if (cfg.reinforce_on_query) {
|
|
292
|
+
const new_sal = clamp_f((m.salience || 0.5) + 0.5, 0, 1);
|
|
293
|
+
await (0, db_1.run_async)(`update ${db_1.memories_table} set salience=?,last_seen_at=? where id=?`, [new_sal, (0, utils_1.now)(), mem_id]);
|
|
294
|
+
updated = true;
|
|
295
|
+
}
|
|
296
|
+
if (updated) {
|
|
297
|
+
// Use stderr for debug output to avoid breaking MCP JSON-RPC protocol
|
|
298
|
+
console.error(`[decay-2.0] regenerated/reinforced memory ${mem_id}`);
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
exports.on_query_hit = on_query_hit;
|