ahok-skill 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc +8 -0
- package/Dockerfile +59 -0
- package/RAW_SKILL.md +219 -0
- package/README.md +277 -0
- package/SKILL.md +58 -0
- package/bin/opm.js +268 -0
- package/data/openmemory.sqlite +0 -0
- package/data/openmemory.sqlite-shm +0 -0
- package/data/openmemory.sqlite-wal +0 -0
- package/dist/ai/graph.js +293 -0
- package/dist/ai/mcp.js +397 -0
- package/dist/cli.js +78 -0
- package/dist/core/cfg.js +87 -0
- package/dist/core/db.js +636 -0
- package/dist/core/memory.js +116 -0
- package/dist/core/migrate.js +227 -0
- package/dist/core/models.js +105 -0
- package/dist/core/telemetry.js +57 -0
- package/dist/core/types.js +2 -0
- package/dist/core/vector/postgres.js +52 -0
- package/dist/core/vector/valkey.js +246 -0
- package/dist/core/vector_store.js +2 -0
- package/dist/index.js +44 -0
- package/dist/memory/decay.js +301 -0
- package/dist/memory/embed.js +675 -0
- package/dist/memory/hsg.js +959 -0
- package/dist/memory/reflect.js +131 -0
- package/dist/memory/user_summary.js +99 -0
- package/dist/migrate.js +9 -0
- package/dist/ops/compress.js +255 -0
- package/dist/ops/dynamics.js +189 -0
- package/dist/ops/extract.js +333 -0
- package/dist/ops/ingest.js +214 -0
- package/dist/server/index.js +109 -0
- package/dist/server/middleware/auth.js +137 -0
- package/dist/server/routes/auth.js +186 -0
- package/dist/server/routes/compression.js +108 -0
- package/dist/server/routes/dashboard.js +399 -0
- package/dist/server/routes/docs.js +241 -0
- package/dist/server/routes/dynamics.js +312 -0
- package/dist/server/routes/ide.js +280 -0
- package/dist/server/routes/index.js +33 -0
- package/dist/server/routes/keys.js +132 -0
- package/dist/server/routes/langgraph.js +61 -0
- package/dist/server/routes/memory.js +213 -0
- package/dist/server/routes/sources.js +140 -0
- package/dist/server/routes/system.js +63 -0
- package/dist/server/routes/temporal.js +293 -0
- package/dist/server/routes/users.js +101 -0
- package/dist/server/routes/vercel.js +57 -0
- package/dist/server/server.js +211 -0
- package/dist/server.js +3 -0
- package/dist/sources/base.js +223 -0
- package/dist/sources/github.js +171 -0
- package/dist/sources/google_drive.js +166 -0
- package/dist/sources/google_sheets.js +112 -0
- package/dist/sources/google_slides.js +139 -0
- package/dist/sources/index.js +34 -0
- package/dist/sources/notion.js +165 -0
- package/dist/sources/onedrive.js +143 -0
- package/dist/sources/web_crawler.js +166 -0
- package/dist/temporal_graph/index.js +20 -0
- package/dist/temporal_graph/query.js +240 -0
- package/dist/temporal_graph/store.js +116 -0
- package/dist/temporal_graph/timeline.js +241 -0
- package/dist/temporal_graph/types.js +2 -0
- package/dist/utils/chunking.js +60 -0
- package/dist/utils/index.js +31 -0
- package/dist/utils/keyword.js +94 -0
- package/dist/utils/text.js +120 -0
- package/nodemon.json +7 -0
- package/package.json +50 -0
- package/references/api_reference.md +66 -0
- package/references/examples.md +45 -0
- package/src/ai/graph.ts +363 -0
- package/src/ai/mcp.ts +494 -0
- package/src/cli.ts +94 -0
- package/src/core/cfg.ts +110 -0
- package/src/core/db.ts +1052 -0
- package/src/core/memory.ts +99 -0
- package/src/core/migrate.ts +302 -0
- package/src/core/models.ts +107 -0
- package/src/core/telemetry.ts +47 -0
- package/src/core/types.ts +130 -0
- package/src/core/vector/postgres.ts +61 -0
- package/src/core/vector/valkey.ts +261 -0
- package/src/core/vector_store.ts +9 -0
- package/src/index.ts +5 -0
- package/src/memory/decay.ts +427 -0
- package/src/memory/embed.ts +707 -0
- package/src/memory/hsg.ts +1245 -0
- package/src/memory/reflect.ts +158 -0
- package/src/memory/user_summary.ts +110 -0
- package/src/migrate.ts +8 -0
- package/src/ops/compress.ts +296 -0
- package/src/ops/dynamics.ts +272 -0
- package/src/ops/extract.ts +360 -0
- package/src/ops/ingest.ts +286 -0
- package/src/server/index.ts +159 -0
- package/src/server/middleware/auth.ts +156 -0
- package/src/server/routes/auth.ts +223 -0
- package/src/server/routes/compression.ts +106 -0
- package/src/server/routes/dashboard.ts +420 -0
- package/src/server/routes/docs.ts +380 -0
- package/src/server/routes/dynamics.ts +516 -0
- package/src/server/routes/ide.ts +283 -0
- package/src/server/routes/index.ts +32 -0
- package/src/server/routes/keys.ts +131 -0
- package/src/server/routes/langgraph.ts +71 -0
- package/src/server/routes/memory.ts +440 -0
- package/src/server/routes/sources.ts +111 -0
- package/src/server/routes/system.ts +68 -0
- package/src/server/routes/temporal.ts +335 -0
- package/src/server/routes/users.ts +111 -0
- package/src/server/routes/vercel.ts +55 -0
- package/src/server/server.js +215 -0
- package/src/server.ts +1 -0
- package/src/sources/base.ts +257 -0
- package/src/sources/github.ts +156 -0
- package/src/sources/google_drive.ts +144 -0
- package/src/sources/google_sheets.ts +85 -0
- package/src/sources/google_slides.ts +115 -0
- package/src/sources/index.ts +19 -0
- package/src/sources/notion.ts +148 -0
- package/src/sources/onedrive.ts +131 -0
- package/src/sources/web_crawler.ts +161 -0
- package/src/temporal_graph/index.ts +4 -0
- package/src/temporal_graph/query.ts +299 -0
- package/src/temporal_graph/store.ts +156 -0
- package/src/temporal_graph/timeline.ts +319 -0
- package/src/temporal_graph/types.ts +41 -0
- package/src/utils/chunking.ts +66 -0
- package/src/utils/index.ts +25 -0
- package/src/utils/keyword.ts +137 -0
- package/src/utils/text.ts +115 -0
- package/tests/test_api_workspace_management.ts +413 -0
- package/tests/test_bulk_delete.ts +267 -0
- package/tests/test_omnibus.ts +166 -0
- package/tests/test_workspace_management.ts +278 -0
- package/tests/verify.ts +104 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,675 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.getEmbeddingInfo = exports.getEmbeddingProvider = exports.embed = exports.bufferToVector = exports.vectorToBuffer = exports.cosineSimilarity = exports.emb_dim = void 0;
|
|
37
|
+
exports.embedForSector = embedForSector;
|
|
38
|
+
exports.embedQueryForAllSectors = embedQueryForAllSectors;
|
|
39
|
+
exports.gen_syn_emb = gen_syn_emb;
|
|
40
|
+
exports.embedMultiSector = embedMultiSector;
|
|
41
|
+
const cfg_1 = require("../core/cfg");
|
|
42
|
+
const models_1 = require("../core/models");
|
|
43
|
+
const hsg_1 = require("./hsg");
|
|
44
|
+
const db_1 = require("../core/db");
|
|
45
|
+
const text_1 = require("../utils/text");
|
|
46
|
+
const client_bedrock_runtime_1 = require("@aws-sdk/client-bedrock-runtime");
|
|
47
|
+
let gem_q = Promise.resolve();
|
|
48
|
+
const emb_dim = () => cfg_1.env.vec_dim;
|
|
49
|
+
exports.emb_dim = emb_dim;
|
|
50
|
+
// Fetch with timeout to prevent hanging requests and enable fallback chain
|
|
51
|
+
const EMBED_TIMEOUT_MS = Number(process.env.OM_EMBED_TIMEOUT_MS) || 30000;
|
|
52
|
+
async function fetchWithTimeout(url, options) {
|
|
53
|
+
const controller = new AbortController();
|
|
54
|
+
const timeoutId = setTimeout(() => controller.abort(), EMBED_TIMEOUT_MS);
|
|
55
|
+
try {
|
|
56
|
+
return await fetch(url, { ...options, signal: controller.signal });
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
clearTimeout(timeoutId);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const compress_vec = (v, td) => {
|
|
63
|
+
if (v.length <= td)
|
|
64
|
+
return v;
|
|
65
|
+
const c = new Float32Array(td), bs = v.length / td;
|
|
66
|
+
for (let i = 0; i < td; i++) {
|
|
67
|
+
const s = Math.floor(i * bs), e = Math.floor((i + 1) * bs);
|
|
68
|
+
let sum = 0, cnt = 0;
|
|
69
|
+
for (let j = s; j < e && j < v.length; j++) {
|
|
70
|
+
sum += v[j];
|
|
71
|
+
cnt++;
|
|
72
|
+
}
|
|
73
|
+
c[i] = cnt > 0 ? sum / cnt : 0;
|
|
74
|
+
}
|
|
75
|
+
let n = 0;
|
|
76
|
+
for (let i = 0; i < td; i++)
|
|
77
|
+
n += c[i] * c[i];
|
|
78
|
+
n = Math.sqrt(n);
|
|
79
|
+
if (n > 0)
|
|
80
|
+
for (let i = 0; i < td; i++)
|
|
81
|
+
c[i] /= n;
|
|
82
|
+
return Array.from(c);
|
|
83
|
+
};
|
|
84
|
+
const fuse_vecs = (syn, sem) => {
|
|
85
|
+
const synLength = syn.length;
|
|
86
|
+
const semLength = sem.length;
|
|
87
|
+
const totalLength = synLength + semLength;
|
|
88
|
+
const f = new Array(totalLength);
|
|
89
|
+
let sumOfSquares = 0;
|
|
90
|
+
for (let i = 0; i < synLength; i++) {
|
|
91
|
+
const val = syn[i] * 0.6;
|
|
92
|
+
f[i] = val;
|
|
93
|
+
sumOfSquares += val * val;
|
|
94
|
+
}
|
|
95
|
+
for (let i = 0; i < semLength; i++) {
|
|
96
|
+
const val = sem[i] * 0.4;
|
|
97
|
+
f[synLength + i] = val;
|
|
98
|
+
sumOfSquares += val * val;
|
|
99
|
+
}
|
|
100
|
+
if (sumOfSquares > 0) {
|
|
101
|
+
const norm = Math.sqrt(sumOfSquares);
|
|
102
|
+
for (let i = 0; i < totalLength; i++) {
|
|
103
|
+
f[i] /= norm;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return f;
|
|
107
|
+
};
|
|
108
|
+
async function embedForSector(t, s) {
|
|
109
|
+
console.error(`[EMBED] Provider: ${cfg_1.env.emb_kind}, Tier: ${cfg_1.tier}, Sector: ${s}`);
|
|
110
|
+
if (!hsg_1.sector_configs[s])
|
|
111
|
+
throw new Error(`Unknown sector: ${s}`);
|
|
112
|
+
if (cfg_1.tier === "hybrid")
|
|
113
|
+
return gen_syn_emb(t, s);
|
|
114
|
+
if (cfg_1.tier === "smart" && cfg_1.env.emb_kind !== "synthetic") {
|
|
115
|
+
const syn = gen_syn_emb(t, s), sem = await get_sem_emb(t, s), comp = compress_vec(sem, 128);
|
|
116
|
+
return fuse_vecs(syn, comp);
|
|
117
|
+
}
|
|
118
|
+
if (cfg_1.tier === "fast")
|
|
119
|
+
return gen_syn_emb(t, s);
|
|
120
|
+
return await get_sem_emb(t, s);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Batch embed query text for ALL sectors in one API call.
|
|
124
|
+
* This significantly improves query performance by reducing 5 sequential
|
|
125
|
+
* API calls to a single batched call (~4.5x faster for deep tier).
|
|
126
|
+
*/
|
|
127
|
+
async function embedQueryForAllSectors(query, sectors) {
|
|
128
|
+
// For hybrid/fast tiers, use synthetic embeddings (already fast)
|
|
129
|
+
if (cfg_1.tier === "hybrid" || cfg_1.tier === "fast") {
|
|
130
|
+
const result = {};
|
|
131
|
+
for (const s of sectors)
|
|
132
|
+
result[s] = gen_syn_emb(query, s);
|
|
133
|
+
return result;
|
|
134
|
+
}
|
|
135
|
+
// For deep/smart tiers with Gemini, batch all sectors in ONE API call
|
|
136
|
+
if (cfg_1.env.emb_kind === "gemini" && cfg_1.env.gemini_key) {
|
|
137
|
+
try {
|
|
138
|
+
const txts = {};
|
|
139
|
+
for (const s of sectors)
|
|
140
|
+
txts[s] = query;
|
|
141
|
+
return await emb_gemini(txts);
|
|
142
|
+
}
|
|
143
|
+
catch (e) {
|
|
144
|
+
console.error(`[EMBED] Gemini batch failed, falling back to sequential: ${e}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
// Fallback: sequential embedding for each sector
|
|
148
|
+
const result = {};
|
|
149
|
+
for (const s of sectors)
|
|
150
|
+
result[s] = await embedForSector(query, s);
|
|
151
|
+
return result;
|
|
152
|
+
}
|
|
153
|
+
// Embed with a specific provider (throws on failure)
|
|
154
|
+
async function embed_with_provider(provider, t, s) {
|
|
155
|
+
switch (provider) {
|
|
156
|
+
case "openai":
|
|
157
|
+
return await emb_openai(t, s);
|
|
158
|
+
case "gemini":
|
|
159
|
+
return (await emb_gemini({ [s]: t }))[s];
|
|
160
|
+
case "ollama":
|
|
161
|
+
return await emb_ollama(t, s);
|
|
162
|
+
case "aws":
|
|
163
|
+
return await emb_aws(t, s);
|
|
164
|
+
case "local":
|
|
165
|
+
return await emb_local(t, s);
|
|
166
|
+
case "synthetic":
|
|
167
|
+
return gen_syn_emb(t, s);
|
|
168
|
+
default:
|
|
169
|
+
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// Get semantic embedding with configurable fallback chain
|
|
173
|
+
async function get_sem_emb(t, s) {
|
|
174
|
+
// Deduplicate providers to avoid wasteful retries (e.g., gemini,gemini,synthetic)
|
|
175
|
+
const providers = [...new Set([cfg_1.env.emb_kind, ...cfg_1.env.embedding_fallback])];
|
|
176
|
+
for (let i = 0; i < providers.length; i++) {
|
|
177
|
+
const provider = providers[i];
|
|
178
|
+
try {
|
|
179
|
+
const result = await embed_with_provider(provider, t, s);
|
|
180
|
+
if (i > 0) {
|
|
181
|
+
console.error(`[EMBED] Fallback to ${provider} succeeded for sector: ${s}`);
|
|
182
|
+
}
|
|
183
|
+
return result;
|
|
184
|
+
}
|
|
185
|
+
catch (e) {
|
|
186
|
+
const errMsg = e instanceof Error ? e.message : String(e);
|
|
187
|
+
const nextProvider = providers[i + 1];
|
|
188
|
+
if (nextProvider) {
|
|
189
|
+
console.error(`[EMBED] ${provider} failed: ${errMsg}, trying ${nextProvider}`);
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
console.error(`[EMBED] All providers failed. Last error (${provider}): ${errMsg}. Using synthetic.`);
|
|
193
|
+
return gen_syn_emb(t, s);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
// Fallback if providers array is empty (shouldn't happen with defaults)
|
|
198
|
+
return gen_syn_emb(t, s);
|
|
199
|
+
}
|
|
200
|
+
// Batch embedding with fallback chain support (for simple mode)
|
|
201
|
+
async function emb_batch_with_fallback(txts) {
|
|
202
|
+
const providers = [...new Set([cfg_1.env.emb_kind, ...cfg_1.env.embedding_fallback])];
|
|
203
|
+
for (let i = 0; i < providers.length; i++) {
|
|
204
|
+
const provider = providers[i];
|
|
205
|
+
try {
|
|
206
|
+
let result;
|
|
207
|
+
switch (provider) {
|
|
208
|
+
case "gemini":
|
|
209
|
+
result = await emb_gemini(txts);
|
|
210
|
+
break;
|
|
211
|
+
case "openai":
|
|
212
|
+
result = await emb_batch_openai(txts);
|
|
213
|
+
break;
|
|
214
|
+
default:
|
|
215
|
+
// For providers without batch support, embed each sector individually
|
|
216
|
+
result = {};
|
|
217
|
+
for (const [s, t] of Object.entries(txts)) {
|
|
218
|
+
result[s] = await embed_with_provider(provider, t, s);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
if (i > 0) {
|
|
222
|
+
console.error(`[EMBED] Fallback to ${provider} succeeded for batch`);
|
|
223
|
+
}
|
|
224
|
+
return result;
|
|
225
|
+
}
|
|
226
|
+
catch (e) {
|
|
227
|
+
const errMsg = e instanceof Error ? e.message : String(e);
|
|
228
|
+
const nextProvider = providers[i + 1];
|
|
229
|
+
if (nextProvider) {
|
|
230
|
+
console.error(`[EMBED] ${provider} batch failed: ${errMsg}, trying ${nextProvider}`);
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
console.error(`[EMBED] All providers failed for batch. Last error (${provider}): ${errMsg}. Using synthetic.`);
|
|
234
|
+
// Fall back to synthetic for all sectors
|
|
235
|
+
const result = {};
|
|
236
|
+
for (const [s, t] of Object.entries(txts)) {
|
|
237
|
+
result[s] = gen_syn_emb(t, s);
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
// Fallback if providers array is empty
|
|
244
|
+
const result = {};
|
|
245
|
+
for (const [s, t] of Object.entries(txts)) {
|
|
246
|
+
result[s] = gen_syn_emb(t, s);
|
|
247
|
+
}
|
|
248
|
+
return result;
|
|
249
|
+
}
|
|
250
|
+
async function emb_openai(t, s) {
|
|
251
|
+
if (!cfg_1.env.openai_key)
|
|
252
|
+
throw new Error("OpenAI key missing");
|
|
253
|
+
const m = (0, models_1.get_model)(s, "openai");
|
|
254
|
+
const r = await fetchWithTimeout(`${cfg_1.env.openai_base_url.replace(/\/$/, "")}/embeddings`, {
|
|
255
|
+
method: "POST",
|
|
256
|
+
headers: {
|
|
257
|
+
"content-type": "application/json",
|
|
258
|
+
authorization: `Bearer ${cfg_1.env.openai_key}`,
|
|
259
|
+
},
|
|
260
|
+
body: JSON.stringify({
|
|
261
|
+
input: t,
|
|
262
|
+
model: cfg_1.env.openai_model || m,
|
|
263
|
+
dimensions: cfg_1.env.vec_dim,
|
|
264
|
+
}),
|
|
265
|
+
});
|
|
266
|
+
if (!r.ok)
|
|
267
|
+
throw new Error(`OpenAI: ${r.status}`);
|
|
268
|
+
return (await r.json()).data[0].embedding;
|
|
269
|
+
}
|
|
270
|
+
async function emb_batch_openai(txts) {
|
|
271
|
+
if (!cfg_1.env.openai_key)
|
|
272
|
+
throw new Error("OpenAI key missing");
|
|
273
|
+
const secs = Object.keys(txts), m = (0, models_1.get_model)("semantic", "openai");
|
|
274
|
+
const r = await fetchWithTimeout(`${cfg_1.env.openai_base_url.replace(/\/$/, "")}/embeddings`, {
|
|
275
|
+
method: "POST",
|
|
276
|
+
headers: {
|
|
277
|
+
"content-type": "application/json",
|
|
278
|
+
authorization: `Bearer ${cfg_1.env.openai_key}`,
|
|
279
|
+
},
|
|
280
|
+
body: JSON.stringify({
|
|
281
|
+
input: Object.values(txts),
|
|
282
|
+
model: cfg_1.env.openai_model || m,
|
|
283
|
+
dimensions: cfg_1.env.vec_dim,
|
|
284
|
+
}),
|
|
285
|
+
});
|
|
286
|
+
if (!r.ok)
|
|
287
|
+
throw new Error(`OpenAI batch: ${r.status}`);
|
|
288
|
+
const d = (await r.json()), out = {};
|
|
289
|
+
secs.forEach((s, i) => (out[s] = d.data[i].embedding));
|
|
290
|
+
return out;
|
|
291
|
+
}
|
|
292
|
+
const task_map = {
|
|
293
|
+
episodic: "RETRIEVAL_DOCUMENT",
|
|
294
|
+
semantic: "SEMANTIC_SIMILARITY",
|
|
295
|
+
procedural: "RETRIEVAL_DOCUMENT",
|
|
296
|
+
emotional: "CLASSIFICATION",
|
|
297
|
+
reflective: "SEMANTIC_SIMILARITY",
|
|
298
|
+
};
|
|
299
|
+
async function emb_gemini(txts) {
|
|
300
|
+
if (!cfg_1.env.gemini_key)
|
|
301
|
+
throw new Error("Gemini key missing");
|
|
302
|
+
const prom = gem_q.then(async () => {
|
|
303
|
+
const url = `https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:batchEmbedContents?key=${cfg_1.env.gemini_key}`;
|
|
304
|
+
for (let a = 0; a < 3; a++) {
|
|
305
|
+
try {
|
|
306
|
+
const reqs = Object.entries(txts).map(([s, t]) => ({
|
|
307
|
+
model: "models/text-embedding-004",
|
|
308
|
+
content: { parts: [{ text: t }] },
|
|
309
|
+
taskType: task_map[s] || task_map.semantic,
|
|
310
|
+
}));
|
|
311
|
+
const r = await fetchWithTimeout(url, {
|
|
312
|
+
method: "POST",
|
|
313
|
+
headers: { "content-type": "application/json" },
|
|
314
|
+
body: JSON.stringify({ requests: reqs }),
|
|
315
|
+
});
|
|
316
|
+
if (!r.ok) {
|
|
317
|
+
if (r.status === 429) {
|
|
318
|
+
const d = Math.min(parseInt(r.headers.get("retry-after") || "2") *
|
|
319
|
+
1000, 1000 * Math.pow(2, a));
|
|
320
|
+
console.error(`[EMBED] Gemini rate limit (${a + 1}/3), waiting ${d}ms`);
|
|
321
|
+
await new Promise((x) => setTimeout(x, d));
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
throw new Error(`Gemini: ${r.status}`);
|
|
325
|
+
}
|
|
326
|
+
const data = (await r.json()), out = {};
|
|
327
|
+
let i = 0;
|
|
328
|
+
for (const s of Object.keys(txts))
|
|
329
|
+
out[s] = resize_vec(data.embeddings[i++].values, cfg_1.env.vec_dim);
|
|
330
|
+
await new Promise((x) => setTimeout(x, 1500));
|
|
331
|
+
return out;
|
|
332
|
+
}
|
|
333
|
+
catch (e) {
|
|
334
|
+
const errMsg = e instanceof Error ? e.message : String(e);
|
|
335
|
+
if (a === 2) {
|
|
336
|
+
throw new Error(`Gemini failed after 3 attempts: ${errMsg}`);
|
|
337
|
+
}
|
|
338
|
+
console.error(`[EMBED] Gemini error (${a + 1}/3): ${errMsg}`);
|
|
339
|
+
await new Promise((x) => setTimeout(x, 1000 * Math.pow(2, a)));
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
throw new Error("Gemini: exhausted retries");
|
|
343
|
+
});
|
|
344
|
+
gem_q = prom.catch(() => { });
|
|
345
|
+
return prom;
|
|
346
|
+
}
|
|
347
|
+
async function emb_ollama(t, s) {
|
|
348
|
+
const m = (0, models_1.get_model)(s, "ollama");
|
|
349
|
+
const r = await fetchWithTimeout(`${cfg_1.env.ollama_url}/api/embeddings`, {
|
|
350
|
+
method: "POST",
|
|
351
|
+
headers: { "content-type": "application/json" },
|
|
352
|
+
body: JSON.stringify({ model: m, prompt: t }),
|
|
353
|
+
});
|
|
354
|
+
if (!r.ok)
|
|
355
|
+
throw new Error(`Ollama: ${r.status}`);
|
|
356
|
+
return resize_vec((await r.json()).embedding, cfg_1.env.vec_dim);
|
|
357
|
+
}
|
|
358
|
+
async function emb_aws(t, s) {
|
|
359
|
+
if (!cfg_1.env.AWS_REGION)
|
|
360
|
+
throw new Error("AWS_REGION missing");
|
|
361
|
+
if (!cfg_1.env.AWS_ACCESS_KEY_ID)
|
|
362
|
+
throw new Error("AWS_ACCESS_KEY_ID missing");
|
|
363
|
+
if (!cfg_1.env.AWS_SECRET_ACCESS_KEY)
|
|
364
|
+
throw new Error("AWS_SECRET_ACCESS_KEY missing");
|
|
365
|
+
const m = (0, models_1.get_model)(s, "aws");
|
|
366
|
+
const client = new client_bedrock_runtime_1.BedrockRuntimeClient({ region: process.env.AWS_REGION });
|
|
367
|
+
const dim = [256, 512, 1024].find((x) => x >= cfg_1.env.vec_dim) ?? 1024;
|
|
368
|
+
const params = {
|
|
369
|
+
modelId: m,
|
|
370
|
+
contentType: "application/json",
|
|
371
|
+
accept: "*/*",
|
|
372
|
+
body: JSON.stringify({
|
|
373
|
+
inputText: t,
|
|
374
|
+
dimensions: dim,
|
|
375
|
+
}),
|
|
376
|
+
};
|
|
377
|
+
const command = new client_bedrock_runtime_1.InvokeModelCommand(params);
|
|
378
|
+
try {
|
|
379
|
+
const response = await client.send(command);
|
|
380
|
+
const jsonString = new TextDecoder().decode(response.body);
|
|
381
|
+
const parsedResponse = JSON.parse(jsonString);
|
|
382
|
+
return resize_vec(parsedResponse.embedding, cfg_1.env.vec_dim);
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
throw new Error(`AWS: ${error}`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
async function emb_local(t, s) {
|
|
389
|
+
if (!cfg_1.env.local_model_path) {
|
|
390
|
+
console.error("[EMBED] Local model missing, using synthetic");
|
|
391
|
+
return gen_syn_emb(t, s);
|
|
392
|
+
}
|
|
393
|
+
try {
|
|
394
|
+
const { createHash } = await Promise.resolve().then(() => __importStar(require("crypto")));
|
|
395
|
+
const h = createHash("sha256")
|
|
396
|
+
.update(t + s)
|
|
397
|
+
.digest(), e = [];
|
|
398
|
+
for (let i = 0; i < cfg_1.env.vec_dim; i++) {
|
|
399
|
+
const b1 = h[i % h.length], b2 = h[(i + 1) % h.length];
|
|
400
|
+
e.push(((b1 * 256 + b2) / 65535) * 2 - 1);
|
|
401
|
+
}
|
|
402
|
+
const n = Math.sqrt(e.reduce((sum, v) => sum + v * v, 0));
|
|
403
|
+
return e.map((v) => v / n);
|
|
404
|
+
}
|
|
405
|
+
catch {
|
|
406
|
+
console.error("[EMBED] Local embedding failed, using synthetic");
|
|
407
|
+
return gen_syn_emb(t, s);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
const h1 = (v) => {
|
|
411
|
+
let h = 0x811c9dc5 | 0;
|
|
412
|
+
for (let i = 0; i < v.length; i++)
|
|
413
|
+
h = Math.imul(h ^ v.charCodeAt(i), 16777619);
|
|
414
|
+
return h >>> 0;
|
|
415
|
+
};
|
|
416
|
+
const h2 = (v, sd) => {
|
|
417
|
+
let h = sd | 0;
|
|
418
|
+
for (let i = 0; i < v.length; i++) {
|
|
419
|
+
h = Math.imul(h ^ v.charCodeAt(i), 0x5bd1e995);
|
|
420
|
+
h = (h >>> 13) ^ h;
|
|
421
|
+
}
|
|
422
|
+
return h >>> 0;
|
|
423
|
+
};
|
|
424
|
+
const add_feat = (vec, dim, k, w) => {
|
|
425
|
+
const h = h1(k), h_2 = h2(k, 0xdeadbeef), val = w * (1 - ((h & 1) << 1));
|
|
426
|
+
if (dim > 0 && (dim & (dim - 1)) === 0) {
|
|
427
|
+
vec[h & (dim - 1)] += val;
|
|
428
|
+
vec[h_2 & (dim - 1)] += val * 0.5;
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
vec[h % dim] += val;
|
|
432
|
+
vec[h_2 % dim] += val * 0.5;
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
const add_pos_feat = (vec, dim, pos, w) => {
|
|
436
|
+
const idx = pos % dim, ang = pos / Math.pow(10000, (2 * idx) / dim);
|
|
437
|
+
vec[idx] += w * Math.sin(ang);
|
|
438
|
+
vec[(idx + 1) % dim] += w * Math.cos(ang);
|
|
439
|
+
};
|
|
440
|
+
const sec_wts = {
|
|
441
|
+
episodic: 1.3,
|
|
442
|
+
semantic: 1.0,
|
|
443
|
+
procedural: 1.2,
|
|
444
|
+
emotional: 1.4,
|
|
445
|
+
reflective: 0.9,
|
|
446
|
+
};
|
|
447
|
+
const norm_v = (v) => {
|
|
448
|
+
let n = 0;
|
|
449
|
+
for (let i = 0; i < v.length; i++)
|
|
450
|
+
n += v[i] * v[i];
|
|
451
|
+
if (n === 0)
|
|
452
|
+
return;
|
|
453
|
+
const inv = 1 / Math.sqrt(n);
|
|
454
|
+
for (let i = 0; i < v.length; i++)
|
|
455
|
+
v[i] *= inv;
|
|
456
|
+
};
|
|
457
|
+
function gen_syn_emb(t, s) {
|
|
458
|
+
const d = cfg_1.env.vec_dim || 768, v = new Float32Array(d).fill(0), ct = (0, text_1.canonical_tokens_from_text)(t);
|
|
459
|
+
if (!ct.length) {
|
|
460
|
+
const x = 1 / Math.sqrt(d);
|
|
461
|
+
return Array.from({ length: d }, () => x);
|
|
462
|
+
}
|
|
463
|
+
const et = Array.from((0, text_1.add_synonym_tokens)(ct)), tc = new Map(), el = et.length;
|
|
464
|
+
for (let i = 0; i < el; i++) {
|
|
465
|
+
const tok = et[i];
|
|
466
|
+
tc.set(tok, (tc.get(tok) || 0) + 1);
|
|
467
|
+
}
|
|
468
|
+
const sw = sec_wts[s] || 1.0, dl = Math.log(1 + el);
|
|
469
|
+
for (const [tok, c] of tc) {
|
|
470
|
+
const tf = c / el, idf = Math.log(1 + el / c), w = (tf * idf + 1) * sw;
|
|
471
|
+
add_feat(v, d, `${s}|tok|${tok}`, w);
|
|
472
|
+
if (tok.length >= 3)
|
|
473
|
+
for (let i = 0; i < tok.length - 2; i++)
|
|
474
|
+
add_feat(v, d, `${s}|c3|${tok.slice(i, i + 3)}`, w * 0.4);
|
|
475
|
+
if (tok.length >= 4)
|
|
476
|
+
for (let i = 0; i < tok.length - 3; i++)
|
|
477
|
+
add_feat(v, d, `${s}|c4|${tok.slice(i, i + 4)}`, w * 0.3);
|
|
478
|
+
}
|
|
479
|
+
for (let i = 0; i < ct.length - 1; i++) {
|
|
480
|
+
const a = ct[i], b = ct[i + 1];
|
|
481
|
+
if (a && b) {
|
|
482
|
+
const pw = 1.0 / (1.0 + i * 0.1);
|
|
483
|
+
add_feat(v, d, `${s}|bi|${a}_${b}`, 1.4 * sw * pw);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
for (let i = 0; i < ct.length - 2; i++) {
|
|
487
|
+
const a = ct[i], b = ct[i + 1], c = ct[i + 2];
|
|
488
|
+
if (a && b && c)
|
|
489
|
+
add_feat(v, d, `${s}|tri|${a}_${b}_${c}`, 1.0 * sw);
|
|
490
|
+
}
|
|
491
|
+
for (let i = 0; i < Math.min(ct.length - 2, 20); i++) {
|
|
492
|
+
const a = ct[i], c = ct[i + 2];
|
|
493
|
+
if (a && c)
|
|
494
|
+
add_feat(v, d, `${s}|skip|${a}_${c}`, 0.7 * sw);
|
|
495
|
+
}
|
|
496
|
+
for (let i = 0; i < Math.min(ct.length, 50); i++)
|
|
497
|
+
add_pos_feat(v, d, i, (0.5 * sw) / dl);
|
|
498
|
+
const lb = Math.min(Math.floor(Math.log2(el + 1)), 10);
|
|
499
|
+
add_feat(v, d, `${s}|len|${lb}`, 0.6 * sw);
|
|
500
|
+
const dens = tc.size / el, db = Math.floor(dens * 10);
|
|
501
|
+
add_feat(v, d, `${s}|dens|${db}`, 0.5 * sw);
|
|
502
|
+
norm_v(v);
|
|
503
|
+
return Array.from(v);
|
|
504
|
+
}
|
|
505
|
+
const resize_vec = (v, t) => {
|
|
506
|
+
if (v.length === t)
|
|
507
|
+
return v;
|
|
508
|
+
if (v.length > t)
|
|
509
|
+
return v.slice(0, t);
|
|
510
|
+
return [...v, ...Array(t - v.length).fill(0)];
|
|
511
|
+
};
|
|
512
|
+
async function embedMultiSector(id, txt, secs, chunks) {
|
|
513
|
+
const r = [];
|
|
514
|
+
await db_1.q.ins_log.run(id, "multi-sector", "pending", Date.now(), null);
|
|
515
|
+
for (let a = 0; a < 3; a++) {
|
|
516
|
+
try {
|
|
517
|
+
const simp = cfg_1.env.embed_mode === "simple";
|
|
518
|
+
if (simp &&
|
|
519
|
+
(cfg_1.env.emb_kind === "gemini" || cfg_1.env.emb_kind === "openai")) {
|
|
520
|
+
console.error(`[EMBED] Simple mode (1 batch for ${secs.length} sectors)`);
|
|
521
|
+
const tb = {};
|
|
522
|
+
secs.forEach((s) => (tb[s] = txt));
|
|
523
|
+
// Use batch embedding with fallback support
|
|
524
|
+
const b = await emb_batch_with_fallback(tb);
|
|
525
|
+
Object.entries(b).forEach(([s, v]) => r.push({ sector: s, vector: v, dim: v.length }));
|
|
526
|
+
}
|
|
527
|
+
else {
|
|
528
|
+
console.error(`[EMBED] Advanced mode (${secs.length} calls)`);
|
|
529
|
+
const par = cfg_1.env.adv_embed_parallel && cfg_1.env.emb_kind !== "gemini";
|
|
530
|
+
if (par) {
|
|
531
|
+
const p = secs.map(async (s) => {
|
|
532
|
+
let v;
|
|
533
|
+
if (chunks && chunks.length > 1) {
|
|
534
|
+
const cv = [];
|
|
535
|
+
for (const c of chunks)
|
|
536
|
+
cv.push(await embedForSector(c.text, s));
|
|
537
|
+
v = agg_chunks(cv);
|
|
538
|
+
}
|
|
539
|
+
else
|
|
540
|
+
v = await embedForSector(txt, s);
|
|
541
|
+
return { sector: s, vector: v, dim: v.length };
|
|
542
|
+
});
|
|
543
|
+
r.push(...(await Promise.all(p)));
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
for (let i = 0; i < secs.length; i++) {
|
|
547
|
+
const s = secs[i];
|
|
548
|
+
let v;
|
|
549
|
+
if (chunks && chunks.length > 1) {
|
|
550
|
+
const cv = [];
|
|
551
|
+
for (const c of chunks)
|
|
552
|
+
cv.push(await embedForSector(c.text, s));
|
|
553
|
+
v = agg_chunks(cv);
|
|
554
|
+
}
|
|
555
|
+
else
|
|
556
|
+
v = await embedForSector(txt, s);
|
|
557
|
+
r.push({ sector: s, vector: v, dim: v.length });
|
|
558
|
+
if (cfg_1.env.embed_delay_ms > 0 && i < secs.length - 1)
|
|
559
|
+
await new Promise((x) => setTimeout(x, cfg_1.env.embed_delay_ms));
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
await db_1.q.upd_log.run("completed", null, id);
|
|
564
|
+
return r;
|
|
565
|
+
}
|
|
566
|
+
catch (e) {
|
|
567
|
+
if (a === 2) {
|
|
568
|
+
await db_1.q.upd_log.run("failed", e instanceof Error ? e.message : String(e), id);
|
|
569
|
+
throw e;
|
|
570
|
+
}
|
|
571
|
+
await new Promise((x) => setTimeout(x, 1000 * Math.pow(2, a)));
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
throw new Error("Embedding failed after retries");
|
|
575
|
+
}
|
|
576
|
+
const agg_chunks = (vecs) => {
|
|
577
|
+
if (!vecs.length)
|
|
578
|
+
throw new Error("No vectors");
|
|
579
|
+
if (vecs.length === 1)
|
|
580
|
+
return vecs[0];
|
|
581
|
+
const d = vecs[0].length, r = Array(d).fill(0);
|
|
582
|
+
for (const v of vecs)
|
|
583
|
+
for (let i = 0; i < d; i++)
|
|
584
|
+
r[i] += v[i];
|
|
585
|
+
return r.map((x) => x / vecs.length);
|
|
586
|
+
};
|
|
587
|
+
const cosineSimilarity = (a, b) => {
|
|
588
|
+
if (a.length !== b.length)
|
|
589
|
+
return 0;
|
|
590
|
+
let dot = 0, na = 0, nb = 0;
|
|
591
|
+
for (let i = 0; i < a.length; i++) {
|
|
592
|
+
dot += a[i] * b[i];
|
|
593
|
+
na += a[i] * a[i];
|
|
594
|
+
nb += b[i] * b[i];
|
|
595
|
+
}
|
|
596
|
+
return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
|
|
597
|
+
};
|
|
598
|
+
exports.cosineSimilarity = cosineSimilarity;
|
|
599
|
+
const vectorToBuffer = (v) => {
|
|
600
|
+
const b = Buffer.allocUnsafe(v.length * 4);
|
|
601
|
+
for (let i = 0; i < v.length; i++)
|
|
602
|
+
b.writeFloatLE(v[i], i * 4);
|
|
603
|
+
return b;
|
|
604
|
+
};
|
|
605
|
+
exports.vectorToBuffer = vectorToBuffer;
|
|
606
|
+
const bufferToVector = (b) => {
|
|
607
|
+
const v = [];
|
|
608
|
+
for (let i = 0; i < b.length; i += 4)
|
|
609
|
+
v.push(b.readFloatLE(i));
|
|
610
|
+
return v;
|
|
611
|
+
};
|
|
612
|
+
exports.bufferToVector = bufferToVector;
|
|
613
|
+
const embed = (t) => embedForSector(t, "semantic");
|
|
614
|
+
exports.embed = embed;
|
|
615
|
+
const getEmbeddingProvider = () => cfg_1.env.emb_kind;
|
|
616
|
+
exports.getEmbeddingProvider = getEmbeddingProvider;
|
|
617
|
+
const getEmbeddingInfo = () => {
|
|
618
|
+
const i = {
|
|
619
|
+
provider: cfg_1.env.emb_kind,
|
|
620
|
+
fallback_chain: cfg_1.env.embedding_fallback,
|
|
621
|
+
dimensions: cfg_1.env.vec_dim,
|
|
622
|
+
mode: cfg_1.env.embed_mode,
|
|
623
|
+
batch_support: cfg_1.env.embed_mode === "simple" &&
|
|
624
|
+
(cfg_1.env.emb_kind === "gemini" || cfg_1.env.emb_kind === "openai"),
|
|
625
|
+
advanced_parallel: cfg_1.env.adv_embed_parallel,
|
|
626
|
+
embed_delay_ms: cfg_1.env.embed_delay_ms,
|
|
627
|
+
};
|
|
628
|
+
if (cfg_1.env.emb_kind === "openai") {
|
|
629
|
+
i.configured = !!cfg_1.env.openai_key;
|
|
630
|
+
i.base_url = cfg_1.env.openai_base_url;
|
|
631
|
+
i.model_override = cfg_1.env.openai_model || null;
|
|
632
|
+
i.batch_api = cfg_1.env.embed_mode === "simple";
|
|
633
|
+
i.models = {
|
|
634
|
+
episodic: (0, models_1.get_model)("episodic", "openai"),
|
|
635
|
+
semantic: (0, models_1.get_model)("semantic", "openai"),
|
|
636
|
+
procedural: (0, models_1.get_model)("procedural", "openai"),
|
|
637
|
+
emotional: (0, models_1.get_model)("emotional", "openai"),
|
|
638
|
+
reflective: (0, models_1.get_model)("reflective", "openai"),
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
else if (cfg_1.env.emb_kind === "gemini") {
|
|
642
|
+
i.configured = !!cfg_1.env.gemini_key;
|
|
643
|
+
i.batch_api = cfg_1.env.embed_mode === "simple";
|
|
644
|
+
i.model = "embedding-001";
|
|
645
|
+
}
|
|
646
|
+
else if (cfg_1.env.emb_kind === "aws") {
|
|
647
|
+
i.configured =
|
|
648
|
+
!!cfg_1.env.AWS_REGION &&
|
|
649
|
+
!!cfg_1.env.AWS_ACCESS_KEY_ID &&
|
|
650
|
+
!!cfg_1.env.AWS_SECRET_ACCESS_KEY;
|
|
651
|
+
i.batch_api = cfg_1.env.embed_mode === "simple";
|
|
652
|
+
i.model = "amazon.titan-embed-text-v2:0";
|
|
653
|
+
}
|
|
654
|
+
else if (cfg_1.env.emb_kind === "ollama") {
|
|
655
|
+
i.configured = true;
|
|
656
|
+
i.url = cfg_1.env.ollama_url;
|
|
657
|
+
i.models = {
|
|
658
|
+
episodic: (0, models_1.get_model)("episodic", "ollama"),
|
|
659
|
+
semantic: (0, models_1.get_model)("semantic", "ollama"),
|
|
660
|
+
procedural: (0, models_1.get_model)("procedural", "ollama"),
|
|
661
|
+
emotional: (0, models_1.get_model)("emotional", "ollama"),
|
|
662
|
+
reflective: (0, models_1.get_model)("reflective", "ollama"),
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
else if (cfg_1.env.emb_kind === "local") {
|
|
666
|
+
i.configured = !!cfg_1.env.local_model_path;
|
|
667
|
+
i.path = cfg_1.env.local_model_path;
|
|
668
|
+
}
|
|
669
|
+
else {
|
|
670
|
+
i.configured = true;
|
|
671
|
+
i.type = "synthetic";
|
|
672
|
+
}
|
|
673
|
+
return i;
|
|
674
|
+
};
|
|
675
|
+
exports.getEmbeddingInfo = getEmbeddingInfo;
|