ahok-skill 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc +8 -0
- package/Dockerfile +59 -0
- package/RAW_SKILL.md +219 -0
- package/README.md +277 -0
- package/SKILL.md +58 -0
- package/bin/opm.js +268 -0
- package/data/openmemory.sqlite +0 -0
- package/data/openmemory.sqlite-shm +0 -0
- package/data/openmemory.sqlite-wal +0 -0
- package/dist/ai/graph.js +293 -0
- package/dist/ai/mcp.js +397 -0
- package/dist/cli.js +78 -0
- package/dist/core/cfg.js +87 -0
- package/dist/core/db.js +636 -0
- package/dist/core/memory.js +116 -0
- package/dist/core/migrate.js +227 -0
- package/dist/core/models.js +105 -0
- package/dist/core/telemetry.js +57 -0
- package/dist/core/types.js +2 -0
- package/dist/core/vector/postgres.js +52 -0
- package/dist/core/vector/valkey.js +246 -0
- package/dist/core/vector_store.js +2 -0
- package/dist/index.js +44 -0
- package/dist/memory/decay.js +301 -0
- package/dist/memory/embed.js +675 -0
- package/dist/memory/hsg.js +959 -0
- package/dist/memory/reflect.js +131 -0
- package/dist/memory/user_summary.js +99 -0
- package/dist/migrate.js +9 -0
- package/dist/ops/compress.js +255 -0
- package/dist/ops/dynamics.js +189 -0
- package/dist/ops/extract.js +333 -0
- package/dist/ops/ingest.js +214 -0
- package/dist/server/index.js +109 -0
- package/dist/server/middleware/auth.js +137 -0
- package/dist/server/routes/auth.js +186 -0
- package/dist/server/routes/compression.js +108 -0
- package/dist/server/routes/dashboard.js +399 -0
- package/dist/server/routes/docs.js +241 -0
- package/dist/server/routes/dynamics.js +312 -0
- package/dist/server/routes/ide.js +280 -0
- package/dist/server/routes/index.js +33 -0
- package/dist/server/routes/keys.js +132 -0
- package/dist/server/routes/langgraph.js +61 -0
- package/dist/server/routes/memory.js +213 -0
- package/dist/server/routes/sources.js +140 -0
- package/dist/server/routes/system.js +63 -0
- package/dist/server/routes/temporal.js +293 -0
- package/dist/server/routes/users.js +101 -0
- package/dist/server/routes/vercel.js +57 -0
- package/dist/server/server.js +211 -0
- package/dist/server.js +3 -0
- package/dist/sources/base.js +223 -0
- package/dist/sources/github.js +171 -0
- package/dist/sources/google_drive.js +166 -0
- package/dist/sources/google_sheets.js +112 -0
- package/dist/sources/google_slides.js +139 -0
- package/dist/sources/index.js +34 -0
- package/dist/sources/notion.js +165 -0
- package/dist/sources/onedrive.js +143 -0
- package/dist/sources/web_crawler.js +166 -0
- package/dist/temporal_graph/index.js +20 -0
- package/dist/temporal_graph/query.js +240 -0
- package/dist/temporal_graph/store.js +116 -0
- package/dist/temporal_graph/timeline.js +241 -0
- package/dist/temporal_graph/types.js +2 -0
- package/dist/utils/chunking.js +60 -0
- package/dist/utils/index.js +31 -0
- package/dist/utils/keyword.js +94 -0
- package/dist/utils/text.js +120 -0
- package/nodemon.json +7 -0
- package/package.json +50 -0
- package/references/api_reference.md +66 -0
- package/references/examples.md +45 -0
- package/src/ai/graph.ts +363 -0
- package/src/ai/mcp.ts +494 -0
- package/src/cli.ts +94 -0
- package/src/core/cfg.ts +110 -0
- package/src/core/db.ts +1052 -0
- package/src/core/memory.ts +99 -0
- package/src/core/migrate.ts +302 -0
- package/src/core/models.ts +107 -0
- package/src/core/telemetry.ts +47 -0
- package/src/core/types.ts +130 -0
- package/src/core/vector/postgres.ts +61 -0
- package/src/core/vector/valkey.ts +261 -0
- package/src/core/vector_store.ts +9 -0
- package/src/index.ts +5 -0
- package/src/memory/decay.ts +427 -0
- package/src/memory/embed.ts +707 -0
- package/src/memory/hsg.ts +1245 -0
- package/src/memory/reflect.ts +158 -0
- package/src/memory/user_summary.ts +110 -0
- package/src/migrate.ts +8 -0
- package/src/ops/compress.ts +296 -0
- package/src/ops/dynamics.ts +272 -0
- package/src/ops/extract.ts +360 -0
- package/src/ops/ingest.ts +286 -0
- package/src/server/index.ts +159 -0
- package/src/server/middleware/auth.ts +156 -0
- package/src/server/routes/auth.ts +223 -0
- package/src/server/routes/compression.ts +106 -0
- package/src/server/routes/dashboard.ts +420 -0
- package/src/server/routes/docs.ts +380 -0
- package/src/server/routes/dynamics.ts +516 -0
- package/src/server/routes/ide.ts +283 -0
- package/src/server/routes/index.ts +32 -0
- package/src/server/routes/keys.ts +131 -0
- package/src/server/routes/langgraph.ts +71 -0
- package/src/server/routes/memory.ts +440 -0
- package/src/server/routes/sources.ts +111 -0
- package/src/server/routes/system.ts +68 -0
- package/src/server/routes/temporal.ts +335 -0
- package/src/server/routes/users.ts +111 -0
- package/src/server/routes/vercel.ts +55 -0
- package/src/server/server.js +215 -0
- package/src/server.ts +1 -0
- package/src/sources/base.ts +257 -0
- package/src/sources/github.ts +156 -0
- package/src/sources/google_drive.ts +144 -0
- package/src/sources/google_sheets.ts +85 -0
- package/src/sources/google_slides.ts +115 -0
- package/src/sources/index.ts +19 -0
- package/src/sources/notion.ts +148 -0
- package/src/sources/onedrive.ts +131 -0
- package/src/sources/web_crawler.ts +161 -0
- package/src/temporal_graph/index.ts +4 -0
- package/src/temporal_graph/query.ts +299 -0
- package/src/temporal_graph/store.ts +156 -0
- package/src/temporal_graph/timeline.ts +319 -0
- package/src/temporal_graph/types.ts +41 -0
- package/src/utils/chunking.ts +66 -0
- package/src/utils/index.ts +25 -0
- package/src/utils/keyword.ts +137 -0
- package/src/utils/text.ts +115 -0
- package/tests/test_api_workspace_management.ts +413 -0
- package/tests/test_bulk_delete.ts +267 -0
- package/tests/test_omnibus.ts +166 -0
- package/tests/test_workspace_management.ts +278 -0
- package/tests/verify.ts +104 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.extractPDF = extractPDF;
|
|
40
|
+
exports.extractDOCX = extractDOCX;
|
|
41
|
+
exports.extractHTML = extractHTML;
|
|
42
|
+
exports.extractURL = extractURL;
|
|
43
|
+
exports.extractAudio = extractAudio;
|
|
44
|
+
exports.extractVideo = extractVideo;
|
|
45
|
+
exports.extractText = extractText;
|
|
46
|
+
const child_process_1 = require("child_process");
|
|
47
|
+
const util_1 = require("util");
|
|
48
|
+
const mammoth_1 = __importDefault(require("mammoth"));
|
|
49
|
+
const fs = __importStar(require("fs"));
|
|
50
|
+
const path = __importStar(require("path"));
|
|
51
|
+
const os = __importStar(require("os"));
|
|
52
|
+
const fluent_ffmpeg_1 = __importDefault(require("fluent-ffmpeg"));
|
|
53
|
+
const openai_1 = __importDefault(require("openai"));
|
|
54
|
+
const TurndownService = require("turndown");
|
|
55
|
+
const execAsync = (0, util_1.promisify)(child_process_1.exec);
|
|
56
|
+
function estimateTokens(text) {
|
|
57
|
+
return Math.ceil(text.length / 4);
|
|
58
|
+
}
|
|
59
|
+
async function extractPDF(buffer) {
|
|
60
|
+
const { PDFParse } = await Promise.resolve().then(() => __importStar(require("pdf-parse")));
|
|
61
|
+
const parser = new PDFParse({ data: buffer });
|
|
62
|
+
const textResult = await parser.getText();
|
|
63
|
+
const infoResult = await parser.getInfo();
|
|
64
|
+
return {
|
|
65
|
+
text: textResult.text,
|
|
66
|
+
metadata: {
|
|
67
|
+
content_type: "pdf",
|
|
68
|
+
char_count: textResult.text.length,
|
|
69
|
+
estimated_tokens: estimateTokens(textResult.text),
|
|
70
|
+
extraction_method: "pdf-parse",
|
|
71
|
+
pages: textResult.total,
|
|
72
|
+
info: infoResult,
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
async function extractDOCX(buffer) {
|
|
77
|
+
const result = await mammoth_1.default.extractRawText({ buffer });
|
|
78
|
+
return {
|
|
79
|
+
text: result.value,
|
|
80
|
+
metadata: {
|
|
81
|
+
content_type: "docx",
|
|
82
|
+
char_count: result.value.length,
|
|
83
|
+
estimated_tokens: estimateTokens(result.value),
|
|
84
|
+
extraction_method: "mammoth",
|
|
85
|
+
messages: result.messages,
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
async function extractHTML(html) {
|
|
90
|
+
const turndown = new TurndownService({
|
|
91
|
+
headingStyle: "atx",
|
|
92
|
+
codeBlockStyle: "fenced",
|
|
93
|
+
});
|
|
94
|
+
const markdown = turndown.turndown(html);
|
|
95
|
+
return {
|
|
96
|
+
text: markdown,
|
|
97
|
+
metadata: {
|
|
98
|
+
content_type: "html",
|
|
99
|
+
char_count: markdown.length,
|
|
100
|
+
estimated_tokens: estimateTokens(markdown),
|
|
101
|
+
extraction_method: "turndown",
|
|
102
|
+
original_html_length: html.length,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
async function extractURL(url) {
|
|
107
|
+
const response = await fetch(url);
|
|
108
|
+
if (!response.ok) {
|
|
109
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
110
|
+
}
|
|
111
|
+
const html = await response.text();
|
|
112
|
+
const turndown = new TurndownService({
|
|
113
|
+
headingStyle: "atx",
|
|
114
|
+
codeBlockStyle: "fenced",
|
|
115
|
+
});
|
|
116
|
+
const markdown = turndown.turndown(html);
|
|
117
|
+
return {
|
|
118
|
+
text: markdown,
|
|
119
|
+
metadata: {
|
|
120
|
+
content_type: "url",
|
|
121
|
+
char_count: markdown.length,
|
|
122
|
+
estimated_tokens: estimateTokens(markdown),
|
|
123
|
+
extraction_method: "node-fetch+turndown",
|
|
124
|
+
source_url: url,
|
|
125
|
+
fetched_at: new Date().toISOString(),
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
async function extractAudio(buffer, mimeType) {
|
|
130
|
+
const apiKey = process.env.OPENAI_API_KEY || process.env.OM_OPENAI_API_KEY;
|
|
131
|
+
if (!apiKey) {
|
|
132
|
+
throw new Error("OpenAI API key required for audio transcription. Set OPENAI_API_KEY in .env");
|
|
133
|
+
}
|
|
134
|
+
// Check file size (Whisper API limit is 25MB)
|
|
135
|
+
const maxSize = 25 * 1024 * 1024; // 25MB
|
|
136
|
+
if (buffer.length > maxSize) {
|
|
137
|
+
throw new Error(`Audio file too large: ${(buffer.length / 1024 / 1024).toFixed(2)}MB. Maximum size is 25MB.`);
|
|
138
|
+
}
|
|
139
|
+
// Create temporary file for Whisper API
|
|
140
|
+
const tempDir = os.tmpdir();
|
|
141
|
+
const ext = getAudioExtension(mimeType);
|
|
142
|
+
const tempFilePath = path.join(tempDir, `audio-${Date.now()}${ext}`);
|
|
143
|
+
try {
|
|
144
|
+
// Write buffer to temp file
|
|
145
|
+
fs.writeFileSync(tempFilePath, buffer);
|
|
146
|
+
// Initialize OpenAI client
|
|
147
|
+
const openai = new openai_1.default({ apiKey });
|
|
148
|
+
// Transcribe audio using Whisper
|
|
149
|
+
const transcription = await openai.audio.transcriptions.create({
|
|
150
|
+
file: fs.createReadStream(tempFilePath),
|
|
151
|
+
model: "whisper-1",
|
|
152
|
+
response_format: "verbose_json",
|
|
153
|
+
});
|
|
154
|
+
const text = transcription.text;
|
|
155
|
+
return {
|
|
156
|
+
text,
|
|
157
|
+
metadata: {
|
|
158
|
+
content_type: "audio",
|
|
159
|
+
char_count: text.length,
|
|
160
|
+
estimated_tokens: estimateTokens(text),
|
|
161
|
+
extraction_method: "whisper",
|
|
162
|
+
audio_format: ext.replace(".", ""),
|
|
163
|
+
file_size_bytes: buffer.length,
|
|
164
|
+
file_size_mb: (buffer.length / 1024 / 1024).toFixed(2),
|
|
165
|
+
duration_seconds: transcription.duration || null,
|
|
166
|
+
language: transcription.language || null,
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
console.error("[EXTRACT] Audio transcription failed:", error);
|
|
172
|
+
throw new Error(`Audio transcription failed: ${error.message}`);
|
|
173
|
+
}
|
|
174
|
+
finally {
|
|
175
|
+
// Clean up temp file
|
|
176
|
+
try {
|
|
177
|
+
if (fs.existsSync(tempFilePath)) {
|
|
178
|
+
fs.unlinkSync(tempFilePath);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
catch (e) {
|
|
182
|
+
console.warn("[EXTRACT] Failed to clean up temp file:", e);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
async function extractVideo(buffer) {
|
|
187
|
+
// Create temporary files for video and audio
|
|
188
|
+
const tempDir = os.tmpdir();
|
|
189
|
+
const videoPath = path.join(tempDir, `video-${Date.now()}.mp4`);
|
|
190
|
+
const audioPath = path.join(tempDir, `audio-${Date.now()}.mp3`);
|
|
191
|
+
try {
|
|
192
|
+
// Write video buffer to temp file
|
|
193
|
+
fs.writeFileSync(videoPath, buffer);
|
|
194
|
+
// Extract audio using ffmpeg
|
|
195
|
+
await new Promise((resolve, reject) => {
|
|
196
|
+
(0, fluent_ffmpeg_1.default)(videoPath)
|
|
197
|
+
.output(audioPath)
|
|
198
|
+
.noVideo()
|
|
199
|
+
.audioCodec("libmp3lame")
|
|
200
|
+
.on("end", () => resolve())
|
|
201
|
+
.on("error", (err) => reject(err))
|
|
202
|
+
.run();
|
|
203
|
+
});
|
|
204
|
+
// Read extracted audio
|
|
205
|
+
const audioBuffer = fs.readFileSync(audioPath);
|
|
206
|
+
// Transcribe extracted audio
|
|
207
|
+
const result = await extractAudio(audioBuffer, "audio/mpeg");
|
|
208
|
+
// Update metadata to reflect video source
|
|
209
|
+
result.metadata.content_type = "video";
|
|
210
|
+
result.metadata.extraction_method = "ffmpeg+whisper";
|
|
211
|
+
result.metadata.video_file_size_bytes = buffer.length;
|
|
212
|
+
result.metadata.video_file_size_mb = (buffer.length /
|
|
213
|
+
1024 /
|
|
214
|
+
1024).toFixed(2);
|
|
215
|
+
return result;
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
if (error.message?.includes("ffmpeg")) {
|
|
219
|
+
throw new Error("FFmpeg not found. Please install FFmpeg to process video files. Visit: https://ffmpeg.org/download.html");
|
|
220
|
+
}
|
|
221
|
+
console.error("[EXTRACT] Video processing failed:", error);
|
|
222
|
+
throw new Error(`Video processing failed: ${error.message}`);
|
|
223
|
+
}
|
|
224
|
+
finally {
|
|
225
|
+
// Clean up temp files
|
|
226
|
+
try {
|
|
227
|
+
if (fs.existsSync(videoPath))
|
|
228
|
+
fs.unlinkSync(videoPath);
|
|
229
|
+
if (fs.existsSync(audioPath))
|
|
230
|
+
fs.unlinkSync(audioPath);
|
|
231
|
+
}
|
|
232
|
+
catch (e) {
|
|
233
|
+
console.warn("[EXTRACT] Failed to clean up temp files:", e);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
function getAudioExtension(mimeType) {
|
|
238
|
+
const mimeMap = {
|
|
239
|
+
"audio/mpeg": ".mp3",
|
|
240
|
+
"audio/mp3": ".mp3",
|
|
241
|
+
"audio/wav": ".wav",
|
|
242
|
+
"audio/wave": ".wav",
|
|
243
|
+
"audio/x-wav": ".wav",
|
|
244
|
+
"audio/mp4": ".m4a",
|
|
245
|
+
"audio/m4a": ".m4a",
|
|
246
|
+
"audio/x-m4a": ".m4a",
|
|
247
|
+
"audio/webm": ".webm",
|
|
248
|
+
"audio/ogg": ".ogg",
|
|
249
|
+
};
|
|
250
|
+
return mimeMap[mimeType.toLowerCase()] || ".mp3";
|
|
251
|
+
}
|
|
252
|
+
async function extractText(contentType, data) {
|
|
253
|
+
const type = contentType.toLowerCase();
|
|
254
|
+
// Audio formats
|
|
255
|
+
if (type === "mp3" ||
|
|
256
|
+
type === "audio" ||
|
|
257
|
+
type === "audio/mpeg" ||
|
|
258
|
+
type === "audio/mp3" ||
|
|
259
|
+
type === "audio/wav" ||
|
|
260
|
+
type === "audio/wave" ||
|
|
261
|
+
type === "audio/x-wav" ||
|
|
262
|
+
type === "wav" ||
|
|
263
|
+
type === "m4a" ||
|
|
264
|
+
type === "audio/mp4" ||
|
|
265
|
+
type === "audio/m4a" ||
|
|
266
|
+
type === "audio/x-m4a" ||
|
|
267
|
+
type === "webm" ||
|
|
268
|
+
type === "audio/webm" ||
|
|
269
|
+
type === "ogg" ||
|
|
270
|
+
type === "audio/ogg") {
|
|
271
|
+
const buffer = Buffer.isBuffer(data)
|
|
272
|
+
? data
|
|
273
|
+
: Buffer.from(data, "base64");
|
|
274
|
+
return extractAudio(buffer, type.startsWith("audio/") ? type : `audio/${type}`);
|
|
275
|
+
}
|
|
276
|
+
// Video formats
|
|
277
|
+
if (type === "mp4" ||
|
|
278
|
+
type === "video" ||
|
|
279
|
+
type === "video/mp4" ||
|
|
280
|
+
type === "video/webm" ||
|
|
281
|
+
type === "video/mpeg" ||
|
|
282
|
+
type === "avi" ||
|
|
283
|
+
type === "video/avi" ||
|
|
284
|
+
type === "mov" ||
|
|
285
|
+
type === "video/quicktime") {
|
|
286
|
+
const buffer = Buffer.isBuffer(data)
|
|
287
|
+
? data
|
|
288
|
+
: Buffer.from(data, "base64");
|
|
289
|
+
return extractVideo(buffer);
|
|
290
|
+
}
|
|
291
|
+
switch (type) {
|
|
292
|
+
case "pdf":
|
|
293
|
+
return extractPDF(Buffer.isBuffer(data)
|
|
294
|
+
? data
|
|
295
|
+
: Buffer.from(data, "base64"));
|
|
296
|
+
case "docx":
|
|
297
|
+
case "doc":
|
|
298
|
+
return extractDOCX(Buffer.isBuffer(data)
|
|
299
|
+
? data
|
|
300
|
+
: Buffer.from(data, "base64"));
|
|
301
|
+
case "html":
|
|
302
|
+
case "htm":
|
|
303
|
+
return extractHTML(data.toString());
|
|
304
|
+
case "md":
|
|
305
|
+
case "markdown": {
|
|
306
|
+
const text = data.toString();
|
|
307
|
+
return {
|
|
308
|
+
text,
|
|
309
|
+
metadata: {
|
|
310
|
+
content_type: "markdown",
|
|
311
|
+
char_count: text.length,
|
|
312
|
+
estimated_tokens: estimateTokens(text),
|
|
313
|
+
extraction_method: "passthrough",
|
|
314
|
+
},
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
case "txt":
|
|
318
|
+
case "text": {
|
|
319
|
+
const text = data.toString();
|
|
320
|
+
return {
|
|
321
|
+
text,
|
|
322
|
+
metadata: {
|
|
323
|
+
content_type: "txt",
|
|
324
|
+
char_count: text.length,
|
|
325
|
+
estimated_tokens: estimateTokens(text),
|
|
326
|
+
extraction_method: "passthrough",
|
|
327
|
+
},
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
default:
|
|
331
|
+
throw new Error(`Unsupported content type: ${contentType}`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.ingestDocument = ingestDocument;
|
|
37
|
+
exports.ingestURL = ingestURL;
|
|
38
|
+
const hsg_1 = require("../memory/hsg");
|
|
39
|
+
const db_1 = require("../core/db");
|
|
40
|
+
const utils_1 = require("../utils");
|
|
41
|
+
const extract_1 = require("./extract");
|
|
42
|
+
const LG = 8000, SEC = 3000;
|
|
43
|
+
const split = (t, sz) => {
|
|
44
|
+
if (t.length <= sz)
|
|
45
|
+
return [t];
|
|
46
|
+
const secs = [];
|
|
47
|
+
const paras = t.split(/\n\n+/);
|
|
48
|
+
let cur = "";
|
|
49
|
+
for (const p of paras) {
|
|
50
|
+
if (cur.length + p.length > sz && cur.length > 0) {
|
|
51
|
+
secs.push(cur.trim());
|
|
52
|
+
cur = p;
|
|
53
|
+
}
|
|
54
|
+
else
|
|
55
|
+
cur += (cur ? "\n\n" : "") + p;
|
|
56
|
+
}
|
|
57
|
+
if (cur.trim())
|
|
58
|
+
secs.push(cur.trim());
|
|
59
|
+
return secs;
|
|
60
|
+
};
|
|
61
|
+
const mkRoot = async (txt, ex, meta, user_id) => {
|
|
62
|
+
const sum = txt.length > 500 ? txt.slice(0, 500) + "..." : txt;
|
|
63
|
+
const cnt = `[Document: ${ex.metadata.content_type.toUpperCase()}]\n\n${sum}\n\n[Full content split across ${Math.ceil(txt.length / SEC)} sections]`;
|
|
64
|
+
const id = (0, utils_1.rid)(), ts = (0, utils_1.now)();
|
|
65
|
+
await db_1.transaction.begin();
|
|
66
|
+
try {
|
|
67
|
+
await db_1.q.ins_mem.run(id, cnt, "reflective", (0, utils_1.j)([]), (0, utils_1.j)({
|
|
68
|
+
...meta,
|
|
69
|
+
...ex.metadata,
|
|
70
|
+
is_root: true,
|
|
71
|
+
ingestion_strategy: "root-child",
|
|
72
|
+
ingested_at: ts,
|
|
73
|
+
}), ts, ts, ts, 1.0, 0.1, 1, user_id || "anonymous", null);
|
|
74
|
+
await db_1.transaction.commit();
|
|
75
|
+
return id;
|
|
76
|
+
}
|
|
77
|
+
catch (e) {
|
|
78
|
+
console.error("[ERROR] Root failed:", e);
|
|
79
|
+
await db_1.transaction.rollback();
|
|
80
|
+
throw e;
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
const mkChild = async (txt, idx, tot, rid, meta, user_id) => {
|
|
84
|
+
const r = await (0, hsg_1.add_hsg_memory)(txt, (0, utils_1.j)([]), {
|
|
85
|
+
...meta,
|
|
86
|
+
is_child: true,
|
|
87
|
+
section_index: idx,
|
|
88
|
+
total_sections: tot,
|
|
89
|
+
parent_id: rid,
|
|
90
|
+
}, user_id || undefined);
|
|
91
|
+
return r.id;
|
|
92
|
+
};
|
|
93
|
+
const link = async (rid, cid, idx, user_id) => {
|
|
94
|
+
const ts = (0, utils_1.now)();
|
|
95
|
+
await db_1.transaction.begin();
|
|
96
|
+
try {
|
|
97
|
+
await db_1.q.ins_waypoint.run(rid, cid, user_id || "anonymous", 1.0, ts, ts);
|
|
98
|
+
await db_1.transaction.commit();
|
|
99
|
+
console.log(`[INGEST] Linked: ${rid.slice(0, 8)} -> ${cid.slice(0, 8)} (section ${idx})`);
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
await db_1.transaction.rollback();
|
|
103
|
+
console.error(`[INGEST] Link failed for section ${idx}:`, e);
|
|
104
|
+
throw e;
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
async function ingestDocument(t, data, meta, cfg, user_id) {
|
|
108
|
+
const th = cfg?.lg_thresh || LG, sz = cfg?.sec_sz || SEC;
|
|
109
|
+
const ex = await (0, extract_1.extractText)(t, data);
|
|
110
|
+
const { text, metadata: exMeta } = ex;
|
|
111
|
+
const useRC = cfg?.force_root || exMeta.estimated_tokens > th;
|
|
112
|
+
if (!useRC) {
|
|
113
|
+
const r = await (0, hsg_1.add_hsg_memory)(text, (0, utils_1.j)([]), {
|
|
114
|
+
...meta,
|
|
115
|
+
...exMeta,
|
|
116
|
+
ingestion_strategy: "single",
|
|
117
|
+
ingested_at: (0, utils_1.now)(),
|
|
118
|
+
}, user_id || undefined);
|
|
119
|
+
return {
|
|
120
|
+
root_memory_id: r.id,
|
|
121
|
+
child_count: 0,
|
|
122
|
+
total_tokens: exMeta.estimated_tokens,
|
|
123
|
+
strategy: "single",
|
|
124
|
+
extraction: exMeta,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
const secs = split(text, sz);
|
|
128
|
+
console.log(`[INGEST] Document: ${exMeta.estimated_tokens} tokens`);
|
|
129
|
+
console.log(`[INGEST] Splitting into ${secs.length} sections`);
|
|
130
|
+
let rid;
|
|
131
|
+
const cids = [];
|
|
132
|
+
try {
|
|
133
|
+
rid = await mkRoot(text, ex, meta, user_id);
|
|
134
|
+
console.log(`[INGEST] Root memory created: ${rid}`);
|
|
135
|
+
for (let i = 0; i < secs.length; i++) {
|
|
136
|
+
try {
|
|
137
|
+
const cid = await mkChild(secs[i], i, secs.length, rid, meta, user_id);
|
|
138
|
+
cids.push(cid);
|
|
139
|
+
await link(rid, cid, i, user_id);
|
|
140
|
+
console.log(`[INGEST] Section ${i + 1}/${secs.length} processed: ${cid}`);
|
|
141
|
+
}
|
|
142
|
+
catch (e) {
|
|
143
|
+
console.error(`[INGEST] Section ${i + 1}/${secs.length} failed:`, e);
|
|
144
|
+
throw e;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
console.log(`[INGEST] Completed: ${cids.length} sections linked to ${rid}`);
|
|
148
|
+
return {
|
|
149
|
+
root_memory_id: rid,
|
|
150
|
+
child_count: secs.length,
|
|
151
|
+
total_tokens: exMeta.estimated_tokens,
|
|
152
|
+
strategy: "root-child",
|
|
153
|
+
extraction: exMeta,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
catch (e) {
|
|
157
|
+
console.error("[INGEST] Document ingestion failed:", e);
|
|
158
|
+
throw e;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
async function ingestURL(url, meta, cfg, user_id) {
|
|
162
|
+
const { extractURL } = await Promise.resolve().then(() => __importStar(require("./extract")));
|
|
163
|
+
const ex = await extractURL(url);
|
|
164
|
+
const th = cfg?.lg_thresh || LG, sz = cfg?.sec_sz || SEC;
|
|
165
|
+
const useRC = cfg?.force_root || ex.metadata.estimated_tokens > th;
|
|
166
|
+
if (!useRC) {
|
|
167
|
+
const r = await (0, hsg_1.add_hsg_memory)(ex.text, (0, utils_1.j)([]), {
|
|
168
|
+
...meta,
|
|
169
|
+
...ex.metadata,
|
|
170
|
+
ingestion_strategy: "single",
|
|
171
|
+
ingested_at: (0, utils_1.now)(),
|
|
172
|
+
}, user_id || undefined);
|
|
173
|
+
return {
|
|
174
|
+
root_memory_id: r.id,
|
|
175
|
+
child_count: 0,
|
|
176
|
+
total_tokens: ex.metadata.estimated_tokens,
|
|
177
|
+
strategy: "single",
|
|
178
|
+
extraction: ex.metadata,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
const secs = split(ex.text, sz);
|
|
182
|
+
console.log(`[INGEST] URL: ${ex.metadata.estimated_tokens} tokens`);
|
|
183
|
+
console.log(`[INGEST] Splitting into ${secs.length} sections`);
|
|
184
|
+
let rid;
|
|
185
|
+
const cids = [];
|
|
186
|
+
try {
|
|
187
|
+
rid = await mkRoot(ex.text, ex, { ...meta, source_url: url }, user_id);
|
|
188
|
+
console.log(`[INGEST] Root memory for URL: ${rid}`);
|
|
189
|
+
for (let i = 0; i < secs.length; i++) {
|
|
190
|
+
try {
|
|
191
|
+
const cid = await mkChild(secs[i], i, secs.length, rid, { ...meta, source_url: url }, user_id);
|
|
192
|
+
cids.push(cid);
|
|
193
|
+
await link(rid, cid, i, user_id);
|
|
194
|
+
console.log(`[INGEST] URL section ${i + 1}/${secs.length} processed: ${cid}`);
|
|
195
|
+
}
|
|
196
|
+
catch (e) {
|
|
197
|
+
console.error(`[INGEST] URL section ${i + 1}/${secs.length} failed:`, e);
|
|
198
|
+
throw e;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
console.log(`[INGEST] URL completed: ${cids.length} sections linked to ${rid}`);
|
|
202
|
+
return {
|
|
203
|
+
root_memory_id: rid,
|
|
204
|
+
child_count: secs.length,
|
|
205
|
+
total_tokens: ex.metadata.estimated_tokens,
|
|
206
|
+
strategy: "root-child",
|
|
207
|
+
extraction: ex.metadata,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
catch (e) {
|
|
211
|
+
console.error("[INGEST] URL ingestion failed:", e);
|
|
212
|
+
throw e;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const server = require("./server.js");
|
|
4
|
+
const cfg_1 = require("../core/cfg");
|
|
5
|
+
const hsg_1 = require("../memory/hsg");
|
|
6
|
+
const mcp_1 = require("../ai/mcp");
|
|
7
|
+
const routes_1 = require("./routes");
|
|
8
|
+
const auth_1 = require("./middleware/auth");
|
|
9
|
+
const reflect_1 = require("../memory/reflect");
|
|
10
|
+
const user_summary_1 = require("../memory/user_summary");
|
|
11
|
+
const telemetry_1 = require("../core/telemetry");
|
|
12
|
+
const dashboard_1 = require("./routes/dashboard");
|
|
13
|
+
const ASC = ` ____ __ __
|
|
14
|
+
/ __ \\ | \\/ |
|
|
15
|
+
| | | |_ __ ___ _ __ | \\ / | ___ _ __ ___ ___ _ __ _ _
|
|
16
|
+
| | | | '_ \\ / _ \\ '_ \\| |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
|
|
17
|
+
| |__| | |_) | __/ | | | | | | __/ | | | | | (_) | | | |_| |
|
|
18
|
+
\\____/| .__/ \\___|_| |_|_| |_|\\___|_| |_| |_|\\___/|_| \\__, |
|
|
19
|
+
| | __/ |
|
|
20
|
+
|_| |___/ `;
|
|
21
|
+
const app = server({ max_payload_size: cfg_1.env.max_payload_size });
|
|
22
|
+
console.log(ASC);
|
|
23
|
+
console.log(`[CONFIG] Vector Dimension: ${cfg_1.env.vec_dim}`);
|
|
24
|
+
console.log(`[CONFIG] Cache Segments: ${cfg_1.env.cache_segments}`);
|
|
25
|
+
console.log(`[CONFIG] Max Active Queries: ${cfg_1.env.max_active}`);
|
|
26
|
+
// Warn about configuration mismatch that causes embedding incompatibility
|
|
27
|
+
if (cfg_1.env.emb_kind !== "synthetic" && (cfg_1.tier === "hybrid" || cfg_1.tier === "fast")) {
|
|
28
|
+
console.warn(`[CONFIG] ⚠️ WARNING: Embedding configuration mismatch detected!\n` +
|
|
29
|
+
` OM_EMBEDDINGS=${cfg_1.env.emb_kind} but OM_TIER=${cfg_1.tier}\n` +
|
|
30
|
+
` Storage will use ${cfg_1.env.emb_kind} embeddings, but queries will use synthetic embeddings.\n` +
|
|
31
|
+
` This causes semantic search to fail. Set OM_TIER=deep to fix.`);
|
|
32
|
+
}
|
|
33
|
+
app.use((0, dashboard_1.req_tracker_mw)());
|
|
34
|
+
app.use((req, res, next) => {
|
|
35
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
36
|
+
res.setHeader("Access-Control-Allow-Methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS");
|
|
37
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type,Authorization,x-api-key");
|
|
38
|
+
if (req.method === "OPTIONS") {
|
|
39
|
+
res.status(200).end();
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
next();
|
|
43
|
+
});
|
|
44
|
+
app.use(auth_1.authenticate_api_request);
|
|
45
|
+
if (process.env.OM_LOG_AUTH === "true") {
|
|
46
|
+
app.use(auth_1.log_authenticated_request);
|
|
47
|
+
}
|
|
48
|
+
const path = require("path");
|
|
49
|
+
(0, routes_1.routes)(app);
|
|
50
|
+
// Serve dashboard static files
|
|
51
|
+
const publicDir = path.join(__dirname, "../..", "public");
|
|
52
|
+
app.use(app.serverStatic("/", publicDir));
|
|
53
|
+
// Fallback for SPA routing
|
|
54
|
+
app.get("*", (req, res, next) => {
|
|
55
|
+
if (req.path.startsWith("/api") || req.path.startsWith("/auth")) {
|
|
56
|
+
return next();
|
|
57
|
+
}
|
|
58
|
+
res.setHeader("Content-Type", "text/html");
|
|
59
|
+
const fs = require("fs");
|
|
60
|
+
const indexPath = path.join(publicDir, "index.html");
|
|
61
|
+
if (fs.existsSync(indexPath)) {
|
|
62
|
+
res.send(fs.readFileSync(indexPath, "utf8"));
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
next();
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
(0, mcp_1.mcp)(app);
|
|
69
|
+
if (cfg_1.env.mode === "langgraph") {
|
|
70
|
+
console.log("[MODE] LangGraph integration enabled");
|
|
71
|
+
}
|
|
72
|
+
const decayIntervalMs = cfg_1.env.decay_interval_minutes * 60 * 1000;
|
|
73
|
+
console.log(`[DECAY] Interval: ${cfg_1.env.decay_interval_minutes} minutes (${decayIntervalMs / 1000}s)`);
|
|
74
|
+
setInterval(async () => {
|
|
75
|
+
console.log("[DECAY] Running HSG decay process...");
|
|
76
|
+
try {
|
|
77
|
+
const result = await (0, hsg_1.run_decay_process)();
|
|
78
|
+
console.log(`[DECAY] Completed: ${result.decayed}/${result.processed} memories updated`);
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
console.error("[DECAY] Process failed:", error);
|
|
82
|
+
}
|
|
83
|
+
}, decayIntervalMs);
|
|
84
|
+
setInterval(async () => {
|
|
85
|
+
console.log("[PRUNE] Pruning weak waypoints...");
|
|
86
|
+
try {
|
|
87
|
+
const pruned = await (0, hsg_1.prune_weak_waypoints)();
|
|
88
|
+
console.log(`[PRUNE] Completed: ${pruned} waypoints removed`);
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
console.error("[PRUNE] Failed:", error);
|
|
92
|
+
}
|
|
93
|
+
}, 7 * 24 * 60 * 60 * 1000);
|
|
94
|
+
setTimeout(() => {
|
|
95
|
+
(0, hsg_1.run_decay_process)()
|
|
96
|
+
.then((result) => {
|
|
97
|
+
console.log(`[INIT] Initial decay: ${result.decayed}/${result.processed} memories updated`);
|
|
98
|
+
})
|
|
99
|
+
.catch(console.error);
|
|
100
|
+
}, 3000);
|
|
101
|
+
(0, reflect_1.start_reflection)();
|
|
102
|
+
(0, user_summary_1.start_user_summary_reflection)();
|
|
103
|
+
console.log(`[SERVER] Starting on port ${cfg_1.env.port}`);
|
|
104
|
+
app.listen(cfg_1.env.port, () => {
|
|
105
|
+
console.log(`[SERVER] Running on http://localhost:${cfg_1.env.port}`);
|
|
106
|
+
(0, telemetry_1.sendTelemetry)().catch(() => {
|
|
107
|
+
// ignore telemetry failures
|
|
108
|
+
});
|
|
109
|
+
});
|