@qearlyao/familiar 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/config.example.toml +2 -2
- package/dist/agent/payload-normalizers.js +52 -0
- package/dist/agent/session-helpers.js +86 -0
- package/dist/agent/tool-descriptions.js +4 -0
- package/dist/agent/tools.js +30 -0
- package/dist/agent/transcript-log.js +93 -0
- package/dist/agent/types.js +1 -0
- package/dist/agent-core.js +82 -0
- package/dist/agent-work-queue.js +55 -0
- package/dist/agent.js +91 -322
- package/dist/browser-tools.js +80 -28
- package/dist/chat-log.js +15 -3
- package/dist/cli.js +36 -6
- package/dist/config/enums.js +35 -0
- package/dist/config/interpolate.js +15 -0
- package/dist/config/model-refs.js +11 -0
- package/dist/config/readers.js +116 -0
- package/dist/config/sections.js +113 -0
- package/dist/config/types.js +1 -0
- package/dist/config-registry.js +26 -7
- package/dist/config.js +8 -271
- package/dist/discord/channel.js +32 -0
- package/dist/discord/chunking.js +163 -0
- package/dist/discord/client.js +44 -0
- package/dist/discord/commands.js +181 -0
- package/dist/discord/inbound.js +44 -0
- package/dist/discord/send.js +106 -0
- package/dist/discord/turn.js +55 -0
- package/dist/discord.js +266 -1186
- package/dist/ids.js +11 -0
- package/dist/image-gen.js +90 -10
- package/dist/index.js +1 -0
- package/dist/memory/index/store.js +21 -17
- package/dist/memory/index/vector-codec.js +2 -2
- package/dist/memory/lcm/context-transformer.js +6 -2
- package/dist/memory/lcm/segment-manager.js +6 -2
- package/dist/memory/lcm/store/index-ids.js +6 -0
- package/dist/memory/lcm/store/inserts.js +31 -0
- package/dist/memory/lcm/store/normalizers.js +91 -0
- package/dist/memory/lcm/store/row-mappers.js +114 -0
- package/dist/memory/lcm/store/row-types.js +1 -0
- package/dist/memory/lcm/store/serialization.js +37 -0
- package/dist/memory/lcm/store/snapshots.js +73 -0
- package/dist/memory/lcm/store.js +20 -360
- package/dist/owner-identity.js +29 -0
- package/dist/runtime-manager.js +51 -0
- package/dist/runtime.js +89 -41
- package/dist/scheduler-runner.js +243 -0
- package/dist/scheduler.js +1 -1
- package/dist/service.js +1 -0
- package/dist/settings.js +3 -0
- package/dist/util/fs.js +1 -1
- package/dist/web/event-hub.js +246 -0
- package/dist/{web-http.js → web/http.js} +19 -5
- package/dist/web/memes.js +25 -0
- package/dist/web/messages.js +345 -0
- package/dist/web/multipart.js +80 -0
- package/dist/web/payloads.js +34 -0
- package/dist/{web-static.js → web/static.js} +19 -14
- package/dist/web/stream.js +69 -0
- package/dist/web-tools/cache.js +42 -0
- package/dist/web-tools/config.js +16 -0
- package/dist/web-tools/fetch-providers.js +119 -0
- package/dist/web-tools/format.js +88 -0
- package/dist/web-tools/http.js +81 -0
- package/dist/web-tools/routing.js +29 -0
- package/dist/web-tools/safety.js +73 -0
- package/dist/web-tools/search-providers.js +277 -0
- package/dist/web-tools/types.js +54 -0
- package/dist/web-tools/util.js +23 -0
- package/dist/web-tools.js +9 -798
- package/dist/web.js +416 -984
- package/npm-shrinkwrap.json +242 -201
- package/package.json +4 -4
- package/web/dist/assets/index-CSkxUQCr.js +63 -0
- package/web/dist/assets/index-DllM6RqL.css +2 -0
- package/web/dist/index.html +6 -3
- package/web/dist/assets/index-B23WT77N.js +0 -63
- package/web/dist/assets/index-D3MotFzN.css +0 -2
- /package/dist/{web-auth.js → web/auth.js} +0 -0
- /package/dist/{web-events.js → web/events.js} +0 -0
- /package/dist/{web-types.js → web/types.js} +0 -0
package/dist/ids.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
export function toUnixMs(ts) {
|
|
3
|
+
const parsed = ts ? Date.parse(ts) : NaN;
|
|
4
|
+
return Number.isFinite(parsed) ? parsed : Date.now();
|
|
5
|
+
}
|
|
6
|
+
export function eventId() {
|
|
7
|
+
return `evt_${randomUUID()}`;
|
|
8
|
+
}
|
|
9
|
+
export function messageId(prefix = "msg") {
|
|
10
|
+
return `${prefix}_${randomUUID()}`;
|
|
11
|
+
}
|
package/dist/image-gen.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { lstat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { homedir } from "node:os";
|
|
3
4
|
import { basename, isAbsolute, relative, resolve } from "node:path";
|
|
4
5
|
import { findEnvKeys, generateImages, getEnvApiKey, getImageModels, getImageProviders, } from "@earendil-works/pi-ai";
|
|
5
6
|
import { Type } from "typebox";
|
|
@@ -13,9 +14,10 @@ const OPENROUTER_IMAGE_BASE_URL = "https://openrouter.ai/api/v1";
|
|
|
13
14
|
const imageGenSchema = Type.Object({
|
|
14
15
|
prompt: Type.String({ description: "Image generation prompt." }),
|
|
15
16
|
referenceImages: Type.Optional(Type.Array(Type.String(), {
|
|
16
|
-
description: "Optional. Image attachment IDs or names, or workspace-relative image file paths, to use as visual references. Prefer IDs from the attachment tags when available.",
|
|
17
|
+
description: "Optional. Image attachment IDs or names, or workspace-relative, absolute, or ~/ image file paths, to use as visual references. Prefer IDs from the attachment tags when available.",
|
|
17
18
|
})),
|
|
18
19
|
}, { additionalProperties: false });
|
|
20
|
+
const MAX_REMOTE_IMAGE_BYTES = 12 * 1024 * 1024;
|
|
19
21
|
function formatImageGenNotice(name) {
|
|
20
22
|
return `${IMAGE_GEN_NOTICE_PREFIX} ${name}`;
|
|
21
23
|
}
|
|
@@ -116,7 +118,7 @@ function recoveredImageFromBase64(value) {
|
|
|
116
118
|
data,
|
|
117
119
|
};
|
|
118
120
|
}
|
|
119
|
-
function
|
|
121
|
+
function recoveredInlineImageFromText(text) {
|
|
120
122
|
const trimmed = text.trim();
|
|
121
123
|
const dataUrlMatch = trimmed.match(/^data:(image\/[^;]+);base64,([A-Za-z0-9+/]+={0,2})$/);
|
|
122
124
|
if (dataUrlMatch)
|
|
@@ -127,7 +129,80 @@ function recoveredImageFromText(text) {
|
|
|
127
129
|
}
|
|
128
130
|
return recoveredImageFromBase64(trimmed);
|
|
129
131
|
}
|
|
130
|
-
function
|
|
132
|
+
function imageUrlFromMarkdownText(text) {
|
|
133
|
+
const match = text.match(/!\[[^\]]*]\((https?:\/\/[^)\s]+)\)/i);
|
|
134
|
+
if (!match?.[1])
|
|
135
|
+
return undefined;
|
|
136
|
+
try {
|
|
137
|
+
const url = new URL(match[1]);
|
|
138
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
139
|
+
return undefined;
|
|
140
|
+
return url;
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
async function readBoundedResponseBody(response, maxBytes) {
|
|
147
|
+
const reader = response.body?.getReader();
|
|
148
|
+
if (!reader) {
|
|
149
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
150
|
+
return buffer.byteLength > maxBytes ? undefined : buffer;
|
|
151
|
+
}
|
|
152
|
+
const chunks = [];
|
|
153
|
+
let total = 0;
|
|
154
|
+
try {
|
|
155
|
+
for (;;) {
|
|
156
|
+
const { done, value } = await reader.read();
|
|
157
|
+
if (done)
|
|
158
|
+
break;
|
|
159
|
+
const chunk = Buffer.from(value);
|
|
160
|
+
total += chunk.byteLength;
|
|
161
|
+
if (total > maxBytes)
|
|
162
|
+
return undefined;
|
|
163
|
+
chunks.push(chunk);
|
|
164
|
+
}
|
|
165
|
+
return Buffer.concat(chunks);
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
reader.releaseLock();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
async function recoveredImageFromRemoteUrl(url, options) {
|
|
172
|
+
try {
|
|
173
|
+
const response = await fetch(url, { signal: options.signal });
|
|
174
|
+
if (!response.ok)
|
|
175
|
+
return undefined;
|
|
176
|
+
const contentLength = Number(response.headers.get("content-length") ?? 0);
|
|
177
|
+
if (contentLength > MAX_REMOTE_IMAGE_BYTES)
|
|
178
|
+
return undefined;
|
|
179
|
+
const bytes = await readBoundedResponseBody(response, MAX_REMOTE_IMAGE_BYTES);
|
|
180
|
+
if (!bytes)
|
|
181
|
+
return undefined;
|
|
182
|
+
const detectedMimeType = sniffImageMimeType(bytes);
|
|
183
|
+
if (!detectedMimeType)
|
|
184
|
+
return undefined;
|
|
185
|
+
return {
|
|
186
|
+
mimeType: detectedMimeType,
|
|
187
|
+
data: bytes.toString("base64"),
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
if (options.signal?.aborted)
|
|
192
|
+
throw error;
|
|
193
|
+
return undefined;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async function recoveredImageFromText(text, options) {
|
|
197
|
+
const inlineImage = recoveredInlineImageFromText(text);
|
|
198
|
+
if (inlineImage)
|
|
199
|
+
return inlineImage;
|
|
200
|
+
const url = imageUrlFromMarkdownText(text);
|
|
201
|
+
if (!url)
|
|
202
|
+
return undefined;
|
|
203
|
+
return recoveredImageFromRemoteUrl(url, options);
|
|
204
|
+
}
|
|
205
|
+
async function normalizeCompatibleImageText(result, options) {
|
|
131
206
|
if (result.output.some((item) => item.type === "image"))
|
|
132
207
|
return result;
|
|
133
208
|
const output = [];
|
|
@@ -136,7 +211,7 @@ function normalizeCompatibleImageText(result) {
|
|
|
136
211
|
output.push(item);
|
|
137
212
|
continue;
|
|
138
213
|
}
|
|
139
|
-
const recovered = recoveredImageFromText(item.text);
|
|
214
|
+
const recovered = await recoveredImageFromText(item.text, options);
|
|
140
215
|
if (!recovered) {
|
|
141
216
|
output.push(item);
|
|
142
217
|
continue;
|
|
@@ -148,7 +223,11 @@ function normalizeCompatibleImageText(result) {
|
|
|
148
223
|
return { ...result, output };
|
|
149
224
|
}
|
|
150
225
|
function resolveWorkspaceReferencePath(config, rawRef) {
|
|
151
|
-
|
|
226
|
+
if (rawRef === "~" || rawRef.startsWith("~/"))
|
|
227
|
+
return resolve(homedir(), rawRef.slice(2));
|
|
228
|
+
if (isAbsolute(rawRef))
|
|
229
|
+
return resolve(rawRef);
|
|
230
|
+
const path = resolve(config.workspacePath, rawRef);
|
|
152
231
|
const workspaceRelative = relative(config.workspacePath, path);
|
|
153
232
|
if (!workspaceRelative || workspaceRelative.startsWith("..") || isAbsolute(workspaceRelative)) {
|
|
154
233
|
throw new Error(`Reference image path must be inside the workspace: ${rawRef}`);
|
|
@@ -304,13 +383,14 @@ async function writeGeneratedImages(config, mediaSink, result) {
|
|
|
304
383
|
async function tryGenerateImages(config, ref, prompt, references, workspaceRefs, signal, generate) {
|
|
305
384
|
const model = resolveImageModel(config, ref);
|
|
306
385
|
const context = await buildImageContext(model, prompt, references, workspaceRefs, config);
|
|
386
|
+
const result = await generate(model, context, {
|
|
387
|
+
apiKey: resolveImageModelApiKey(config, model),
|
|
388
|
+
signal,
|
|
389
|
+
timeoutMs: config.imageGen.timeoutMs,
|
|
390
|
+
});
|
|
307
391
|
return {
|
|
308
392
|
model,
|
|
309
|
-
result:
|
|
310
|
-
apiKey: resolveImageModelApiKey(config, model),
|
|
311
|
-
signal,
|
|
312
|
-
timeoutMs: config.imageGen.timeoutMs,
|
|
313
|
-
})),
|
|
393
|
+
result: await normalizeCompatibleImageText(result, { signal }),
|
|
314
394
|
};
|
|
315
395
|
}
|
|
316
396
|
function attemptDetails(model, result) {
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { createFamiliarAgent } from "./agent.js";
|
|
2
|
+
export { createAgentCore } from "./agent-core.js";
|
|
2
3
|
export { buildRecordBase, chatChannelKey, chatLogPath, createChatLog, } from "./chat-log.js";
|
|
3
4
|
export { loadConfig } from "./config.js";
|
|
4
5
|
export { startDiscordDaemon } from "./discord.js";
|
|
@@ -11,6 +11,11 @@ export class MemoryIndexStore {
|
|
|
11
11
|
embeddingProvider;
|
|
12
12
|
embeddingModel;
|
|
13
13
|
embeddingDimensions;
|
|
14
|
+
vectorCapabilityValue;
|
|
15
|
+
findChunkIdByHashStmt;
|
|
16
|
+
insertChunkStmt;
|
|
17
|
+
insertFtsStmt;
|
|
18
|
+
insertMemoryVecStmt;
|
|
14
19
|
constructor(options) {
|
|
15
20
|
if (!options.db && !options.path)
|
|
16
21
|
throw new Error("MemoryIndexStore requires a db or path");
|
|
@@ -32,6 +37,16 @@ export class MemoryIndexStore {
|
|
|
32
37
|
embeddingModel: this.embeddingModel,
|
|
33
38
|
embeddingDimensions: this.embeddingDimensions,
|
|
34
39
|
});
|
|
40
|
+
this.vectorCapabilityValue = readMeta(this.db, "vector_capability") === "sqlite-vec" ? "sqlite-vec" : "blob-js";
|
|
41
|
+
this.findChunkIdByHashStmt = this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?");
|
|
42
|
+
this.insertChunkStmt = this.db.prepare(`INSERT INTO memory_chunks (
|
|
43
|
+
content_hash, corpus, text_full, snippet, token_count, metadata_json, embedding_model,
|
|
44
|
+
embedding_dimensions, embedding
|
|
45
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
46
|
+
this.insertFtsStmt = this.db.prepare("INSERT INTO memory_fts(rowid, text_full, snippet) VALUES (?, ?, ?)");
|
|
47
|
+
if (this.vectorCapabilityValue === "sqlite-vec") {
|
|
48
|
+
this.insertMemoryVecStmt = this.db.prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)");
|
|
49
|
+
}
|
|
35
50
|
}
|
|
36
51
|
static open(config) {
|
|
37
52
|
return new MemoryIndexStore({
|
|
@@ -80,9 +95,7 @@ export class MemoryIndexStore {
|
|
|
80
95
|
this.insertSourceMapping(preloadedId, item);
|
|
81
96
|
continue;
|
|
82
97
|
}
|
|
83
|
-
const existing = this.
|
|
84
|
-
.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
|
|
85
|
-
.get(item.contentHash);
|
|
98
|
+
const existing = this.findChunkIdByHashStmt.get(item.contentHash);
|
|
86
99
|
if (existing)
|
|
87
100
|
this.insertSourceMapping(existing.id, item);
|
|
88
101
|
}
|
|
@@ -333,7 +346,7 @@ export class MemoryIndexStore {
|
|
|
333
346
|
};
|
|
334
347
|
}
|
|
335
348
|
vectorCapability() {
|
|
336
|
-
return
|
|
349
|
+
return this.vectorCapabilityValue;
|
|
337
350
|
}
|
|
338
351
|
vectorRowCount() {
|
|
339
352
|
try {
|
|
@@ -377,26 +390,17 @@ export class MemoryIndexStore {
|
|
|
377
390
|
? { id: knownId }
|
|
378
391
|
: knownMissingHashes?.has(item.contentHash)
|
|
379
392
|
? undefined
|
|
380
|
-
: this.
|
|
393
|
+
: this.findChunkIdByHashStmt.get(item.contentHash);
|
|
381
394
|
if (existing) {
|
|
382
395
|
knownIds.set(item.contentHash, existing.id);
|
|
383
396
|
this.insertSourceMapping(existing.id, item);
|
|
384
397
|
return existing.id;
|
|
385
398
|
}
|
|
386
|
-
const result = this.
|
|
387
|
-
.prepare(`INSERT INTO memory_chunks (
|
|
388
|
-
content_hash, corpus, text_full, snippet, token_count, metadata_json, embedding_model,
|
|
389
|
-
embedding_dimensions, embedding
|
|
390
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
391
|
-
.run(item.contentHash, item.corpus, item.text, item.snippet, item.tokenCount, item.metadataJson, this.embeddingModel, this.embeddingDimensions, encodeVector(item.embedding));
|
|
399
|
+
const result = this.insertChunkStmt.run(item.contentHash, item.corpus, item.text, item.snippet, item.tokenCount, item.metadataJson, this.embeddingModel, this.embeddingDimensions, encodeVector(item.embedding));
|
|
392
400
|
const id = Number(result.lastInsertRowid);
|
|
393
|
-
this.
|
|
394
|
-
.prepare("INSERT INTO memory_fts(rowid, text_full, snippet) VALUES (?, ?, ?)")
|
|
395
|
-
.run(id, item.text, item.snippet);
|
|
401
|
+
this.insertFtsStmt.run(id, item.text, item.snippet);
|
|
396
402
|
if (this.vectorCapability() === "sqlite-vec") {
|
|
397
|
-
this.
|
|
398
|
-
.prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
|
|
399
|
-
.run(id, encodeVector(item.embedding));
|
|
403
|
+
this.insertMemoryVecStmt.run(id, encodeVector(item.embedding));
|
|
400
404
|
}
|
|
401
405
|
knownIds.set(item.contentHash, id);
|
|
402
406
|
this.insertSourceMapping(id, item);
|
|
@@ -15,8 +15,8 @@ export function cosineDistance(a, b) {
|
|
|
15
15
|
let aNorm = 0;
|
|
16
16
|
let bNorm = 0;
|
|
17
17
|
for (let index = 0; index < a.length; index++) {
|
|
18
|
-
const av = a[index]
|
|
19
|
-
const bv = b[index]
|
|
18
|
+
const av = a[index];
|
|
19
|
+
const bv = b[index];
|
|
20
20
|
dot += av * bv;
|
|
21
21
|
aNorm += av * av;
|
|
22
22
|
bNorm += bv * bv;
|
|
@@ -176,8 +176,12 @@ export class LcmContextTransformer {
|
|
|
176
176
|
tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
|
|
177
177
|
await this.condenseRuntimeSummaries({ state, sessionKey: input.sessionKey, signal: input.signal });
|
|
178
178
|
};
|
|
179
|
-
|
|
180
|
-
|
|
179
|
+
// Serialize compactions per state: run after the prior one settles. Keep the shared
|
|
180
|
+
// compactionQueue non-rejecting so a failure here can't poison the next caller's link,
|
|
181
|
+
// and await the real work promise so this caller still surfaces its own error.
|
|
182
|
+
const settled = input.state.compactionQueue.then(run);
|
|
183
|
+
input.state.compactionQueue = settled.catch(() => undefined);
|
|
184
|
+
await settled;
|
|
181
185
|
return { compacted, tokensSaved };
|
|
182
186
|
}
|
|
183
187
|
async persistRuntimeSummary(input) {
|
|
@@ -19,8 +19,12 @@ export class LcmSegmentManager {
|
|
|
19
19
|
}
|
|
20
20
|
subscribeRuntime(runtime, sessionId) {
|
|
21
21
|
const unsubscribe = runtime.subscribe((record) => {
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
// Serialize projections; the prior link is always non-rejecting (see the .catch
|
|
23
|
+
// below), so each projection runs once after it. Count + log failures in the same
|
|
24
|
+
// chain and recover, so the stored queue never carries a rejection forward.
|
|
25
|
+
this.projectionQueue = this.projectionQueue
|
|
26
|
+
.then(() => this.projectRuntimeRecord(runtime, record, sessionId))
|
|
27
|
+
.catch((error) => {
|
|
24
28
|
this.projectionFailures += 1;
|
|
25
29
|
console.error(`memory projection failed for ${runtime.channelKey}`, error);
|
|
26
30
|
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { jsonOrNull, sourceRecordIdToString } from "./serialization.js";
|
|
2
|
+
export function insertRecordPrepared(db, normalized) {
|
|
3
|
+
const inserted = db
|
|
4
|
+
.prepare(`INSERT INTO lcm_records (
|
|
5
|
+
record_key, segment_id, kind, text_full, happened_at, session_id, channel_key,
|
|
6
|
+
channel_id, job_id, source_type, source_path, source_line, source_record_id,
|
|
7
|
+
source_message_id, source_ref, attachments_json, metadata_json, parts_json
|
|
8
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
9
|
+
.run(normalized.recordKey, normalized.segmentId, normalized.kind, normalized.text, normalized.happenedAt, normalized.sessionId, normalized.channelKey, normalized.channelId, normalized.jobId, normalized.source.sourceType, normalized.source.sourcePath ?? null, normalized.source.sourceLine ?? null, sourceRecordIdToString(normalized.source.sourceRecordId), normalized.source.sourceMessageId ?? null, normalized.source.sourceRef ?? null, jsonOrNull(normalized.attachments), jsonOrNull(normalized.metadata), jsonOrNull(normalized.parts));
|
|
10
|
+
const id = Number(inserted.lastInsertRowid);
|
|
11
|
+
if (normalized.kind !== "boundary") {
|
|
12
|
+
db.prepare("INSERT INTO lcm_records_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
|
|
13
|
+
}
|
|
14
|
+
const row = db.prepare("SELECT * FROM lcm_records WHERE id = ?").get(id);
|
|
15
|
+
if (!row)
|
|
16
|
+
throw new Error(`Failed to read inserted LCM record: ${id}`);
|
|
17
|
+
return row;
|
|
18
|
+
}
|
|
19
|
+
export function insertSummaryPrepared(db, normalized, insertEdges) {
|
|
20
|
+
const inserted = db
|
|
21
|
+
.prepare(`INSERT INTO lcm_summaries (
|
|
22
|
+
summary_key, segment_id, depth, status, text_full, pinned,
|
|
23
|
+
covers_from_record_id, covers_to_record_id, snapshot_json, source_type, source_path,
|
|
24
|
+
source_line, source_record_id, source_message_id, source_ref, metadata_json
|
|
25
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
26
|
+
.run(normalized.summaryKey, normalized.segmentId, normalized.depth, normalized.status, normalized.text, normalized.pinned ? 1 : 0, normalized.coversFromRecordId, normalized.coversToRecordId, null, normalized.source.sourceType, normalized.source.sourcePath ?? null, normalized.source.sourceLine ?? null, sourceRecordIdToString(normalized.source.sourceRecordId), normalized.source.sourceMessageId ?? null, normalized.source.sourceRef ?? null, jsonOrNull(normalized.metadata));
|
|
27
|
+
const id = Number(inserted.lastInsertRowid);
|
|
28
|
+
db.prepare("INSERT INTO lcm_summaries_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
|
|
29
|
+
insertEdges(id, normalized.sourceItems, normalized.parents);
|
|
30
|
+
return id;
|
|
31
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { stableHash } from "./serialization.js";
|
|
2
|
+
export function normalizeSource(source) {
|
|
3
|
+
return {
|
|
4
|
+
sourceType: source.sourceType,
|
|
5
|
+
sourcePath: source.sourcePath ?? null,
|
|
6
|
+
sourceLine: source.sourceLine ?? null,
|
|
7
|
+
sourceRecordId: source.sourceRecordId ?? null,
|
|
8
|
+
sourceMessageId: source.sourceMessageId ?? null,
|
|
9
|
+
sourceRef: source.sourceRef ?? null,
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export function normalizeRecordInput(input) {
|
|
13
|
+
const text = (input.text ?? "").trim();
|
|
14
|
+
if (!text && input.kind !== "boundary")
|
|
15
|
+
throw new Error("LCM record text must not be empty");
|
|
16
|
+
const source = normalizeSource(input.source);
|
|
17
|
+
const happenedAt = input.happenedAt ?? new Date().toISOString();
|
|
18
|
+
const parts = input.parts?.length ? input.parts : null;
|
|
19
|
+
const normalizedText = text || "Session boundary";
|
|
20
|
+
return {
|
|
21
|
+
segmentId: input.segmentId,
|
|
22
|
+
kind: input.kind,
|
|
23
|
+
text: normalizedText,
|
|
24
|
+
parts,
|
|
25
|
+
happenedAt,
|
|
26
|
+
sessionId: input.sessionId ?? null,
|
|
27
|
+
channelKey: input.channelKey ?? null,
|
|
28
|
+
channelId: input.channelId ?? null,
|
|
29
|
+
jobId: input.jobId ?? null,
|
|
30
|
+
source,
|
|
31
|
+
attachments: input.attachments?.length ? input.attachments : null,
|
|
32
|
+
metadata: input.metadata ?? null,
|
|
33
|
+
recordKey: computeLcmRecordKey({ ...input, happenedAt }),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
export function normalizeSummaryInput(input) {
|
|
37
|
+
const text = (input.text ?? "").trim();
|
|
38
|
+
const source = normalizeSource(input.source);
|
|
39
|
+
const status = input.status ?? (text ? "ready" : "placeholder");
|
|
40
|
+
const normalizedText = text || "";
|
|
41
|
+
return {
|
|
42
|
+
segmentId: input.segmentId,
|
|
43
|
+
depth: input.depth,
|
|
44
|
+
status,
|
|
45
|
+
text: normalizedText,
|
|
46
|
+
pinned: input.pinned ?? false,
|
|
47
|
+
coversFromRecordId: input.coversFromRecordId ?? null,
|
|
48
|
+
coversToRecordId: input.coversToRecordId ?? null,
|
|
49
|
+
source,
|
|
50
|
+
sourceItems: input.sourceItems ?? [],
|
|
51
|
+
parents: input.parents ?? [],
|
|
52
|
+
metadata: input.metadata ?? null,
|
|
53
|
+
summaryKey: stableHash({
|
|
54
|
+
segmentId: input.segmentId,
|
|
55
|
+
depth: input.depth,
|
|
56
|
+
status,
|
|
57
|
+
text: normalizedText,
|
|
58
|
+
coversFromRecordId: input.coversFromRecordId ?? null,
|
|
59
|
+
coversToRecordId: input.coversToRecordId ?? null,
|
|
60
|
+
source,
|
|
61
|
+
parents: input.parents ?? [],
|
|
62
|
+
}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
export function computeLcmRecordKey(input) {
|
|
66
|
+
const text = (input.text ?? "").trim();
|
|
67
|
+
if (!text && input.kind !== "boundary")
|
|
68
|
+
throw new Error("LCM record text must not be empty");
|
|
69
|
+
const parts = input.parts?.length ? input.parts : null;
|
|
70
|
+
return stableHash({
|
|
71
|
+
segmentId: input.segmentId,
|
|
72
|
+
kind: input.kind,
|
|
73
|
+
text: text || "Session boundary",
|
|
74
|
+
parts,
|
|
75
|
+
happenedAt: input.happenedAt ?? new Date().toISOString(),
|
|
76
|
+
source: normalizeSource(input.source),
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
export function dedupeSummaryParentIds(values) {
|
|
80
|
+
const seen = new Set();
|
|
81
|
+
const result = [];
|
|
82
|
+
for (const value of values) {
|
|
83
|
+
if (!Number.isInteger(value) || value <= 0)
|
|
84
|
+
throw new Error("LCM summary parents must be positive integer ids");
|
|
85
|
+
if (seen.has(value))
|
|
86
|
+
continue;
|
|
87
|
+
seen.add(value);
|
|
88
|
+
result.push(value);
|
|
89
|
+
}
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { parseJsonArray, parseJsonObject } from "./serialization.js";
|
|
2
|
+
function sourceFromRow(row) {
|
|
3
|
+
return {
|
|
4
|
+
sourceType: row.source_type,
|
|
5
|
+
sourcePath: row.source_path,
|
|
6
|
+
sourceLine: row.source_line,
|
|
7
|
+
sourceRecordId: row.source_record_id,
|
|
8
|
+
sourceMessageId: row.source_message_id,
|
|
9
|
+
sourceRef: row.source_ref,
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export function segmentFromRow(row) {
|
|
13
|
+
return {
|
|
14
|
+
id: row.id,
|
|
15
|
+
status: row.status,
|
|
16
|
+
sessionId: row.session_id,
|
|
17
|
+
channelKey: row.channel_key,
|
|
18
|
+
startedAt: row.started_at,
|
|
19
|
+
closedAt: row.closed_at,
|
|
20
|
+
rawPrunedAt: row.raw_pruned_at,
|
|
21
|
+
boundarySource: parseJsonObject(row.boundary_source_json),
|
|
22
|
+
metadata: parseJsonObject(row.metadata_json),
|
|
23
|
+
createdAt: row.created_at,
|
|
24
|
+
updatedAt: row.updated_at,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export function recordFromRow(row) {
|
|
28
|
+
return {
|
|
29
|
+
id: row.id,
|
|
30
|
+
recordKey: row.record_key,
|
|
31
|
+
segmentId: row.segment_id,
|
|
32
|
+
kind: row.kind,
|
|
33
|
+
text: row.text_full,
|
|
34
|
+
parts: parseJsonArray(row.parts_json),
|
|
35
|
+
happenedAt: row.happened_at,
|
|
36
|
+
sessionId: row.session_id,
|
|
37
|
+
channelKey: row.channel_key,
|
|
38
|
+
channelId: row.channel_id,
|
|
39
|
+
jobId: row.job_id,
|
|
40
|
+
source: sourceFromRow(row),
|
|
41
|
+
attachments: parseJsonArray(row.attachments_json),
|
|
42
|
+
metadata: parseJsonObject(row.metadata_json),
|
|
43
|
+
createdAt: row.created_at,
|
|
44
|
+
updatedAt: row.updated_at,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function summaryFromRow(row, parents = []) {
|
|
48
|
+
return {
|
|
49
|
+
id: row.id,
|
|
50
|
+
summaryKey: row.summary_key,
|
|
51
|
+
segmentId: row.segment_id,
|
|
52
|
+
depth: row.depth,
|
|
53
|
+
status: row.status,
|
|
54
|
+
text: row.text_full,
|
|
55
|
+
pinned: row.pinned === 1,
|
|
56
|
+
coversFromRecordId: row.covers_from_record_id,
|
|
57
|
+
coversToRecordId: row.covers_to_record_id,
|
|
58
|
+
source: sourceFromRow(row),
|
|
59
|
+
metadata: parseJsonObject(row.metadata_json),
|
|
60
|
+
snapshot: parseJsonArray(row.snapshot_json),
|
|
61
|
+
parents,
|
|
62
|
+
createdAt: row.created_at,
|
|
63
|
+
updatedAt: row.updated_at,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export function summarySourceFromRow(row) {
|
|
67
|
+
return {
|
|
68
|
+
summaryId: row.summary_id,
|
|
69
|
+
ord: row.ord,
|
|
70
|
+
recordId: row.record_id,
|
|
71
|
+
sourceSummaryId: row.source_summary_id,
|
|
72
|
+
sourceRef: row.source_ref,
|
|
73
|
+
snapshot: parseJsonObject(row.snapshot_json),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
export function contextItemFromRow(row) {
|
|
77
|
+
if (row.item_type === "raw") {
|
|
78
|
+
if (row.record_id === null)
|
|
79
|
+
throw new Error(`Invalid raw LCM context item at ordinal ${row.ordinal}`);
|
|
80
|
+
return {
|
|
81
|
+
sessionKey: row.session_key,
|
|
82
|
+
ordinal: row.ordinal,
|
|
83
|
+
type: "raw",
|
|
84
|
+
recordId: row.record_id,
|
|
85
|
+
summaryId: null,
|
|
86
|
+
fingerprint: row.fingerprint,
|
|
87
|
+
happenedAt: row.happened_at,
|
|
88
|
+
updatedAt: row.updated_at,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (row.item_type === "summary") {
|
|
92
|
+
if (row.summary_id === null)
|
|
93
|
+
throw new Error(`Invalid summary LCM context item at ordinal ${row.ordinal}`);
|
|
94
|
+
return {
|
|
95
|
+
sessionKey: row.session_key,
|
|
96
|
+
ordinal: row.ordinal,
|
|
97
|
+
type: "summary",
|
|
98
|
+
recordId: null,
|
|
99
|
+
summaryId: row.summary_id,
|
|
100
|
+
fingerprint: row.fingerprint,
|
|
101
|
+
happenedAt: row.happened_at,
|
|
102
|
+
updatedAt: row.updated_at,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
throw new Error(`Unknown LCM context item type: ${row.item_type}`);
|
|
106
|
+
}
|
|
107
|
+
export function sessionStateFromRow(row) {
|
|
108
|
+
return {
|
|
109
|
+
sessionKey: row.session_key,
|
|
110
|
+
compactionDebt: row.compaction_debt,
|
|
111
|
+
cacheTouchedAt: row.cache_touched_at,
|
|
112
|
+
updatedAt: row.updated_at,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { isRecord } from "../../../util/guards.js";
|
|
3
|
+
export function jsonOrNull(value) {
|
|
4
|
+
if (value === null || value === undefined)
|
|
5
|
+
return null;
|
|
6
|
+
return JSON.stringify(value);
|
|
7
|
+
}
|
|
8
|
+
export function parseJsonObject(value) {
|
|
9
|
+
if (!value)
|
|
10
|
+
return null;
|
|
11
|
+
try {
|
|
12
|
+
const parsed = JSON.parse(value);
|
|
13
|
+
return isRecord(parsed) ? parsed : null;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function parseJsonArray(value) {
|
|
20
|
+
if (!value)
|
|
21
|
+
return null;
|
|
22
|
+
try {
|
|
23
|
+
const parsed = JSON.parse(value);
|
|
24
|
+
return Array.isArray(parsed) ? parsed : null;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function stableHash(value) {
|
|
31
|
+
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
32
|
+
}
|
|
33
|
+
export function sourceRecordIdToString(value) {
|
|
34
|
+
if (value === null || value === undefined)
|
|
35
|
+
return null;
|
|
36
|
+
return String(value);
|
|
37
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { parseJsonArray, parseJsonObject } from "./serialization.js";
|
|
2
|
+
const SUMMARY_SNAPSHOT_TEXT_LIMIT = 4 * 1024;
|
|
3
|
+
const SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX = "…[truncated]";
|
|
4
|
+
export function buildSummarySnapshot(db, summary) {
|
|
5
|
+
const rows = db
|
|
6
|
+
.prepare(`SELECT * FROM lcm_records
|
|
7
|
+
WHERE segment_id = ?
|
|
8
|
+
AND id BETWEEN ? AND ?
|
|
9
|
+
ORDER BY happened_at, id`)
|
|
10
|
+
.all(summary.segment_id, summary.covers_from_record_id, summary.covers_to_record_id);
|
|
11
|
+
return rows.map(snapshotRecordFromRow);
|
|
12
|
+
}
|
|
13
|
+
export function buildSummaryParentSnapshot(db, summaryId, visiting) {
|
|
14
|
+
if (visiting.has(summaryId))
|
|
15
|
+
throw new Error(`Cycle detected in LCM summary parents at ${summaryId}`);
|
|
16
|
+
visiting.add(summaryId);
|
|
17
|
+
const row = db.prepare("SELECT * FROM lcm_summaries WHERE id = ?").get(summaryId);
|
|
18
|
+
if (!row)
|
|
19
|
+
throw new Error(`LCM summary does not exist: ${summaryId}`);
|
|
20
|
+
let snapshot = parseJsonArray(row.snapshot_json);
|
|
21
|
+
if (!snapshot &&
|
|
22
|
+
row.covers_from_record_id !== null &&
|
|
23
|
+
row.covers_to_record_id !== null &&
|
|
24
|
+
db
|
|
25
|
+
.prepare("SELECT 1 FROM lcm_records WHERE segment_id = ? AND id BETWEEN ? AND ? LIMIT 1")
|
|
26
|
+
.get(row.segment_id, row.covers_from_record_id, row.covers_to_record_id)) {
|
|
27
|
+
snapshot = buildSummarySnapshot(db, row);
|
|
28
|
+
}
|
|
29
|
+
const parentRows = db
|
|
30
|
+
.prepare(`SELECT parent_summary_id
|
|
31
|
+
FROM lcm_summary_parents
|
|
32
|
+
WHERE summary_id = ?
|
|
33
|
+
ORDER BY ord, parent_summary_id`)
|
|
34
|
+
.all(summaryId);
|
|
35
|
+
const parents = parentRows.map((parent) => buildSummaryParentSnapshot(db, parent.parent_summary_id, visiting));
|
|
36
|
+
visiting.delete(summaryId);
|
|
37
|
+
return {
|
|
38
|
+
summaryId: row.id,
|
|
39
|
+
depth: row.depth,
|
|
40
|
+
text: row.text_full,
|
|
41
|
+
coversFromRecordId: row.covers_from_record_id,
|
|
42
|
+
coversToRecordId: row.covers_to_record_id,
|
|
43
|
+
snapshot,
|
|
44
|
+
parents,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function snapshotRecordFromRow(row) {
|
|
48
|
+
const metadata = parseJsonObject(row.metadata_json);
|
|
49
|
+
return {
|
|
50
|
+
id: row.id,
|
|
51
|
+
kind: row.kind,
|
|
52
|
+
happened_at: row.happened_at,
|
|
53
|
+
role: snapshotRole(row.kind, metadata),
|
|
54
|
+
text: truncateSummarySnapshotText(row.text_full),
|
|
55
|
+
parts: parseJsonArray(row.parts_json),
|
|
56
|
+
attachments: parseJsonArray(row.attachments_json),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
export function snapshotRole(kind, metadata) {
|
|
60
|
+
if (typeof metadata?.role === "string" && metadata.role.trim())
|
|
61
|
+
return metadata.role;
|
|
62
|
+
if (kind === "user" || kind === "assistant")
|
|
63
|
+
return kind;
|
|
64
|
+
if (kind === "tool")
|
|
65
|
+
return "tool";
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
export function truncateSummarySnapshotText(text) {
|
|
69
|
+
if (text.length <= SUMMARY_SNAPSHOT_TEXT_LIMIT)
|
|
70
|
+
return text;
|
|
71
|
+
const retainedLength = Math.max(0, SUMMARY_SNAPSHOT_TEXT_LIMIT - SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX.length);
|
|
72
|
+
return `${text.slice(0, retainedLength)}${SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX}`;
|
|
73
|
+
}
|