@dudousxd/nestjs-agent-store-drizzle 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -3
- package/dist/index.cjs +808 -32
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1965 -534
- package/dist/index.d.ts +1965 -534
- package/dist/index.js +792 -21
- package/dist/index.js.map +1 -1
- package/package.json +6 -4
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@ var __defProp = Object.defineProperty;
|
|
|
2
2
|
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
3
|
|
|
4
4
|
// src/schema.ts
|
|
5
|
-
import { index, integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
5
|
+
import { index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
|
|
6
6
|
var agentThread = sqliteTable("agent_thread", {
|
|
7
7
|
id: text("id").primaryKey(),
|
|
8
8
|
actorRef: text("actor_ref").notNull(),
|
|
@@ -12,6 +12,8 @@ var agentThread = sqliteTable("agent_thread", {
|
|
|
12
12
|
mode: "boolean"
|
|
13
13
|
}).notNull().default(false),
|
|
14
14
|
activeStreamId: text("active_stream_id"),
|
|
15
|
+
/** The agent a new turn on this thread defaults to when the caller names none. */
|
|
16
|
+
defaultAgent: text("default_agent"),
|
|
15
17
|
createdAt: integer("created_at", {
|
|
16
18
|
mode: "timestamp_ms"
|
|
17
19
|
}).notNull(),
|
|
@@ -37,6 +39,9 @@ var agentMessage = sqliteTable("agent_message", {
|
|
|
37
39
|
toolResults: text("tool_results", {
|
|
38
40
|
mode: "json"
|
|
39
41
|
}).$type(),
|
|
42
|
+
attachments: text("attachments", {
|
|
43
|
+
mode: "json"
|
|
44
|
+
}).$type(),
|
|
40
45
|
followUps: text("follow_ups", {
|
|
41
46
|
mode: "json"
|
|
42
47
|
}).$type(),
|
|
@@ -44,6 +49,7 @@ var agentMessage = sqliteTable("agent_message", {
|
|
|
44
49
|
mode: "json"
|
|
45
50
|
}).$type(),
|
|
46
51
|
agentName: text("agent_name"),
|
|
52
|
+
runId: text("run_id"),
|
|
47
53
|
createdAt: integer("created_at", {
|
|
48
54
|
mode: "timestamp_ms"
|
|
49
55
|
}).notNull()
|
|
@@ -75,7 +81,9 @@ var agentToolCall = sqliteTable("agent_tool_call", {
|
|
|
75
81
|
}),
|
|
76
82
|
/** The run (turn) this call belongs to, for a trace deep-link; null for a pre-rollout row. */
|
|
77
83
|
runId: text("run_id")
|
|
78
|
-
})
|
|
84
|
+
}, (table) => [
|
|
85
|
+
index("agent_tool_call_message_idx").on(table.messageId)
|
|
86
|
+
]);
|
|
79
87
|
var agentTokenUsage = sqliteTable("agent_token_usage", {
|
|
80
88
|
id: text("id").primaryKey(),
|
|
81
89
|
threadId: text("thread_id").notNull().references(() => agentThread.id, {
|
|
@@ -134,17 +142,79 @@ var agentRun = sqliteTable("agent_run", {
|
|
|
134
142
|
mode: "timestamp_ms"
|
|
135
143
|
}),
|
|
136
144
|
/** sha256 hex of the run's resolved (pre-RAG) system prompt; null for a pre-existing run. */
|
|
137
|
-
promptHash: text("prompt_hash")
|
|
145
|
+
promptHash: text("prompt_hash"),
|
|
146
|
+
/**
|
|
147
|
+
* The run that delegated this one; null for a turn nobody delegated. The durable runtime's
|
|
148
|
+
* journal holds the same edge, but only there — this column is what lets a reader of run ROWS
|
|
149
|
+
* build the delegation tree and roll a child's cost up to the turn that asked for it, including
|
|
150
|
+
* for a detached child, which outlives its parent's turn and so is paired by nothing in the
|
|
151
|
+
* transcript.
|
|
152
|
+
*/
|
|
153
|
+
parentRunId: text("parent_run_id")
|
|
138
154
|
}, (table) => [
|
|
139
155
|
index("agent_run_started_idx").on(table.startedAt)
|
|
140
156
|
]);
|
|
157
|
+
var agentMemory = sqliteTable("agent_memory", {
|
|
158
|
+
id: text("id").primaryKey(),
|
|
159
|
+
/** The opaque scope token. Unique with `key`: one fact per key per scope. */
|
|
160
|
+
scope: text("scope").notNull(),
|
|
161
|
+
/** What the fact is ABOUT. Two scopes sharing a key are one question answered twice. */
|
|
162
|
+
key: text("key").notNull(),
|
|
163
|
+
/** The fact itself, one line, as the model reads it. */
|
|
164
|
+
text: text("text").notNull(),
|
|
165
|
+
originAuthor: text("origin_author").$type().notNull(),
|
|
166
|
+
/** A pointer that is allowed to dangle — see the table note. */
|
|
167
|
+
originThreadId: text("origin_thread_id"),
|
|
168
|
+
originRunId: text("origin_run_id"),
|
|
169
|
+
originActorRef: text("origin_actor_ref"),
|
|
170
|
+
/** Always-on: carried in every prompt. Set by a host's console, never by a write. */
|
|
171
|
+
pinned: integer("pinned", {
|
|
172
|
+
mode: "boolean"
|
|
173
|
+
}).notNull().default(false),
|
|
174
|
+
createdAt: integer("created_at", {
|
|
175
|
+
mode: "timestamp_ms"
|
|
176
|
+
}).notNull(),
|
|
177
|
+
updatedAt: integer("updated_at", {
|
|
178
|
+
mode: "timestamp_ms"
|
|
179
|
+
}).notNull()
|
|
180
|
+
}, (table) => [
|
|
181
|
+
uniqueIndex("agent_memory_scope_key_uq").on(table.scope, table.key)
|
|
182
|
+
]);
|
|
183
|
+
var ragIngestionLog = sqliteTable("rag_ingestion_log", {
|
|
184
|
+
/** The vector-store document id (chunk ids are `${documentId}#<n>`). */
|
|
185
|
+
documentId: text("document_id").primaryKey(),
|
|
186
|
+
status: text("status").$type().notNull(),
|
|
187
|
+
collection: text("collection"),
|
|
188
|
+
ownerType: text("owner_type"),
|
|
189
|
+
ownerId: text("owner_id"),
|
|
190
|
+
/** Citation-facing origin — the media record's path. */
|
|
191
|
+
source: text("source"),
|
|
192
|
+
mimeType: text("mime_type"),
|
|
193
|
+
size: integer("size"),
|
|
194
|
+
/** Chunk count on `ingested`; null otherwise. */
|
|
195
|
+
chunks: integer("chunks"),
|
|
196
|
+
/** Skip reason on `skipped` (`unsupported-type` | `too-large` | `empty-text`); null otherwise. */
|
|
197
|
+
reason: text("reason"),
|
|
198
|
+
/** Error message on `failed`; null otherwise. */
|
|
199
|
+
error: text("error"),
|
|
200
|
+
createdAt: integer("created_at", {
|
|
201
|
+
mode: "timestamp_ms"
|
|
202
|
+
}).notNull(),
|
|
203
|
+
updatedAt: integer("updated_at", {
|
|
204
|
+
mode: "timestamp_ms"
|
|
205
|
+
}).notNull()
|
|
206
|
+
}, (table) => [
|
|
207
|
+
index("rag_ingestion_log_collection_idx").on(table.collection, table.updatedAt)
|
|
208
|
+
]);
|
|
141
209
|
var agentSchema = {
|
|
142
210
|
agentThread,
|
|
143
211
|
agentMessage,
|
|
144
212
|
agentToolCall,
|
|
145
213
|
agentTokenUsage,
|
|
146
214
|
agentModelPricing,
|
|
147
|
-
agentRun
|
|
215
|
+
agentRun,
|
|
216
|
+
agentMemory,
|
|
217
|
+
ragIngestionLog
|
|
148
218
|
};
|
|
149
219
|
|
|
150
220
|
// src/ensure-schema.ts
|
|
@@ -157,6 +227,7 @@ var statements = [
|
|
|
157
227
|
title TEXT NOT NULL,
|
|
158
228
|
transient INTEGER NOT NULL DEFAULT 0,
|
|
159
229
|
active_stream_id TEXT,
|
|
230
|
+
default_agent TEXT,
|
|
160
231
|
created_at INTEGER NOT NULL,
|
|
161
232
|
updated_at INTEGER NOT NULL,
|
|
162
233
|
deleted_at INTEGER
|
|
@@ -170,9 +241,11 @@ var statements = [
|
|
|
170
241
|
content TEXT NOT NULL,
|
|
171
242
|
tool_calls TEXT,
|
|
172
243
|
tool_results TEXT,
|
|
244
|
+
attachments TEXT,
|
|
173
245
|
follow_ups TEXT,
|
|
174
246
|
usage TEXT,
|
|
175
247
|
agent_name TEXT,
|
|
248
|
+
run_id TEXT,
|
|
176
249
|
created_at INTEGER NOT NULL
|
|
177
250
|
)`,
|
|
178
251
|
`CREATE INDEX IF NOT EXISTS agent_message_thread_created_idx
|
|
@@ -192,6 +265,8 @@ var statements = [
|
|
|
192
265
|
executed_at INTEGER,
|
|
193
266
|
run_id TEXT
|
|
194
267
|
)`,
|
|
268
|
+
`CREATE INDEX IF NOT EXISTS agent_tool_call_message_idx
|
|
269
|
+
ON agent_tool_call (message_id)`,
|
|
195
270
|
`CREATE TABLE IF NOT EXISTS agent_token_usage (
|
|
196
271
|
id TEXT PRIMARY KEY NOT NULL,
|
|
197
272
|
thread_id TEXT NOT NULL REFERENCES agent_thread(id) ON DELETE CASCADE,
|
|
@@ -230,20 +305,93 @@ var statements = [
|
|
|
230
305
|
retries INTEGER NOT NULL DEFAULT 0,
|
|
231
306
|
started_at INTEGER NOT NULL,
|
|
232
307
|
settled_at INTEGER,
|
|
233
|
-
prompt_hash TEXT
|
|
308
|
+
prompt_hash TEXT,
|
|
309
|
+
parent_run_id TEXT
|
|
234
310
|
)`,
|
|
235
311
|
`CREATE INDEX IF NOT EXISTS agent_run_started_idx
|
|
236
|
-
ON agent_run (started_at)
|
|
312
|
+
ON agent_run (started_at)`,
|
|
313
|
+
// `key` and `text` are quoted: both are keywords in at least one engine this DDL is read by, and
|
|
314
|
+
// they are the SPI's own field names, which is worth more here than dodging the quoting.
|
|
315
|
+
`CREATE TABLE IF NOT EXISTS agent_memory (
|
|
316
|
+
id TEXT PRIMARY KEY NOT NULL,
|
|
317
|
+
scope TEXT NOT NULL,
|
|
318
|
+
"key" TEXT NOT NULL,
|
|
319
|
+
"text" TEXT NOT NULL,
|
|
320
|
+
origin_author TEXT NOT NULL,
|
|
321
|
+
origin_thread_id TEXT,
|
|
322
|
+
origin_run_id TEXT,
|
|
323
|
+
origin_actor_ref TEXT,
|
|
324
|
+
pinned INTEGER NOT NULL DEFAULT 0,
|
|
325
|
+
created_at INTEGER NOT NULL,
|
|
326
|
+
updated_at INTEGER NOT NULL
|
|
327
|
+
)`,
|
|
328
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS agent_memory_scope_key_uq
|
|
329
|
+
ON agent_memory (scope, "key")`,
|
|
330
|
+
`CREATE TABLE IF NOT EXISTS rag_ingestion_log (
|
|
331
|
+
document_id TEXT PRIMARY KEY NOT NULL,
|
|
332
|
+
status TEXT NOT NULL,
|
|
333
|
+
collection TEXT,
|
|
334
|
+
owner_type TEXT,
|
|
335
|
+
owner_id TEXT,
|
|
336
|
+
source TEXT,
|
|
337
|
+
mime_type TEXT,
|
|
338
|
+
size INTEGER,
|
|
339
|
+
chunks INTEGER,
|
|
340
|
+
reason TEXT,
|
|
341
|
+
error TEXT,
|
|
342
|
+
created_at INTEGER NOT NULL,
|
|
343
|
+
updated_at INTEGER NOT NULL
|
|
344
|
+
)`,
|
|
345
|
+
`CREATE INDEX IF NOT EXISTS rag_ingestion_log_collection_idx
|
|
346
|
+
ON rag_ingestion_log (collection, updated_at)`
|
|
347
|
+
];
|
|
348
|
+
var additiveColumns = [
|
|
349
|
+
{
|
|
350
|
+
table: "agent_message",
|
|
351
|
+
column: "run_id",
|
|
352
|
+
ddl: "ALTER TABLE agent_message ADD COLUMN run_id TEXT"
|
|
353
|
+
},
|
|
354
|
+
{
|
|
355
|
+
table: "agent_message",
|
|
356
|
+
column: "attachments",
|
|
357
|
+
ddl: "ALTER TABLE agent_message ADD COLUMN attachments TEXT"
|
|
358
|
+
},
|
|
359
|
+
{
|
|
360
|
+
table: "agent_thread",
|
|
361
|
+
column: "default_agent",
|
|
362
|
+
ddl: "ALTER TABLE agent_thread ADD COLUMN default_agent TEXT"
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
table: "agent_run",
|
|
366
|
+
column: "parent_run_id",
|
|
367
|
+
ddl: "ALTER TABLE agent_run ADD COLUMN parent_run_id TEXT"
|
|
368
|
+
}
|
|
237
369
|
];
|
|
238
370
|
async function ensureAgentSchema(db) {
|
|
239
371
|
for (const statement of statements) {
|
|
240
372
|
await db.run(sql.raw(statement));
|
|
241
373
|
}
|
|
374
|
+
for (const { table, column, ddl } of additiveColumns) {
|
|
375
|
+
const existing = await db.all(sql.raw(`PRAGMA table_info(${table})`));
|
|
376
|
+
if (!existing.some((row) => row.name === column)) {
|
|
377
|
+
await db.run(sql.raw(ddl));
|
|
378
|
+
}
|
|
379
|
+
}
|
|
242
380
|
}
|
|
243
381
|
__name(ensureAgentSchema, "ensureAgentSchema");
|
|
244
382
|
|
|
245
383
|
// src/drizzle-agent-store.ts
|
|
246
|
-
import { and, asc, desc, eq, gte, inArray, isNull, lte, sql as sql2 } from "drizzle-orm";
|
|
384
|
+
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, sql as sql2 } from "drizzle-orm";
|
|
385
|
+
var TURN_MESSAGE_COLUMNS = {
|
|
386
|
+
id: agentMessage.id,
|
|
387
|
+
role: agentMessage.role,
|
|
388
|
+
content: agentMessage.content,
|
|
389
|
+
agentName: agentMessage.agentName,
|
|
390
|
+
toolCalls: agentMessage.toolCalls,
|
|
391
|
+
toolResults: agentMessage.toolResults,
|
|
392
|
+
attachments: agentMessage.attachments,
|
|
393
|
+
createdAt: agentMessage.createdAt
|
|
394
|
+
};
|
|
247
395
|
var DrizzleAgentStore = class {
|
|
248
396
|
static {
|
|
249
397
|
__name(this, "DrizzleAgentStore");
|
|
@@ -261,6 +409,7 @@ var DrizzleAgentStore = class {
|
|
|
261
409
|
title: input.title ?? "New chat",
|
|
262
410
|
transient: input.transient ?? false,
|
|
263
411
|
activeStreamId: null,
|
|
412
|
+
defaultAgent: null,
|
|
264
413
|
createdAt: now,
|
|
265
414
|
updatedAt: now,
|
|
266
415
|
deletedAt: null
|
|
@@ -283,6 +432,50 @@ var DrizzleAgentStore = class {
|
|
|
283
432
|
} : {}
|
|
284
433
|
};
|
|
285
434
|
}
|
|
435
|
+
/**
|
|
436
|
+
* The window a turn reads off a thread: its newest `messageLimit` messages (oldest first), the
|
|
437
|
+
* title, the default agent, and whether the thread has EVER been answered.
|
|
438
|
+
*
|
|
439
|
+
* {@link getThread} is the wrong read for a turn. It materializes the transcript — every message,
|
|
440
|
+
* every attachment, every tool output the thread ever recorded — and the run then journals what it
|
|
441
|
+
* loaded, so a long thread pays for its whole history on every turn and again on every replay.
|
|
442
|
+
*
|
|
443
|
+
* `hasAssistantMessage` is answered over the WHOLE thread, not the returned page: it answers "has
|
|
444
|
+
* this conversation been answered before?", and a thread whose window happens to hold only the
|
|
445
|
+
* user's last questions has still been answered. `null` when the thread is unknown or
|
|
446
|
+
* soft-deleted, same as {@link getThread}.
|
|
447
|
+
*/
|
|
448
|
+
async loadThreadForTurn(query) {
|
|
449
|
+
const [thread] = await this.db.select({
|
|
450
|
+
title: agentThread.title,
|
|
451
|
+
defaultAgent: agentThread.defaultAgent
|
|
452
|
+
}).from(agentThread).where(and(eq(agentThread.id, query.threadId), isNull(agentThread.deletedAt)));
|
|
453
|
+
if (thread === void 0) {
|
|
454
|
+
return null;
|
|
455
|
+
}
|
|
456
|
+
const messages = await this.loadTurnWindow(query.threadId, query.messageLimit);
|
|
457
|
+
const [answered] = await this.db.select({
|
|
458
|
+
id: agentMessage.id
|
|
459
|
+
}).from(agentMessage).where(and(eq(agentMessage.threadId, query.threadId), eq(agentMessage.role, "assistant"))).limit(1);
|
|
460
|
+
return {
|
|
461
|
+
title: thread.title,
|
|
462
|
+
defaultAgent: thread.defaultAgent,
|
|
463
|
+
hasAssistantMessage: answered !== void 0,
|
|
464
|
+
messages
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* The newest `limit` messages of a thread, oldest first, projected to the columns a model turn
|
|
469
|
+
* reads. Read newest-first so the LIMIT is the database's job, then reversed for the prompt.
|
|
470
|
+
*/
|
|
471
|
+
async loadTurnWindow(threadId, limit) {
|
|
472
|
+
if (limit !== void 0 && limit <= 0) {
|
|
473
|
+
return [];
|
|
474
|
+
}
|
|
475
|
+
const window = this.db.select(TURN_MESSAGE_COLUMNS).from(agentMessage).where(eq(agentMessage.threadId, threadId)).orderBy(desc(agentMessage.createdAt), desc(agentMessage.id));
|
|
476
|
+
const rows = limit === void 0 ? await window : await window.limit(limit);
|
|
477
|
+
return rows.reverse().map((row) => this.toStoredMessage(row));
|
|
478
|
+
}
|
|
286
479
|
async listThreads(actorRef, limit = 50) {
|
|
287
480
|
const threads = await this.db.select().from(agentThread).where(and(eq(agentThread.actorRef, actorRef), eq(agentThread.transient, false), isNull(agentThread.deletedAt))).orderBy(desc(agentThread.updatedAt)).limit(limit);
|
|
288
481
|
const summaries = [];
|
|
@@ -313,6 +506,7 @@ var DrizzleAgentStore = class {
|
|
|
313
506
|
title: source.title,
|
|
314
507
|
transient: false,
|
|
315
508
|
activeStreamId: null,
|
|
509
|
+
defaultAgent: source.defaultAgent,
|
|
316
510
|
createdAt: now,
|
|
317
511
|
updatedAt: now,
|
|
318
512
|
deletedAt: null
|
|
@@ -326,9 +520,14 @@ var DrizzleAgentStore = class {
|
|
|
326
520
|
content: message.content,
|
|
327
521
|
toolCalls: message.toolCalls,
|
|
328
522
|
toolResults: message.toolResults,
|
|
523
|
+
attachments: message.attachments,
|
|
329
524
|
followUps: message.followUps,
|
|
330
525
|
usage: message.usage,
|
|
331
526
|
agentName: message.agentName,
|
|
527
|
+
// The copy is the same message, so it keeps the run that wrote it. A reader asking which
|
|
528
|
+
// turn produced this text gets the truthful answer; the run's own thread is still the
|
|
529
|
+
// original, so a run-scoped read never picks the fork's rows up.
|
|
530
|
+
runId: message.runId,
|
|
332
531
|
createdAt: message.createdAt
|
|
333
532
|
})));
|
|
334
533
|
}
|
|
@@ -348,9 +547,10 @@ var DrizzleAgentStore = class {
|
|
|
348
547
|
}
|
|
349
548
|
async runForToolCall(toolCallId) {
|
|
350
549
|
const [row] = await this.db.select({
|
|
550
|
+
runId: agentToolCall.runId,
|
|
351
551
|
activeStreamId: agentThread.activeStreamId
|
|
352
552
|
}).from(agentToolCall).innerJoin(agentMessage, eq(agentToolCall.messageId, agentMessage.id)).innerJoin(agentThread, eq(agentMessage.threadId, agentThread.id)).where(eq(agentToolCall.id, toolCallId));
|
|
353
|
-
return row?.activeStreamId ?? null;
|
|
553
|
+
return row?.runId ?? row?.activeStreamId ?? null;
|
|
354
554
|
}
|
|
355
555
|
async ownerOfActiveStream(runId) {
|
|
356
556
|
const [thread] = await this.db.select({
|
|
@@ -375,7 +575,43 @@ var DrizzleAgentStore = class {
|
|
|
375
575
|
activeStreamId: runId
|
|
376
576
|
}).where(eq(agentThread.id, threadId));
|
|
377
577
|
}
|
|
378
|
-
/**
|
|
578
|
+
/**
|
|
579
|
+
* Patch a thread's `title` and/or `defaultAgent`. Each field is only touched when PRESENT in
|
|
580
|
+
* `patch` — `defaultAgent: null` clears it (falls back to the app default), while an omitted
|
|
581
|
+
* `defaultAgent` leaves whatever is stored untouched. A no-op (including `updatedAt`) when the
|
|
582
|
+
* patch is empty or the thread doesn't exist.
|
|
583
|
+
*/
|
|
584
|
+
async updateThread(threadId, patch) {
|
|
585
|
+
const updates = {};
|
|
586
|
+
if (patch.title !== void 0) {
|
|
587
|
+
updates.title = patch.title;
|
|
588
|
+
}
|
|
589
|
+
if (patch.defaultAgent !== void 0) {
|
|
590
|
+
updates.defaultAgent = patch.defaultAgent;
|
|
591
|
+
}
|
|
592
|
+
if (Object.keys(updates).length === 0) {
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
updates.updatedAt = /* @__PURE__ */ new Date();
|
|
596
|
+
await this.db.update(agentThread).set(updates).where(eq(agentThread.id, threadId));
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* The thread's default agent, projected — the caller wants one nullable scalar to decide which
|
|
600
|
+
* agent answers the next turn, and {@link getThread} would materialize the whole transcript to
|
|
601
|
+
* hand it over. `null` when the thread is unknown, soft-deleted, or has no default set.
|
|
602
|
+
*/
|
|
603
|
+
async defaultAgentForThread(threadId) {
|
|
604
|
+
const [thread] = await this.db.select({
|
|
605
|
+
defaultAgent: agentThread.defaultAgent
|
|
606
|
+
}).from(agentThread).where(and(eq(agentThread.id, threadId), isNull(agentThread.deletedAt)));
|
|
607
|
+
return thread?.defaultAgent ?? null;
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Persist the start of a run (turn). Replay-safe: called under a durable localStep.
|
|
611
|
+
*
|
|
612
|
+
* Takes the SPI's own {@link RecordRunStartInput} rather than a hand-copied shape: a field added
|
|
613
|
+
* to the input is otherwise accepted and dropped, silently, by every adapter that re-declares it.
|
|
614
|
+
*/
|
|
379
615
|
async recordRunStart(run) {
|
|
380
616
|
const runRow = {
|
|
381
617
|
id: run.runId,
|
|
@@ -389,7 +625,8 @@ var DrizzleAgentStore = class {
|
|
|
389
625
|
retries: 0,
|
|
390
626
|
startedAt: /* @__PURE__ */ new Date(),
|
|
391
627
|
settledAt: null,
|
|
392
|
-
promptHash: run.promptHash ?? null
|
|
628
|
+
promptHash: run.promptHash ?? null,
|
|
629
|
+
parentRunId: run.parentRunId ?? null
|
|
393
630
|
};
|
|
394
631
|
await this.db.insert(agentRun).values(runRow);
|
|
395
632
|
}
|
|
@@ -429,9 +666,11 @@ var DrizzleAgentStore = class {
|
|
|
429
666
|
content: input.content,
|
|
430
667
|
toolCalls: input.toolCalls ?? null,
|
|
431
668
|
toolResults: input.toolResults ?? null,
|
|
669
|
+
attachments: input.attachments ?? null,
|
|
432
670
|
followUps: input.followUps ?? null,
|
|
433
671
|
usage: input.usage ?? null,
|
|
434
672
|
agentName: input.agentName ?? null,
|
|
673
|
+
runId: input.runId ?? null,
|
|
435
674
|
createdAt: now
|
|
436
675
|
};
|
|
437
676
|
await this.db.insert(agentMessage).values(message);
|
|
@@ -440,6 +679,11 @@ var DrizzleAgentStore = class {
|
|
|
440
679
|
}).where(eq(agentThread.id, input.threadId));
|
|
441
680
|
return this.toStoredMessage(message);
|
|
442
681
|
}
|
|
682
|
+
async setMessageToolResults(messageId, results) {
|
|
683
|
+
await this.db.update(agentMessage).set({
|
|
684
|
+
toolResults: results
|
|
685
|
+
}).where(eq(agentMessage.id, messageId));
|
|
686
|
+
}
|
|
443
687
|
async truncateFrom(threadId, messageId) {
|
|
444
688
|
const messages = await this.db.select().from(agentMessage).where(eq(agentMessage.threadId, threadId)).orderBy(asc(agentMessage.createdAt), asc(agentMessage.id));
|
|
445
689
|
const cutoff = messages.findIndex((message) => message.id === messageId);
|
|
@@ -488,6 +732,39 @@ var DrizzleAgentStore = class {
|
|
|
488
732
|
}
|
|
489
733
|
await this.db.update(agentToolCall).set(updates).where(eq(agentToolCall.id, input.toolCallId));
|
|
490
734
|
}
|
|
735
|
+
/**
|
|
736
|
+
* Of `mediaIds`, the ones a surviving message in one of this actor's threads still carries.
|
|
737
|
+
*
|
|
738
|
+
* Reads the `attachments` JSON back out and matches in memory rather than pushing the match into
|
|
739
|
+
* SQL: the column holds an array of objects, and every dialect this adapter's sibling targets
|
|
740
|
+
* spells that query differently (`jsonb` containment, `json_table`, `json_each`), while none of
|
|
741
|
+
* them can use an index for it anyway. The scan is bounded by ONE actor's attachment-bearing
|
|
742
|
+
* messages, which is a small set — attachments are rare, and a message without one is excluded
|
|
743
|
+
* by the `is not null` before any JSON is parsed.
|
|
744
|
+
*
|
|
745
|
+
* Soft-deleted threads are deliberately included: their message rows survive, so the bytes they
|
|
746
|
+
* point at are still reachable from stored state and are not garbage.
|
|
747
|
+
*/
|
|
748
|
+
async referencedMediaIds(actorRef, mediaIds) {
|
|
749
|
+
if (mediaIds.length === 0) {
|
|
750
|
+
return [];
|
|
751
|
+
}
|
|
752
|
+
const rows = await this.db.select({
|
|
753
|
+
attachments: agentMessage.attachments
|
|
754
|
+
}).from(agentMessage).innerJoin(agentThread, eq(agentMessage.threadId, agentThread.id)).where(and(eq(agentThread.actorRef, actorRef), isNotNull(agentMessage.attachments)));
|
|
755
|
+
const wanted = new Set(mediaIds);
|
|
756
|
+
const found = /* @__PURE__ */ new Set();
|
|
757
|
+
for (const row of rows) {
|
|
758
|
+
for (const attachment of row.attachments ?? []) {
|
|
759
|
+
if (wanted.has(attachment.mediaId)) {
|
|
760
|
+
found.add(attachment.mediaId);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
return [
|
|
765
|
+
...wanted
|
|
766
|
+
].filter((mediaId) => found.has(mediaId));
|
|
767
|
+
}
|
|
491
768
|
async recordUsage(input) {
|
|
492
769
|
await this.db.insert(agentTokenUsage).values({
|
|
493
770
|
id: crypto.randomUUID(),
|
|
@@ -522,6 +799,7 @@ var DrizzleAgentStore = class {
|
|
|
522
799
|
transient: thread.transient,
|
|
523
800
|
createdAt: thread.createdAt.toISOString(),
|
|
524
801
|
updatedAt: thread.updatedAt.toISOString(),
|
|
802
|
+
defaultAgent: thread.defaultAgent,
|
|
525
803
|
...lastContent !== void 0 ? {
|
|
526
804
|
lastMessagePreview: lastContent.slice(0, 120)
|
|
527
805
|
} : {}
|
|
@@ -539,6 +817,9 @@ var DrizzleAgentStore = class {
|
|
|
539
817
|
...message.toolResults != null ? {
|
|
540
818
|
toolResults: message.toolResults
|
|
541
819
|
} : {},
|
|
820
|
+
...message.attachments != null ? {
|
|
821
|
+
attachments: message.attachments
|
|
822
|
+
} : {},
|
|
542
823
|
...message.followUps != null ? {
|
|
543
824
|
followUps: message.followUps
|
|
544
825
|
} : {},
|
|
@@ -547,6 +828,9 @@ var DrizzleAgentStore = class {
|
|
|
547
828
|
} : {},
|
|
548
829
|
...message.agentName != null ? {
|
|
549
830
|
agentName: message.agentName
|
|
831
|
+
} : {},
|
|
832
|
+
...message.runId != null ? {
|
|
833
|
+
runId: message.runId
|
|
550
834
|
} : {}
|
|
551
835
|
};
|
|
552
836
|
}
|
|
@@ -580,7 +864,8 @@ __name(isToolKind, "isToolKind");
|
|
|
580
864
|
var RUN_STATUSES = [
|
|
581
865
|
"running",
|
|
582
866
|
"completed",
|
|
583
|
-
"failed"
|
|
867
|
+
"failed",
|
|
868
|
+
"cancelled"
|
|
584
869
|
];
|
|
585
870
|
function isRunStatus(value) {
|
|
586
871
|
return RUN_STATUSES.includes(value);
|
|
@@ -621,7 +906,8 @@ function toRecentRunRow(run) {
|
|
|
621
906
|
errorMessage: run.errorMessage ?? null,
|
|
622
907
|
retries: run.retries,
|
|
623
908
|
startedAt: run.startedAt.toISOString(),
|
|
624
|
-
promptHash: run.promptHash ?? null
|
|
909
|
+
promptHash: run.promptHash ?? null,
|
|
910
|
+
parentRunId: run.parentRunId ?? null
|
|
625
911
|
};
|
|
626
912
|
}
|
|
627
913
|
__name(toRecentRunRow, "toRecentRunRow");
|
|
@@ -1201,8 +1487,150 @@ var DrizzleGovernanceQueries = class {
|
|
|
1201
1487
|
}
|
|
1202
1488
|
};
|
|
1203
1489
|
|
|
1490
|
+
// src/drizzle-memory-provider.ts
|
|
1491
|
+
import { actorScope } from "@dudousxd/nestjs-agent-core";
|
|
1492
|
+
import { and as and3, eq as eq3, inArray as inArray3 } from "drizzle-orm";
|
|
1493
|
+
function toRecord(row) {
|
|
1494
|
+
return {
|
|
1495
|
+
id: row.id,
|
|
1496
|
+
key: row.key,
|
|
1497
|
+
text: row.text,
|
|
1498
|
+
scope: row.scope,
|
|
1499
|
+
origin: {
|
|
1500
|
+
author: row.originAuthor,
|
|
1501
|
+
...row.originThreadId !== null ? {
|
|
1502
|
+
threadId: row.originThreadId
|
|
1503
|
+
} : {},
|
|
1504
|
+
...row.originRunId !== null ? {
|
|
1505
|
+
runId: row.originRunId
|
|
1506
|
+
} : {},
|
|
1507
|
+
...row.originActorRef !== null ? {
|
|
1508
|
+
actorRef: row.originActorRef
|
|
1509
|
+
} : {}
|
|
1510
|
+
},
|
|
1511
|
+
updatedAt: row.updatedAt.toISOString(),
|
|
1512
|
+
pinned: row.pinned
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
__name(toRecord, "toRecord");
|
|
1516
|
+
var DrizzleMemoryProvider = class {
|
|
1517
|
+
static {
|
|
1518
|
+
__name(this, "DrizzleMemoryProvider");
|
|
1519
|
+
}
|
|
1520
|
+
db;
|
|
1521
|
+
constructor(db) {
|
|
1522
|
+
this.db = db;
|
|
1523
|
+
}
|
|
1524
|
+
/**
|
|
1525
|
+
* Every memory visible at `scopes` — asked on EVERY turn, so it is one indexed `scope in (…)`
|
|
1526
|
+
* against the leading column of the (`scope`, `key`) unique index.
|
|
1527
|
+
*
|
|
1528
|
+
* The scope filter is in the QUERY. The library drops out-of-scope records it is handed, but that
|
|
1529
|
+
* is a backstop: a memory held for another actor or another tenant is never selected here, so it
|
|
1530
|
+
* is unreachable rather than outranked.
|
|
1531
|
+
*/
|
|
1532
|
+
async list({ scopes }) {
|
|
1533
|
+
if (scopes.length === 0) {
|
|
1534
|
+
return [];
|
|
1535
|
+
}
|
|
1536
|
+
const rows = await this.db.select().from(agentMemory).where(inArray3(agentMemory.scope, [
|
|
1537
|
+
...scopes
|
|
1538
|
+
]));
|
|
1539
|
+
return rows.map(toRecord);
|
|
1540
|
+
}
|
|
1541
|
+
/**
|
|
1542
|
+
* Delete one, and only from the actor's OWN scope.
|
|
1543
|
+
*
|
|
1544
|
+
* `memoryForgetVerdict` already holds the library's endpoint to that rule; the `where` repeats it
|
|
1545
|
+
* because `forget` takes an id and nothing else, so a caller reaching this provider by any other
|
|
1546
|
+
* path would otherwise delete a tenant's or the deployment's memory by knowing one id.
|
|
1547
|
+
*
|
|
1548
|
+
* `false` where nothing matched — which covers "no such id" and "not yours" identically, so the
|
|
1549
|
+
* answer cannot be used to find out which memories exist about other people.
|
|
1550
|
+
*/
|
|
1551
|
+
async forget({ id, ctx }) {
|
|
1552
|
+
const deleted = await this.db.delete(agentMemory).where(and3(eq3(agentMemory.id, id), eq3(agentMemory.scope, actorScope(ctx.actor)))).returning({
|
|
1553
|
+
id: agentMemory.id
|
|
1554
|
+
});
|
|
1555
|
+
return deleted.length > 0;
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* Upsert on (`scope`, `key`) — one statement, against the unique index, rather than a read
|
|
1559
|
+
* followed by a write: two turns concluding the same key at once would otherwise race to a
|
|
1560
|
+
* duplicate-key insert and one of them would fail the `remember` tool.
|
|
1561
|
+
*
|
|
1562
|
+
* The conflict `set` names exactly what a rewrite may change, so `pinned`, `createdAt` and `id`
|
|
1563
|
+
* are left alone by construction rather than by an exclusion anyone could forget to extend.
|
|
1564
|
+
* Dropping `pinned` would silently unpin a row the next time the agent restated the fact;
|
|
1565
|
+
* rewriting `createdAt` would lose when the belief was first formed; reassigning `id` would
|
|
1566
|
+
* invalidate the delete handle a person was already shown.
|
|
1567
|
+
*/
|
|
1568
|
+
async write({ key, text: text2, scope, origin, ctx }) {
|
|
1569
|
+
assertAuthorMayWriteScope({
|
|
1570
|
+
scope,
|
|
1571
|
+
origin,
|
|
1572
|
+
ctx
|
|
1573
|
+
});
|
|
1574
|
+
const now = /* @__PURE__ */ new Date();
|
|
1575
|
+
const columns = {
|
|
1576
|
+
originAuthor: origin.author,
|
|
1577
|
+
originThreadId: origin.threadId ?? null,
|
|
1578
|
+
originRunId: origin.runId ?? null,
|
|
1579
|
+
originActorRef: origin.actorRef ?? null
|
|
1580
|
+
};
|
|
1581
|
+
const [row] = await this.db.insert(agentMemory).values({
|
|
1582
|
+
id: crypto.randomUUID(),
|
|
1583
|
+
scope,
|
|
1584
|
+
key,
|
|
1585
|
+
text: text2,
|
|
1586
|
+
...columns,
|
|
1587
|
+
pinned: false,
|
|
1588
|
+
createdAt: now,
|
|
1589
|
+
updatedAt: now
|
|
1590
|
+
}).onConflictDoUpdate({
|
|
1591
|
+
target: [
|
|
1592
|
+
agentMemory.scope,
|
|
1593
|
+
agentMemory.key
|
|
1594
|
+
],
|
|
1595
|
+
set: {
|
|
1596
|
+
text: text2,
|
|
1597
|
+
...columns,
|
|
1598
|
+
updatedAt: now
|
|
1599
|
+
}
|
|
1600
|
+
}).returning();
|
|
1601
|
+
if (row === void 0) {
|
|
1602
|
+
throw new Error(`DrizzleMemoryProvider: the upsert of "${key}" at "${scope}" returned no row; the driver does not support RETURNING on conflict.`);
|
|
1603
|
+
}
|
|
1604
|
+
return toRecord(row);
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* Make one memory always-on, or stop it being one. The operator act the SPI has no method for, on
|
|
1608
|
+
* purpose: a pin grants a fact a permanent place in every future prompt, so nothing an agent can
|
|
1609
|
+
* reach may set it. Authorize it in your own console — this takes no `ScopeContext` because it is
|
|
1610
|
+
* not an actor's act on their own memory.
|
|
1611
|
+
*
|
|
1612
|
+
* `false` where there is no such id.
|
|
1613
|
+
*/
|
|
1614
|
+
async pin({ id, pinned }) {
|
|
1615
|
+
const updated = await this.db.update(agentMemory).set({
|
|
1616
|
+
pinned,
|
|
1617
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1618
|
+
}).where(eq3(agentMemory.id, id)).returning({
|
|
1619
|
+
id: agentMemory.id
|
|
1620
|
+
});
|
|
1621
|
+
return updated.length > 0;
|
|
1622
|
+
}
|
|
1623
|
+
};
|
|
1624
|
+
function assertAuthorMayWriteScope({ scope, origin, ctx }) {
|
|
1625
|
+
const own = actorScope(ctx.actor);
|
|
1626
|
+
if (origin.author !== "human" && scope !== own) {
|
|
1627
|
+
throw new Error(`DrizzleMemoryProvider: refusing to write an agent-authored memory at "${scope}"; an agent may write only at "${own}"`);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
__name(assertAuthorMayWriteScope, "assertAuthorMayWriteScope");
|
|
1631
|
+
|
|
1204
1632
|
// src/drizzle-pricing-store.ts
|
|
1205
|
-
import { and as
|
|
1633
|
+
import { and as and4, eq as eq4 } from "drizzle-orm";
|
|
1206
1634
|
var DrizzlePricingStore = class {
|
|
1207
1635
|
static {
|
|
1208
1636
|
__name(this, "DrizzlePricingStore");
|
|
@@ -1214,7 +1642,7 @@ var DrizzlePricingStore = class {
|
|
|
1214
1642
|
async upsertModelPrice(input) {
|
|
1215
1643
|
await this.db.update(agentModelPricing).set({
|
|
1216
1644
|
isCurrent: false
|
|
1217
|
-
}).where(
|
|
1645
|
+
}).where(and4(eq4(agentModelPricing.modelId, input.modelId), eq4(agentModelPricing.isCurrent, true)));
|
|
1218
1646
|
await this.db.insert(agentModelPricing).values({
|
|
1219
1647
|
id: crypto.randomUUID(),
|
|
1220
1648
|
modelId: input.modelId,
|
|
@@ -1227,7 +1655,7 @@ var DrizzlePricingStore = class {
|
|
|
1227
1655
|
});
|
|
1228
1656
|
}
|
|
1229
1657
|
async listCurrentPrices() {
|
|
1230
|
-
const rows = await this.db.select().from(agentModelPricing).where(
|
|
1658
|
+
const rows = await this.db.select().from(agentModelPricing).where(eq4(agentModelPricing.isCurrent, true));
|
|
1231
1659
|
return rows.map((row) => ({
|
|
1232
1660
|
modelId: row.modelId,
|
|
1233
1661
|
inputPricePer1m: row.inputPricePer1m,
|
|
@@ -1243,25 +1671,352 @@ var DrizzlePricingStore = class {
|
|
|
1243
1671
|
}
|
|
1244
1672
|
};
|
|
1245
1673
|
|
|
1674
|
+
// src/drizzle-rag-ingestion-log.ts
|
|
1675
|
+
import { subscribe, unsubscribe } from "node:diagnostics_channel";
|
|
1676
|
+
import { Injectable, Logger } from "@nestjs/common";
|
|
1677
|
+
import { and as and5, asc as asc3, count as count2, desc as desc3, eq as eq5, gt, lt, or } from "drizzle-orm";
|
|
1678
|
+
function _ts_decorate(decorators, target, key, desc4) {
|
|
1679
|
+
var c = arguments.length, r = c < 3 ? target : desc4 === null ? desc4 = Object.getOwnPropertyDescriptor(target, key) : desc4, d;
|
|
1680
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc4);
|
|
1681
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1682
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1683
|
+
}
|
|
1684
|
+
__name(_ts_decorate, "_ts_decorate");
|
|
1685
|
+
function _ts_metadata(k, v) {
|
|
1686
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
1687
|
+
}
|
|
1688
|
+
__name(_ts_metadata, "_ts_metadata");
|
|
1689
|
+
var CHANNELS = [
|
|
1690
|
+
{
|
|
1691
|
+
name: "aviary:rag:media.ingested",
|
|
1692
|
+
status: "ingested"
|
|
1693
|
+
},
|
|
1694
|
+
{
|
|
1695
|
+
name: "aviary:rag:media.skipped",
|
|
1696
|
+
status: "skipped"
|
|
1697
|
+
},
|
|
1698
|
+
{
|
|
1699
|
+
name: "aviary:rag:media.failed",
|
|
1700
|
+
status: "failed"
|
|
1701
|
+
},
|
|
1702
|
+
{
|
|
1703
|
+
name: "aviary:rag:media.removed",
|
|
1704
|
+
status: "removed"
|
|
1705
|
+
}
|
|
1706
|
+
];
|
|
1707
|
+
function str(value) {
|
|
1708
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
1709
|
+
}
|
|
1710
|
+
__name(str, "str");
|
|
1711
|
+
function num(value) {
|
|
1712
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1713
|
+
}
|
|
1714
|
+
__name(num, "num");
|
|
1715
|
+
var RAG_INGESTION_LOG_PAGE_ORDER = [
|
|
1716
|
+
desc3(ragIngestionLog.updatedAt),
|
|
1717
|
+
asc3(ragIngestionLog.documentId)
|
|
1718
|
+
];
|
|
1719
|
+
var DEFAULT_PAGE_SIZE = 200;
|
|
1720
|
+
var DEFAULT_BATCH_SIZE = 200;
|
|
1721
|
+
function batches(batchSize) {
|
|
1722
|
+
const requested = Math.trunc(batchSize ?? DEFAULT_BATCH_SIZE);
|
|
1723
|
+
return Number.isFinite(requested) && requested >= 1 ? requested : DEFAULT_BATCH_SIZE;
|
|
1724
|
+
}
|
|
1725
|
+
__name(batches, "batches");
|
|
1726
|
+
var CURSOR_COLUMNS = {
|
|
1727
|
+
documentId: ragIngestionLog.documentId,
|
|
1728
|
+
updatedAt: ragIngestionLog.updatedAt
|
|
1729
|
+
};
|
|
1730
|
+
var DrizzleRagIngestionLog = class _DrizzleRagIngestionLog {
|
|
1731
|
+
static {
|
|
1732
|
+
__name(this, "DrizzleRagIngestionLog");
|
|
1733
|
+
}
|
|
1734
|
+
db;
|
|
1735
|
+
logger = new Logger(_DrizzleRagIngestionLog.name);
|
|
1736
|
+
teardowns = [];
|
|
1737
|
+
inFlight = /* @__PURE__ */ new Set();
|
|
1738
|
+
constructor(db) {
|
|
1739
|
+
this.db = db;
|
|
1740
|
+
}
|
|
1741
|
+
onModuleInit() {
|
|
1742
|
+
for (const channel of CHANNELS) {
|
|
1743
|
+
const listener = /* @__PURE__ */ __name((message) => {
|
|
1744
|
+
const payload = message?.payload;
|
|
1745
|
+
if (payload !== void 0) {
|
|
1746
|
+
this.track(this.record(channel.status, payload));
|
|
1747
|
+
}
|
|
1748
|
+
}, "listener");
|
|
1749
|
+
subscribe(channel.name, listener);
|
|
1750
|
+
this.teardowns.push(() => unsubscribe(channel.name, listener));
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
async onModuleDestroy() {
|
|
1754
|
+
for (const teardown of this.teardowns) {
|
|
1755
|
+
teardown();
|
|
1756
|
+
}
|
|
1757
|
+
this.teardowns = [];
|
|
1758
|
+
await this.settle();
|
|
1759
|
+
}
|
|
1760
|
+
/** Await every in-flight write — for graceful shutdown and deterministic tests. */
|
|
1761
|
+
async settle() {
|
|
1762
|
+
await Promise.allSettled([
|
|
1763
|
+
...this.inFlight
|
|
1764
|
+
]);
|
|
1765
|
+
}
|
|
1766
|
+
/** The latest recorded outcome per document, newest first. */
|
|
1767
|
+
async list(query = {}) {
|
|
1768
|
+
const rows = this.db.select().from(ragIngestionLog).where(this.where(query)).orderBy(...RAG_INGESTION_LOG_PAGE_ORDER).limit(query.limit ?? DEFAULT_PAGE_SIZE);
|
|
1769
|
+
return query.offset === void 0 ? rows : rows.offset(query.offset);
|
|
1770
|
+
}
|
|
1771
|
+
/**
|
|
1772
|
+
* The same page plus the unpaginated total, so a caller can tell "these are all of them" from
|
|
1773
|
+
* "these are the first N". A list that silently truncates reads as complete when it isn't.
|
|
1774
|
+
*
|
|
1775
|
+
* Offset paging is only sound over a table nobody is deleting from underneath you: a row removed
|
|
1776
|
+
* behind the cursor shifts everything after it back by one, so the next page starts one row late
|
|
1777
|
+
* and that row is never seen. Use {@link iterate} for any sweep that deletes.
|
|
1778
|
+
*/
|
|
1779
|
+
async listPage(query = {}) {
|
|
1780
|
+
const [totalRow] = await this.db.select({
|
|
1781
|
+
value: count2()
|
|
1782
|
+
}).from(ragIngestionLog).where(this.where(query));
|
|
1783
|
+
return {
|
|
1784
|
+
rows: await this.list(query),
|
|
1785
|
+
total: totalRow?.value ?? 0
|
|
1786
|
+
};
|
|
1787
|
+
}
|
|
1788
|
+
/**
|
|
1789
|
+
* Every matching row, newest first, streamed in batches — the sweep-safe way to walk this table.
|
|
1790
|
+
*
|
|
1791
|
+
* Paging is *keyset*, not offset: each batch asks for the rows that sort strictly after the last
|
|
1792
|
+
* row yielded, using `(updatedAt, documentId)` — one point in
|
|
1793
|
+
* {@link RAG_INGESTION_LOG_PAGE_ORDER}, which is total because it ends in the primary key. A
|
|
1794
|
+
* cursor made of column values does not move when rows leave the table, which is what makes this
|
|
1795
|
+
* delete-safe *by construction* rather than by the caller carefully advancing an offset by only
|
|
1796
|
+
* the rows it kept.
|
|
1797
|
+
*
|
|
1798
|
+
* **The guarantee, stated once:** a row is yielded exactly when — at the moment the batch that
|
|
1799
|
+
* would contain it is fetched — the row exists, matches `where`, and sorts strictly after the
|
|
1800
|
+
* last row already yielded. Everything below is a consequence of that one sentence.
|
|
1801
|
+
*
|
|
1802
|
+
* What you can rely on:
|
|
1803
|
+
* - **Deleting rows while iterating is safe.** Delete any row you like — the one just yielded, the
|
|
1804
|
+
* whole batch, rows you have not reached yet — and every row that is still there when the sweep
|
|
1805
|
+
* reaches its position is visited exactly once. Nothing is skipped and nothing is repeated,
|
|
1806
|
+
* because deleting a row moves no *other* row's `(updatedAt, documentId)`. (Rows you delete
|
|
1807
|
+
* *ahead* of the cursor are simply never visited, which is the point of deleting them.)
|
|
1808
|
+
* - **Ties are safe.** A bulk upload stamps a whole batch with the same `updatedAt`; the
|
|
1809
|
+
* `documentId` tiebreaker keeps consecutive batches disjoint even when every row ties.
|
|
1810
|
+
* - **Each row is yielded at most once**, for as long as `updatedAt` only ever moves forward —
|
|
1811
|
+
* which is what `record()` does, since it always stamps `now`.
|
|
1812
|
+
*
|
|
1813
|
+
* What this is **not**: a snapshot, or a repeatable read. The sweep observes the table as it is
|
|
1814
|
+
* when each batch is fetched, so:
|
|
1815
|
+
* - A row **inserted** mid-sweep by `record()` is stamped `now`. The order is newest-first, so it
|
|
1816
|
+
* sorts ahead of a cursor the sweep has already moved past, and is not visited. A sweep
|
|
1817
|
+
* therefore never sees documents ingested after it started — the behaviour an orphan sweep
|
|
1818
|
+
* wants, since a document that arrived while you were sweeping is not an orphan. (A *direct*
|
|
1819
|
+
* insert with a backdated `updatedAt` lands ahead of the cursor and **is** visited; nothing in
|
|
1820
|
+
* this class writes one.)
|
|
1821
|
+
* - A row **re-ingested** mid-sweep has its `updatedAt` bumped to `now`, moving it ahead of the
|
|
1822
|
+
* cursor for the same reason. If the sweep had already passed it, it is not visited twice; if it
|
|
1823
|
+
* had not, it is **missed for this pass**. That is the honest limit: against a concurrent
|
|
1824
|
+
* writer `iterate` guarantees "at most once", not "at least once". Sweeps that must not miss a
|
|
1825
|
+
* concurrently-rewritten row should be idempotent and run again.
|
|
1826
|
+
* - Only a row whose `updatedAt` moves *backwards* — which needs a direct write — can be yielded
|
|
1827
|
+
* twice.
|
|
1828
|
+
* - `collection` / `status` are re-evaluated per batch, so a row that stops matching mid-sweep
|
|
1829
|
+
* stops being visited.
|
|
1830
|
+
*
|
|
1831
|
+
* There is no `limit`/`offset` here on purpose: `where` says *which* rows, `options` says how the
|
|
1832
|
+
* sweep is run. A sweep that stops early stops by `break`ing out of the `for await`.
|
|
1833
|
+
*/
|
|
1834
|
+
iterate(where = {}, options = {}) {
|
|
1835
|
+
return this.sweep(where, batches(options.batchSize), options.after, (filter, limit) => this.db.select().from(ragIngestionLog).where(filter).orderBy(...RAG_INGESTION_LOG_PAGE_ORDER).limit(limit));
|
|
1836
|
+
}
|
|
1837
|
+
/**
|
|
1838
|
+
* Just the document ids of the matching rows, in {@link RAG_INGESTION_LOG_PAGE_ORDER}.
|
|
1839
|
+
*
|
|
1840
|
+
* Returns **all** of them — no 200-row cap, unlike `list` — by sweeping on {@link iterate}'s
|
|
1841
|
+
* keyset internally, so it is correct against concurrent deletes for the same reason `iterate`
|
|
1842
|
+
* is. What the cap protects against is hydrating an unbounded number of wide rows (`error` is a
|
|
1843
|
+
* TEXT column); this selects two columns — the id, plus the `updatedAt` the cursor needs — and
|
|
1844
|
+
* keeps only the ids, so the peak footprint is one batch of two-column rows plus a list of
|
|
1845
|
+
* strings whose size *is* the answer the caller asked for.
|
|
1846
|
+
*
|
|
1847
|
+
* For callers that need the id set of a collection and nothing else: an orphan sweep unioning the
|
|
1848
|
+
* log against the vector store's own document list.
|
|
1849
|
+
*/
|
|
1850
|
+
async listDocumentIds(where = {}, options = {}) {
|
|
1851
|
+
const ids = [];
|
|
1852
|
+
const rows = this.sweep(where, batches(options.batchSize), void 0, (filter, limit) => this.db.select(CURSOR_COLUMNS).from(ragIngestionLog).where(filter).orderBy(...RAG_INGESTION_LOG_PAGE_ORDER).limit(limit));
|
|
1853
|
+
for await (const row of rows) {
|
|
1854
|
+
ids.push(row.documentId);
|
|
1855
|
+
}
|
|
1856
|
+
return ids;
|
|
1857
|
+
}
|
|
1858
|
+
/** The latest recorded outcome for one document, or null if it was never attempted. */
|
|
1859
|
+
async get(documentId) {
|
|
1860
|
+
const [row] = await this.db.select().from(ragIngestionLog).where(eq5(ragIngestionLog.documentId, documentId));
|
|
1861
|
+
return row ?? null;
|
|
1862
|
+
}
|
|
1863
|
+
/** Forget one document's record. Returns whether a row was actually removed. */
|
|
1864
|
+
async remove(documentId) {
|
|
1865
|
+
const removed = await this.db.delete(ragIngestionLog).where(eq5(ragIngestionLog.documentId, documentId)).returning({
|
|
1866
|
+
documentId: ragIngestionLog.documentId
|
|
1867
|
+
});
|
|
1868
|
+
return removed.length > 0;
|
|
1869
|
+
}
|
|
1870
|
+
/** Forget every record for a collection — for when the collection itself is deleted. */
|
|
1871
|
+
async removeByCollection(collection) {
|
|
1872
|
+
const removed = await this.db.delete(ragIngestionLog).where(eq5(ragIngestionLog.collection, collection)).returning({
|
|
1873
|
+
documentId: ragIngestionLog.documentId
|
|
1874
|
+
});
|
|
1875
|
+
return removed.length;
|
|
1876
|
+
}
|
|
1877
|
+
/**
|
|
1878
|
+
* The keyset sweep both {@link iterate} and {@link listDocumentIds} are made of. `page` decides
|
|
1879
|
+
* how wide a row is — whole rows for `iterate`, two columns for `listDocumentIds` — and the
|
|
1880
|
+
* `R extends RagIngestionLogCursor` bound says the only thing the sweep needs back: a row it can
|
|
1881
|
+
* read the next cursor off.
|
|
1882
|
+
*
|
|
1883
|
+
* The cursor advances **per yielded row**, not per batch, so a consumer that `break`s mid-batch
|
|
1884
|
+
* leaves it on the last row it actually saw. It is two column values read off that row, so a row
|
|
1885
|
+
* leaving the table cannot move it — the whole reason this is delete-safe by construction.
|
|
1886
|
+
*/
|
|
1887
|
+
async *sweep(where, batchSize, after, page) {
|
|
1888
|
+
let cursor = after;
|
|
1889
|
+
for (; ; ) {
|
|
1890
|
+
const rows = await page(this.keyset(where, cursor), batchSize);
|
|
1891
|
+
if (rows.length === 0) {
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
for (const row of rows) {
|
|
1895
|
+
yield row;
|
|
1896
|
+
cursor = {
|
|
1897
|
+
updatedAt: row.updatedAt,
|
|
1898
|
+
documentId: row.documentId
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
where(query) {
|
|
1904
|
+
return and5(query.collection !== void 0 ? eq5(ragIngestionLog.collection, query.collection) : void 0, query.status !== void 0 ? eq5(ragIngestionLog.status, query.status) : void 0);
|
|
1905
|
+
}
|
|
1906
|
+
/**
|
|
1907
|
+
* The coordinates filter AND'd with "strictly after `cursor` in
|
|
1908
|
+
* {@link RAG_INGESTION_LOG_PAGE_ORDER}".
|
|
1909
|
+
*
|
|
1910
|
+
* The order is `updatedAt desc, documentId asc`, so "after" means an older timestamp, or the same
|
|
1911
|
+
* timestamp and a larger id. Both halves are needed: the `<` alone would re-yield every row tied
|
|
1912
|
+
* with the cursor, and dropping the tie branch would skip them instead.
|
|
1913
|
+
*/
|
|
1914
|
+
keyset(where, cursor) {
|
|
1915
|
+
const base = this.where(where);
|
|
1916
|
+
if (cursor === void 0) {
|
|
1917
|
+
return base;
|
|
1918
|
+
}
|
|
1919
|
+
return and5(base, or(lt(ragIngestionLog.updatedAt, cursor.updatedAt), and5(eq5(ragIngestionLog.updatedAt, cursor.updatedAt), gt(ragIngestionLog.documentId, cursor.documentId))));
|
|
1920
|
+
}
|
|
1921
|
+
/**
|
|
1922
|
+
* Upsert one outcome. Keyed by document id so the row always reflects the *current* state — a
|
|
1923
|
+
* retry that succeeds overwrites the failure it replaces, rather than leaving a stale error next
|
|
1924
|
+
* to a working document.
|
|
1925
|
+
*/
|
|
1926
|
+
async record(status, payload) {
|
|
1927
|
+
const documentId = str(payload.mediaId);
|
|
1928
|
+
if (documentId === null) {
|
|
1929
|
+
return;
|
|
1930
|
+
}
|
|
1931
|
+
try {
|
|
1932
|
+
const now = /* @__PURE__ */ new Date();
|
|
1933
|
+
const collection = str(payload.collection);
|
|
1934
|
+
const ownerType = str(payload.ownerType);
|
|
1935
|
+
const ownerId = str(payload.ownerId);
|
|
1936
|
+
const source = str(payload.source);
|
|
1937
|
+
const mimeType = str(payload.mimeType);
|
|
1938
|
+
const size = num(payload.size);
|
|
1939
|
+
const outcome = {
|
|
1940
|
+
status,
|
|
1941
|
+
// The three outcome-specific columns are exclusive: null out the ones this status doesn't
|
|
1942
|
+
// own on every write, so a successful retry clears the previous attempt's error rather than
|
|
1943
|
+
// leaving it next to a working document. Always present → always in the conflict `set`.
|
|
1944
|
+
chunks: status === "ingested" ? num(payload.chunks) : null,
|
|
1945
|
+
reason: status === "skipped" ? str(payload.reason) : null,
|
|
1946
|
+
error: status === "failed" ? str(payload.error) : null,
|
|
1947
|
+
updatedAt: now,
|
|
1948
|
+
...collection !== null ? {
|
|
1949
|
+
collection
|
|
1950
|
+
} : {},
|
|
1951
|
+
...ownerType !== null ? {
|
|
1952
|
+
ownerType
|
|
1953
|
+
} : {},
|
|
1954
|
+
...ownerId !== null ? {
|
|
1955
|
+
ownerId
|
|
1956
|
+
} : {},
|
|
1957
|
+
...source !== null ? {
|
|
1958
|
+
source
|
|
1959
|
+
} : {},
|
|
1960
|
+
...mimeType !== null ? {
|
|
1961
|
+
mimeType
|
|
1962
|
+
} : {},
|
|
1963
|
+
...size !== null ? {
|
|
1964
|
+
size
|
|
1965
|
+
} : {}
|
|
1966
|
+
};
|
|
1967
|
+
await this.db.insert(ragIngestionLog).values({
|
|
1968
|
+
documentId,
|
|
1969
|
+
createdAt: now,
|
|
1970
|
+
...outcome
|
|
1971
|
+
}).onConflictDoUpdate({
|
|
1972
|
+
target: ragIngestionLog.documentId,
|
|
1973
|
+
set: outcome
|
|
1974
|
+
});
|
|
1975
|
+
} catch (error) {
|
|
1976
|
+
this.logger.warn(`Could not record RAG ingestion outcome for ${documentId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
track(promise) {
|
|
1980
|
+
const tracked = promise.finally(() => {
|
|
1981
|
+
this.inFlight.delete(tracked);
|
|
1982
|
+
});
|
|
1983
|
+
this.inFlight.add(tracked);
|
|
1984
|
+
}
|
|
1985
|
+
};
|
|
1986
|
+
DrizzleRagIngestionLog = _ts_decorate([
|
|
1987
|
+
Injectable(),
|
|
1988
|
+
_ts_metadata("design:type", Function),
|
|
1989
|
+
_ts_metadata("design:paramtypes", [
|
|
1990
|
+
typeof AgentDrizzleDb === "undefined" ? Object : AgentDrizzleDb
|
|
1991
|
+
])
|
|
1992
|
+
], DrizzleRagIngestionLog);
|
|
1993
|
+
|
|
1246
1994
|
// src/drizzle-agent-store.module.ts
|
|
1247
1995
|
import { AGENT_GOVERNANCE_QUERIES, AGENT_PRICING_STORE, AGENT_STORE } from "@dudousxd/nestjs-agent-core";
|
|
1248
1996
|
import { Module } from "@nestjs/common";
|
|
1249
|
-
function
|
|
1250
|
-
var c = arguments.length, r = c < 3 ? target :
|
|
1251
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key,
|
|
1997
|
+
function _ts_decorate2(decorators, target, key, desc4) {
|
|
1998
|
+
var c = arguments.length, r = c < 3 ? target : desc4 === null ? desc4 = Object.getOwnPropertyDescriptor(target, key) : desc4, d;
|
|
1999
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc4);
|
|
1252
2000
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1253
2001
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1254
2002
|
}
|
|
1255
|
-
__name(
|
|
2003
|
+
__name(_ts_decorate2, "_ts_decorate");
|
|
1256
2004
|
var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
1257
2005
|
static {
|
|
1258
2006
|
__name(this, "DrizzleAgentStoreModule");
|
|
1259
2007
|
}
|
|
1260
2008
|
static forRoot(options) {
|
|
2009
|
+
const ragProviders = options.ragIngestionLog === false ? [] : [
|
|
2010
|
+
{
|
|
2011
|
+
provide: DrizzleRagIngestionLog,
|
|
2012
|
+
useFactory: /* @__PURE__ */ __name(() => new DrizzleRagIngestionLog(options.db), "useFactory")
|
|
2013
|
+
}
|
|
2014
|
+
];
|
|
1261
2015
|
return {
|
|
1262
2016
|
module: _DrizzleAgentStoreModule,
|
|
1263
2017
|
global: true,
|
|
1264
2018
|
providers: [
|
|
2019
|
+
...ragProviders,
|
|
1265
2020
|
{
|
|
1266
2021
|
provide: DrizzleAgentStore,
|
|
1267
2022
|
useFactory: /* @__PURE__ */ __name(() => new DrizzleAgentStore(options.db), "useFactory")
|
|
@@ -1270,6 +2025,13 @@ var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
|
1270
2025
|
provide: AGENT_STORE,
|
|
1271
2026
|
useExisting: DrizzleAgentStore
|
|
1272
2027
|
},
|
|
2028
|
+
// Exported, never bound to AGENT_MEMORY: that token's PRESENCE is what turns memory on, so
|
|
2029
|
+
// binding it here would switch the feature on for every host that installs the store. A
|
|
2030
|
+
// host opts in by naming this provider in `AgentModule.forRoot({ memory: { provider } })`.
|
|
2031
|
+
{
|
|
2032
|
+
provide: DrizzleMemoryProvider,
|
|
2033
|
+
useFactory: /* @__PURE__ */ __name(() => new DrizzleMemoryProvider(options.db), "useFactory")
|
|
2034
|
+
},
|
|
1273
2035
|
{
|
|
1274
2036
|
provide: DrizzlePricingStore,
|
|
1275
2037
|
useFactory: /* @__PURE__ */ __name(() => new DrizzlePricingStore(options.db), "useFactory")
|
|
@@ -1296,8 +2058,12 @@ var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
|
1296
2058
|
}
|
|
1297
2059
|
],
|
|
1298
2060
|
exports: [
|
|
2061
|
+
...options.ragIngestionLog === false ? [] : [
|
|
2062
|
+
DrizzleRagIngestionLog
|
|
2063
|
+
],
|
|
1299
2064
|
DrizzleAgentStore,
|
|
1300
2065
|
AGENT_STORE,
|
|
2066
|
+
DrizzleMemoryProvider,
|
|
1301
2067
|
DrizzleGovernanceQueries,
|
|
1302
2068
|
AGENT_GOVERNANCE_QUERIES,
|
|
1303
2069
|
DrizzlePricingStore,
|
|
@@ -1306,14 +2072,18 @@ var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
|
1306
2072
|
};
|
|
1307
2073
|
}
|
|
1308
2074
|
};
|
|
1309
|
-
DrizzleAgentStoreModule =
|
|
2075
|
+
DrizzleAgentStoreModule = _ts_decorate2([
|
|
1310
2076
|
Module({})
|
|
1311
2077
|
], DrizzleAgentStoreModule);
|
|
1312
2078
|
export {
|
|
1313
2079
|
DrizzleAgentStore,
|
|
1314
2080
|
DrizzleAgentStoreModule,
|
|
1315
2081
|
DrizzleGovernanceQueries,
|
|
2082
|
+
DrizzleMemoryProvider,
|
|
1316
2083
|
DrizzlePricingStore,
|
|
2084
|
+
DrizzleRagIngestionLog,
|
|
2085
|
+
RAG_INGESTION_LOG_PAGE_ORDER,
|
|
2086
|
+
agentMemory,
|
|
1317
2087
|
agentMessage,
|
|
1318
2088
|
agentModelPricing,
|
|
1319
2089
|
agentRun,
|
|
@@ -1321,6 +2091,7 @@ export {
|
|
|
1321
2091
|
agentThread,
|
|
1322
2092
|
agentTokenUsage,
|
|
1323
2093
|
agentToolCall,
|
|
1324
|
-
ensureAgentSchema
|
|
2094
|
+
ensureAgentSchema,
|
|
2095
|
+
ragIngestionLog
|
|
1325
2096
|
};
|
|
1326
2097
|
//# sourceMappingURL=index.js.map
|