@dudousxd/nestjs-agent-store-drizzle 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -1
- package/dist/index.cjs +602 -24
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1641 -425
- package/dist/index.d.ts +1641 -425
- package/dist/index.js +585 -12
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@ var __defProp = Object.defineProperty;
|
|
|
2
2
|
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
3
|
|
|
4
4
|
// src/schema.ts
|
|
5
|
-
import { index, integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
5
|
+
import { index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
|
|
6
6
|
var agentThread = sqliteTable("agent_thread", {
|
|
7
7
|
id: text("id").primaryKey(),
|
|
8
8
|
actorRef: text("actor_ref").notNull(),
|
|
@@ -154,13 +154,67 @@ var agentRun = sqliteTable("agent_run", {
|
|
|
154
154
|
}, (table) => [
|
|
155
155
|
index("agent_run_started_idx").on(table.startedAt)
|
|
156
156
|
]);
|
|
157
|
+
var agentMemory = sqliteTable("agent_memory", {
|
|
158
|
+
id: text("id").primaryKey(),
|
|
159
|
+
/** The opaque scope token. Unique with `key`: one fact per key per scope. */
|
|
160
|
+
scope: text("scope").notNull(),
|
|
161
|
+
/** What the fact is ABOUT. Two scopes sharing a key are one question answered twice. */
|
|
162
|
+
key: text("key").notNull(),
|
|
163
|
+
/** The fact itself, one line, as the model reads it. */
|
|
164
|
+
text: text("text").notNull(),
|
|
165
|
+
originAuthor: text("origin_author").$type().notNull(),
|
|
166
|
+
/** A pointer that is allowed to dangle — see the table note. */
|
|
167
|
+
originThreadId: text("origin_thread_id"),
|
|
168
|
+
originRunId: text("origin_run_id"),
|
|
169
|
+
originActorRef: text("origin_actor_ref"),
|
|
170
|
+
/** Always-on: carried in every prompt. Set by a host's console, never by a write. */
|
|
171
|
+
pinned: integer("pinned", {
|
|
172
|
+
mode: "boolean"
|
|
173
|
+
}).notNull().default(false),
|
|
174
|
+
createdAt: integer("created_at", {
|
|
175
|
+
mode: "timestamp_ms"
|
|
176
|
+
}).notNull(),
|
|
177
|
+
updatedAt: integer("updated_at", {
|
|
178
|
+
mode: "timestamp_ms"
|
|
179
|
+
}).notNull()
|
|
180
|
+
}, (table) => [
|
|
181
|
+
uniqueIndex("agent_memory_scope_key_uq").on(table.scope, table.key)
|
|
182
|
+
]);
|
|
183
|
+
var ragIngestionLog = sqliteTable("rag_ingestion_log", {
|
|
184
|
+
/** The vector-store document id (chunk ids are `${documentId}#<n>`). */
|
|
185
|
+
documentId: text("document_id").primaryKey(),
|
|
186
|
+
status: text("status").$type().notNull(),
|
|
187
|
+
collection: text("collection"),
|
|
188
|
+
ownerType: text("owner_type"),
|
|
189
|
+
ownerId: text("owner_id"),
|
|
190
|
+
/** Citation-facing origin — the media record's path. */
|
|
191
|
+
source: text("source"),
|
|
192
|
+
mimeType: text("mime_type"),
|
|
193
|
+
size: integer("size"),
|
|
194
|
+
/** Chunk count on `ingested`; null otherwise. */
|
|
195
|
+
chunks: integer("chunks"),
|
|
196
|
+
/** Skip reason on `skipped` (`unsupported-type` | `too-large` | `empty-text`); null otherwise. */
|
|
197
|
+
reason: text("reason"),
|
|
198
|
+
/** Error message on `failed`; null otherwise. */
|
|
199
|
+
error: text("error"),
|
|
200
|
+
createdAt: integer("created_at", {
|
|
201
|
+
mode: "timestamp_ms"
|
|
202
|
+
}).notNull(),
|
|
203
|
+
updatedAt: integer("updated_at", {
|
|
204
|
+
mode: "timestamp_ms"
|
|
205
|
+
}).notNull()
|
|
206
|
+
}, (table) => [
|
|
207
|
+
index("rag_ingestion_log_collection_idx").on(table.collection, table.updatedAt)
|
|
208
|
+
]);
|
|
157
209
|
var agentSchema = {
|
|
158
210
|
agentThread,
|
|
159
211
|
agentMessage,
|
|
160
212
|
agentToolCall,
|
|
161
213
|
agentTokenUsage,
|
|
162
214
|
agentModelPricing,
|
|
163
|
-
agentRun
|
|
215
|
+
agentRun,
|
|
216
|
+
agentMemory,
|
|
217
|
+
ragIngestionLog
|
|
164
218
|
};
|
|
165
219
|
|
|
166
220
|
// src/ensure-schema.ts
|
|
@@ -255,7 +309,41 @@ var statements = [
|
|
|
255
309
|
parent_run_id TEXT
|
|
256
310
|
)`,
|
|
257
311
|
`CREATE INDEX IF NOT EXISTS agent_run_started_idx
|
|
258
|
-
ON agent_run (started_at)
|
|
312
|
+
ON agent_run (started_at)`,
|
|
313
|
+
// `key` and `text` are quoted: both are keywords in at least one engine this DDL is read by, and
|
|
314
|
+
// they are the SPI's own field names, which is worth more here than dodging the quoting.
|
|
315
|
+
`CREATE TABLE IF NOT EXISTS agent_memory (
|
|
316
|
+
id TEXT PRIMARY KEY NOT NULL,
|
|
317
|
+
scope TEXT NOT NULL,
|
|
318
|
+
"key" TEXT NOT NULL,
|
|
319
|
+
"text" TEXT NOT NULL,
|
|
320
|
+
origin_author TEXT NOT NULL,
|
|
321
|
+
origin_thread_id TEXT,
|
|
322
|
+
origin_run_id TEXT,
|
|
323
|
+
origin_actor_ref TEXT,
|
|
324
|
+
pinned INTEGER NOT NULL DEFAULT 0,
|
|
325
|
+
created_at INTEGER NOT NULL,
|
|
326
|
+
updated_at INTEGER NOT NULL
|
|
327
|
+
)`,
|
|
328
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS agent_memory_scope_key_uq
|
|
329
|
+
ON agent_memory (scope, "key")`,
|
|
330
|
+
`CREATE TABLE IF NOT EXISTS rag_ingestion_log (
|
|
331
|
+
document_id TEXT PRIMARY KEY NOT NULL,
|
|
332
|
+
status TEXT NOT NULL,
|
|
333
|
+
collection TEXT,
|
|
334
|
+
owner_type TEXT,
|
|
335
|
+
owner_id TEXT,
|
|
336
|
+
source TEXT,
|
|
337
|
+
mime_type TEXT,
|
|
338
|
+
size INTEGER,
|
|
339
|
+
chunks INTEGER,
|
|
340
|
+
reason TEXT,
|
|
341
|
+
error TEXT,
|
|
342
|
+
created_at INTEGER NOT NULL,
|
|
343
|
+
updated_at INTEGER NOT NULL
|
|
344
|
+
)`,
|
|
345
|
+
`CREATE INDEX IF NOT EXISTS rag_ingestion_log_collection_idx
|
|
346
|
+
ON rag_ingestion_log (collection, updated_at)`
|
|
259
347
|
];
|
|
260
348
|
var additiveColumns = [
|
|
261
349
|
{
|
|
@@ -1399,8 +1487,150 @@ var DrizzleGovernanceQueries = class {
|
|
|
1399
1487
|
}
|
|
1400
1488
|
};
|
|
1401
1489
|
|
|
1490
|
+
// src/drizzle-memory-provider.ts
|
|
1491
|
+
import { actorScope } from "@dudousxd/nestjs-agent-core";
|
|
1492
|
+
import { and as and3, eq as eq3, inArray as inArray3 } from "drizzle-orm";
|
|
1493
|
+
function toRecord(row) {
|
|
1494
|
+
return {
|
|
1495
|
+
id: row.id,
|
|
1496
|
+
key: row.key,
|
|
1497
|
+
text: row.text,
|
|
1498
|
+
scope: row.scope,
|
|
1499
|
+
origin: {
|
|
1500
|
+
author: row.originAuthor,
|
|
1501
|
+
...row.originThreadId !== null ? {
|
|
1502
|
+
threadId: row.originThreadId
|
|
1503
|
+
} : {},
|
|
1504
|
+
...row.originRunId !== null ? {
|
|
1505
|
+
runId: row.originRunId
|
|
1506
|
+
} : {},
|
|
1507
|
+
...row.originActorRef !== null ? {
|
|
1508
|
+
actorRef: row.originActorRef
|
|
1509
|
+
} : {}
|
|
1510
|
+
},
|
|
1511
|
+
updatedAt: row.updatedAt.toISOString(),
|
|
1512
|
+
pinned: row.pinned
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
__name(toRecord, "toRecord");
|
|
1516
|
+
var DrizzleMemoryProvider = class {
|
|
1517
|
+
static {
|
|
1518
|
+
__name(this, "DrizzleMemoryProvider");
|
|
1519
|
+
}
|
|
1520
|
+
db;
|
|
1521
|
+
constructor(db) {
|
|
1522
|
+
this.db = db;
|
|
1523
|
+
}
|
|
1524
|
+
/**
|
|
1525
|
+
* Every memory visible at `scopes` — asked on EVERY turn, so it is one indexed `scope in (…)`
|
|
1526
|
+
* against the leading column of the (`scope`, `key`) unique index.
|
|
1527
|
+
*
|
|
1528
|
+
* The scope filter is in the QUERY. The library drops out-of-scope records it is handed, but that
|
|
1529
|
+
* is a backstop: a memory held for another actor or another tenant is never selected here, so it
|
|
1530
|
+
* is unreachable rather than outranked.
|
|
1531
|
+
*/
|
|
1532
|
+
async list({ scopes }) {
|
|
1533
|
+
if (scopes.length === 0) {
|
|
1534
|
+
return [];
|
|
1535
|
+
}
|
|
1536
|
+
const rows = await this.db.select().from(agentMemory).where(inArray3(agentMemory.scope, [
|
|
1537
|
+
...scopes
|
|
1538
|
+
]));
|
|
1539
|
+
return rows.map(toRecord);
|
|
1540
|
+
}
|
|
1541
|
+
/**
|
|
1542
|
+
* Delete one, and only from the actor's OWN scope.
|
|
1543
|
+
*
|
|
1544
|
+
* `memoryForgetVerdict` already holds the library's endpoint to that rule; the `where` repeats it
|
|
1545
|
+
* because `forget` takes an id and nothing else, so a caller reaching this provider by any other
|
|
1546
|
+
* path would otherwise delete a tenant's or the deployment's memory by knowing one id.
|
|
1547
|
+
*
|
|
1548
|
+
* `false` where nothing matched — which covers "no such id" and "not yours" identically, so the
|
|
1549
|
+
* answer cannot be used to find out which memories exist about other people.
|
|
1550
|
+
*/
|
|
1551
|
+
async forget({ id, ctx }) {
|
|
1552
|
+
const deleted = await this.db.delete(agentMemory).where(and3(eq3(agentMemory.id, id), eq3(agentMemory.scope, actorScope(ctx.actor)))).returning({
|
|
1553
|
+
id: agentMemory.id
|
|
1554
|
+
});
|
|
1555
|
+
return deleted.length > 0;
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* Upsert on (`scope`, `key`) — one statement, against the unique index, rather than a read
|
|
1559
|
+
* followed by a write: two turns concluding the same key at once would otherwise race to a
|
|
1560
|
+
* duplicate-key insert and one of them would fail the `remember` tool.
|
|
1561
|
+
*
|
|
1562
|
+
* The conflict `set` names exactly what a rewrite may change, so `pinned`, `createdAt` and `id`
|
|
1563
|
+
* are left alone by construction rather than by an exclusion anyone could forget to extend.
|
|
1564
|
+
* Dropping `pinned` would silently unpin a row the next time the agent restated the fact;
|
|
1565
|
+
* rewriting `createdAt` would lose when the belief was first formed; reassigning `id` would
|
|
1566
|
+
* invalidate the delete handle a person was already shown.
|
|
1567
|
+
*/
|
|
1568
|
+
async write({ key, text: text2, scope, origin, ctx }) {
|
|
1569
|
+
assertAuthorMayWriteScope({
|
|
1570
|
+
scope,
|
|
1571
|
+
origin,
|
|
1572
|
+
ctx
|
|
1573
|
+
});
|
|
1574
|
+
const now = /* @__PURE__ */ new Date();
|
|
1575
|
+
const columns = {
|
|
1576
|
+
originAuthor: origin.author,
|
|
1577
|
+
originThreadId: origin.threadId ?? null,
|
|
1578
|
+
originRunId: origin.runId ?? null,
|
|
1579
|
+
originActorRef: origin.actorRef ?? null
|
|
1580
|
+
};
|
|
1581
|
+
const [row] = await this.db.insert(agentMemory).values({
|
|
1582
|
+
id: crypto.randomUUID(),
|
|
1583
|
+
scope,
|
|
1584
|
+
key,
|
|
1585
|
+
text: text2,
|
|
1586
|
+
...columns,
|
|
1587
|
+
pinned: false,
|
|
1588
|
+
createdAt: now,
|
|
1589
|
+
updatedAt: now
|
|
1590
|
+
}).onConflictDoUpdate({
|
|
1591
|
+
target: [
|
|
1592
|
+
agentMemory.scope,
|
|
1593
|
+
agentMemory.key
|
|
1594
|
+
],
|
|
1595
|
+
set: {
|
|
1596
|
+
text: text2,
|
|
1597
|
+
...columns,
|
|
1598
|
+
updatedAt: now
|
|
1599
|
+
}
|
|
1600
|
+
}).returning();
|
|
1601
|
+
if (row === void 0) {
|
|
1602
|
+
throw new Error(`DrizzleMemoryProvider: the upsert of "${key}" at "${scope}" returned no row; the driver does not support RETURNING on conflict.`);
|
|
1603
|
+
}
|
|
1604
|
+
return toRecord(row);
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* Make one memory always-on, or stop it being one. The operator act the SPI has no method for, on
|
|
1608
|
+
* purpose: a pin grants a fact a permanent place in every future prompt, so nothing an agent can
|
|
1609
|
+
* reach may set it. Authorize it in your own console — this takes no `ScopeContext` because it is
|
|
1610
|
+
* not an actor's act on their own memory.
|
|
1611
|
+
*
|
|
1612
|
+
* `false` where there is no such id.
|
|
1613
|
+
*/
|
|
1614
|
+
async pin({ id, pinned }) {
|
|
1615
|
+
const updated = await this.db.update(agentMemory).set({
|
|
1616
|
+
pinned,
|
|
1617
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1618
|
+
}).where(eq3(agentMemory.id, id)).returning({
|
|
1619
|
+
id: agentMemory.id
|
|
1620
|
+
});
|
|
1621
|
+
return updated.length > 0;
|
|
1622
|
+
}
|
|
1623
|
+
};
|
|
1624
|
+
function assertAuthorMayWriteScope({ scope, origin, ctx }) {
|
|
1625
|
+
const own = actorScope(ctx.actor);
|
|
1626
|
+
if (origin.author !== "human" && scope !== own) {
|
|
1627
|
+
throw new Error(`DrizzleMemoryProvider: refusing to write an agent-authored memory at "${scope}"; an agent may write only at "${own}"`);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
__name(assertAuthorMayWriteScope, "assertAuthorMayWriteScope");
|
|
1631
|
+
|
|
1402
1632
|
// src/drizzle-pricing-store.ts
|
|
1403
|
-
import { and as
|
|
1633
|
+
import { and as and4, eq as eq4 } from "drizzle-orm";
|
|
1404
1634
|
var DrizzlePricingStore = class {
|
|
1405
1635
|
static {
|
|
1406
1636
|
__name(this, "DrizzlePricingStore");
|
|
@@ -1412,7 +1642,7 @@ var DrizzlePricingStore = class {
|
|
|
1412
1642
|
async upsertModelPrice(input) {
|
|
1413
1643
|
await this.db.update(agentModelPricing).set({
|
|
1414
1644
|
isCurrent: false
|
|
1415
|
-
}).where(
|
|
1645
|
+
}).where(and4(eq4(agentModelPricing.modelId, input.modelId), eq4(agentModelPricing.isCurrent, true)));
|
|
1416
1646
|
await this.db.insert(agentModelPricing).values({
|
|
1417
1647
|
id: crypto.randomUUID(),
|
|
1418
1648
|
modelId: input.modelId,
|
|
@@ -1425,7 +1655,7 @@ var DrizzlePricingStore = class {
|
|
|
1425
1655
|
});
|
|
1426
1656
|
}
|
|
1427
1657
|
async listCurrentPrices() {
|
|
1428
|
-
const rows = await this.db.select().from(agentModelPricing).where(
|
|
1658
|
+
const rows = await this.db.select().from(agentModelPricing).where(eq4(agentModelPricing.isCurrent, true));
|
|
1429
1659
|
return rows.map((row) => ({
|
|
1430
1660
|
modelId: row.modelId,
|
|
1431
1661
|
inputPricePer1m: row.inputPricePer1m,
|
|
@@ -1441,25 +1671,352 @@ var DrizzlePricingStore = class {
|
|
|
1441
1671
|
}
|
|
1442
1672
|
};
|
|
1443
1673
|
|
|
1674
|
+
// src/drizzle-rag-ingestion-log.ts
|
|
1675
|
+
import { subscribe, unsubscribe } from "node:diagnostics_channel";
|
|
1676
|
+
import { Injectable, Logger } from "@nestjs/common";
|
|
1677
|
+
import { and as and5, asc as asc3, count as count2, desc as desc3, eq as eq5, gt, lt, or } from "drizzle-orm";
|
|
1678
|
+
function _ts_decorate(decorators, target, key, desc4) {
|
|
1679
|
+
var c = arguments.length, r = c < 3 ? target : desc4 === null ? desc4 = Object.getOwnPropertyDescriptor(target, key) : desc4, d;
|
|
1680
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc4);
|
|
1681
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1682
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1683
|
+
}
|
|
1684
|
+
__name(_ts_decorate, "_ts_decorate");
|
|
1685
|
+
function _ts_metadata(k, v) {
|
|
1686
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
1687
|
+
}
|
|
1688
|
+
__name(_ts_metadata, "_ts_metadata");
|
|
1689
|
+
var CHANNELS = [
|
|
1690
|
+
{
|
|
1691
|
+
name: "aviary:rag:media.ingested",
|
|
1692
|
+
status: "ingested"
|
|
1693
|
+
},
|
|
1694
|
+
{
|
|
1695
|
+
name: "aviary:rag:media.skipped",
|
|
1696
|
+
status: "skipped"
|
|
1697
|
+
},
|
|
1698
|
+
{
|
|
1699
|
+
name: "aviary:rag:media.failed",
|
|
1700
|
+
status: "failed"
|
|
1701
|
+
},
|
|
1702
|
+
{
|
|
1703
|
+
name: "aviary:rag:media.removed",
|
|
1704
|
+
status: "removed"
|
|
1705
|
+
}
|
|
1706
|
+
];
|
|
1707
|
+
function str(value) {
|
|
1708
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
1709
|
+
}
|
|
1710
|
+
__name(str, "str");
|
|
1711
|
+
function num(value) {
|
|
1712
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1713
|
+
}
|
|
1714
|
+
__name(num, "num");
|
|
1715
|
+
var RAG_INGESTION_LOG_PAGE_ORDER = [
|
|
1716
|
+
desc3(ragIngestionLog.updatedAt),
|
|
1717
|
+
asc3(ragIngestionLog.documentId)
|
|
1718
|
+
];
|
|
1719
|
+
var DEFAULT_PAGE_SIZE = 200;
|
|
1720
|
+
var DEFAULT_BATCH_SIZE = 200;
|
|
1721
|
+
function batches(batchSize) {
|
|
1722
|
+
const requested = Math.trunc(batchSize ?? DEFAULT_BATCH_SIZE);
|
|
1723
|
+
return Number.isFinite(requested) && requested >= 1 ? requested : DEFAULT_BATCH_SIZE;
|
|
1724
|
+
}
|
|
1725
|
+
__name(batches, "batches");
|
|
1726
|
+
var CURSOR_COLUMNS = {
|
|
1727
|
+
documentId: ragIngestionLog.documentId,
|
|
1728
|
+
updatedAt: ragIngestionLog.updatedAt
|
|
1729
|
+
};
|
|
1730
|
+
var DrizzleRagIngestionLog = class _DrizzleRagIngestionLog {
|
|
1731
|
+
static {
|
|
1732
|
+
__name(this, "DrizzleRagIngestionLog");
|
|
1733
|
+
}
|
|
1734
|
+
db;
|
|
1735
|
+
logger = new Logger(_DrizzleRagIngestionLog.name);
|
|
1736
|
+
teardowns = [];
|
|
1737
|
+
inFlight = /* @__PURE__ */ new Set();
|
|
1738
|
+
constructor(db) {
|
|
1739
|
+
this.db = db;
|
|
1740
|
+
}
|
|
1741
|
+
onModuleInit() {
|
|
1742
|
+
for (const channel of CHANNELS) {
|
|
1743
|
+
const listener = /* @__PURE__ */ __name((message) => {
|
|
1744
|
+
const payload = message?.payload;
|
|
1745
|
+
if (payload !== void 0) {
|
|
1746
|
+
this.track(this.record(channel.status, payload));
|
|
1747
|
+
}
|
|
1748
|
+
}, "listener");
|
|
1749
|
+
subscribe(channel.name, listener);
|
|
1750
|
+
this.teardowns.push(() => unsubscribe(channel.name, listener));
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
async onModuleDestroy() {
|
|
1754
|
+
for (const teardown of this.teardowns) {
|
|
1755
|
+
teardown();
|
|
1756
|
+
}
|
|
1757
|
+
this.teardowns = [];
|
|
1758
|
+
await this.settle();
|
|
1759
|
+
}
|
|
1760
|
+
/** Await every in-flight write — for graceful shutdown and deterministic tests. */
|
|
1761
|
+
async settle() {
|
|
1762
|
+
await Promise.allSettled([
|
|
1763
|
+
...this.inFlight
|
|
1764
|
+
]);
|
|
1765
|
+
}
|
|
1766
|
+
/** The latest recorded outcome per document, newest first. */
|
|
1767
|
+
async list(query = {}) {
|
|
1768
|
+
const rows = this.db.select().from(ragIngestionLog).where(this.where(query)).orderBy(...RAG_INGESTION_LOG_PAGE_ORDER).limit(query.limit ?? DEFAULT_PAGE_SIZE);
|
|
1769
|
+
return query.offset === void 0 ? rows : rows.offset(query.offset);
|
|
1770
|
+
}
|
|
1771
|
+
/**
|
|
1772
|
+
* The same page plus the unpaginated total, so a caller can tell "these are all of them" from
|
|
1773
|
+
* "these are the first N". A list that silently truncates reads as complete when it isn't.
|
|
1774
|
+
*
|
|
1775
|
+
* Offset paging is only sound over a table nobody is deleting from underneath you: a row removed
|
|
1776
|
+
* behind the cursor shifts everything after it back by one, so the next page starts one row late
|
|
1777
|
+
* and that row is never seen. Use {@link iterate} for any sweep that deletes.
|
|
1778
|
+
*/
|
|
1779
|
+
async listPage(query = {}) {
|
|
1780
|
+
const [totalRow] = await this.db.select({
|
|
1781
|
+
value: count2()
|
|
1782
|
+
}).from(ragIngestionLog).where(this.where(query));
|
|
1783
|
+
return {
|
|
1784
|
+
rows: await this.list(query),
|
|
1785
|
+
total: totalRow?.value ?? 0
|
|
1786
|
+
};
|
|
1787
|
+
}
|
|
1788
|
+
/**
|
|
1789
|
+
* Every matching row, newest first, streamed in batches — the sweep-safe way to walk this table.
|
|
1790
|
+
*
|
|
1791
|
+
* Paging is *keyset*, not offset: each batch asks for the rows that sort strictly after the last
|
|
1792
|
+
* row yielded, using `(updatedAt, documentId)` — one point in
|
|
1793
|
+
* {@link RAG_INGESTION_LOG_PAGE_ORDER}, which is total because it ends in the primary key. A
|
|
1794
|
+
* cursor made of column values does not move when rows leave the table, which is what makes this
|
|
1795
|
+
* delete-safe *by construction* rather than by the caller carefully advancing an offset by only
|
|
1796
|
+
* the rows it kept.
|
|
1797
|
+
*
|
|
1798
|
+
* **The guarantee, stated once:** a row is yielded exactly when — at the moment the batch that
|
|
1799
|
+
* would contain it is fetched — the row exists, matches `where`, and sorts strictly after the
|
|
1800
|
+
* last row already yielded. Everything below is a consequence of that one sentence.
|
|
1801
|
+
*
|
|
1802
|
+
* What you can rely on:
|
|
1803
|
+
* - **Deleting rows while iterating is safe.** Delete any row you like — the one just yielded, the
|
|
1804
|
+
* whole batch, rows you have not reached yet — and every row that is still there when the sweep
|
|
1805
|
+
* reaches its position is visited exactly once. Nothing is skipped and nothing is repeated,
|
|
1806
|
+
* because deleting a row moves no *other* row's `(updatedAt, documentId)`. (Rows you delete
|
|
1807
|
+
* *ahead* of the cursor are simply never visited, which is the point of deleting them.)
|
|
1808
|
+
* - **Ties are safe.** A bulk upload stamps a whole batch with the same `updatedAt`; the
|
|
1809
|
+
* `documentId` tiebreaker keeps consecutive batches disjoint even when every row ties.
|
|
1810
|
+
* - **Each row is yielded at most once**, for as long as `updatedAt` only ever moves forward —
|
|
1811
|
+
* which is what `record()` does, since it always stamps `now`.
|
|
1812
|
+
*
|
|
1813
|
+
* What this is **not**: a snapshot, or a repeatable read. The sweep observes the table as it is
|
|
1814
|
+
* when each batch is fetched, so:
|
|
1815
|
+
* - A row **inserted** mid-sweep by `record()` is stamped `now`. The order is newest-first, so it
|
|
1816
|
+
* sorts ahead of a cursor the sweep has already moved past, and is not visited. A sweep
|
|
1817
|
+
* therefore never sees documents ingested after it started — the behaviour an orphan sweep
|
|
1818
|
+
* wants, since a document that arrived while you were sweeping is not an orphan. (A *direct*
|
|
1819
|
+
* insert with a backdated `updatedAt` lands ahead of the cursor and **is** visited; nothing in
|
|
1820
|
+
* this class writes one.)
|
|
1821
|
+
* - A row **re-ingested** mid-sweep has its `updatedAt` bumped to `now`, moving it ahead of the
|
|
1822
|
+
* cursor for the same reason. If the sweep had already passed it, it is not visited twice; if it
|
|
1823
|
+
* had not, it is **missed for this pass**. That is the honest limit: against a concurrent
|
|
1824
|
+
* writer `iterate` guarantees "at most once", not "at least once". Sweeps that must not miss a
|
|
1825
|
+
* concurrently-rewritten row should be idempotent and run again.
|
|
1826
|
+
* - Only a row whose `updatedAt` moves *backwards* — which needs a direct write — can be yielded
|
|
1827
|
+
* twice.
|
|
1828
|
+
* - `collection` / `status` are re-evaluated per batch, so a row that stops matching mid-sweep
|
|
1829
|
+
* stops being visited.
|
|
1830
|
+
*
|
|
1831
|
+
* There is no `limit`/`offset` here on purpose: `where` says *which* rows, `options` says how the
|
|
1832
|
+
* sweep is run. A sweep that stops early stops by `break`ing out of the `for await`.
|
|
1833
|
+
*/
|
|
1834
|
+
iterate(where = {}, options = {}) {
|
|
1835
|
+
return this.sweep(where, batches(options.batchSize), options.after, (filter, limit) => this.db.select().from(ragIngestionLog).where(filter).orderBy(...RAG_INGESTION_LOG_PAGE_ORDER).limit(limit));
|
|
1836
|
+
}
|
|
1837
|
+
/**
|
|
1838
|
+
* Just the document ids of the matching rows, in {@link RAG_INGESTION_LOG_PAGE_ORDER}.
|
|
1839
|
+
*
|
|
1840
|
+
* Returns **all** of them — no 200-row cap, unlike `list` — by sweeping on {@link iterate}'s
|
|
1841
|
+
* keyset internally, so it is correct against concurrent deletes for the same reason `iterate`
|
|
1842
|
+
* is. What the cap protects against is hydrating an unbounded number of wide rows (`error` is a
|
|
1843
|
+
* TEXT column); this selects two columns — the id, plus the `updatedAt` the cursor needs — and
|
|
1844
|
+
* keeps only the ids, so the peak footprint is one batch of two-column rows plus a list of
|
|
1845
|
+
* strings whose size *is* the answer the caller asked for.
|
|
1846
|
+
*
|
|
1847
|
+
* For callers that need the id set of a collection and nothing else: an orphan sweep unioning the
|
|
1848
|
+
* log against the vector store's own document list.
|
|
1849
|
+
*/
|
|
1850
|
+
async listDocumentIds(where = {}, options = {}) {
|
|
1851
|
+
const ids = [];
|
|
1852
|
+
const rows = this.sweep(where, batches(options.batchSize), void 0, (filter, limit) => this.db.select(CURSOR_COLUMNS).from(ragIngestionLog).where(filter).orderBy(...RAG_INGESTION_LOG_PAGE_ORDER).limit(limit));
|
|
1853
|
+
for await (const row of rows) {
|
|
1854
|
+
ids.push(row.documentId);
|
|
1855
|
+
}
|
|
1856
|
+
return ids;
|
|
1857
|
+
}
|
|
1858
|
+
/** The latest recorded outcome for one document, or null if it was never attempted. */
|
|
1859
|
+
async get(documentId) {
|
|
1860
|
+
const [row] = await this.db.select().from(ragIngestionLog).where(eq5(ragIngestionLog.documentId, documentId));
|
|
1861
|
+
return row ?? null;
|
|
1862
|
+
}
|
|
1863
|
+
/** Forget one document's record. Returns whether a row was actually removed. */
|
|
1864
|
+
async remove(documentId) {
|
|
1865
|
+
const removed = await this.db.delete(ragIngestionLog).where(eq5(ragIngestionLog.documentId, documentId)).returning({
|
|
1866
|
+
documentId: ragIngestionLog.documentId
|
|
1867
|
+
});
|
|
1868
|
+
return removed.length > 0;
|
|
1869
|
+
}
|
|
1870
|
+
/** Forget every record for a collection — for when the collection itself is deleted. */
|
|
1871
|
+
async removeByCollection(collection) {
|
|
1872
|
+
const removed = await this.db.delete(ragIngestionLog).where(eq5(ragIngestionLog.collection, collection)).returning({
|
|
1873
|
+
documentId: ragIngestionLog.documentId
|
|
1874
|
+
});
|
|
1875
|
+
return removed.length;
|
|
1876
|
+
}
|
|
1877
|
+
/**
|
|
1878
|
+
* The keyset sweep both {@link iterate} and {@link listDocumentIds} are made of. `page` decides
|
|
1879
|
+
* how wide a row is — whole rows for `iterate`, two columns for `listDocumentIds` — and the
|
|
1880
|
+
* `R extends RagIngestionLogCursor` bound says the only thing the sweep needs back: a row it can
|
|
1881
|
+
* read the next cursor off.
|
|
1882
|
+
*
|
|
1883
|
+
* The cursor advances **per yielded row**, not per batch, so a consumer that `break`s mid-batch
|
|
1884
|
+
* leaves it on the last row it actually saw. It is two column values read off that row, so a row
|
|
1885
|
+
* leaving the table cannot move it — the whole reason this is delete-safe by construction.
|
|
1886
|
+
*/
|
|
1887
|
+
async *sweep(where, batchSize, after, page) {
|
|
1888
|
+
let cursor = after;
|
|
1889
|
+
for (; ; ) {
|
|
1890
|
+
const rows = await page(this.keyset(where, cursor), batchSize);
|
|
1891
|
+
if (rows.length === 0) {
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
for (const row of rows) {
|
|
1895
|
+
yield row;
|
|
1896
|
+
cursor = {
|
|
1897
|
+
updatedAt: row.updatedAt,
|
|
1898
|
+
documentId: row.documentId
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
where(query) {
|
|
1904
|
+
return and5(query.collection !== void 0 ? eq5(ragIngestionLog.collection, query.collection) : void 0, query.status !== void 0 ? eq5(ragIngestionLog.status, query.status) : void 0);
|
|
1905
|
+
}
|
|
1906
|
+
/**
|
|
1907
|
+
* The coordinates filter AND'd with "strictly after `cursor` in
|
|
1908
|
+
* {@link RAG_INGESTION_LOG_PAGE_ORDER}".
|
|
1909
|
+
*
|
|
1910
|
+
* The order is `updatedAt desc, documentId asc`, so "after" means an older timestamp, or the same
|
|
1911
|
+
* timestamp and a larger id. Both halves are needed: the `<` alone would re-yield every row tied
|
|
1912
|
+
* with the cursor, and dropping the tie branch would skip them instead.
|
|
1913
|
+
*/
|
|
1914
|
+
keyset(where, cursor) {
|
|
1915
|
+
const base = this.where(where);
|
|
1916
|
+
if (cursor === void 0) {
|
|
1917
|
+
return base;
|
|
1918
|
+
}
|
|
1919
|
+
return and5(base, or(lt(ragIngestionLog.updatedAt, cursor.updatedAt), and5(eq5(ragIngestionLog.updatedAt, cursor.updatedAt), gt(ragIngestionLog.documentId, cursor.documentId))));
|
|
1920
|
+
}
|
|
1921
|
+
/**
|
|
1922
|
+
* Upsert one outcome. Keyed by document id so the row always reflects the *current* state — a
|
|
1923
|
+
* retry that succeeds overwrites the failure it replaces, rather than leaving a stale error next
|
|
1924
|
+
* to a working document.
|
|
1925
|
+
*/
|
|
1926
|
+
async record(status, payload) {
|
|
1927
|
+
const documentId = str(payload.mediaId);
|
|
1928
|
+
if (documentId === null) {
|
|
1929
|
+
return;
|
|
1930
|
+
}
|
|
1931
|
+
try {
|
|
1932
|
+
const now = /* @__PURE__ */ new Date();
|
|
1933
|
+
const collection = str(payload.collection);
|
|
1934
|
+
const ownerType = str(payload.ownerType);
|
|
1935
|
+
const ownerId = str(payload.ownerId);
|
|
1936
|
+
const source = str(payload.source);
|
|
1937
|
+
const mimeType = str(payload.mimeType);
|
|
1938
|
+
const size = num(payload.size);
|
|
1939
|
+
const outcome = {
|
|
1940
|
+
status,
|
|
1941
|
+
// The three outcome-specific columns are exclusive: null out the ones this status doesn't
|
|
1942
|
+
// own on every write, so a successful retry clears the previous attempt's error rather than
|
|
1943
|
+
// leaving it next to a working document. Always present → always in the conflict `set`.
|
|
1944
|
+
chunks: status === "ingested" ? num(payload.chunks) : null,
|
|
1945
|
+
reason: status === "skipped" ? str(payload.reason) : null,
|
|
1946
|
+
error: status === "failed" ? str(payload.error) : null,
|
|
1947
|
+
updatedAt: now,
|
|
1948
|
+
...collection !== null ? {
|
|
1949
|
+
collection
|
|
1950
|
+
} : {},
|
|
1951
|
+
...ownerType !== null ? {
|
|
1952
|
+
ownerType
|
|
1953
|
+
} : {},
|
|
1954
|
+
...ownerId !== null ? {
|
|
1955
|
+
ownerId
|
|
1956
|
+
} : {},
|
|
1957
|
+
...source !== null ? {
|
|
1958
|
+
source
|
|
1959
|
+
} : {},
|
|
1960
|
+
...mimeType !== null ? {
|
|
1961
|
+
mimeType
|
|
1962
|
+
} : {},
|
|
1963
|
+
...size !== null ? {
|
|
1964
|
+
size
|
|
1965
|
+
} : {}
|
|
1966
|
+
};
|
|
1967
|
+
await this.db.insert(ragIngestionLog).values({
|
|
1968
|
+
documentId,
|
|
1969
|
+
createdAt: now,
|
|
1970
|
+
...outcome
|
|
1971
|
+
}).onConflictDoUpdate({
|
|
1972
|
+
target: ragIngestionLog.documentId,
|
|
1973
|
+
set: outcome
|
|
1974
|
+
});
|
|
1975
|
+
} catch (error) {
|
|
1976
|
+
this.logger.warn(`Could not record RAG ingestion outcome for ${documentId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
track(promise) {
|
|
1980
|
+
const tracked = promise.finally(() => {
|
|
1981
|
+
this.inFlight.delete(tracked);
|
|
1982
|
+
});
|
|
1983
|
+
this.inFlight.add(tracked);
|
|
1984
|
+
}
|
|
1985
|
+
};
|
|
1986
|
+
DrizzleRagIngestionLog = _ts_decorate([
|
|
1987
|
+
Injectable(),
|
|
1988
|
+
_ts_metadata("design:type", Function),
|
|
1989
|
+
_ts_metadata("design:paramtypes", [
|
|
1990
|
+
typeof AgentDrizzleDb === "undefined" ? Object : AgentDrizzleDb
|
|
1991
|
+
])
|
|
1992
|
+
], DrizzleRagIngestionLog);
|
|
1993
|
+
|
|
1444
1994
|
// src/drizzle-agent-store.module.ts
|
|
1445
1995
|
import { AGENT_GOVERNANCE_QUERIES, AGENT_PRICING_STORE, AGENT_STORE } from "@dudousxd/nestjs-agent-core";
|
|
1446
1996
|
import { Module } from "@nestjs/common";
|
|
1447
|
-
function
|
|
1448
|
-
var c = arguments.length, r = c < 3 ? target :
|
|
1449
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key,
|
|
1997
|
+
function _ts_decorate2(decorators, target, key, desc4) {
|
|
1998
|
+
var c = arguments.length, r = c < 3 ? target : desc4 === null ? desc4 = Object.getOwnPropertyDescriptor(target, key) : desc4, d;
|
|
1999
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc4);
|
|
1450
2000
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1451
2001
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1452
2002
|
}
|
|
1453
|
-
__name(
|
|
2003
|
+
__name(_ts_decorate2, "_ts_decorate");
|
|
1454
2004
|
var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
1455
2005
|
static {
|
|
1456
2006
|
__name(this, "DrizzleAgentStoreModule");
|
|
1457
2007
|
}
|
|
1458
2008
|
static forRoot(options) {
|
|
2009
|
+
const ragProviders = options.ragIngestionLog === false ? [] : [
|
|
2010
|
+
{
|
|
2011
|
+
provide: DrizzleRagIngestionLog,
|
|
2012
|
+
useFactory: /* @__PURE__ */ __name(() => new DrizzleRagIngestionLog(options.db), "useFactory")
|
|
2013
|
+
}
|
|
2014
|
+
];
|
|
1459
2015
|
return {
|
|
1460
2016
|
module: _DrizzleAgentStoreModule,
|
|
1461
2017
|
global: true,
|
|
1462
2018
|
providers: [
|
|
2019
|
+
...ragProviders,
|
|
1463
2020
|
{
|
|
1464
2021
|
provide: DrizzleAgentStore,
|
|
1465
2022
|
useFactory: /* @__PURE__ */ __name(() => new DrizzleAgentStore(options.db), "useFactory")
|
|
@@ -1468,6 +2025,13 @@ var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
|
1468
2025
|
provide: AGENT_STORE,
|
|
1469
2026
|
useExisting: DrizzleAgentStore
|
|
1470
2027
|
},
|
|
2028
|
+
// Exported, never bound to AGENT_MEMORY: that token's PRESENCE is what turns memory on, so
|
|
2029
|
+
// binding it here would switch the feature on for every host that installs the store. A
|
|
2030
|
+
// host opts in by naming this provider in `AgentModule.forRoot({ memory: { provider } })`.
|
|
2031
|
+
{
|
|
2032
|
+
provide: DrizzleMemoryProvider,
|
|
2033
|
+
useFactory: /* @__PURE__ */ __name(() => new DrizzleMemoryProvider(options.db), "useFactory")
|
|
2034
|
+
},
|
|
1471
2035
|
{
|
|
1472
2036
|
provide: DrizzlePricingStore,
|
|
1473
2037
|
useFactory: /* @__PURE__ */ __name(() => new DrizzlePricingStore(options.db), "useFactory")
|
|
@@ -1494,8 +2058,12 @@ var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
|
1494
2058
|
}
|
|
1495
2059
|
],
|
|
1496
2060
|
exports: [
|
|
2061
|
+
...options.ragIngestionLog === false ? [] : [
|
|
2062
|
+
DrizzleRagIngestionLog
|
|
2063
|
+
],
|
|
1497
2064
|
DrizzleAgentStore,
|
|
1498
2065
|
AGENT_STORE,
|
|
2066
|
+
DrizzleMemoryProvider,
|
|
1499
2067
|
DrizzleGovernanceQueries,
|
|
1500
2068
|
AGENT_GOVERNANCE_QUERIES,
|
|
1501
2069
|
DrizzlePricingStore,
|
|
@@ -1504,14 +2072,18 @@ var DrizzleAgentStoreModule = class _DrizzleAgentStoreModule {
|
|
|
1504
2072
|
};
|
|
1505
2073
|
}
|
|
1506
2074
|
};
|
|
1507
|
-
DrizzleAgentStoreModule =
|
|
2075
|
+
DrizzleAgentStoreModule = _ts_decorate2([
|
|
1508
2076
|
Module({})
|
|
1509
2077
|
], DrizzleAgentStoreModule);
|
|
1510
2078
|
export {
|
|
1511
2079
|
DrizzleAgentStore,
|
|
1512
2080
|
DrizzleAgentStoreModule,
|
|
1513
2081
|
DrizzleGovernanceQueries,
|
|
2082
|
+
DrizzleMemoryProvider,
|
|
1514
2083
|
DrizzlePricingStore,
|
|
2084
|
+
DrizzleRagIngestionLog,
|
|
2085
|
+
RAG_INGESTION_LOG_PAGE_ORDER,
|
|
2086
|
+
agentMemory,
|
|
1515
2087
|
agentMessage,
|
|
1516
2088
|
agentModelPricing,
|
|
1517
2089
|
agentRun,
|
|
@@ -1519,6 +2091,7 @@ export {
|
|
|
1519
2091
|
agentThread,
|
|
1520
2092
|
agentTokenUsage,
|
|
1521
2093
|
agentToolCall,
|
|
1522
|
-
ensureAgentSchema
|
|
2094
|
+
ensureAgentSchema,
|
|
2095
|
+
ragIngestionLog
|
|
1523
2096
|
};
|
|
1524
2097
|
//# sourceMappingURL=index.js.map
|