@ponraaj/dimple-sdk 0.0.0-alpha.8 → 0.0.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/dimple.d.ts +4 -0
- package/dist/index.js +1155 -754
- package/dist/internal/store/db-backend.d.ts +21 -2
- package/dist/internal/store/service.d.ts +28 -1
- package/dist/internal/store/topics/routing.d.ts +33 -0
- package/dist/internal/workflow/signals.d.ts +20 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -16,7 +16,7 @@ var __export = (target, all) => {
|
|
|
16
16
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
17
17
|
|
|
18
18
|
// src/make.ts
|
|
19
|
-
import { Effect as
|
|
19
|
+
import { Effect as Effect51, Exit as Exit8, Layer as Layer26, ManagedRuntime, Scope as Scope5 } from "effect";
|
|
20
20
|
|
|
21
21
|
// ../core/src/core.ts
|
|
22
22
|
import { Context, Schema } from "effect";
|
|
@@ -29,10 +29,10 @@ class CoreConfigError extends Schema.TaggedErrorClass()("core.ConfigError", {
|
|
|
29
29
|
class Service extends Context.Service()("@dimple/core/Core") {
|
|
30
30
|
}
|
|
31
31
|
// ../core/src/layer.ts
|
|
32
|
-
import { Effect as
|
|
32
|
+
import { Effect as Effect40, Layer as Layer15, Schema as Schema24 } from "effect";
|
|
33
33
|
|
|
34
34
|
// ../store/src/store.ts
|
|
35
|
-
import { Config as Config4, Effect as
|
|
35
|
+
import { Config as Config4, Effect as Effect27, Layer as Layer10, Schema as Schema19 } from "effect";
|
|
36
36
|
import { eq as eq8, and as and8, desc as desc2, asc as asc5, inArray as inArray4, ne as ne4, or, isNull as isNull2, like, sql as sql7 } from "drizzle-orm";
|
|
37
37
|
|
|
38
38
|
// ../store/src/db-backend.ts
|
|
@@ -113,6 +113,11 @@ var EMBEDDED_MIGRATIONS = [
|
|
|
113
113
|
name: "20260807074034_ttl_secs_nullable",
|
|
114
114
|
hash: "be3a8d0e64b0224021d869c1c7e217e40b6eb868615cc4c86a6717f5b7211721",
|
|
115
115
|
sql: "PRAGMA foreign_keys=OFF;--> statement-breakpoint\nCREATE TABLE `__new_memories` (\n\t`id` text PRIMARY KEY,\n\t`logical_key` text NOT NULL,\n\t`content` text NOT NULL,\n\t`content_type` text DEFAULT 'text/plain' NOT NULL,\n\t`state` text DEFAULT 'active' NOT NULL,\n\t`created_at` integer NOT NULL,\n\t`ttl_secs` integer DEFAULT 604800,\n\t`namespace` text DEFAULT 'default' NOT NULL,\n\t`metadata` text,\n\t`embedding` F32_BLOB(384),\n\t`embedding_model` text\n);\n--> statement-breakpoint\nINSERT INTO `__new_memories`(`id`, `logical_key`, `content`, `content_type`, `state`, `created_at`, `ttl_secs`, `namespace`, `metadata`, `embedding`, `embedding_model`) SELECT `id`, `logical_key`, `content`, `content_type`, `state`, `created_at`, `ttl_secs`, `namespace`, `metadata`, `embedding`, `embedding_model` FROM `memories`;--> statement-breakpoint\nDROP TABLE `memories`;--> statement-breakpoint\nALTER TABLE `__new_memories` RENAME TO `memories`;--> statement-breakpoint\nPRAGMA foreign_keys=ON;--> statement-breakpoint\nCREATE INDEX `idx_memories_ns_created` ON `memories` (`namespace`,`created_at`);--> statement-breakpoint\nCREATE INDEX `idx_memories_state_created` ON `memories` (`state`,`created_at`);--> statement-breakpoint\nCREATE UNIQUE INDEX `idx_memories_active_key` ON `memories` (`namespace`,`logical_key`) WHERE state = 'active';"
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
name: "20260812122840_perf_indexes",
|
|
119
|
+
hash: "cc45ad830f7101fae454e9d5b43cfdfb922dbee7a5e5b7d9b9b81ec610d21d8e",
|
|
120
|
+
sql: "CREATE INDEX `idx_memories_created` ON `memories` (`created_at`);--> statement-breakpoint\nCREATE INDEX `ae_status_source` ON `artifact_edges` (`status`,`source`,`expired_at`,`created_at`);"
|
|
116
121
|
}
|
|
117
122
|
];
|
|
118
123
|
|
|
@@ -160,6 +165,7 @@ var memories = sqliteTable("memories", {
|
|
|
160
165
|
}, (table) => [
|
|
161
166
|
index("idx_memories_ns_created").on(table.namespace, table.createdAt),
|
|
162
167
|
index("idx_memories_state_created").on(table.state, table.createdAt),
|
|
168
|
+
index("idx_memories_created").on(table.createdAt),
|
|
163
169
|
uniqueIndex("idx_memories_active_key").on(table.namespace, table.logicalKey).where(sql`state = 'active'`)
|
|
164
170
|
]);
|
|
165
171
|
var memoryEvents = sqliteTable("memory_events", {
|
|
@@ -343,7 +349,8 @@ var artifactEdges = sqliteTable3("artifact_edges", {
|
|
|
343
349
|
}, (t) => [
|
|
344
350
|
primaryKey3({ columns: [t.fromMemoryId, t.toMemoryId, t.edgeKind] }),
|
|
345
351
|
index3("ae_from").on(t.fromMemoryId, t.status),
|
|
346
|
-
index3("ae_to").on(t.toMemoryId, t.status)
|
|
352
|
+
index3("ae_to").on(t.toMemoryId, t.status),
|
|
353
|
+
index3("ae_status_source").on(t.status, t.source, t.expiredAt, t.createdAt)
|
|
347
354
|
]);
|
|
348
355
|
var ArtifactEdgeInsert = createInsertSchema3(artifactEdges);
|
|
349
356
|
var ArtifactEdgeSelect = createSelectSchema3(artifactEdges);
|
|
@@ -1083,7 +1090,14 @@ var runEmbeddedMigrations = async (client) => {
|
|
|
1083
1090
|
var makeClientDb = (url) => {
|
|
1084
1091
|
const client = createClient({ url });
|
|
1085
1092
|
const db = drizzle({ client });
|
|
1093
|
+
const rawExecute = client.execute.bind(client);
|
|
1094
|
+
const rawExecuteMultiple = client.executeMultiple.bind(client);
|
|
1095
|
+
const rawBatch = client.batch.bind(client);
|
|
1086
1096
|
const rawTransaction = client.transaction.bind(client);
|
|
1097
|
+
const local = url.startsWith("file:");
|
|
1098
|
+
if (local) {
|
|
1099
|
+
rawExecute("PRAGMA busy_timeout=5000");
|
|
1100
|
+
}
|
|
1087
1101
|
let txTail = Promise.resolve();
|
|
1088
1102
|
let releaseSlot;
|
|
1089
1103
|
const acquire = () => {
|
|
@@ -1100,14 +1114,18 @@ var makeClientDb = (url) => {
|
|
|
1100
1114
|
releaseSlot?.();
|
|
1101
1115
|
releaseSlot = undefined;
|
|
1102
1116
|
};
|
|
1103
|
-
const rawExecute = client.execute.bind(client);
|
|
1104
|
-
const rawExecuteMultiple = client.executeMultiple.bind(client);
|
|
1105
|
-
const rawBatch = client.batch.bind(client);
|
|
1106
1117
|
const chained = (run) => {
|
|
1107
1118
|
const prev = acquire();
|
|
1108
1119
|
return prev.then(run, run).finally(release);
|
|
1109
1120
|
};
|
|
1110
|
-
|
|
1121
|
+
const WRITE_RE = /^\s*(INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|REPLACE|PRAGMA|BEGIN|COMMIT|ROLLBACK|VACUUM|REINDEX|ANALYZE|ATTACH|DETACH)\b/i;
|
|
1122
|
+
const isWrite = (sql3) => WRITE_RE.test(sql3);
|
|
1123
|
+
client.execute = (...args) => {
|
|
1124
|
+
const statement = args[0];
|
|
1125
|
+
const sql3 = typeof statement === "string" ? statement : statement.sql ?? "";
|
|
1126
|
+
const run = () => rawExecute(...args);
|
|
1127
|
+
return local && !isWrite(sql3) ? run() : chained(run);
|
|
1128
|
+
};
|
|
1111
1129
|
client.executeMultiple = (sql3) => chained(() => rawExecuteMultiple(sql3));
|
|
1112
1130
|
client.batch = (statements, mode) => chained(() => rawBatch(statements, mode));
|
|
1113
1131
|
client.transaction = async (mode) => {
|
|
@@ -1245,9 +1263,9 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1245
1263
|
topics,
|
|
1246
1264
|
artifactEdges
|
|
1247
1265
|
},
|
|
1248
|
-
ftsSearch: (query, namespace, limit, topicKeys, modelKey, contentType) => Effect9.tryPromise({
|
|
1266
|
+
ftsSearch: (query, namespace, limit, topicKeys, modelKey, contentType, includeEmbedding) => Effect9.tryPromise({
|
|
1249
1267
|
try: () => client.execute({
|
|
1250
|
-
sql: `SELECT m
|
|
1268
|
+
sql: `SELECT m.id, m.logical_key, m.content, m.content_type, m.state, m.created_at, m.ttl_secs, m.namespace, m.metadata, m.embedding_model${includeEmbedding ? ", m.embedding" : ""}, f.rank
|
|
1251
1269
|
FROM memories_fts f
|
|
1252
1270
|
JOIN memories m ON m.rowid = f.rowid
|
|
1253
1271
|
WHERE f.content MATCH ? AND m.namespace = ? AND m.state = 'active'${topicFilter(topicKeys)}${contentTypeFilter(contentType)}
|
|
@@ -1263,7 +1281,7 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1263
1281
|
}).then((rs) => rs.rows),
|
|
1264
1282
|
catch: (cause) => new StoreError({ message: `DbBackend.ftsSearch: ${cause}` })
|
|
1265
1283
|
}),
|
|
1266
|
-
vectorSearch: (query, namespace, limit, model, topicKeys, contentType) => Effect9.gen(function* () {
|
|
1284
|
+
vectorSearch: (query, namespace, limit, model, topicKeys, contentType, includeEmbedding) => Effect9.gen(function* () {
|
|
1267
1285
|
for (const n of query) {
|
|
1268
1286
|
if (!Number.isFinite(n)) {
|
|
1269
1287
|
yield* new StoreError({
|
|
@@ -1274,26 +1292,50 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1274
1292
|
const tableName = vectorTableName(model);
|
|
1275
1293
|
const vecJson = JSON.stringify([...query]);
|
|
1276
1294
|
const convertFn = VECTOR_CONVERT_FNS[model.vectorType ?? "float32"] ?? "vector32";
|
|
1277
|
-
const
|
|
1278
|
-
|
|
1295
|
+
const whereClause = `WHERE m.namespace = ? AND m.state = 'active'${topicFilter(topicKeys)}${contentTypeFilter(contentType)}`;
|
|
1296
|
+
const selectColumns = `m.id, m.logical_key, m.content, m.content_type, m.state, m.created_at, m.ttl_secs, m.namespace, m.metadata, m.embedding_model${includeEmbedding ? ", m.embedding" : ""}`;
|
|
1297
|
+
const baseArgs = [
|
|
1279
1298
|
namespace,
|
|
1280
1299
|
...topicKeys && topicKeys.length > 0 ? [...topicKeys, model.id] : [],
|
|
1281
|
-
...typeof contentType === "string" ? [contentType] : contentType ?? []
|
|
1282
|
-
limit
|
|
1300
|
+
...typeof contentType === "string" ? [contentType] : contentType ?? []
|
|
1283
1301
|
];
|
|
1302
|
+
const indexName = `idx_${tableName}_vec`;
|
|
1303
|
+
const annSql = `
|
|
1304
|
+
SELECT ${selectColumns},
|
|
1305
|
+
vector_distance_cos(v.embedding, ${convertFn}(?)) AS distance
|
|
1306
|
+
FROM vector_top_k(?, ${convertFn}(?), ?) AS k
|
|
1307
|
+
JOIN ${tableName} v ON v.rowid = k.id
|
|
1308
|
+
JOIN memories m ON m.id = v.memory_id
|
|
1309
|
+
${whereClause}
|
|
1310
|
+
ORDER BY distance, m.id
|
|
1311
|
+
LIMIT ?
|
|
1312
|
+
`;
|
|
1313
|
+
const scanSql = `
|
|
1314
|
+
SELECT ${selectColumns},
|
|
1315
|
+
vector_distance_cos(v.embedding, ${convertFn}(?)) AS distance
|
|
1316
|
+
FROM memories m
|
|
1317
|
+
JOIN ${tableName} v ON v.memory_id = m.id
|
|
1318
|
+
${whereClause}
|
|
1319
|
+
ORDER BY distance, m.id
|
|
1320
|
+
LIMIT ?
|
|
1321
|
+
`;
|
|
1322
|
+
const annK = Math.max(limit * 16, 64);
|
|
1323
|
+
const runAnn = () => client.execute({
|
|
1324
|
+
sql: annSql,
|
|
1325
|
+
args: [vecJson, indexName, vecJson, annK, ...baseArgs, limit]
|
|
1326
|
+
}).then((rs) => rs.rows);
|
|
1327
|
+
const runScan = () => client.execute({
|
|
1328
|
+
sql: scanSql,
|
|
1329
|
+
args: [vecJson, ...baseArgs, limit]
|
|
1330
|
+
}).then((rs) => rs.rows);
|
|
1284
1331
|
const rows = yield* Effect9.tryPromise({
|
|
1285
|
-
try: () =>
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
ORDER BY distance
|
|
1293
|
-
LIMIT ?
|
|
1294
|
-
`,
|
|
1295
|
-
args
|
|
1296
|
-
}).then((rs) => rs.rows),
|
|
1332
|
+
try: () => runAnn().catch((cause) => {
|
|
1333
|
+
const message = String(cause.message ?? cause);
|
|
1334
|
+
const absent = message.includes("no such table") || message.includes("no such index") || message.includes("vector_top_k");
|
|
1335
|
+
if (absent)
|
|
1336
|
+
return runScan();
|
|
1337
|
+
throw cause;
|
|
1338
|
+
}),
|
|
1297
1339
|
catch: (cause) => new StoreError({ message: `DbBackend.vectorSearch: ${cause}` })
|
|
1298
1340
|
}).pipe(Effect9.catchTag("Store.StoreError", (error) => error.message.includes("no such table") ? Effect9.succeed([]) : Effect9.fail(error)));
|
|
1299
1341
|
return rows;
|
|
@@ -1473,7 +1515,7 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1473
1515
|
return results;
|
|
1474
1516
|
}),
|
|
1475
1517
|
ensureVectorTable: (model) => Effect9.tryPromise({
|
|
1476
|
-
try: () => {
|
|
1518
|
+
try: async () => {
|
|
1477
1519
|
const tableName = vectorTableName(model);
|
|
1478
1520
|
const vecType = model.vectorType ?? "float32";
|
|
1479
1521
|
const sqlType = VECTOR_SQL_TYPES[vecType] ?? "F32_BLOB";
|
|
@@ -1485,7 +1527,12 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1485
1527
|
model_id TEXT NOT NULL
|
|
1486
1528
|
);
|
|
1487
1529
|
`;
|
|
1488
|
-
|
|
1530
|
+
await client.executeMultiple(sql3);
|
|
1531
|
+
if (vecType === "float32") {
|
|
1532
|
+
try {
|
|
1533
|
+
await client.execute(`CREATE INDEX IF NOT EXISTS idx_${tableName}_vec ON ${tableName}(libsql_vector_idx(embedding))`);
|
|
1534
|
+
} catch {}
|
|
1535
|
+
}
|
|
1489
1536
|
},
|
|
1490
1537
|
catch: (cause) => new StoreError({ message: `DbBackend.ensureVectorTable: ${cause}` })
|
|
1491
1538
|
}),
|
|
@@ -1501,6 +1548,24 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1501
1548
|
},
|
|
1502
1549
|
catch: (cause) => new StoreError({ message: `DbBackend.insertVector: ${cause}` })
|
|
1503
1550
|
}),
|
|
1551
|
+
insertVectors: (entries, model) => Effect9.tryPromise({
|
|
1552
|
+
try: async () => {
|
|
1553
|
+
if (entries.length === 0)
|
|
1554
|
+
return;
|
|
1555
|
+
const tableName = vectorTableName(model);
|
|
1556
|
+
const convertFn = VECTOR_CONVERT_FNS[model.vectorType ?? "float32"] ?? "vector32";
|
|
1557
|
+
const values = entries.map(() => `(?, ${convertFn}(?), ?)`).join(", ");
|
|
1558
|
+
const args = [];
|
|
1559
|
+
for (const entry of entries) {
|
|
1560
|
+
args.push(entry.memoryId, JSON.stringify([...entry.vector]), model.id);
|
|
1561
|
+
}
|
|
1562
|
+
return client.execute({
|
|
1563
|
+
sql: `INSERT INTO ${tableName} (memory_id, embedding, model_id) VALUES ${values} ON CONFLICT(memory_id) DO UPDATE SET embedding = excluded.embedding, model_id = excluded.model_id`,
|
|
1564
|
+
args
|
|
1565
|
+
});
|
|
1566
|
+
},
|
|
1567
|
+
catch: (cause) => new StoreError({ message: `DbBackend.insertVectors: ${cause}` })
|
|
1568
|
+
}),
|
|
1504
1569
|
deleteVector: () => Effect9.succeed(undefined),
|
|
1505
1570
|
ensureTopicVectorTable: (model) => Effect9.tryPromise({
|
|
1506
1571
|
try: () => {
|
|
@@ -1528,6 +1593,23 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1528
1593
|
},
|
|
1529
1594
|
catch: (cause) => new StoreError({ message: `DbBackend.upsertTopicVector: ${cause}` })
|
|
1530
1595
|
}),
|
|
1596
|
+
upsertTopicVectors: (entries, model) => Effect9.tryPromise({
|
|
1597
|
+
try: async () => {
|
|
1598
|
+
if (entries.length === 0)
|
|
1599
|
+
return;
|
|
1600
|
+
const tableName = topicVectorTableName(model);
|
|
1601
|
+
const values = entries.map((e) => "(?, vector32(?), ?)").join(", ");
|
|
1602
|
+
const args = [];
|
|
1603
|
+
for (const entry of entries) {
|
|
1604
|
+
args.push(entry.topicKey, JSON.stringify([...entry.sumVector]), model.id);
|
|
1605
|
+
}
|
|
1606
|
+
return client.execute({
|
|
1607
|
+
sql: `INSERT OR REPLACE INTO ${tableName} (topic_key, sum_vector, model_key) VALUES ${values}`,
|
|
1608
|
+
args
|
|
1609
|
+
});
|
|
1610
|
+
},
|
|
1611
|
+
catch: (cause) => new StoreError({ message: `DbBackend.upsertTopicVectors: ${cause}` })
|
|
1612
|
+
}),
|
|
1531
1613
|
deleteTopicVectors: (model) => Effect9.tryPromise({
|
|
1532
1614
|
try: () => {
|
|
1533
1615
|
const tableName = topicVectorTableName(model);
|
|
@@ -1549,6 +1631,20 @@ var makeBackend = (url, client, db, migrationFolder) => ({
|
|
|
1549
1631
|
sumVector: decodeVectorBlob(row.sum_vector)
|
|
1550
1632
|
}));
|
|
1551
1633
|
}),
|
|
1634
|
+
fetchTopicVector: (topicKey, model) => Effect9.gen(function* () {
|
|
1635
|
+
const tableName = topicVectorTableName(model);
|
|
1636
|
+
const rows = yield* Effect9.tryPromise({
|
|
1637
|
+
try: () => client.execute({
|
|
1638
|
+
sql: `SELECT topic_key, sum_vector FROM ${tableName} WHERE topic_key = ?`,
|
|
1639
|
+
args: [topicKey]
|
|
1640
|
+
}).then((rs) => rs.rows),
|
|
1641
|
+
catch: (cause) => new StoreError({ message: `DbBackend.fetchTopicVector: ${cause}` })
|
|
1642
|
+
});
|
|
1643
|
+
const row = rows[0];
|
|
1644
|
+
if (!row)
|
|
1645
|
+
return;
|
|
1646
|
+
return { topicKey: row.topic_key, sumVector: decodeVectorBlob(row.sum_vector) };
|
|
1647
|
+
}),
|
|
1552
1648
|
fetchMemoryVectors: (memoryIds, model) => Effect9.gen(function* () {
|
|
1553
1649
|
if (memoryIds.length === 0)
|
|
1554
1650
|
return [];
|
|
@@ -2142,6 +2238,131 @@ var routeInsert = Effect11.fn("topics.routeInsert")(function* (memoryId, vector,
|
|
|
2142
2238
|
}
|
|
2143
2239
|
return { topicKey, overflow: leafCf.count > (config.maxLeafItems ?? 128) };
|
|
2144
2240
|
});
|
|
2241
|
+
var routeInsertMany = Effect11.fn("topics.routeInsertMany")(function* (entries, model, config) {
|
|
2242
|
+
const violations = validateConfig(config);
|
|
2243
|
+
if (violations.length > 0) {
|
|
2244
|
+
return yield* new InvariantViolation({ topicKey: ROOT_TOPIC_KEY, violations });
|
|
2245
|
+
}
|
|
2246
|
+
for (const entry of entries) {
|
|
2247
|
+
if (entry.vector.length !== model.dimensions) {
|
|
2248
|
+
return yield* new ModelMismatch({
|
|
2249
|
+
expected: `${model.id} (${model.dimensions} dims)`,
|
|
2250
|
+
actual: `${entry.vector.length} dims`
|
|
2251
|
+
});
|
|
2252
|
+
}
|
|
2253
|
+
}
|
|
2254
|
+
if (entries.length === 0)
|
|
2255
|
+
return [];
|
|
2256
|
+
const backend = yield* Service2;
|
|
2257
|
+
const db = backend.db;
|
|
2258
|
+
const now = yield* Clock.currentTimeMillis;
|
|
2259
|
+
const maxLeafItems = config.maxLeafItems ?? 128;
|
|
2260
|
+
const rootRows = yield* Effect11.tryPromise({
|
|
2261
|
+
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and(eq(topics.topicKey, ROOT_TOPIC_KEY), eq(topics.modelKey, model.id))).limit(1),
|
|
2262
|
+
catch: (cause) => cause
|
|
2263
|
+
}).pipe(routingFailure("topics.routeInsertMany"));
|
|
2264
|
+
if (rootRows.length === 0) {
|
|
2265
|
+
return yield* new TopicNotFound({ topicKey: ROOT_TOPIC_KEY });
|
|
2266
|
+
}
|
|
2267
|
+
const vectorRows = yield* backend.fetchTopicVectors(model).pipe(routingFailure("topics.routeInsertMany"));
|
|
2268
|
+
const sumByTopic = new Map(vectorRows.map((r) => [r.topicKey, new Float32Array(r.sumVector)]));
|
|
2269
|
+
const zeros = new Float32Array(model.dimensions);
|
|
2270
|
+
const sumOf = (topicKey) => sumByTopic.get(topicKey) ?? zeros;
|
|
2271
|
+
const result = yield* Effect11.tryPromise({
|
|
2272
|
+
try: () => db.transaction(async (tx) => {
|
|
2273
|
+
const topicRows = await tx.select({
|
|
2274
|
+
topicKey: topics.topicKey,
|
|
2275
|
+
kind: topics.kind,
|
|
2276
|
+
status: topics.status,
|
|
2277
|
+
depth: topics.depth,
|
|
2278
|
+
memberCount: topics.memberCount,
|
|
2279
|
+
sumSquares: topics.sumSquares
|
|
2280
|
+
}).from(topics).where(eq(topics.modelKey, model.id)).orderBy(asc(topics.topicKey));
|
|
2281
|
+
const relations = await tx.select({
|
|
2282
|
+
parentKey: topicRelations.parentKey,
|
|
2283
|
+
childKey: topicRelations.childKey
|
|
2284
|
+
}).from(topicRelations).where(and(eq(topicRelations.modelKey, model.id), eq(topicRelations.relationKind, "parent_child")));
|
|
2285
|
+
const topicByKey = new Map(topicRows.map((r) => [r.topicKey, r]));
|
|
2286
|
+
const childrenByParent = new Map;
|
|
2287
|
+
const parentOf = new Map;
|
|
2288
|
+
for (const rel of relations) {
|
|
2289
|
+
const list = childrenByParent.get(rel.parentKey);
|
|
2290
|
+
if (list)
|
|
2291
|
+
list.push(rel.childKey);
|
|
2292
|
+
else
|
|
2293
|
+
childrenByParent.set(rel.parentKey, [rel.childKey]);
|
|
2294
|
+
parentOf.set(rel.childKey, rel.parentKey);
|
|
2295
|
+
}
|
|
2296
|
+
for (const children of childrenByParent.values()) {
|
|
2297
|
+
children.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
2298
|
+
}
|
|
2299
|
+
const membershipRows = [];
|
|
2300
|
+
const topicDelta = new Map;
|
|
2301
|
+
const outcomes = [];
|
|
2302
|
+
for (const entry of entries) {
|
|
2303
|
+
const x = new Float32Array(entry.vector);
|
|
2304
|
+
const descent = descendTree(topicByKey, childrenByParent, sumOf, x, config);
|
|
2305
|
+
if (!descent.ok)
|
|
2306
|
+
return Promise.reject(descent.error);
|
|
2307
|
+
const topicKey = descent.topicKey;
|
|
2308
|
+
const leafRow = topicByKey.get(topicKey);
|
|
2309
|
+
if (!leafRow) {
|
|
2310
|
+
return Promise.reject(new Error(`topics.routeInsertMany: topic row missing for ${topicKey}`));
|
|
2311
|
+
}
|
|
2312
|
+
const currentSum = sumOf(topicKey);
|
|
2313
|
+
const leafCf = cfInsert({ count: leafRow.memberCount, ls: currentSum, ss: leafRow.sumSquares }, x);
|
|
2314
|
+
leafRow.memberCount = leafCf.count;
|
|
2315
|
+
leafRow.sumSquares = leafCf.ss;
|
|
2316
|
+
sumByTopic.set(topicKey, leafCf.ls);
|
|
2317
|
+
membershipRows.push({
|
|
2318
|
+
topicKey,
|
|
2319
|
+
memoryId: entry.memoryId,
|
|
2320
|
+
modelKey: model.id,
|
|
2321
|
+
updatedAt: now
|
|
2322
|
+
});
|
|
2323
|
+
const acc = topicDelta.get(topicKey);
|
|
2324
|
+
if (acc) {
|
|
2325
|
+
acc.count += 1;
|
|
2326
|
+
acc.ss = leafCf.ss;
|
|
2327
|
+
} else {
|
|
2328
|
+
topicDelta.set(topicKey, { count: 1, ss: leafCf.ss });
|
|
2329
|
+
}
|
|
2330
|
+
let ancestorKey = parentOf.get(topicKey);
|
|
2331
|
+
while (ancestorKey !== undefined) {
|
|
2332
|
+
const row = topicByKey.get(ancestorKey);
|
|
2333
|
+
if (!row) {
|
|
2334
|
+
return Promise.reject(new Error(`topics.routeInsertMany: ancestor row missing for ${ancestorKey}`));
|
|
2335
|
+
}
|
|
2336
|
+
row.memberCount += 1;
|
|
2337
|
+
const ancestorSum = addToSum(sumOf(ancestorKey), x);
|
|
2338
|
+
sumByTopic.set(ancestorKey, ancestorSum);
|
|
2339
|
+
const accA = topicDelta.get(ancestorKey);
|
|
2340
|
+
if (accA)
|
|
2341
|
+
accA.count += 1;
|
|
2342
|
+
else
|
|
2343
|
+
topicDelta.set(ancestorKey, { count: 1, ss: row.sumSquares });
|
|
2344
|
+
ancestorKey = parentOf.get(ancestorKey);
|
|
2345
|
+
}
|
|
2346
|
+
outcomes.push({
|
|
2347
|
+
memoryId: entry.memoryId,
|
|
2348
|
+
topicKey,
|
|
2349
|
+
overflow: leafCf.count > maxLeafItems
|
|
2350
|
+
});
|
|
2351
|
+
}
|
|
2352
|
+
await tx.insert(topicMemberships).values(membershipRows);
|
|
2353
|
+
for (const [key, delta] of topicDelta) {
|
|
2354
|
+
await tx.update(topics).set({ memberCount: delta.count, sumSquares: delta.ss, updatedAt: now }).where(and(eq(topics.topicKey, key), eq(topics.modelKey, model.id)));
|
|
2355
|
+
}
|
|
2356
|
+
return { outcomes, topicDelta };
|
|
2357
|
+
}),
|
|
2358
|
+
catch: (cause) => cause
|
|
2359
|
+
}).pipe(routingFailure("topics.routeInsertMany"));
|
|
2360
|
+
yield* backend.upsertTopicVectors(Array.from(result.topicDelta, ([topicKey]) => ({
|
|
2361
|
+
topicKey,
|
|
2362
|
+
sumVector: Array.from(sumByTopic.get(topicKey) ?? zeros)
|
|
2363
|
+
})), model).pipe(routingFailure("topics.routeInsertMany"));
|
|
2364
|
+
return result.outcomes;
|
|
2365
|
+
});
|
|
2145
2366
|
|
|
2146
2367
|
// ../store/src/topics/query.ts
|
|
2147
2368
|
var DEFAULT_BEAM_WIDTH = 3;
|
|
@@ -2237,7 +2458,7 @@ var resolveTopics = Effect12.fn("topics.resolveTopics")(function* (queryVector,
|
|
|
2237
2458
|
});
|
|
2238
2459
|
|
|
2239
2460
|
// ../store/src/topics/lifecycle.ts
|
|
2240
|
-
import { Clock as Clock3, Effect as
|
|
2461
|
+
import { Clock as Clock3, Effect as Effect22 } from "effect";
|
|
2241
2462
|
import { and as and3, eq as eq3, inArray, ne } from "drizzle-orm";
|
|
2242
2463
|
|
|
2243
2464
|
// ../workflow/src/queue.ts
|
|
@@ -2386,7 +2607,18 @@ var queues = [
|
|
|
2386
2607
|
AuditWrite
|
|
2387
2608
|
];
|
|
2388
2609
|
// ../workflow/src/runtime.ts
|
|
2389
|
-
import {
|
|
2610
|
+
import {
|
|
2611
|
+
Cause as Cause2,
|
|
2612
|
+
Clock as Clock2,
|
|
2613
|
+
Context as Context7,
|
|
2614
|
+
Deferred as Deferred2,
|
|
2615
|
+
Duration as Duration2,
|
|
2616
|
+
Effect as Effect19,
|
|
2617
|
+
Exit as Exit6,
|
|
2618
|
+
Layer as Layer7,
|
|
2619
|
+
Option as Option5,
|
|
2620
|
+
Schema as Schema14
|
|
2621
|
+
} from "effect";
|
|
2390
2622
|
import { WorkflowEngine as WorkflowEngine3 } from "effect/unstable/workflow";
|
|
2391
2623
|
import { PersistedQueue as PersistedQueue3, Persistence as PersistenceNamespace3 } from "effect/unstable/persistence";
|
|
2392
2624
|
|
|
@@ -2476,8 +2708,37 @@ var readWorkflowResult = (store, workflow, executionId) => Effect14.gen(function
|
|
|
2476
2708
|
return Option2.none();
|
|
2477
2709
|
});
|
|
2478
2710
|
|
|
2711
|
+
// ../workflow/src/signals.ts
|
|
2712
|
+
import { Deferred, Effect as Effect15, Exit as Exit2 } from "effect";
|
|
2713
|
+
var listeners = new Map;
|
|
2714
|
+
var onExecutionDone = (executionId, deferred) => {
|
|
2715
|
+
let set = listeners.get(executionId);
|
|
2716
|
+
if (set === undefined) {
|
|
2717
|
+
set = new Set;
|
|
2718
|
+
listeners.set(executionId, set);
|
|
2719
|
+
}
|
|
2720
|
+
set.add(deferred);
|
|
2721
|
+
};
|
|
2722
|
+
var offExecutionDone = (executionId, deferred) => {
|
|
2723
|
+
const set = listeners.get(executionId);
|
|
2724
|
+
if (set === undefined)
|
|
2725
|
+
return;
|
|
2726
|
+
set.delete(deferred);
|
|
2727
|
+
if (set.size === 0)
|
|
2728
|
+
listeners.delete(executionId);
|
|
2729
|
+
};
|
|
2730
|
+
var notifyExecutionDone = (executionId) => {
|
|
2731
|
+
const set = listeners.get(executionId);
|
|
2732
|
+
if (set === undefined)
|
|
2733
|
+
return;
|
|
2734
|
+
listeners.delete(executionId);
|
|
2735
|
+
for (const deferred of set) {
|
|
2736
|
+
Effect15.runFork(Deferred.done(deferred, Exit2.void));
|
|
2737
|
+
}
|
|
2738
|
+
};
|
|
2739
|
+
|
|
2479
2740
|
// ../workflow/src/engine.ts
|
|
2480
|
-
import { Effect as
|
|
2741
|
+
import { Effect as Effect18, Exit as Exit5, Fiber, FiberMap, Layer as Layer6, Option as Option4, Schema as Schema13 } from "effect";
|
|
2481
2742
|
import { Workflow as Workflow3, WorkflowEngine as WorkflowEngine2 } from "effect/unstable/workflow";
|
|
2482
2743
|
import { Persistence as PersistenceNamespace2, PersistedQueue as PersistedQueue2 } from "effect/unstable/persistence";
|
|
2483
2744
|
|
|
@@ -2486,8 +2747,8 @@ import {
|
|
|
2486
2747
|
Cause,
|
|
2487
2748
|
Data,
|
|
2488
2749
|
Duration,
|
|
2489
|
-
Effect as
|
|
2490
|
-
Exit as
|
|
2750
|
+
Effect as Effect16,
|
|
2751
|
+
Exit as Exit3,
|
|
2491
2752
|
Latch,
|
|
2492
2753
|
Layer as Layer4,
|
|
2493
2754
|
MutableRef,
|
|
@@ -2500,7 +2761,7 @@ import { SqlClient } from "effect/unstable/sql";
|
|
|
2500
2761
|
|
|
2501
2762
|
class QueueKey extends Data.Class {
|
|
2502
2763
|
}
|
|
2503
|
-
var makeStore =
|
|
2764
|
+
var makeStore = Effect16.fnUntraced(function* (options) {
|
|
2504
2765
|
const sql3 = (yield* SqlClient.SqlClient).withoutTransforms();
|
|
2505
2766
|
const tableName = options?.tableName ?? "effect_queue";
|
|
2506
2767
|
const tableNameSql = sql3(tableName);
|
|
@@ -2533,16 +2794,16 @@ var makeStore = Effect15.fnUntraced(function* (options) {
|
|
|
2533
2794
|
VALUES (${id}, ${name}, ${element}, FALSE, 0, ${sqlNow}, ${sqlNow})
|
|
2534
2795
|
`;
|
|
2535
2796
|
const elementIds = new Set;
|
|
2536
|
-
const refreshLocks =
|
|
2797
|
+
const refreshLocks = Effect16.suspend(() => {
|
|
2537
2798
|
if (elementIds.size === 0)
|
|
2538
|
-
return
|
|
2799
|
+
return Effect16.void;
|
|
2539
2800
|
const ids = Array.from(elementIds);
|
|
2540
2801
|
return sql3`
|
|
2541
2802
|
UPDATE ${tableNameSql}
|
|
2542
2803
|
SET acquired_at = ${sqlNow}
|
|
2543
2804
|
WHERE sequence IN (${sql3.literal(ids.join(","))})
|
|
2544
2805
|
AND acquired_by = ${workerIdSql}
|
|
2545
|
-
`.pipe(
|
|
2806
|
+
`.pipe(Effect16.asVoid);
|
|
2546
2807
|
});
|
|
2547
2808
|
const complete = (sequence, attempts) => {
|
|
2548
2809
|
elementIds.delete(sequence);
|
|
@@ -2551,7 +2812,7 @@ var makeStore = Effect15.fnUntraced(function* (options) {
|
|
|
2551
2812
|
SET acquired_at = NULL, acquired_by = NULL, updated_at = ${sqlNow}, completed = ${sqlTrue}, attempts = ${attempts}
|
|
2552
2813
|
WHERE sequence = ${sequence}
|
|
2553
2814
|
AND acquired_by = ${workerIdSql}
|
|
2554
|
-
`.pipe(
|
|
2815
|
+
`.pipe(Effect16.retry({ times: 5, schedule: Schedule.exponential(100, 1.5) }), Effect16.orDie);
|
|
2555
2816
|
};
|
|
2556
2817
|
const retry = (sequence, attempts, cause) => {
|
|
2557
2818
|
elementIds.delete(sequence);
|
|
@@ -2560,33 +2821,33 @@ var makeStore = Effect15.fnUntraced(function* (options) {
|
|
|
2560
2821
|
SET acquired_at = NULL, acquired_by = NULL, updated_at = ${sqlNow}, attempts = ${attempts}, last_failure = ${Cause.pretty(cause)}
|
|
2561
2822
|
WHERE sequence = ${sequence}
|
|
2562
2823
|
AND acquired_by = ${workerIdSql}
|
|
2563
|
-
`.pipe(
|
|
2824
|
+
`.pipe(Effect16.retry({ times: 5, schedule: Schedule.exponential(100, 1.5) }), Effect16.orDie);
|
|
2564
2825
|
};
|
|
2565
2826
|
const interrupt = (ids) => {
|
|
2566
2827
|
for (const id of ids)
|
|
2567
2828
|
elementIds.delete(id);
|
|
2568
2829
|
if (ids.length === 0)
|
|
2569
|
-
return
|
|
2830
|
+
return Effect16.void;
|
|
2570
2831
|
return sql3`
|
|
2571
2832
|
UPDATE ${tableNameSql}
|
|
2572
2833
|
SET acquired_at = NULL, acquired_by = NULL
|
|
2573
2834
|
WHERE sequence IN (${sql3.literal(ids.join(","))})
|
|
2574
2835
|
AND acquired_by = ${workerIdSql}
|
|
2575
|
-
`.pipe(
|
|
2836
|
+
`.pipe(Effect16.retry({ times: 5, schedule: Schedule.exponential(100, 1.5) }), Effect16.orDie);
|
|
2576
2837
|
};
|
|
2577
|
-
yield* refreshLocks.pipe(
|
|
2838
|
+
yield* refreshLocks.pipe(Effect16.tapCause(Effect16.logWarning), Effect16.retry(Schedule.spaced(500)), Effect16.schedule(Schedule.fixed(lockRefreshInterval)), Effect16.annotateLogs({
|
|
2578
2839
|
package: "@dimple/workflow",
|
|
2579
2840
|
module: "QueueStore",
|
|
2580
2841
|
fiber: "refreshLocks"
|
|
2581
|
-
}),
|
|
2842
|
+
}), Effect16.forkScoped);
|
|
2582
2843
|
const mailboxes = yield* RcMap.make({
|
|
2583
|
-
lookup:
|
|
2844
|
+
lookup: Effect16.fnUntraced(function* ({ maxAttempts, name }) {
|
|
2584
2845
|
const queue = yield* Queue.make();
|
|
2585
2846
|
const takers = MutableRef.make(0);
|
|
2586
2847
|
const pollLatch = Latch.makeUnsafe();
|
|
2587
2848
|
const takenLatch = Latch.makeUnsafe();
|
|
2588
|
-
yield*
|
|
2589
|
-
const poll = (size) =>
|
|
2849
|
+
yield* Effect16.addFinalizer(() => Effect16.flatMap(Queue.clear(queue), (elements) => elements.length === 0 ? Effect16.void : interrupt(Array.from(elements, (e) => e.sequence))));
|
|
2850
|
+
const poll = (size) => Effect16.suspend(() => sql3`
|
|
2590
2851
|
UPDATE ${tableNameSql}
|
|
2591
2852
|
SET acquired_at = ${sqlNow}, acquired_by = ${workerIdSql}
|
|
2592
2853
|
WHERE queue_name = ${name}
|
|
@@ -2594,14 +2855,14 @@ var makeStore = Effect15.fnUntraced(function* (options) {
|
|
|
2594
2855
|
AND attempts < ${maxAttempts}
|
|
2595
2856
|
AND (acquired_at IS NULL OR acquired_at < ${expiresAt})
|
|
2596
2857
|
RETURNING sequence, id, queue_name, element, attempts
|
|
2597
|
-
`.pipe(
|
|
2598
|
-
yield*
|
|
2858
|
+
`.pipe(Effect16.map((rows) => rows)));
|
|
2859
|
+
yield* Effect16.gen(function* () {
|
|
2599
2860
|
while (true) {
|
|
2600
2861
|
yield* pollLatch.await;
|
|
2601
|
-
yield*
|
|
2862
|
+
yield* Effect16.yieldNow;
|
|
2602
2863
|
const results = takers.current === 0 ? [] : yield* poll(takers.current);
|
|
2603
2864
|
if (results.length === 0) {
|
|
2604
|
-
yield*
|
|
2865
|
+
yield* Effect16.sleep(pollInterval);
|
|
2605
2866
|
continue;
|
|
2606
2867
|
}
|
|
2607
2868
|
takenLatch.closeUnsafe();
|
|
@@ -2610,24 +2871,24 @@ var makeStore = Effect15.fnUntraced(function* (options) {
|
|
|
2610
2871
|
}
|
|
2611
2872
|
yield* Queue.offerAll(queue, results);
|
|
2612
2873
|
yield* takenLatch.await;
|
|
2613
|
-
yield*
|
|
2874
|
+
yield* Effect16.yieldNow;
|
|
2614
2875
|
}
|
|
2615
|
-
}).pipe(
|
|
2876
|
+
}).pipe(Effect16.sandbox, Effect16.retry(Schedule.spaced(500)), Effect16.forkScoped, Effect16.interruptible);
|
|
2616
2877
|
return { queue, takers, pollLatch, takenLatch };
|
|
2617
2878
|
}),
|
|
2618
2879
|
idleTimeToLive: Duration.seconds(30)
|
|
2619
2880
|
});
|
|
2620
2881
|
return PersistedQueue.PersistedQueueStore.of({
|
|
2621
|
-
offer: ({ element, id, name }) =>
|
|
2882
|
+
offer: ({ element, id, name }) => Effect16.catchCause(Effect16.suspend(() => offer(id, name, JSON.stringify(element))), (cause) => Effect16.fail(new PersistedQueue.PersistedQueueError({
|
|
2622
2883
|
message: "Failed to offer element to persisted queue",
|
|
2623
2884
|
cause
|
|
2624
2885
|
}))),
|
|
2625
|
-
take: ({ maxAttempts, name }) =>
|
|
2886
|
+
take: ({ maxAttempts, name }) => Effect16.uninterruptibleMask((restore) => RcMap.get(mailboxes, new QueueKey({ name, maxAttempts })).pipe(Effect16.flatMap(({ pollLatch, queue, takenLatch, takers }) => {
|
|
2626
2887
|
takers.current++;
|
|
2627
2888
|
if (takers.current === 1) {
|
|
2628
2889
|
pollLatch.openUnsafe();
|
|
2629
2890
|
}
|
|
2630
|
-
return
|
|
2891
|
+
return Effect16.tap(restore(Queue.take(queue)), () => Effect16.sync(() => {
|
|
2631
2892
|
takers.current--;
|
|
2632
2893
|
if (takers.current === 0) {
|
|
2633
2894
|
pollLatch.closeUnsafe();
|
|
@@ -2636,10 +2897,10 @@ var makeStore = Effect15.fnUntraced(function* (options) {
|
|
|
2636
2897
|
takenLatch.openUnsafe();
|
|
2637
2898
|
}
|
|
2638
2899
|
}));
|
|
2639
|
-
}),
|
|
2900
|
+
}), Effect16.scoped, restore, Effect16.tap((element) => Effect16.addFinalizer(Exit3.match({
|
|
2640
2901
|
onFailure: (cause) => Cause.hasInterruptsOnly(cause) ? interrupt([element.sequence]) : retry(element.sequence, element.attempts + 1, cause),
|
|
2641
2902
|
onSuccess: () => complete(element.sequence, element.attempts + 1)
|
|
2642
|
-
}))),
|
|
2903
|
+
}))), Effect16.map((element) => ({ ...element, element: JSON.parse(element.element) }))))
|
|
2643
2904
|
});
|
|
2644
2905
|
});
|
|
2645
2906
|
var storeSqliteLayer = Layer4.effect(PersistedQueue.PersistedQueueStore, makeStore());
|
|
@@ -2656,8 +2917,8 @@ __export(exports_LibsqlClient, {
|
|
|
2656
2917
|
import * as Libsql from "@libsql/client";
|
|
2657
2918
|
import * as Config3 from "effect/Config";
|
|
2658
2919
|
import * as Context6 from "effect/Context";
|
|
2659
|
-
import * as
|
|
2660
|
-
import * as
|
|
2920
|
+
import * as Effect17 from "effect/Effect";
|
|
2921
|
+
import * as Exit4 from "effect/Exit";
|
|
2661
2922
|
import * as Layer5 from "effect/Layer";
|
|
2662
2923
|
import * as Option3 from "effect/Option";
|
|
2663
2924
|
import * as Redacted from "effect/Redacted";
|
|
@@ -2676,7 +2937,7 @@ var classifyError = (cause, message, operation) => classifySqliteError(cause, {
|
|
|
2676
2937
|
var TypeId = "~@effect/sql-libsql/LibsqlClient";
|
|
2677
2938
|
var LibsqlClient = /* @__PURE__ */ Context6.Service("@effect/sql-libsql/LibsqlClient");
|
|
2678
2939
|
var LibsqlTransaction = /* @__PURE__ */ Context6.Service("@effect/sql-libsql/LibsqlClient/LibsqlTransaction");
|
|
2679
|
-
var make4 = (options) =>
|
|
2940
|
+
var make4 = (options) => Effect17.gen(function* () {
|
|
2680
2941
|
const compiler = Statement.makeCompilerSqlite(options.transformQueryNames);
|
|
2681
2942
|
const transformRows = options.transformResultNames ? Statement.defaultTransforms(options.transformResultNames).array : undefined;
|
|
2682
2943
|
const spanAttributes = [...options.spanAttributes ? Object.entries(options.spanAttributes) : [], [ATTR_DB_SYSTEM_NAME, "sqlite"]];
|
|
@@ -2687,7 +2948,7 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2687
2948
|
this.sdk = sdk;
|
|
2688
2949
|
}
|
|
2689
2950
|
run(sql3, params = []) {
|
|
2690
|
-
return
|
|
2951
|
+
return Effect17.map(Effect17.tryPromise({
|
|
2691
2952
|
try: () => this.sdk.execute({
|
|
2692
2953
|
sql: sql3,
|
|
2693
2954
|
args: params
|
|
@@ -2698,7 +2959,7 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2698
2959
|
}), (results) => results.rows);
|
|
2699
2960
|
}
|
|
2700
2961
|
runRaw(sql3, params = []) {
|
|
2701
|
-
return
|
|
2962
|
+
return Effect17.tryPromise({
|
|
2702
2963
|
try: () => this.sdk.execute({
|
|
2703
2964
|
sql: sql3,
|
|
2704
2965
|
args: params
|
|
@@ -2709,13 +2970,13 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2709
2970
|
});
|
|
2710
2971
|
}
|
|
2711
2972
|
execute(sql3, params, transformRows2) {
|
|
2712
|
-
return transformRows2 ?
|
|
2973
|
+
return transformRows2 ? Effect17.map(this.run(sql3, params), transformRows2) : this.run(sql3, params);
|
|
2713
2974
|
}
|
|
2714
2975
|
executeRaw(sql3, params) {
|
|
2715
2976
|
return this.runRaw(sql3, params);
|
|
2716
2977
|
}
|
|
2717
2978
|
executeValues(sql3, params) {
|
|
2718
|
-
return
|
|
2979
|
+
return Effect17.map(this.run(sql3, params), (rows) => rows.map((row) => Array.from(row)));
|
|
2719
2980
|
}
|
|
2720
2981
|
executeValuesUnprepared(sql3, params) {
|
|
2721
2982
|
return this.executeValues(sql3, params);
|
|
@@ -2727,7 +2988,7 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2727
2988
|
return Stream.die("executeStream not implemented");
|
|
2728
2989
|
}
|
|
2729
2990
|
get beginTransaction() {
|
|
2730
|
-
return
|
|
2991
|
+
return Effect17.map(Effect17.tryPromise({
|
|
2731
2992
|
try: () => this.sdk.transaction("write"),
|
|
2732
2993
|
catch: (cause) => new SqlError2({
|
|
2733
2994
|
reason: classifyError(cause, "Failed to begin transaction", "beginTransaction")
|
|
@@ -2735,7 +2996,7 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2735
2996
|
}), (tx) => new LibsqlConnectionImpl(tx));
|
|
2736
2997
|
}
|
|
2737
2998
|
get commit() {
|
|
2738
|
-
return
|
|
2999
|
+
return Effect17.tryPromise({
|
|
2739
3000
|
try: () => this.sdk.commit(),
|
|
2740
3001
|
catch: (cause) => new SqlError2({
|
|
2741
3002
|
reason: classifyError(cause, "Failed to commit transaction", "commitTransaction")
|
|
@@ -2743,7 +3004,7 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2743
3004
|
});
|
|
2744
3005
|
}
|
|
2745
3006
|
get rollback() {
|
|
2746
|
-
return
|
|
3007
|
+
return Effect17.tryPromise({
|
|
2747
3008
|
try: () => this.sdk.rollback(),
|
|
2748
3009
|
catch: (cause) => new SqlError2({
|
|
2749
3010
|
reason: classifyError(cause, "Failed to rollback transaction", "rollbackTransaction")
|
|
@@ -2751,33 +3012,33 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2751
3012
|
});
|
|
2752
3013
|
}
|
|
2753
3014
|
}
|
|
2754
|
-
const connection = "liveClient" in options ? new LibsqlConnectionImpl(options.liveClient) : yield*
|
|
3015
|
+
const connection = "liveClient" in options ? new LibsqlConnectionImpl(options.liveClient) : yield* Effect17.map(Effect17.acquireRelease(Effect17.sync(() => Libsql.createClient({
|
|
2755
3016
|
...options,
|
|
2756
3017
|
authToken: Redacted.isRedacted(options.authToken) ? Redacted.value(options.authToken) : options.authToken,
|
|
2757
3018
|
encryptionKey: Redacted.isRedacted(options.encryptionKey) ? Redacted.value(options.encryptionKey) : options.encryptionKey,
|
|
2758
3019
|
url: options.url.toString(),
|
|
2759
3020
|
syncUrl: options.syncUrl?.toString()
|
|
2760
|
-
})), (sdk) =>
|
|
3021
|
+
})), (sdk) => Effect17.sync(() => sdk.close())), (sdk) => new LibsqlConnectionImpl(sdk));
|
|
2761
3022
|
const semaphore = yield* Semaphore.make(1);
|
|
2762
3023
|
const withTransaction = Client.makeWithTransaction({
|
|
2763
3024
|
transactionService: LibsqlTransaction,
|
|
2764
3025
|
spanAttributes,
|
|
2765
|
-
acquireConnection:
|
|
3026
|
+
acquireConnection: Effect17.uninterruptibleMask(Effect17.fnUntraced(function* (restore) {
|
|
2766
3027
|
const scope = Scope3.makeUnsafe();
|
|
2767
3028
|
yield* restore(semaphore.take(1));
|
|
2768
3029
|
yield* Scope3.addFinalizer(scope, semaphore.release(1));
|
|
2769
|
-
const conn = yield* connection.beginTransaction.pipe(
|
|
3030
|
+
const conn = yield* connection.beginTransaction.pipe(Effect17.tapCause((cause) => Scope3.close(scope, Exit4.failCause(cause))));
|
|
2770
3031
|
return [scope, conn];
|
|
2771
3032
|
})),
|
|
2772
|
-
begin: () =>
|
|
3033
|
+
begin: () => Effect17.void,
|
|
2773
3034
|
savepoint: (conn, id) => conn.executeRaw(`SAVEPOINT effect_sql_${id};`, []),
|
|
2774
3035
|
commit: (conn) => conn.commit,
|
|
2775
3036
|
rollback: (conn) => conn.rollback,
|
|
2776
3037
|
rollbackSavepoint: (conn, id) => conn.executeRaw(`ROLLBACK TO SAVEPOINT effect_sql_${id};`, [])
|
|
2777
3038
|
});
|
|
2778
|
-
const acquirer =
|
|
2779
|
-
onNone: () => semaphore.withPermits(1)(
|
|
2780
|
-
onSome: ([conn]) =>
|
|
3039
|
+
const acquirer = Effect17.flatMap(Effect17.serviceOption(LibsqlTransaction), Option3.match({
|
|
3040
|
+
onNone: () => semaphore.withPermits(1)(Effect17.succeed(connection)),
|
|
3041
|
+
onSome: ([conn]) => Effect17.succeed(conn)
|
|
2781
3042
|
}));
|
|
2782
3043
|
return Object.assign(yield* Client.make({
|
|
2783
3044
|
acquirer,
|
|
@@ -2791,21 +3052,21 @@ var make4 = (options) => Effect16.gen(function* () {
|
|
|
2791
3052
|
sdk: connection.sdk
|
|
2792
3053
|
});
|
|
2793
3054
|
});
|
|
2794
|
-
var layerConfig2 = (config) => Layer5.effectContext(Config3.unwrap(config).pipe(
|
|
2795
|
-
var layer5 = (config) => Layer5.effectContext(
|
|
3055
|
+
var layerConfig2 = (config) => Layer5.effectContext(Config3.unwrap(config).pipe(Effect17.flatMap(make4), Effect17.map((client) => Context6.make(LibsqlClient, client).pipe(Context6.add(Client.SqlClient, client))))).pipe(Layer5.provide(Reactivity.layer));
|
|
3056
|
+
var layer5 = (config) => Layer5.effectContext(Effect17.map(make4(config), (client) => Context6.make(LibsqlClient, client).pipe(Context6.add(Client.SqlClient, client)))).pipe(Layer5.provide(Reactivity.layer));
|
|
2796
3057
|
// ../workflow/src/engine.ts
|
|
2797
3058
|
var Persistence = PersistenceNamespace2.Persistence;
|
|
2798
3059
|
var PersistedQueueFactory = PersistedQueue2.PersistedQueueFactory;
|
|
2799
|
-
var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(
|
|
3060
|
+
var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect18.gen(function* () {
|
|
2800
3061
|
const persistence = yield* Persistence;
|
|
2801
3062
|
const queueFactory = yield* PersistedQueueFactory;
|
|
2802
|
-
const scope = yield*
|
|
3063
|
+
const scope = yield* Effect18.scope;
|
|
2803
3064
|
const store = yield* persistence.make({ storeId: STORE_ID });
|
|
2804
3065
|
const resumeQueue = yield* queueFactory.make({ name: RESUME_QUEUE, schema: ResumeSignal });
|
|
2805
3066
|
const workflows = new Map;
|
|
2806
3067
|
const executions = new Map;
|
|
2807
3068
|
const clocks = yield* FiberMap.make();
|
|
2808
|
-
const resume = (state) =>
|
|
3069
|
+
const resume = (state) => Effect18.gen(function* () {
|
|
2809
3070
|
const exit = state.fiber?.pollUnsafe();
|
|
2810
3071
|
if (exit && exit._tag === "Success" && exit.value._tag === "Complete")
|
|
2811
3072
|
return;
|
|
@@ -2817,31 +3078,32 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(fu
|
|
|
2817
3078
|
const instance = WorkflowEngine2.WorkflowInstance.initial(state.instance.workflow, state.instance.executionId);
|
|
2818
3079
|
instance.interrupted = state.instance.interrupted;
|
|
2819
3080
|
state.instance = instance;
|
|
2820
|
-
state.fiber = yield* state.execute(state.payload, state.instance.executionId).pipe(
|
|
3081
|
+
state.fiber = yield* state.execute(state.payload, state.instance.executionId).pipe(Effect18.onExit(() => {
|
|
2821
3082
|
if (!instance.interrupted)
|
|
2822
|
-
return
|
|
3083
|
+
return Effect18.void;
|
|
2823
3084
|
instance.suspended = false;
|
|
2824
|
-
return
|
|
2825
|
-
}), Workflow3.intoResult,
|
|
3085
|
+
return Effect18.withFiber((fiber) => Effect18.interruptible(Fiber.interrupt(fiber)));
|
|
3086
|
+
}), Workflow3.intoResult, Effect18.provideService(WorkflowEngine2.WorkflowInstance, instance), Effect18.provideService(WorkflowEngine2.WorkflowEngine, engine), Effect18.tap((result) => {
|
|
2826
3087
|
if (result._tag !== "Complete")
|
|
2827
|
-
return
|
|
2828
|
-
return
|
|
3088
|
+
return Effect18.void;
|
|
3089
|
+
return Effect18.gen(function* () {
|
|
2829
3090
|
const entry2 = workflows.get(state.instance.workflow._tag);
|
|
2830
3091
|
if (!entry2)
|
|
2831
3092
|
return;
|
|
2832
|
-
const encoded = yield*
|
|
2833
|
-
yield* setEncodedExit(store, resultKey(state.instance.executionId),
|
|
3093
|
+
const encoded = yield* Effect18.orDie(Schema13.encodeEffect(resultCodecFor(entry2.workflow))(result));
|
|
3094
|
+
yield* setEncodedExit(store, resultKey(state.instance.executionId), Exit5.succeed(encoded));
|
|
3095
|
+
notifyExecutionDone(state.instance.executionId);
|
|
2834
3096
|
});
|
|
2835
|
-
}),
|
|
3097
|
+
}), Effect18.tap((result) => {
|
|
2836
3098
|
if (!state.parent || result._tag !== "Complete")
|
|
2837
|
-
return
|
|
3099
|
+
return Effect18.void;
|
|
2838
3100
|
const parent = executions.get(state.parent);
|
|
2839
3101
|
if (parent)
|
|
2840
|
-
return
|
|
2841
|
-
return
|
|
2842
|
-
}),
|
|
3102
|
+
return Effect18.forkIn(resume(parent), scope);
|
|
3103
|
+
return Effect18.void;
|
|
3104
|
+
}), Effect18.forkIn(entry.scope));
|
|
2843
3105
|
});
|
|
2844
|
-
const resumeExecution = (executionId) =>
|
|
3106
|
+
const resumeExecution = (executionId) => Effect18.gen(function* () {
|
|
2845
3107
|
const state = executions.get(executionId);
|
|
2846
3108
|
if (state) {
|
|
2847
3109
|
yield* resume(state);
|
|
@@ -2864,22 +3126,22 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(fu
|
|
|
2864
3126
|
}
|
|
2865
3127
|
});
|
|
2866
3128
|
const engine = WorkflowEngine2.makeUnsafe({
|
|
2867
|
-
register:
|
|
3129
|
+
register: Effect18.fnUntraced(function* (workflow, execute) {
|
|
2868
3130
|
workflows.set(workflow._tag, {
|
|
2869
3131
|
workflow,
|
|
2870
3132
|
execute,
|
|
2871
|
-
scope: yield*
|
|
3133
|
+
scope: yield* Effect18.scope
|
|
2872
3134
|
});
|
|
2873
3135
|
}),
|
|
2874
3136
|
execute: executeEncoded(workflows, executions, store, resume),
|
|
2875
|
-
interrupt:
|
|
3137
|
+
interrupt: Effect18.fnUntraced(function* (_workflow, executionId) {
|
|
2876
3138
|
const state = executions.get(executionId);
|
|
2877
3139
|
if (!state)
|
|
2878
3140
|
return;
|
|
2879
3141
|
state.instance.interrupted = true;
|
|
2880
3142
|
yield* resume(state);
|
|
2881
3143
|
}),
|
|
2882
|
-
interruptUnsafe:
|
|
3144
|
+
interruptUnsafe: Effect18.fnUntraced(function* (_workflow, executionId) {
|
|
2883
3145
|
const state = executions.get(executionId);
|
|
2884
3146
|
if (!state)
|
|
2885
3147
|
return;
|
|
@@ -2889,36 +3151,36 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(fu
|
|
|
2889
3151
|
}
|
|
2890
3152
|
}),
|
|
2891
3153
|
resume: (_workflow, executionId) => resumeExecution(executionId),
|
|
2892
|
-
activityExecute:
|
|
3154
|
+
activityExecute: Effect18.fnUntraced(function* (activity, attempt) {
|
|
2893
3155
|
const instance = yield* WorkflowEngine2.WorkflowInstance;
|
|
2894
3156
|
const key = activityKey(instance.executionId, activity.name, attempt);
|
|
2895
3157
|
const persisted = yield* getEncodedExit(store, key);
|
|
2896
|
-
if (persisted &&
|
|
3158
|
+
if (persisted && Exit5.isSuccess(persisted)) {
|
|
2897
3159
|
const entry = workflows.get(instance.workflow._tag);
|
|
2898
3160
|
if (!entry)
|
|
2899
|
-
return yield*
|
|
2900
|
-
return yield*
|
|
3161
|
+
return yield* Effect18.die(`activity replay: unknown workflow ${instance.workflow._tag}`);
|
|
3162
|
+
return yield* Effect18.orDie(Schema13.decodeUnknownEffect(resultCodecFor(entry.workflow))(persisted.value));
|
|
2901
3163
|
}
|
|
2902
3164
|
const activityInstance = WorkflowEngine2.WorkflowInstance.initial(instance.workflow, instance.executionId);
|
|
2903
3165
|
activityInstance.interrupted = instance.interrupted;
|
|
2904
|
-
const result = yield* activity.executeEncoded.pipe(Workflow3.intoResult,
|
|
3166
|
+
const result = yield* activity.executeEncoded.pipe(Workflow3.intoResult, Effect18.provideService(WorkflowEngine2.WorkflowInstance, activityInstance), Effect18.exit);
|
|
2905
3167
|
if (result._tag === "Success" && result.value._tag !== "Suspended") {
|
|
2906
3168
|
const entry = workflows.get(instance.workflow._tag);
|
|
2907
3169
|
if (entry) {
|
|
2908
|
-
const encoded = yield*
|
|
2909
|
-
yield* setEncodedExit(store, key,
|
|
3170
|
+
const encoded = yield* Effect18.orDie(Schema13.encodeEffect(resultCodecFor(entry.workflow))(result.value));
|
|
3171
|
+
yield* setEncodedExit(store, key, Exit5.succeed(encoded));
|
|
2910
3172
|
}
|
|
2911
3173
|
}
|
|
2912
3174
|
return yield* result;
|
|
2913
3175
|
}),
|
|
2914
|
-
poll: (_workflow, executionId) =>
|
|
3176
|
+
poll: (_workflow, executionId) => Effect18.gen(function* () {
|
|
2915
3177
|
const state = executions.get(executionId);
|
|
2916
3178
|
if (!state) {
|
|
2917
3179
|
const persisted = yield* getEncodedExit(store, resultKey(executionId));
|
|
2918
|
-
if (persisted &&
|
|
3180
|
+
if (persisted && Exit5.isSuccess(persisted)) {
|
|
2919
3181
|
const workflow = yield* workflowTagFor(workflows, store, executionId);
|
|
2920
3182
|
if (workflow) {
|
|
2921
|
-
return Option4.some(yield*
|
|
3183
|
+
return Option4.some(yield* Effect18.orDie(Schema13.decodeUnknownEffect(resultCodecFor(workflow))(persisted.value)));
|
|
2922
3184
|
}
|
|
2923
3185
|
}
|
|
2924
3186
|
return Option4.none();
|
|
@@ -2928,9 +3190,9 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(fu
|
|
|
2928
3190
|
return Option4.none();
|
|
2929
3191
|
if (exit._tag === "Success")
|
|
2930
3192
|
return Option4.some(exit.value);
|
|
2931
|
-
return yield*
|
|
3193
|
+
return yield* Effect18.die(exit.cause);
|
|
2932
3194
|
}),
|
|
2933
|
-
deferredResult:
|
|
3195
|
+
deferredResult: Effect18.fnUntraced(function* (deferred) {
|
|
2934
3196
|
const instance = yield* WorkflowEngine2.WorkflowInstance;
|
|
2935
3197
|
const persisted = yield* getEncodedExit(store, deferredKey(instance.executionId, deferred.name));
|
|
2936
3198
|
if (persisted) {
|
|
@@ -2938,26 +3200,26 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(fu
|
|
|
2938
3200
|
}
|
|
2939
3201
|
return Option4.none();
|
|
2940
3202
|
}),
|
|
2941
|
-
deferredDone: (options) =>
|
|
3203
|
+
deferredDone: (options) => Effect18.gen(function* () {
|
|
2942
3204
|
yield* setEncodedExit(store, deferredKey(options.executionId, options.deferredName), options.exit);
|
|
2943
|
-
yield* resumeQueue.offer({ executionId: options.executionId }, { id: options.executionId }).pipe(
|
|
3205
|
+
yield* resumeQueue.offer({ executionId: options.executionId }, { id: options.executionId }).pipe(Effect18.orDie);
|
|
2944
3206
|
yield* resumeExecution(options.executionId);
|
|
2945
3207
|
}),
|
|
2946
3208
|
scheduleClock: (workflow, options) => engine.deferredDone(options.clock.deferred, {
|
|
2947
3209
|
workflowName: workflow._tag,
|
|
2948
3210
|
executionId: options.executionId,
|
|
2949
3211
|
deferredName: options.clock.deferred.name,
|
|
2950
|
-
exit:
|
|
2951
|
-
}).pipe(
|
|
3212
|
+
exit: Exit5.void
|
|
3213
|
+
}).pipe(Effect18.delay(options.clock.duration), FiberMap.run(clocks, `${options.executionId}/${options.clock.name}`, {
|
|
2952
3214
|
onlyIfMissing: true
|
|
2953
|
-
}),
|
|
3215
|
+
}), Effect18.asVoid)
|
|
2954
3216
|
});
|
|
2955
3217
|
return engine;
|
|
2956
3218
|
}));
|
|
2957
|
-
var executeEncoded = (workflows, executions, store, resume) =>
|
|
3219
|
+
var executeEncoded = (workflows, executions, store, resume) => Effect18.fnUntraced(function* (workflow, options) {
|
|
2958
3220
|
const entry = workflows.get(workflow._tag);
|
|
2959
3221
|
if (!entry) {
|
|
2960
|
-
return yield*
|
|
3222
|
+
return yield* Effect18.orDie(Effect18.fail(`Workflow ${workflow._tag} is not registered`));
|
|
2961
3223
|
}
|
|
2962
3224
|
let state = executions.get(options.executionId);
|
|
2963
3225
|
if (!state) {
|
|
@@ -2969,24 +3231,24 @@ var executeEncoded = (workflows, executions, store, resume) => Effect17.fnUntrac
|
|
|
2969
3231
|
parent: options.parent?.executionId
|
|
2970
3232
|
};
|
|
2971
3233
|
executions.set(options.executionId, state);
|
|
2972
|
-
yield* setEncodedExit(store, recordKey(options.executionId),
|
|
3234
|
+
yield* setEncodedExit(store, recordKey(options.executionId), Exit5.succeed({ workflowTag: workflow._tag, payload: options.payload }));
|
|
2973
3235
|
yield* resume(state);
|
|
2974
3236
|
}
|
|
2975
3237
|
if (options.discard)
|
|
2976
|
-
return yield*
|
|
3238
|
+
return yield* Effect18.void;
|
|
2977
3239
|
if (!state.fiber) {
|
|
2978
|
-
return yield*
|
|
3240
|
+
return yield* Effect18.orDie(Effect18.fail(`Workflow ${workflow._tag}: execution ${options.executionId} has no fiber`));
|
|
2979
3241
|
}
|
|
2980
3242
|
return yield* Fiber.join(state.fiber);
|
|
2981
3243
|
});
|
|
2982
|
-
var loadRecord = (store, executionId) =>
|
|
3244
|
+
var loadRecord = (store, executionId) => Effect18.gen(function* () {
|
|
2983
3245
|
const persisted = yield* getEncodedExit(store, recordKey(executionId));
|
|
2984
|
-
if (persisted &&
|
|
2985
|
-
return yield*
|
|
3246
|
+
if (persisted && Exit5.isSuccess(persisted)) {
|
|
3247
|
+
return yield* Effect18.orDie(Schema13.decodeUnknownEffect(ExecutionRecord)(persisted.value));
|
|
2986
3248
|
}
|
|
2987
3249
|
return;
|
|
2988
3250
|
});
|
|
2989
|
-
var workflowTagFor = (workflows, store, executionId) =>
|
|
3251
|
+
var workflowTagFor = (workflows, store, executionId) => Effect18.gen(function* () {
|
|
2990
3252
|
const record = yield* loadRecord(store, executionId);
|
|
2991
3253
|
if (!record)
|
|
2992
3254
|
return;
|
|
@@ -2995,10 +3257,10 @@ var workflowTagFor = (workflows, store, executionId) => Effect17.gen(function* (
|
|
|
2995
3257
|
var ensureDbDir = (url) => url.startsWith("file:") ? import("node:fs").then((fs) => import("node:path").then(({ dirname }) => {
|
|
2996
3258
|
fs.mkdirSync(dirname(url.slice("file:".length)), { recursive: true });
|
|
2997
3259
|
})) : Promise.resolve();
|
|
2998
|
-
var libsqlStack = (options) => Layer6.unwrap(
|
|
3260
|
+
var libsqlStack = (options) => Layer6.unwrap(Effect18.tryPromise({
|
|
2999
3261
|
try: () => ensureDbDir(options.url),
|
|
3000
3262
|
catch: (cause) => new Error(String(cause))
|
|
3001
|
-
}).pipe(
|
|
3263
|
+
}).pipe(Effect18.catch(() => Effect18.void), Effect18.map(() => Layer6.mergeAll(PersistenceNamespace2.layerSql, PersistedQueue2.layer.pipe(Layer6.provide(storeSqliteLayer))).pipe(Layer6.provide(exports_LibsqlClient.layer(options))))));
|
|
3002
3264
|
var layerDurableLibsql = (options) => layerDurable.pipe(Layer6.provide(libsqlStack(options)));
|
|
3003
3265
|
|
|
3004
3266
|
// ../workflow/src/runtime.ts
|
|
@@ -3017,7 +3279,7 @@ var jobsError = (jobId, e) => {
|
|
|
3017
3279
|
return e;
|
|
3018
3280
|
return new QueueError({ message: e instanceof Error ? e.message : String(e) });
|
|
3019
3281
|
};
|
|
3020
|
-
var layer6 = (defs) => Layer7.effect(Service7,
|
|
3282
|
+
var layer6 = (defs) => Layer7.effect(Service7, Effect19.gen(function* () {
|
|
3021
3283
|
const engine = yield* WorkflowEngine3.WorkflowEngine;
|
|
3022
3284
|
const persistence = yield* Persistence2;
|
|
3023
3285
|
const queueFactory = yield* PersistedQueueFactory2;
|
|
@@ -3025,51 +3287,57 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect18.gen(function* () {
|
|
|
3025
3287
|
for (const def of defs) {
|
|
3026
3288
|
yield* def.register(engine);
|
|
3027
3289
|
}
|
|
3028
|
-
const enqueue =
|
|
3029
|
-
const decoded = yield* Schema14.decodeUnknownEffect(def.payloadSchema)(payload).pipe(
|
|
3290
|
+
const enqueue = Effect19.fn("JobRuntime.enqueue")(function* (def, payload) {
|
|
3291
|
+
const decoded = yield* Schema14.decodeUnknownEffect(def.payloadSchema)(payload).pipe(Effect19.mapError((e) => new QueueError({ message: `invalid payload: ${e.message}` })));
|
|
3030
3292
|
const token = def.idempotencyKey(decoded);
|
|
3031
3293
|
const envelope = { token, payload: decoded, traceId: "" };
|
|
3032
|
-
const jobId = yield* def.workflow.executionId(envelope).pipe(
|
|
3033
|
-
yield* def.workflow.execute(envelope, { discard: true }).pipe(
|
|
3294
|
+
const jobId = yield* def.workflow.executionId(envelope).pipe(Effect19.provideService(WorkflowEngine3.WorkflowEngine, engine), Effect19.mapError((e) => new QueueError({ message: e instanceof Error ? e.message : String(e) })));
|
|
3295
|
+
yield* def.workflow.execute(envelope, { discard: true }).pipe(Effect19.provideService(WorkflowEngine3.WorkflowEngine, engine), Effect19.mapError((e) => jobsError(jobId, e)));
|
|
3034
3296
|
return jobId;
|
|
3035
3297
|
});
|
|
3036
|
-
const wait =
|
|
3298
|
+
const wait = Effect19.fn("JobRuntime.wait")(function* (jobId, options) {
|
|
3037
3299
|
const timeout = Duration2.fromInput(options?.timeout ?? DEFAULT_TIMEOUT).pipe(Option5.getOrElse(() => Duration2.minutes(5)));
|
|
3038
3300
|
const pollInterval = Duration2.fromInput(options?.pollInterval ?? DEFAULT_POLL_INTERVAL).pipe(Option5.getOrElse(() => Duration2.millis(250)));
|
|
3039
|
-
const deadline = yield* Clock2.currentTimeMillis.pipe(
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3301
|
+
const deadline = yield* Clock2.currentTimeMillis.pipe(Effect19.map((now) => now + Duration2.toMillis(timeout)));
|
|
3302
|
+
const completionSignal = yield* Deferred2.make();
|
|
3303
|
+
onExecutionDone(jobId, completionSignal);
|
|
3304
|
+
try {
|
|
3305
|
+
while (true) {
|
|
3306
|
+
if (yield* readMarker(store, jobId, CANCELLED)) {
|
|
3307
|
+
return yield* Effect19.fail(new JobCancelled({ jobId }));
|
|
3308
|
+
}
|
|
3309
|
+
const def = yield* findDef(defs, jobId, store);
|
|
3310
|
+
if (def) {
|
|
3311
|
+
const result = yield* readWorkflowResult(store, def.workflow, jobId);
|
|
3312
|
+
if (Option5.isSome(result)) {
|
|
3313
|
+
const r = result.value;
|
|
3314
|
+
if (r._tag === "Complete") {
|
|
3315
|
+
if (Exit6.isSuccess(r.exit))
|
|
3316
|
+
return r.exit.value;
|
|
3317
|
+
const failure = Exit6.getCause(r.exit).pipe(Option5.flatMap(Cause2.findErrorOption), Option5.map((e) => typeof e === "string" ? e : String(e)));
|
|
3318
|
+
return yield* Effect19.fail(new JobFailed({
|
|
3319
|
+
jobId,
|
|
3320
|
+
message: Option5.getOrElse(failure, () => "workflow failed")
|
|
3321
|
+
}));
|
|
3322
|
+
}
|
|
3057
3323
|
}
|
|
3058
3324
|
}
|
|
3325
|
+
const now = yield* Clock2.currentTimeMillis;
|
|
3326
|
+
if (now >= deadline) {
|
|
3327
|
+
yield* writeMarker(store, jobId, TIMED_OUT);
|
|
3328
|
+
return yield* Effect19.fail(new JobTimeout({ jobId, timeout }));
|
|
3329
|
+
}
|
|
3330
|
+
yield* Effect19.race(Deferred2.await(completionSignal).pipe(Effect19.asVoid), Effect19.sleep(pollInterval));
|
|
3059
3331
|
}
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
yield* writeMarker(store, jobId, TIMED_OUT);
|
|
3063
|
-
return yield* Effect18.fail(new JobTimeout({ jobId, timeout }));
|
|
3064
|
-
}
|
|
3065
|
-
yield* Effect18.sleep(pollInterval);
|
|
3332
|
+
} finally {
|
|
3333
|
+
offExecutionDone(jobId, completionSignal);
|
|
3066
3334
|
}
|
|
3067
3335
|
});
|
|
3068
|
-
const submit =
|
|
3336
|
+
const submit = Effect19.fn("JobRuntime.submit")(function* (def, payload) {
|
|
3069
3337
|
const jobId = yield* enqueue(def, payload);
|
|
3070
3338
|
return yield* wait(jobId);
|
|
3071
3339
|
});
|
|
3072
|
-
const status =
|
|
3340
|
+
const status = Effect19.fn("JobRuntime.status")(function* (jobId) {
|
|
3073
3341
|
if (yield* readMarker(store, jobId, CANCELLED))
|
|
3074
3342
|
return "cancelled";
|
|
3075
3343
|
if (yield* readMarker(store, jobId, TIMED_OUT))
|
|
@@ -3080,35 +3348,35 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect18.gen(function* () {
|
|
|
3080
3348
|
if (Option5.isSome(result)) {
|
|
3081
3349
|
const r = result.value;
|
|
3082
3350
|
if (r._tag === "Complete") {
|
|
3083
|
-
return
|
|
3351
|
+
return Exit6.isSuccess(r.exit) ? "succeeded" : "failed";
|
|
3084
3352
|
}
|
|
3085
3353
|
}
|
|
3086
3354
|
}
|
|
3087
3355
|
return "pending";
|
|
3088
3356
|
});
|
|
3089
|
-
const cancel =
|
|
3357
|
+
const cancel = Effect19.fn("JobRuntime.cancel")(function* (jobId) {
|
|
3090
3358
|
yield* writeMarker(store, jobId, CANCELLED);
|
|
3091
3359
|
return true;
|
|
3092
3360
|
});
|
|
3093
|
-
const processPending =
|
|
3361
|
+
const processPending = Effect19.fn("JobRuntime.processPending")(function* (options) {
|
|
3094
3362
|
const resumeQueue = yield* queueFactory.make({ name: RESUME_QUEUE, schema: ResumeSignal });
|
|
3095
3363
|
const window = Duration2.toMillis(Duration2.fromInput(options?.window ?? PROCESS_WINDOW).pipe(Option5.getOrElse(() => Duration2.seconds(30))));
|
|
3096
|
-
const deadline = yield* Clock2.currentTimeMillis.pipe(
|
|
3364
|
+
const deadline = yield* Clock2.currentTimeMillis.pipe(Effect19.map((now) => now + window));
|
|
3097
3365
|
let processed = 0;
|
|
3098
3366
|
while (true) {
|
|
3099
3367
|
const now = yield* Clock2.currentTimeMillis;
|
|
3100
3368
|
if (now >= deadline)
|
|
3101
3369
|
break;
|
|
3102
|
-
const signal = yield* resumeQueue.take((value2, _meta) =>
|
|
3370
|
+
const signal = yield* resumeQueue.take((value2, _meta) => Effect19.succeed(Option5.some(value2)), { maxAttempts: 1 }).pipe(Effect19.mapError((e) => new QueueError({ message: e instanceof Error ? e.message : String(e) })), Effect19.timeoutOrElse({
|
|
3103
3371
|
duration: Duration2.millis(100),
|
|
3104
|
-
orElse: () =>
|
|
3372
|
+
orElse: () => Effect19.succeed(Option5.none())
|
|
3105
3373
|
}));
|
|
3106
3374
|
if (Option5.isNone(signal))
|
|
3107
3375
|
continue;
|
|
3108
3376
|
const executionId = signal.value.executionId;
|
|
3109
3377
|
const def = yield* findDef(defs, executionId, store);
|
|
3110
3378
|
if (def) {
|
|
3111
|
-
yield* engine.resume(def.workflow, executionId).pipe(
|
|
3379
|
+
yield* engine.resume(def.workflow, executionId).pipe(Effect19.orDie);
|
|
3112
3380
|
processed++;
|
|
3113
3381
|
}
|
|
3114
3382
|
}
|
|
@@ -3116,16 +3384,16 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect18.gen(function* () {
|
|
|
3116
3384
|
});
|
|
3117
3385
|
return Service7.of({ enqueue, wait, submit, status, cancel, processPending });
|
|
3118
3386
|
}));
|
|
3119
|
-
var findDef = (defs, executionId, store) =>
|
|
3387
|
+
var findDef = (defs, executionId, store) => Effect19.gen(function* () {
|
|
3120
3388
|
const persisted = yield* getEncodedExit(store, `${executionId}/record`);
|
|
3121
|
-
if (!persisted || !
|
|
3389
|
+
if (!persisted || !Exit6.isSuccess(persisted))
|
|
3122
3390
|
return;
|
|
3123
|
-
const record = yield*
|
|
3391
|
+
const record = yield* Effect19.orDie(Schema14.decodeUnknownEffect(ExecutionRecord)(persisted.value));
|
|
3124
3392
|
return defs.find((d) => d.workflow._tag === record.workflowTag);
|
|
3125
3393
|
});
|
|
3126
3394
|
var layerLibsql = (defs, options) => layer6(defs).pipe(Layer7.provideMerge(Layer7.mergeAll(layerDurableLibsql(options), libsqlStack(options))));
|
|
3127
3395
|
// ../workflow/src/registry.ts
|
|
3128
|
-
import { Context as Context8, Effect as
|
|
3396
|
+
import { Context as Context8, Effect as Effect20, Layer as Layer8, Option as Option6, Ref, Schema as Schema15 } from "effect";
|
|
3129
3397
|
class JobFailure extends Schema15.TaggedErrorClass()("JobFailure", {
|
|
3130
3398
|
jobId: Schema15.String,
|
|
3131
3399
|
message: Schema15.String
|
|
@@ -3134,9 +3402,9 @@ class JobFailure extends Schema15.TaggedErrorClass()("JobFailure", {
|
|
|
3134
3402
|
|
|
3135
3403
|
class Service8 extends Context8.Service()("@dimple/workflow/JobRegistry") {
|
|
3136
3404
|
}
|
|
3137
|
-
var layer7 = Layer8.effect(Service8,
|
|
3405
|
+
var layer7 = Layer8.effect(Service8, Effect20.gen(function* () {
|
|
3138
3406
|
const handlers = yield* Ref.make(new Map);
|
|
3139
|
-
const register =
|
|
3407
|
+
const register = Effect20.fn("JobRegistry.register")(function* (type, handler) {
|
|
3140
3408
|
const existed = yield* Ref.modify(handlers, (map2) => {
|
|
3141
3409
|
if (map2.has(type)) {
|
|
3142
3410
|
return [true, map2];
|
|
@@ -3144,46 +3412,46 @@ var layer7 = Layer8.effect(Service8, Effect19.gen(function* () {
|
|
|
3144
3412
|
return [false, new Map(map2).set(type, handler)];
|
|
3145
3413
|
});
|
|
3146
3414
|
if (existed) {
|
|
3147
|
-
return yield*
|
|
3415
|
+
return yield* Effect20.fail(new RegistryError({ type, message: `Job type already registered: ${type}` }));
|
|
3148
3416
|
}
|
|
3149
3417
|
});
|
|
3150
|
-
const resolve =
|
|
3151
|
-
const handler = yield* Ref.get(handlers).pipe(
|
|
3418
|
+
const resolve = Effect20.fn("JobRegistry.resolve")(function* (type) {
|
|
3419
|
+
const handler = yield* Ref.get(handlers).pipe(Effect20.map((map2) => Option6.fromNullishOr(map2.get(type))));
|
|
3152
3420
|
if (Option6.isNone(handler)) {
|
|
3153
|
-
return yield*
|
|
3421
|
+
return yield* Effect20.fail(new UnknownJobType({ type }));
|
|
3154
3422
|
}
|
|
3155
3423
|
return handler.value;
|
|
3156
3424
|
});
|
|
3157
3425
|
return Service8.of({ register, resolve });
|
|
3158
3426
|
}));
|
|
3159
3427
|
// ../workflow/src/executor.ts
|
|
3160
|
-
import { Cause as Cause3, Duration as Duration3, Effect as
|
|
3428
|
+
import { Cause as Cause3, Duration as Duration3, Effect as Effect21, Layer as Layer9, Schedule as Schedule2 } from "effect";
|
|
3161
3429
|
import { DurableQueue as DurableQueue2, WorkflowEngine as WorkflowEngine4 } from "effect/unstable/workflow";
|
|
3162
|
-
var runHandler = (def, payload, jobId, maxAttempts) =>
|
|
3430
|
+
var runHandler = (def, payload, jobId, maxAttempts) => Effect21.gen(function* () {
|
|
3163
3431
|
const registry = yield* Service8;
|
|
3164
3432
|
const handler = yield* registry.resolve(def.name);
|
|
3165
|
-
yield*
|
|
3433
|
+
yield* Effect21.logInfo("job.started").pipe(Effect21.annotateLogs({
|
|
3166
3434
|
scope: "@dimple/workflow",
|
|
3167
3435
|
event: "job.started",
|
|
3168
3436
|
job: def.name,
|
|
3169
3437
|
idempotencyKey: jobId
|
|
3170
3438
|
}));
|
|
3171
3439
|
const run = () => handler(payload, { jobId });
|
|
3172
|
-
const outcome = yield* (maxAttempts > 1 ? run().pipe(
|
|
3440
|
+
const outcome = yield* (maxAttempts > 1 ? run().pipe(Effect21.retry({
|
|
3173
3441
|
times: maxAttempts - 1,
|
|
3174
3442
|
schedule: Schedule2.spaced(Duration3.millis(250)).pipe(Schedule2.jittered)
|
|
3175
|
-
})) : run()).pipe(
|
|
3443
|
+
})) : run()).pipe(Effect21.exit);
|
|
3176
3444
|
if (outcome._tag === "Failure") {
|
|
3177
|
-
yield*
|
|
3445
|
+
yield* Effect21.logError("job.failed").pipe(Effect21.annotateLogs({
|
|
3178
3446
|
scope: "@dimple/workflow",
|
|
3179
3447
|
event: "job.failed",
|
|
3180
3448
|
job: def.name,
|
|
3181
3449
|
idempotencyKey: jobId,
|
|
3182
3450
|
error: Cause3.pretty(outcome.cause)
|
|
3183
3451
|
}));
|
|
3184
|
-
return yield*
|
|
3452
|
+
return yield* Effect21.failCause(outcome.cause).pipe(Effect21.mapError((e) => e.message));
|
|
3185
3453
|
}
|
|
3186
|
-
yield*
|
|
3454
|
+
yield* Effect21.logInfo("job.completed").pipe(Effect21.annotateLogs({
|
|
3187
3455
|
scope: "@dimple/workflow",
|
|
3188
3456
|
event: "job.completed",
|
|
3189
3457
|
job: def.name,
|
|
@@ -3216,17 +3484,17 @@ var JobsConfig2 = Schema16.Struct({
|
|
|
3216
3484
|
pollIntervalMs: Schema16.optional(Schema16.Number)
|
|
3217
3485
|
});
|
|
3218
3486
|
// ../store/src/topics/lifecycle.ts
|
|
3219
|
-
var routingFailure3 = (tag) => (effect) => effect.pipe(
|
|
3220
|
-
var isActive =
|
|
3487
|
+
var routingFailure3 = (tag) => (effect) => effect.pipe(Effect22.mapError((cause) => new RoutingFailure({ cause: { operation: tag, cause } })));
|
|
3488
|
+
var isActive = Effect22.fn("topics.isActive")(function* (modelKey) {
|
|
3221
3489
|
const backend = yield* Service2;
|
|
3222
3490
|
const db = backend.db;
|
|
3223
|
-
const rows = yield*
|
|
3491
|
+
const rows = yield* Effect22.tryPromise({
|
|
3224
3492
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and3(eq3(topics.modelKey, modelKey), eq3(topics.status, "active"))).limit(1),
|
|
3225
3493
|
catch: (cause) => cause
|
|
3226
3494
|
}).pipe(routingFailure3("topics.isActive"));
|
|
3227
3495
|
return rows.length > 0;
|
|
3228
3496
|
});
|
|
3229
|
-
var setActiveModel =
|
|
3497
|
+
var setActiveModel = Effect22.fn("topics.setActiveModel")(function* (model, config) {
|
|
3230
3498
|
const backend = yield* Service2;
|
|
3231
3499
|
const db = backend.db;
|
|
3232
3500
|
yield* ensureTopicTree(model, config).pipe(routingFailure3("topics.setActiveModel.seed"));
|
|
@@ -3234,14 +3502,14 @@ var setActiveModel = Effect21.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3234
3502
|
let routed = 0;
|
|
3235
3503
|
let alreadyInTree = 0;
|
|
3236
3504
|
if (memoryIds.length > 0) {
|
|
3237
|
-
const topicRows = yield*
|
|
3505
|
+
const topicRows = yield* Effect22.tryPromise({
|
|
3238
3506
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(eq3(topics.modelKey, model.id)),
|
|
3239
3507
|
catch: (cause) => cause
|
|
3240
3508
|
}).pipe(routingFailure3("topics.setActiveModel.topics"));
|
|
3241
3509
|
const modelTopicKeys = topicRows.map((r) => r.topicKey);
|
|
3242
3510
|
if (modelTopicKeys.length > 0) {
|
|
3243
3511
|
for (const memoryId of memoryIds) {
|
|
3244
|
-
const existing = yield*
|
|
3512
|
+
const existing = yield* Effect22.tryPromise({
|
|
3245
3513
|
try: () => db.select({ topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and3(eq3(topicMemberships.memoryId, memoryId), eq3(topicMemberships.modelKey, model.id), inArray(topicMemberships.topicKey, modelTopicKeys))).limit(1),
|
|
3246
3514
|
catch: (cause) => cause
|
|
3247
3515
|
}).pipe(routingFailure3("topics.setActiveModel.membership"));
|
|
@@ -3259,7 +3527,7 @@ var setActiveModel = Effect21.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3259
3527
|
}
|
|
3260
3528
|
}
|
|
3261
3529
|
const auditTimestamp = yield* Clock3.currentTimeMillis;
|
|
3262
|
-
const deactivated = yield*
|
|
3530
|
+
const deactivated = yield* Effect22.tryPromise({
|
|
3263
3531
|
try: () => db.transaction(async (tx) => {
|
|
3264
3532
|
const others = await tx.selectDistinct({ modelKey: topics.modelKey }).from(topics).where(and3(ne(topics.modelKey, model.id), ne(topics.status, "inactive")));
|
|
3265
3533
|
const affected = others.map((r) => r.modelKey).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
@@ -3283,18 +3551,18 @@ var setActiveModel = Effect21.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3283
3551
|
return { routed, alreadyInTree, deactivated };
|
|
3284
3552
|
});
|
|
3285
3553
|
function memoryEmbedHandler(deps) {
|
|
3286
|
-
const run = (payload, ctx) =>
|
|
3554
|
+
const run = (payload, ctx) => Effect22.gen(function* () {
|
|
3287
3555
|
const envelope = payload;
|
|
3288
3556
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
3289
3557
|
if (!model) {
|
|
3290
|
-
return yield*
|
|
3558
|
+
return yield* Effect22.fail(new JobFailure({
|
|
3291
3559
|
jobId: ctx.jobId,
|
|
3292
3560
|
message: `memory.embed: unknown model "${envelope.modelKey}"`
|
|
3293
3561
|
}));
|
|
3294
3562
|
}
|
|
3295
|
-
const content = yield* deps.getMemoryContent(envelope.memoryId).pipe(
|
|
3563
|
+
const content = yield* deps.getMemoryContent(envelope.memoryId).pipe(Effect22.catchTag("Store.NotFound", () => Effect22.succeed(undefined)), Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: load content: ${e}` })));
|
|
3296
3564
|
if (content === undefined) {
|
|
3297
|
-
yield*
|
|
3565
|
+
yield* Effect22.logDebug("embed.stale-reoffer").pipe(Effect22.annotateLogs({
|
|
3298
3566
|
scope: "@dimple/store",
|
|
3299
3567
|
event: "embed.stale-reoffer",
|
|
3300
3568
|
memoryId: envelope.memoryId,
|
|
@@ -3302,20 +3570,20 @@ function memoryEmbedHandler(deps) {
|
|
|
3302
3570
|
}));
|
|
3303
3571
|
return;
|
|
3304
3572
|
}
|
|
3305
|
-
const vector = yield* deps.embedOne(content, model).pipe(
|
|
3573
|
+
const vector = yield* deps.embedOne(content, model).pipe(Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: embed: ${e}` })));
|
|
3306
3574
|
const backend = yield* Service2;
|
|
3307
3575
|
const db = backend.db;
|
|
3308
|
-
const existing = yield*
|
|
3576
|
+
const existing = yield* Effect22.tryPromise({
|
|
3309
3577
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and3(eq3(topicMemberships.memoryId, envelope.memoryId), eq3(topicMemberships.modelKey, model.id))),
|
|
3310
3578
|
catch: (cause) => new StoreError({ message: `memory.embed: membership check: ${cause}` })
|
|
3311
|
-
}).pipe(
|
|
3312
|
-
yield* ensureTopicTree(model, deps.config).pipe(
|
|
3313
|
-
yield* backend.ensureVectorTable(model).pipe(
|
|
3314
|
-
yield* backend.insertVector(envelope.memoryId, [...vector], model).pipe(
|
|
3579
|
+
}).pipe(Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: membership check: ${e}` })));
|
|
3580
|
+
yield* ensureTopicTree(model, deps.config).pipe(Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: seed tree: ${e}` })));
|
|
3581
|
+
yield* backend.ensureVectorTable(model).pipe(Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: vector table: ${e}` })));
|
|
3582
|
+
yield* backend.insertVector(envelope.memoryId, [...vector], model).pipe(Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: insert vector: ${e}` })));
|
|
3315
3583
|
if (existing.length === 0) {
|
|
3316
|
-
yield* routeInsert(envelope.memoryId, [...vector], model, deps.config).pipe(
|
|
3584
|
+
yield* routeInsert(envelope.memoryId, [...vector], model, deps.config).pipe(Effect22.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: route: ${e}` })));
|
|
3317
3585
|
}
|
|
3318
|
-
yield*
|
|
3586
|
+
yield* Effect22.logInfo("embed.completed").pipe(Effect22.annotateLogs({
|
|
3319
3587
|
scope: "@dimple/store",
|
|
3320
3588
|
event: "embed.completed",
|
|
3321
3589
|
memoryId: envelope.memoryId,
|
|
@@ -3324,11 +3592,11 @@ function memoryEmbedHandler(deps) {
|
|
|
3324
3592
|
}));
|
|
3325
3593
|
return;
|
|
3326
3594
|
});
|
|
3327
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
3595
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect22.provideService(Service2, deps.backend)) : run;
|
|
3328
3596
|
}
|
|
3329
3597
|
|
|
3330
3598
|
// ../store/src/topics/jobs.ts
|
|
3331
|
-
import { Clock as Clock4, Effect as
|
|
3599
|
+
import { Clock as Clock4, Effect as Effect23 } from "effect";
|
|
3332
3600
|
import { and as and4, asc as asc3, eq as eq4, inArray as inArray2, sql as sql4 } from "drizzle-orm";
|
|
3333
3601
|
|
|
3334
3602
|
// ../store/src/topics/topic-vectors.ts
|
|
@@ -3352,23 +3620,28 @@ var DEFAULT_SPLIT_DEFERRED_HARD_LIMIT = 512;
|
|
|
3352
3620
|
var shouldSplit = (leafMemberCount, config) => leafMemberCount > (config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS);
|
|
3353
3621
|
var childBaseKey = (parent, vectors) => {
|
|
3354
3622
|
const dims = vectors.length > 0 ? vectors[0].length : 0;
|
|
3355
|
-
|
|
3623
|
+
let hash = 2166136261;
|
|
3624
|
+
const step = (byte) => {
|
|
3625
|
+
hash ^= byte;
|
|
3626
|
+
hash = Math.imul(hash, 16777619);
|
|
3627
|
+
};
|
|
3628
|
+
for (let i = 0;i < parent.length; i++)
|
|
3629
|
+
step(parent.charCodeAt(i) & 255);
|
|
3630
|
+
step(255);
|
|
3356
3631
|
for (const v of vectors) {
|
|
3357
|
-
const bytes =
|
|
3358
|
-
for (let i = 0;i <
|
|
3359
|
-
bytes
|
|
3360
|
-
}
|
|
3361
|
-
parts.push(bytes.join(","));
|
|
3632
|
+
const bytes = new Uint8Array(v.buffer, v.byteOffset, dims * 4);
|
|
3633
|
+
for (let i = 0;i < bytes.length; i++)
|
|
3634
|
+
step(bytes[i]);
|
|
3362
3635
|
}
|
|
3363
|
-
return `t_${
|
|
3636
|
+
return `t_${(hash >>> 0).toString(16).padStart(8, "0")}`;
|
|
3364
3637
|
};
|
|
3365
|
-
var splitTopic =
|
|
3638
|
+
var splitTopic = Effect23.fn("TopicSplit.splitTopic")(function* (topicKey, model, config) {
|
|
3366
3639
|
const backend = yield* Service2;
|
|
3367
3640
|
const db = backend.db;
|
|
3368
3641
|
yield* backend.ensureTopicVectorTable(model);
|
|
3369
3642
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS;
|
|
3370
3643
|
const hardLimit = config.splitDeferredHardLimit ?? DEFAULT_SPLIT_DEFERRED_HARD_LIMIT;
|
|
3371
|
-
const leafRows = yield*
|
|
3644
|
+
const leafRows = yield* Effect23.tryPromise({
|
|
3372
3645
|
try: () => db.select().from(topics).where(and4(eq4(topics.topicKey, topicKey), eq4(topics.modelKey, model.id))),
|
|
3373
3646
|
catch: (cause) => new StoreError({ message: `splitTopic: load topic: ${cause}` })
|
|
3374
3647
|
});
|
|
@@ -3379,13 +3652,13 @@ var splitTopic = Effect22.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3379
3652
|
if (leaf.modelKey !== model.id) {
|
|
3380
3653
|
return yield* new ModelMismatch({ expected: model.id, actual: leaf.modelKey });
|
|
3381
3654
|
}
|
|
3382
|
-
const memberRows = yield*
|
|
3655
|
+
const memberRows = yield* Effect23.tryPromise({
|
|
3383
3656
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and4(eq4(topicMemberships.topicKey, topicKey), eq4(topicMemberships.modelKey, model.id))).orderBy(asc3(topicMemberships.memoryId)),
|
|
3384
3657
|
catch: (cause) => new StoreError({ message: `splitTopic: load members: ${cause}` })
|
|
3385
3658
|
});
|
|
3386
3659
|
const memberIds = memberRows.map((r) => r.memoryId);
|
|
3387
3660
|
const memberCount = memberIds.length;
|
|
3388
|
-
const growthRows = yield*
|
|
3661
|
+
const growthRows = yield* Effect23.tryPromise({
|
|
3389
3662
|
try: () => db.select().from(topicGrowthState).where(and4(eq4(topicGrowthState.topicKey, topicKey), eq4(topicGrowthState.modelKey, model.id))),
|
|
3390
3663
|
catch: (cause) => new StoreError({ message: `splitTopic: load growth state: ${cause}` })
|
|
3391
3664
|
});
|
|
@@ -3417,7 +3690,7 @@ var splitTopic = Effect22.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3417
3690
|
const forced = memberCount > hardLimit;
|
|
3418
3691
|
if (!plan.accepted && !forced) {
|
|
3419
3692
|
const now2 = yield* Clock4.currentTimeMillis;
|
|
3420
|
-
yield*
|
|
3693
|
+
yield* Effect23.tryPromise({
|
|
3421
3694
|
try: () => db.transaction(async (tx) => {
|
|
3422
3695
|
await tx.insert(topicGrowthState).values({
|
|
3423
3696
|
topicKey,
|
|
@@ -3450,7 +3723,7 @@ var splitTopic = Effect22.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3450
3723
|
};
|
|
3451
3724
|
}
|
|
3452
3725
|
const now = yield* Clock4.currentTimeMillis;
|
|
3453
|
-
const children = yield*
|
|
3726
|
+
const children = yield* Effect23.tryPromise({
|
|
3454
3727
|
try: () => db.transaction(async (tx) => {
|
|
3455
3728
|
const child0Cf = cfFromVectors(groupVectors(0));
|
|
3456
3729
|
const child1Cf = cfFromVectors(groupVectors(1));
|
|
@@ -3571,34 +3844,34 @@ var splitTopic = Effect22.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3571
3844
|
});
|
|
3572
3845
|
return { accepted: true, children: children.keys, childCounts: children.counts };
|
|
3573
3846
|
});
|
|
3574
|
-
var staleSplitNoOp = (notFound, model) =>
|
|
3847
|
+
var staleSplitNoOp = (notFound, model) => Effect23.gen(function* () {
|
|
3575
3848
|
const backend = yield* Service2;
|
|
3576
3849
|
const db = backend.db;
|
|
3577
|
-
const rows = yield*
|
|
3850
|
+
const rows = yield* Effect23.tryPromise({
|
|
3578
3851
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and4(eq4(topics.topicKey, notFound.topicKey), eq4(topics.modelKey, model.id))),
|
|
3579
3852
|
catch: (cause) => new StoreError({ message: `topic.split: stale check: ${cause}` })
|
|
3580
3853
|
});
|
|
3581
3854
|
if (rows.length > 0) {
|
|
3582
3855
|
return { accepted: false, reason: "no-improvement" };
|
|
3583
3856
|
}
|
|
3584
|
-
return yield*
|
|
3857
|
+
return yield* Effect23.fail(notFound);
|
|
3585
3858
|
});
|
|
3586
3859
|
function topicSplitHandler(deps) {
|
|
3587
|
-
const run = (payload, ctx) =>
|
|
3860
|
+
const run = (payload, ctx) => Effect23.gen(function* () {
|
|
3588
3861
|
const envelope = payload;
|
|
3589
3862
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
3590
3863
|
if (!model) {
|
|
3591
|
-
return yield*
|
|
3864
|
+
return yield* Effect23.fail(new JobFailure({
|
|
3592
3865
|
jobId: ctx.jobId,
|
|
3593
3866
|
message: `topic.split: unknown model "${envelope.modelKey}"`
|
|
3594
3867
|
}));
|
|
3595
3868
|
}
|
|
3596
|
-
const outcome = yield* splitTopic(envelope.topicId, model, deps.config).pipe(
|
|
3869
|
+
const outcome = yield* splitTopic(envelope.topicId, model, deps.config).pipe(Effect23.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleSplitNoOp(notFound, model)), Effect23.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
3597
3870
|
if (outcome.accepted && deps.onUnevenSplit) {
|
|
3598
3871
|
const [n0, n1] = outcome.childCounts;
|
|
3599
3872
|
const [k0] = outcome.children;
|
|
3600
3873
|
if (Math.max(n0, n1) > 2 * Math.min(n0, n1)) {
|
|
3601
|
-
yield* deps.onUnevenSplit(k0, envelope.modelKey).pipe(
|
|
3874
|
+
yield* deps.onUnevenSplit(k0, envelope.modelKey).pipe(Effect23.mapError((error) => new JobFailure({
|
|
3602
3875
|
jobId: ctx.jobId,
|
|
3603
3876
|
message: `topic.split: enqueue rebalance: ${error}`
|
|
3604
3877
|
})));
|
|
@@ -3608,14 +3881,14 @@ function topicSplitHandler(deps) {
|
|
|
3608
3881
|
const maxLeafItems = deps.config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS;
|
|
3609
3882
|
for (let i = 0;i < outcome.children.length; i++) {
|
|
3610
3883
|
if (outcome.childCounts[i] > maxLeafItems) {
|
|
3611
|
-
yield* deps.onChildOverflow(outcome.children[i], envelope.modelKey).pipe(
|
|
3884
|
+
yield* deps.onChildOverflow(outcome.children[i], envelope.modelKey).pipe(Effect23.mapError((error) => new JobFailure({
|
|
3612
3885
|
jobId: ctx.jobId,
|
|
3613
3886
|
message: `topic.split: enqueue child split: ${error}`
|
|
3614
3887
|
})));
|
|
3615
3888
|
}
|
|
3616
3889
|
}
|
|
3617
3890
|
}
|
|
3618
|
-
yield*
|
|
3891
|
+
yield* Effect23.logInfo("split.completed").pipe(Effect23.annotateLogs({
|
|
3619
3892
|
scope: "@dimple/store",
|
|
3620
3893
|
event: "split.completed",
|
|
3621
3894
|
topicId: envelope.topicId,
|
|
@@ -3625,16 +3898,16 @@ function topicSplitHandler(deps) {
|
|
|
3625
3898
|
}));
|
|
3626
3899
|
return;
|
|
3627
3900
|
});
|
|
3628
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
3901
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect23.provideService(Service2, deps.backend)) : run;
|
|
3629
3902
|
}
|
|
3630
3903
|
|
|
3631
3904
|
// ../store/src/topics/maintenance.ts
|
|
3632
|
-
import { Clock as Clock5, Effect as
|
|
3905
|
+
import { Clock as Clock5, Effect as Effect24 } from "effect";
|
|
3633
3906
|
import { and as and5, asc as asc4, eq as eq5, ne as ne2 } from "drizzle-orm";
|
|
3634
3907
|
var DEFAULT_MIN_LEAF_ITEMS = 32;
|
|
3635
3908
|
var DEFAULT_MAX_LEAF_ITEMS2 = 128;
|
|
3636
3909
|
var DEFAULT_REBALANCE_COOLDOWN_MS = 60000;
|
|
3637
|
-
var maintenanceError = (tag) => (effect) => effect.pipe(
|
|
3910
|
+
var maintenanceError = (tag) => (effect) => effect.pipe(Effect24.mapError((cause) => new StoreError({ message: `${tag}: ${cause}` })));
|
|
3638
3911
|
var dot2 = (a, b) => {
|
|
3639
3912
|
let sum = 0;
|
|
3640
3913
|
for (let i = 0;i < a.length; i++)
|
|
@@ -3652,12 +3925,12 @@ var resolvedConfig2 = (config) => ({
|
|
|
3652
3925
|
maxLeafItems: config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS2,
|
|
3653
3926
|
rebalanceCooldownMs: config.rebalanceCooldownMs ?? DEFAULT_REBALANCE_COOLDOWN_MS
|
|
3654
3927
|
});
|
|
3655
|
-
var mergeOrBorrow =
|
|
3928
|
+
var mergeOrBorrow = Effect24.fn("topics.mergeOrBorrow")(function* (topicKey, model, config) {
|
|
3656
3929
|
const backend = yield* Service2;
|
|
3657
3930
|
const db = backend.db;
|
|
3658
3931
|
const { minLeafItems, maxLeafItems } = resolvedConfig2(config);
|
|
3659
3932
|
const now = yield* Clock5.currentTimeMillis;
|
|
3660
|
-
const leafRows = yield*
|
|
3933
|
+
const leafRows = yield* Effect24.tryPromise({
|
|
3661
3934
|
try: () => db.select().from(topics).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id))),
|
|
3662
3935
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load topic: ${cause}` })
|
|
3663
3936
|
});
|
|
@@ -3670,7 +3943,7 @@ var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
3670
3943
|
}
|
|
3671
3944
|
if (leaf.memberCount >= minLeafItems)
|
|
3672
3945
|
return { action: "none" };
|
|
3673
|
-
const parentRows = yield*
|
|
3946
|
+
const parentRows = yield* Effect24.tryPromise({
|
|
3674
3947
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.parentKey)),
|
|
3675
3948
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load parent: ${cause}` })
|
|
3676
3949
|
});
|
|
@@ -3678,16 +3951,16 @@ var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
3678
3951
|
return yield* logUnderfilled(db, topicKey, model, now, "no-borrowable");
|
|
3679
3952
|
}
|
|
3680
3953
|
const parentKey = parentRows[0].parentKey;
|
|
3681
|
-
const parentRowRows = yield*
|
|
3954
|
+
const parentRowRows = yield* Effect24.tryPromise({
|
|
3682
3955
|
try: () => db.select({ childCount: topics.childCount }).from(topics).where(and5(eq5(topics.topicKey, parentKey), eq5(topics.modelKey, model.id))),
|
|
3683
3956
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load parent row: ${cause}` })
|
|
3684
3957
|
});
|
|
3685
3958
|
const parentRow = parentRowRows[0];
|
|
3686
|
-
const siblingRows = yield*
|
|
3959
|
+
const siblingRows = yield* Effect24.tryPromise({
|
|
3687
3960
|
try: () => db.select().from(topics).where(and5(eq5(topics.modelKey, model.id), eq5(topics.kind, "leaf"), ne2(topics.status, "inactive"))).orderBy(asc4(topics.topicKey)),
|
|
3688
3961
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load siblings: ${cause}` })
|
|
3689
3962
|
});
|
|
3690
|
-
const relationRows = yield*
|
|
3963
|
+
const relationRows = yield* Effect24.tryPromise({
|
|
3691
3964
|
try: () => db.select({ childKey: topicRelations.childKey }).from(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.childKey)),
|
|
3692
3965
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load relations: ${cause}` })
|
|
3693
3966
|
});
|
|
@@ -3709,7 +3982,7 @@ var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
3709
3982
|
const membersByTopic = new Map;
|
|
3710
3983
|
const vecById = new Map;
|
|
3711
3984
|
for (const sibling of siblingsOf) {
|
|
3712
|
-
const members = yield*
|
|
3985
|
+
const members = yield* Effect24.tryPromise({
|
|
3713
3986
|
try: () => db.select({
|
|
3714
3987
|
memoryId: topicMemberships.memoryId,
|
|
3715
3988
|
manualLock: topicMemberships.manualLock
|
|
@@ -3722,7 +3995,7 @@ var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
3722
3995
|
vecById.set(row.memoryId, new Float32Array(row.vector));
|
|
3723
3996
|
}
|
|
3724
3997
|
const movedIds = new Set;
|
|
3725
|
-
const outcome = yield*
|
|
3998
|
+
const outcome = yield* Effect24.tryPromise({
|
|
3726
3999
|
try: () => db.transaction(async (tx) => {
|
|
3727
4000
|
const leafCf0 = cfOf(leaf);
|
|
3728
4001
|
let leafCf = leafCf0;
|
|
@@ -3838,7 +4111,36 @@ var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
3838
4111
|
await tx.delete(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.childKey, sibling.topicKey), eq5(topicRelations.modelKey, model.id)));
|
|
3839
4112
|
await deleteTopicVectorRow(tx, model, sibling.topicKey);
|
|
3840
4113
|
await writeTopicVector(tx, model, topicKey, mergedCf.ls);
|
|
3841
|
-
|
|
4114
|
+
const remainingChildren = Math.max(0, (parentRow?.childCount ?? 1) - 1);
|
|
4115
|
+
await tx.update(topics).set({ childCount: remainingChildren, updatedAt: now }).where(and5(eq5(topics.topicKey, parentKey), eq5(topics.modelKey, model.id)));
|
|
4116
|
+
if (remainingChildren === 1) {
|
|
4117
|
+
const grandparentRows = await tx.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).limit(1);
|
|
4118
|
+
const grandparentKey = grandparentRows[0]?.parentKey;
|
|
4119
|
+
if (grandparentKey !== undefined) {
|
|
4120
|
+
await tx.delete(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id)));
|
|
4121
|
+
await tx.delete(topicRelations).where(and5(eq5(topicRelations.parentKey, grandparentKey), eq5(topicRelations.childKey, parentKey), eq5(topicRelations.modelKey, model.id)));
|
|
4122
|
+
await tx.insert(topicRelations).values({
|
|
4123
|
+
parentKey: grandparentKey,
|
|
4124
|
+
childKey: topicKey,
|
|
4125
|
+
modelKey: model.id,
|
|
4126
|
+
relationKind: "parent_child",
|
|
4127
|
+
updatedAt: now
|
|
4128
|
+
});
|
|
4129
|
+
await tx.update(topics).set({
|
|
4130
|
+
status: "inactive",
|
|
4131
|
+
memberCount: 0,
|
|
4132
|
+
sumSquares: 0,
|
|
4133
|
+
childCount: 0,
|
|
4134
|
+
updatedAt: now
|
|
4135
|
+
}).where(and5(eq5(topics.topicKey, parentKey), eq5(topics.modelKey, model.id)));
|
|
4136
|
+
await deleteTopicVectorRow(tx, model, parentKey);
|
|
4137
|
+
const grandparentDepthRows = await tx.select({ depth: topics.depth }).from(topics).where(and5(eq5(topics.topicKey, grandparentKey), eq5(topics.modelKey, model.id))).limit(1);
|
|
4138
|
+
await tx.update(topics).set({
|
|
4139
|
+
depth: (grandparentDepthRows[0]?.depth ?? 0) + 1,
|
|
4140
|
+
updatedAt: now
|
|
4141
|
+
}).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id)));
|
|
4142
|
+
}
|
|
4143
|
+
}
|
|
3842
4144
|
await tx.insert(topicGrowthState).values({
|
|
3843
4145
|
topicKey,
|
|
3844
4146
|
modelKey: model.id,
|
|
@@ -3862,12 +4164,12 @@ var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
3862
4164
|
});
|
|
3863
4165
|
return outcome;
|
|
3864
4166
|
});
|
|
3865
|
-
var rebalanceSiblings =
|
|
4167
|
+
var rebalanceSiblings = Effect24.fn("topics.rebalanceSiblings")(function* (topicKey, model, config) {
|
|
3866
4168
|
const backend = yield* Service2;
|
|
3867
4169
|
const db = backend.db;
|
|
3868
4170
|
const { rebalanceCooldownMs } = resolvedConfig2(config);
|
|
3869
4171
|
const now = yield* Clock5.currentTimeMillis;
|
|
3870
|
-
const rows = yield*
|
|
4172
|
+
const rows = yield* Effect24.tryPromise({
|
|
3871
4173
|
try: () => db.select().from(topics).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id))),
|
|
3872
4174
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load topic: ${cause}` })
|
|
3873
4175
|
});
|
|
@@ -3876,19 +4178,19 @@ var rebalanceSiblings = Effect23.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
3876
4178
|
return yield* new TopicNotFound({ topicKey });
|
|
3877
4179
|
if (row.modelKey !== model.id)
|
|
3878
4180
|
return yield* new ModelMismatch({ expected: model.id, actual: row.modelKey });
|
|
3879
|
-
const parentRows = yield*
|
|
4181
|
+
const parentRows = yield* Effect24.tryPromise({
|
|
3880
4182
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.parentKey)),
|
|
3881
4183
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load parent: ${cause}` })
|
|
3882
4184
|
});
|
|
3883
4185
|
if (parentRows.length === 0)
|
|
3884
4186
|
return { action: "none", reason: "balanced" };
|
|
3885
4187
|
const parentKey = parentRows[0].parentKey;
|
|
3886
|
-
const relationRows = yield*
|
|
4188
|
+
const relationRows = yield* Effect24.tryPromise({
|
|
3887
4189
|
try: () => db.select({ childKey: topicRelations.childKey }).from(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.childKey)),
|
|
3888
4190
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load relations: ${cause}` })
|
|
3889
4191
|
});
|
|
3890
4192
|
const siblingKeys = relationRows.map((r) => r.childKey);
|
|
3891
|
-
const topicRows = yield*
|
|
4193
|
+
const topicRows = yield* Effect24.tryPromise({
|
|
3892
4194
|
try: () => db.select().from(topics).where(and5(eq5(topics.modelKey, model.id), eq5(topics.kind, "leaf"), ne2(topics.status, "inactive"))),
|
|
3893
4195
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load siblings: ${cause}` })
|
|
3894
4196
|
});
|
|
@@ -3896,7 +4198,7 @@ var rebalanceSiblings = Effect23.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
3896
4198
|
const siblings = siblingKeys.map((k) => map2.get(k)).filter(Boolean);
|
|
3897
4199
|
if (siblings.length < 2)
|
|
3898
4200
|
return { action: "none", reason: "balanced" };
|
|
3899
|
-
const growthRows = yield*
|
|
4201
|
+
const growthRows = yield* Effect24.tryPromise({
|
|
3900
4202
|
try: () => db.select().from(topicGrowthState).where(and5(eq5(topicGrowthState.topicKey, topicKey), eq5(topicGrowthState.modelKey, model.id))),
|
|
3901
4203
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load growth: ${cause}` })
|
|
3902
4204
|
});
|
|
@@ -3921,7 +4223,7 @@ var rebalanceSiblings = Effect23.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
3921
4223
|
const membersByTopic = new Map;
|
|
3922
4224
|
const vecById = new Map;
|
|
3923
4225
|
for (const sibling of siblings) {
|
|
3924
|
-
const members = yield*
|
|
4226
|
+
const members = yield* Effect24.tryPromise({
|
|
3925
4227
|
try: () => db.select({
|
|
3926
4228
|
memoryId: topicMemberships.memoryId,
|
|
3927
4229
|
manualLock: topicMemberships.manualLock
|
|
@@ -3934,7 +4236,7 @@ var rebalanceSiblings = Effect23.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
3934
4236
|
vecById.set(row2.memoryId, new Float32Array(row2.vector));
|
|
3935
4237
|
}
|
|
3936
4238
|
const movedIds = new Set;
|
|
3937
|
-
const outcome = yield*
|
|
4239
|
+
const outcome = yield* Effect24.tryPromise({
|
|
3938
4240
|
try: () => db.transaction(async (tx) => {
|
|
3939
4241
|
const liveCfs = new Map(siblings.map((s) => [s.topicKey, cfOf(s)]));
|
|
3940
4242
|
const liveList = () => [...liveCfs.entries()].map(([topicKey2, cf]) => ({ topicKey: topicKey2, cf })).sort((a, b) => b.cf.count - a.cf.count || (a.topicKey < b.topicKey ? -1 : 1));
|
|
@@ -4021,7 +4323,7 @@ var loadMembers = async (tx, topicKey, modelKey) => {
|
|
|
4021
4323
|
const rows = await tx.select({ memoryId: topicMemberships.memoryId, manualLock: topicMemberships.manualLock }).from(topicMemberships).where(and5(eq5(topicMemberships.topicKey, topicKey), eq5(topicMemberships.modelKey, modelKey))).orderBy(asc4(topicMemberships.memoryId));
|
|
4022
4324
|
return rows;
|
|
4023
4325
|
};
|
|
4024
|
-
var logUnderfilled = (db, topicKey, model, now, reason) =>
|
|
4326
|
+
var logUnderfilled = (db, topicKey, model, now, reason) => Effect24.tryPromise({
|
|
4025
4327
|
try: () => db.insert(topicChangeLog).values({
|
|
4026
4328
|
topicKey,
|
|
4027
4329
|
operation: "merge",
|
|
@@ -4031,7 +4333,7 @@ var logUnderfilled = (db, topicKey, model, now, reason) => Effect23.tryPromise({
|
|
|
4031
4333
|
createdAt: now
|
|
4032
4334
|
}),
|
|
4033
4335
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: log underfilled: ${cause}` })
|
|
4034
|
-
}).pipe(
|
|
4336
|
+
}).pipe(Effect24.map(() => ({ action: "underfilled", reason })));
|
|
4035
4337
|
var logUnderfilledTx = async (tx, topicKey, model, now, reason) => {
|
|
4036
4338
|
await tx.insert(topicChangeLog).values({
|
|
4037
4339
|
topicKey,
|
|
@@ -4044,61 +4346,61 @@ var logUnderfilledTx = async (tx, topicKey, model, now, reason) => {
|
|
|
4044
4346
|
return { action: "underfilled", reason };
|
|
4045
4347
|
};
|
|
4046
4348
|
function topicMergeHandler(deps) {
|
|
4047
|
-
const run = (payload, ctx) =>
|
|
4349
|
+
const run = (payload, ctx) => Effect24.gen(function* () {
|
|
4048
4350
|
const envelope = payload;
|
|
4049
4351
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4050
4352
|
if (!model) {
|
|
4051
|
-
return yield*
|
|
4353
|
+
return yield* Effect24.fail(new JobFailure({
|
|
4052
4354
|
jobId: ctx.jobId,
|
|
4053
4355
|
message: `topic.merge: unknown model "${envelope.modelKey}"`
|
|
4054
4356
|
}));
|
|
4055
4357
|
}
|
|
4056
|
-
yield* mergeOrBorrow(envelope.topicId, model, deps.config).pipe(
|
|
4358
|
+
yield* mergeOrBorrow(envelope.topicId, model, deps.config).pipe(Effect24.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleNoOp(notFound, model)), Effect24.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4057
4359
|
return;
|
|
4058
4360
|
});
|
|
4059
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4361
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect24.provideService(Service2, deps.backend)) : run;
|
|
4060
4362
|
}
|
|
4061
4363
|
function topicRebalanceHandler(deps) {
|
|
4062
|
-
const run = (payload, ctx) =>
|
|
4364
|
+
const run = (payload, ctx) => Effect24.gen(function* () {
|
|
4063
4365
|
const envelope = payload;
|
|
4064
4366
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4065
4367
|
if (!model) {
|
|
4066
|
-
return yield*
|
|
4368
|
+
return yield* Effect24.fail(new JobFailure({
|
|
4067
4369
|
jobId: ctx.jobId,
|
|
4068
4370
|
message: `topic.rebalance: unknown model "${envelope.modelKey}"`
|
|
4069
4371
|
}));
|
|
4070
4372
|
}
|
|
4071
|
-
yield* rebalanceSiblings(envelope.topicId, model, deps.config).pipe(
|
|
4373
|
+
yield* rebalanceSiblings(envelope.topicId, model, deps.config).pipe(Effect24.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleNoOp(notFound, model)), Effect24.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4072
4374
|
return;
|
|
4073
4375
|
});
|
|
4074
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4376
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect24.provideService(Service2, deps.backend)) : run;
|
|
4075
4377
|
}
|
|
4076
|
-
var staleNoOp = (notFound, model) =>
|
|
4378
|
+
var staleNoOp = (notFound, model) => Effect24.gen(function* () {
|
|
4077
4379
|
const backend = yield* Service2;
|
|
4078
4380
|
const db = backend.db;
|
|
4079
|
-
const rows = yield*
|
|
4381
|
+
const rows = yield* Effect24.tryPromise({
|
|
4080
4382
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and5(eq5(topics.topicKey, notFound.topicKey), eq5(topics.modelKey, model.id))),
|
|
4081
4383
|
catch: (cause) => new StoreError({ message: `maintenance: stale check: ${cause}` })
|
|
4082
4384
|
});
|
|
4083
4385
|
if (rows.length > 0) {
|
|
4084
4386
|
return;
|
|
4085
4387
|
}
|
|
4086
|
-
return yield*
|
|
4087
|
-
}).pipe(
|
|
4388
|
+
return yield* Effect24.fail(new JobFailure({ jobId: notFound.topicKey, message: `topic missing: ${notFound.topicKey}` }));
|
|
4389
|
+
}).pipe(Effect24.mapError((error) => error instanceof JobFailure ? error : new JobFailure({ jobId: notFound.topicKey, message: `${error}` })));
|
|
4088
4390
|
|
|
4089
4391
|
// ../store/src/topics/observability.ts
|
|
4090
|
-
import { Effect as
|
|
4392
|
+
import { Effect as Effect25 } from "effect";
|
|
4091
4393
|
import { and as and6, desc, eq as eq6, inArray as inArray3, ne as ne3, sql as sql5 } from "drizzle-orm";
|
|
4092
4394
|
var DEFAULT_MAX_LEAF_ITEMS3 = 128;
|
|
4093
4395
|
var DEFAULT_SPLIT_DEFERRED_HARD_LIMIT2 = 512;
|
|
4094
4396
|
var DEFAULT_CHANGE_LOG_LIMIT = 50;
|
|
4095
|
-
var routingFailure4 = (tag) => (effect) => effect.pipe(
|
|
4397
|
+
var routingFailure4 = (tag) => (effect) => effect.pipe(Effect25.mapError((cause) => new RoutingFailure({ cause: { operation: tag, cause } })));
|
|
4096
4398
|
var modelTopicKeys = (db, modelKey) => db.select({ topicKey: topics.topicKey }).from(topics).where(eq6(topics.modelKey, modelKey));
|
|
4097
|
-
var topicMetrics =
|
|
4399
|
+
var topicMetrics = Effect25.fn("topics.topicMetrics")(function* (modelKey, config) {
|
|
4098
4400
|
const backend = yield* Service2;
|
|
4099
4401
|
const db = backend.db;
|
|
4100
4402
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS3;
|
|
4101
|
-
const rows = yield*
|
|
4403
|
+
const rows = yield* Effect25.tryPromise({
|
|
4102
4404
|
try: () => db.select({
|
|
4103
4405
|
topicKey: topics.topicKey,
|
|
4104
4406
|
kind: topics.kind,
|
|
@@ -4131,12 +4433,12 @@ var topicMetrics = Effect24.fn("topics.topicMetrics")(function* (modelKey, confi
|
|
|
4131
4433
|
deferredLeaves++;
|
|
4132
4434
|
}
|
|
4133
4435
|
leafMemberCounts.sort((a, b) => a - b);
|
|
4134
|
-
const growth = yield*
|
|
4436
|
+
const growth = yield* Effect25.tryPromise({
|
|
4135
4437
|
try: () => db.select({ total: sql5`coalesce(sum(${topicGrowthState.splitRejectedCount}), 0)` }).from(topicGrowthState).where(eq6(topicGrowthState.modelKey, modelKey)),
|
|
4136
4438
|
catch: (cause) => cause
|
|
4137
4439
|
}).pipe(routingFailure4("topics.topicMetrics.growth"));
|
|
4138
4440
|
const splitRejectedTotal = Number(growth[0]?.total ?? 0);
|
|
4139
|
-
const lastLog = yield*
|
|
4441
|
+
const lastLog = yield* Effect25.tryPromise({
|
|
4140
4442
|
try: () => db.select({
|
|
4141
4443
|
operation: topicChangeLog.operation,
|
|
4142
4444
|
createdAt: topicChangeLog.createdAt
|
|
@@ -4156,11 +4458,11 @@ var topicMetrics = Effect24.fn("topics.topicMetrics")(function* (modelKey, confi
|
|
|
4156
4458
|
lastOperation
|
|
4157
4459
|
};
|
|
4158
4460
|
});
|
|
4159
|
-
var topicChangeLog2 =
|
|
4461
|
+
var topicChangeLog2 = Effect25.fn("topics.topicChangeLog")(function* (modelKey, opts) {
|
|
4160
4462
|
const backend = yield* Service2;
|
|
4161
4463
|
const db = backend.db;
|
|
4162
4464
|
const limit = opts?.limit ?? DEFAULT_CHANGE_LOG_LIMIT;
|
|
4163
|
-
const rows = yield*
|
|
4465
|
+
const rows = yield* Effect25.tryPromise({
|
|
4164
4466
|
try: () => {
|
|
4165
4467
|
const scoped = inArray3(topicChangeLog.topicKey, modelTopicKeys(db, modelKey));
|
|
4166
4468
|
const filtered = opts?.operation ? and6(scoped, eq6(topicChangeLog.operation, opts.operation)) : scoped;
|
|
@@ -4170,14 +4472,14 @@ var topicChangeLog2 = Effect24.fn("topics.topicChangeLog")(function* (modelKey,
|
|
|
4170
4472
|
}).pipe(routingFailure4("topics.topicChangeLog"));
|
|
4171
4473
|
return rows;
|
|
4172
4474
|
});
|
|
4173
|
-
var topicHealth =
|
|
4475
|
+
var topicHealth = Effect25.fn("topics.topicHealth")(function* (modelKey, config) {
|
|
4174
4476
|
const backend = yield* Service2;
|
|
4175
4477
|
const db = backend.db;
|
|
4176
4478
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS3;
|
|
4177
4479
|
const hardLimit = config.splitDeferredHardLimit ?? DEFAULT_SPLIT_DEFERRED_HARD_LIMIT2;
|
|
4178
4480
|
const slack = Math.max(2, Math.ceil(hardLimit / 2) - maxLeafItems);
|
|
4179
4481
|
const leafLimit = maxLeafItems + slack;
|
|
4180
|
-
const rows = yield*
|
|
4482
|
+
const rows = yield* Effect25.tryPromise({
|
|
4181
4483
|
try: () => db.select({
|
|
4182
4484
|
topicKey: topics.topicKey,
|
|
4183
4485
|
kind: topics.kind,
|
|
@@ -4187,7 +4489,7 @@ var topicHealth = Effect24.fn("topics.topicHealth")(function* (modelKey, config)
|
|
|
4187
4489
|
}).from(topics).where(and6(eq6(topics.modelKey, modelKey), ne3(topics.status, "inactive"))),
|
|
4188
4490
|
catch: (cause) => cause
|
|
4189
4491
|
}).pipe(routingFailure4("topics.topicHealth.topics"));
|
|
4190
|
-
const countRows = yield*
|
|
4492
|
+
const countRows = yield* Effect25.tryPromise({
|
|
4191
4493
|
try: () => db.select({ topicKey: topicMemberships.topicKey, count: sql5`count(*)` }).from(topicMemberships).where(eq6(topicMemberships.modelKey, modelKey)).groupBy(topicMemberships.topicKey),
|
|
4192
4494
|
catch: (cause) => cause
|
|
4193
4495
|
}).pipe(routingFailure4("topics.topicHealth.memberships"));
|
|
@@ -4212,30 +4514,30 @@ var topicHealth = Effect24.fn("topics.topicHealth")(function* (modelKey, config)
|
|
|
4212
4514
|
});
|
|
4213
4515
|
|
|
4214
4516
|
// ../store/src/graph/observability.ts
|
|
4215
|
-
import { Effect as
|
|
4517
|
+
import { Effect as Effect26 } from "effect";
|
|
4216
4518
|
import { and as and7, eq as eq7, isNull, sql as sql6 } from "drizzle-orm";
|
|
4217
|
-
var graphCoverage =
|
|
4519
|
+
var graphCoverage = Effect26.fn("GraphCoverage.run")(function* (modelKey) {
|
|
4218
4520
|
const backend = yield* Service2;
|
|
4219
4521
|
const db = backend.db;
|
|
4220
|
-
const totalRows = yield*
|
|
4522
|
+
const totalRows = yield* Effect26.tryPromise({
|
|
4221
4523
|
try: () => db.select({ id: memories.id }).from(memories).where(and7(eq7(memories.state, "active"), eq7(memories.embeddingModel, modelKey))),
|
|
4222
4524
|
catch: (cause) => new StoreError({ message: `graphCoverage: total: ${cause}` })
|
|
4223
4525
|
});
|
|
4224
4526
|
const totalMemories = totalRows.length;
|
|
4225
|
-
const isolatedRows = yield* backend.listIsolatedMemories(0, modelKey).pipe(
|
|
4527
|
+
const isolatedRows = yield* backend.listIsolatedMemories(0, modelKey).pipe(Effect26.mapError((cause) => new StoreError({ message: `graphCoverage: isolated: ${cause}` })));
|
|
4226
4528
|
const isolatedMemories = isolatedRows.length;
|
|
4227
4529
|
const coveredMemories = totalMemories - isolatedMemories;
|
|
4228
4530
|
const coveragePercent = totalMemories === 0 ? 0 : Math.round(coveredMemories / totalMemories * 1000) / 10;
|
|
4229
|
-
const pendingRows = yield*
|
|
4531
|
+
const pendingRows = yield* Effect26.tryPromise({
|
|
4230
4532
|
try: () => db.select({ count: sql6`count(*)` }).from(artifactEdges).where(and7(eq7(artifactEdges.status, "pending"), isNull(artifactEdges.expiredAt))),
|
|
4231
4533
|
catch: (cause) => new StoreError({ message: `graphCoverage: pending: ${cause}` })
|
|
4232
4534
|
});
|
|
4233
4535
|
const pendingCandidates = totalMemories === 0 ? 0 : Number(pendingRows[0]?.count ?? 0);
|
|
4234
|
-
const conceptCount = totalMemories === 0 ? 0 : (yield*
|
|
4536
|
+
const conceptCount = totalMemories === 0 ? 0 : (yield* Effect26.tryPromise({
|
|
4235
4537
|
try: () => db.select({ id: concepts.id }).from(concepts),
|
|
4236
4538
|
catch: (cause) => new StoreError({ message: `graphCoverage: concepts: ${cause}` })
|
|
4237
4539
|
})).length;
|
|
4238
|
-
const annotatedMemories = totalMemories === 0 ? 0 : new Set((yield*
|
|
4540
|
+
const annotatedMemories = totalMemories === 0 ? 0 : new Set((yield* Effect26.tryPromise({
|
|
4239
4541
|
try: () => db.select({ memoryId: memoryConcepts.memoryId }).from(memoryConcepts),
|
|
4240
4542
|
catch: (cause) => new StoreError({ message: `graphCoverage: annotations: ${cause}` })
|
|
4241
4543
|
})).map((row) => row.memoryId)).size;
|
|
@@ -4465,27 +4767,25 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4465
4767
|
const db = backend.db;
|
|
4466
4768
|
const topicsConfig = topicsOptions?.config ?? Schema19.decodeUnknownSync(TopicConfig)({});
|
|
4467
4769
|
const topicsEnabled = topicsOptions?.enabled ?? false;
|
|
4468
|
-
const updateTopicVectorForget = (backendImpl, model, topicKey, x) =>
|
|
4469
|
-
const
|
|
4470
|
-
const current = rows.find((r) => r.topicKey === topicKey);
|
|
4770
|
+
const updateTopicVectorForget = (backendImpl, model, topicKey, x) => Effect27.gen(function* () {
|
|
4771
|
+
const current = yield* backendImpl.fetchTopicVector(topicKey, model).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.forget: load vector: ${cause}` })));
|
|
4471
4772
|
const ls = current ? new Float32Array(current.sumVector) : new Float32Array(model.dimensions);
|
|
4472
4773
|
for (let i = 0;i < ls.length; i++)
|
|
4473
4774
|
ls[i] = ls[i] - x[i];
|
|
4474
|
-
yield* backendImpl.upsertTopicVector(topicKey, Array.from(ls), model).pipe(
|
|
4475
|
-
const relRows = yield*
|
|
4775
|
+
yield* backendImpl.upsertTopicVector(topicKey, Array.from(ls), model).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.forget: write vector: ${cause}` })));
|
|
4776
|
+
const relRows = yield* Effect27.tryPromise({
|
|
4476
4777
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and8(eq8(topicRelations.childKey, topicKey), eq8(topicRelations.modelKey, model.id), eq8(topicRelations.relationKind, "parent_child"))),
|
|
4477
4778
|
catch: (cause) => new StoreError({ message: `Store.forget: load ancestor: ${cause}` })
|
|
4478
4779
|
});
|
|
4479
4780
|
let ancestorKey = relRows[0]?.parentKey;
|
|
4480
4781
|
while (ancestorKey !== undefined) {
|
|
4481
4782
|
const key = ancestorKey;
|
|
4482
|
-
const
|
|
4483
|
-
const a = aRows.find((r) => r.topicKey === key);
|
|
4783
|
+
const a = yield* backendImpl.fetchTopicVector(key, model).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.forget: load vector: ${cause}` })));
|
|
4484
4784
|
const als = a ? new Float32Array(a.sumVector) : new Float32Array(model.dimensions);
|
|
4485
4785
|
for (let i = 0;i < als.length; i++)
|
|
4486
4786
|
als[i] = als[i] - x[i];
|
|
4487
|
-
yield* backendImpl.upsertTopicVector(key, Array.from(als), model).pipe(
|
|
4488
|
-
const up = yield*
|
|
4787
|
+
yield* backendImpl.upsertTopicVector(key, Array.from(als), model).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.forget: write vector: ${cause}` })));
|
|
4788
|
+
const up = yield* Effect27.tryPromise({
|
|
4489
4789
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and8(eq8(topicRelations.childKey, key), eq8(topicRelations.modelKey, model.id), eq8(topicRelations.relationKind, "parent_child"))),
|
|
4490
4790
|
catch: (cause) => new StoreError({ message: `Store.forget: load ancestor: ${cause}` })
|
|
4491
4791
|
});
|
|
@@ -4535,18 +4835,18 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4535
4835
|
};
|
|
4536
4836
|
const seam = auditOptions?.enqueueAudit;
|
|
4537
4837
|
if (seam !== undefined) {
|
|
4538
|
-
return propagateSeam ? seam(row).pipe(
|
|
4838
|
+
return propagateSeam ? seam(row).pipe(Effect27.mapError((cause) => new StoreError({ message: `audit.enqueue-failed: ${cause}` }))) : seam(row).pipe(Effect27.catch((cause) => Effect27.logError("audit.enqueue-failed").pipe(Effect27.annotateLogs({
|
|
4539
4839
|
scope: "@dimple/store",
|
|
4540
4840
|
event: "audit.enqueue-failed",
|
|
4541
4841
|
mode,
|
|
4542
4842
|
cause: String(cause)
|
|
4543
4843
|
}))));
|
|
4544
4844
|
}
|
|
4545
|
-
return
|
|
4845
|
+
return Effect27.sync(() => {
|
|
4546
4846
|
db.insert(retrievalLog2).values(row).catch(() => {});
|
|
4547
4847
|
});
|
|
4548
4848
|
};
|
|
4549
|
-
const logAudit =
|
|
4849
|
+
const logAudit = Effect27.fn("Store.logAudit")(function* (input) {
|
|
4550
4850
|
return yield* logRetrieval(input.query, input.namespace, input.mode, input.resultCount, input.startTime, input.queryProfile, input.sources, input.hashSeed, true);
|
|
4551
4851
|
});
|
|
4552
4852
|
const GRAPH_SUPERSEDE_DEMOTION = graphConfig?.supersedeDemotion ?? 0.5;
|
|
@@ -4554,9 +4854,10 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4554
4854
|
nearDuplicate: 0.9,
|
|
4555
4855
|
conflictLow: 0.6
|
|
4556
4856
|
};
|
|
4857
|
+
const graphEnabled = graphConfig?.enabled !== false;
|
|
4557
4858
|
const GRAPH_CANDIDATE_LIMIT_MAX = 200;
|
|
4558
4859
|
const GRAPH_ISOLATED_EXAMPLES = 10;
|
|
4559
|
-
const fetchResultEdges = (ids) =>
|
|
4860
|
+
const fetchResultEdges = (ids) => Effect27.tryPromise({
|
|
4560
4861
|
try: () => db.select({
|
|
4561
4862
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
4562
4863
|
toMemoryId: artifactEdges.toMemoryId,
|
|
@@ -4565,11 +4866,11 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4565
4866
|
}).from(artifactEdges).where(and8(or(inArray4(artifactEdges.fromMemoryId, ids), inArray4(artifactEdges.toMemoryId, ids)), ne4(artifactEdges.status, "rejected"), isNull2(artifactEdges.expiredAt))),
|
|
4566
4867
|
catch: (cause) => new StoreError({ message: `Store.retrieval post-pass: edges: ${cause}` })
|
|
4567
4868
|
});
|
|
4568
|
-
const fetchResultConcepts = (ids) =>
|
|
4869
|
+
const fetchResultConcepts = (ids) => Effect27.tryPromise({
|
|
4569
4870
|
try: () => db.select({ memoryId: memoryConcepts2.memoryId, conceptId: memoryConcepts2.conceptId }).from(memoryConcepts2).where(inArray4(memoryConcepts2.memoryId, ids)),
|
|
4570
4871
|
catch: (cause) => new StoreError({ message: `Store.retrieval post-pass: concepts: ${cause}` })
|
|
4571
4872
|
});
|
|
4572
|
-
const augmentRetrieval = (results) =>
|
|
4873
|
+
const augmentRetrieval = (results) => Effect27.gen(function* () {
|
|
4573
4874
|
if (results.length === 0)
|
|
4574
4875
|
return results;
|
|
4575
4876
|
const ids = results.map((r) => r.memory.id);
|
|
@@ -4579,6 +4880,9 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4579
4880
|
supersedeDemotion: GRAPH_SUPERSEDE_DEMOTION
|
|
4580
4881
|
});
|
|
4581
4882
|
});
|
|
4883
|
+
const augmentSearchResults = Effect27.fn("Store.augmentSearchResults")(function* (results) {
|
|
4884
|
+
return yield* augmentRetrieval(results);
|
|
4885
|
+
});
|
|
4582
4886
|
const EDGE_KINDS = [
|
|
4583
4887
|
"memory_supports",
|
|
4584
4888
|
"memory_supersedes",
|
|
@@ -4586,7 +4890,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4586
4890
|
"code_calls"
|
|
4587
4891
|
];
|
|
4588
4892
|
const EDGE_SOURCES = ["rule", "llm", "agent"];
|
|
4589
|
-
const write =
|
|
4893
|
+
const write = Effect27.fn("Store.write")(function* (input) {
|
|
4590
4894
|
if (input.content.includes("\x00")) {
|
|
4591
4895
|
return yield* new StoreError({
|
|
4592
4896
|
message: "Store.write: content contains a NUL byte (libSQL would silently truncate it)"
|
|
@@ -4595,7 +4899,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4595
4899
|
const id = crypto.randomUUID();
|
|
4596
4900
|
const now = Date.now();
|
|
4597
4901
|
const metadataBlob = input.typedMetadata !== undefined ? mergeMetadataBlob(input.metadata, input.typedMetadata) : input.metadata ?? null;
|
|
4598
|
-
yield*
|
|
4902
|
+
yield* Effect27.tryPromise({
|
|
4599
4903
|
try: () => db.transaction(async (tx) => {
|
|
4600
4904
|
await tx.insert(memories2).values({
|
|
4601
4905
|
id,
|
|
@@ -4628,7 +4932,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4628
4932
|
embeddingModel: undefined
|
|
4629
4933
|
};
|
|
4630
4934
|
});
|
|
4631
|
-
const writeMany =
|
|
4935
|
+
const writeMany = Effect27.fn("Store.writeMany")(function* (inputs) {
|
|
4632
4936
|
if (inputs.length === 0)
|
|
4633
4937
|
return [];
|
|
4634
4938
|
for (const input of inputs) {
|
|
@@ -4652,7 +4956,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4652
4956
|
metadata: metadataBlob
|
|
4653
4957
|
};
|
|
4654
4958
|
});
|
|
4655
|
-
yield*
|
|
4959
|
+
yield* Effect27.tryPromise({
|
|
4656
4960
|
try: () => db.transaction(async (tx) => {
|
|
4657
4961
|
await tx.insert(memories2).values(rows);
|
|
4658
4962
|
await tx.insert(memoryEvents2).values(rows.map((row) => ({
|
|
@@ -4680,8 +4984,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4680
4984
|
embeddingModel: undefined
|
|
4681
4985
|
}));
|
|
4682
4986
|
});
|
|
4683
|
-
const get =
|
|
4684
|
-
const rows = yield*
|
|
4987
|
+
const get = Effect27.fn("Store.get")(function* (id) {
|
|
4988
|
+
const rows = yield* Effect27.tryPromise({
|
|
4685
4989
|
try: () => db.select(memoryFields).from(memories2).where(eq8(memories2.id, id)).limit(1),
|
|
4686
4990
|
catch: (cause) => new StoreError({ message: `Store.get: ${cause}` })
|
|
4687
4991
|
});
|
|
@@ -4690,8 +4994,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4690
4994
|
return yield* new NotFound({ id });
|
|
4691
4995
|
return fromMemoryRow(row);
|
|
4692
4996
|
});
|
|
4693
|
-
const getByKey =
|
|
4694
|
-
const rows = yield*
|
|
4997
|
+
const getByKey = Effect27.fn("Store.getByKey")(function* (key, namespace) {
|
|
4998
|
+
const rows = yield* Effect27.tryPromise({
|
|
4695
4999
|
try: () => db.select(memoryFields).from(memories2).where(and8(eq8(memories2.logicalKey, key), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).orderBy(desc2(memories2.createdAt)).limit(1),
|
|
4696
5000
|
catch: (cause) => new StoreError({ message: `Store.getByKey: ${cause}` })
|
|
4697
5001
|
});
|
|
@@ -4700,9 +5004,9 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4700
5004
|
return yield* new NotFound({ id: key });
|
|
4701
5005
|
return fromMemoryRow(row);
|
|
4702
5006
|
});
|
|
4703
|
-
const recency =
|
|
5007
|
+
const recency = Effect27.fn("Store.recency")(function* (namespace, limit) {
|
|
4704
5008
|
const start = Date.now();
|
|
4705
|
-
const rows = yield*
|
|
5009
|
+
const rows = yield* Effect27.tryPromise({
|
|
4706
5010
|
try: () => db.select(memoryFields).from(memories2).where(and8(eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).orderBy(desc2(memories2.createdAt)).limit(limit),
|
|
4707
5011
|
catch: (cause) => new StoreError({ message: `Store.recency: ${cause}` })
|
|
4708
5012
|
});
|
|
@@ -4715,9 +5019,9 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4715
5019
|
yield* logRetrieval(namespace, namespace, "recency", results.length, start);
|
|
4716
5020
|
return yield* augmentRetrieval(results);
|
|
4717
5021
|
});
|
|
4718
|
-
const searchFts5 =
|
|
5022
|
+
const searchFts5 = Effect27.fn("Store.searchFts5")(function* (query, namespace, limit, opts) {
|
|
4719
5023
|
const start = Date.now();
|
|
4720
|
-
const rows = yield* backend.ftsSearch(query, namespace, limit, opts?.topicKeys, modelConfig?.id, opts?.contentType);
|
|
5024
|
+
const rows = yield* backend.ftsSearch(query, namespace, limit, opts?.topicKeys, modelConfig?.id, opts?.contentType, opts?.includeEmbedding === true);
|
|
4721
5025
|
if (opts?.audit !== false) {
|
|
4722
5026
|
yield* logRetrieval(query, namespace, "fts", rows.length, start);
|
|
4723
5027
|
}
|
|
@@ -4734,7 +5038,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4734
5038
|
ttlSecs: r.ttl_secs,
|
|
4735
5039
|
namespace: r.namespace,
|
|
4736
5040
|
metadata: r.metadata ?? undefined,
|
|
4737
|
-
embedding: decodeBlobVector(r.embedding),
|
|
5041
|
+
embedding: opts?.includeEmbedding === true ? decodeBlobVector(r.embedding) : undefined,
|
|
4738
5042
|
embeddingModel: r.embedding_model ?? undefined
|
|
4739
5043
|
},
|
|
4740
5044
|
score: -r.rank,
|
|
@@ -4742,9 +5046,11 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4742
5046
|
source: "fts5"
|
|
4743
5047
|
};
|
|
4744
5048
|
});
|
|
5049
|
+
if (opts?.augment === false)
|
|
5050
|
+
return results;
|
|
4745
5051
|
return yield* augmentRetrieval(results);
|
|
4746
5052
|
});
|
|
4747
|
-
const searchVector =
|
|
5053
|
+
const searchVector = Effect27.fn("Store.searchVector")(function* (query, namespace, limit, opts) {
|
|
4748
5054
|
if (!modelConfig) {
|
|
4749
5055
|
return yield* new StoreError({
|
|
4750
5056
|
message: "Store.searchVector: no embedding model configured"
|
|
@@ -4754,13 +5060,13 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4754
5060
|
let allowedMemoryIds;
|
|
4755
5061
|
const conceptIds = opts?.conceptIds;
|
|
4756
5062
|
if (conceptIds !== undefined && conceptIds.length > 0) {
|
|
4757
|
-
const conceptRows = yield*
|
|
5063
|
+
const conceptRows = yield* Effect27.tryPromise({
|
|
4758
5064
|
try: () => db.select({ memoryId: memoryConcepts2.memoryId }).from(memoryConcepts2).where(inArray4(memoryConcepts2.conceptId, conceptIds)),
|
|
4759
5065
|
catch: (cause) => new StoreError({ message: `Store.searchVector: concepts: ${cause}` })
|
|
4760
5066
|
});
|
|
4761
5067
|
allowedMemoryIds = new Set(conceptRows.map((r) => r.memoryId));
|
|
4762
5068
|
}
|
|
4763
|
-
const rows = yield* backend.vectorSearch(query, namespace, limit, modelConfig, opts?.topicKeys, opts?.contentType);
|
|
5069
|
+
const rows = yield* backend.vectorSearch(query, namespace, limit, modelConfig, opts?.topicKeys, opts?.contentType, opts?.includeEmbedding === true);
|
|
4764
5070
|
const allowed = allowedMemoryIds;
|
|
4765
5071
|
const scopedRows = allowed !== undefined ? rows.filter((row) => allowed.has(row.id)) : rows;
|
|
4766
5072
|
if (opts?.audit !== false) {
|
|
@@ -4779,7 +5085,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4779
5085
|
ttlSecs: r.ttl_secs,
|
|
4780
5086
|
namespace: r.namespace,
|
|
4781
5087
|
metadata: r.metadata ?? undefined,
|
|
4782
|
-
embedding: decodeBlobVector(r.embedding),
|
|
5088
|
+
embedding: opts?.includeEmbedding === true ? decodeBlobVector(r.embedding) : undefined,
|
|
4783
5089
|
embeddingModel: r.embedding_model ?? undefined
|
|
4784
5090
|
},
|
|
4785
5091
|
score: 1 - r.distance,
|
|
@@ -4787,11 +5093,13 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4787
5093
|
source: "vector"
|
|
4788
5094
|
};
|
|
4789
5095
|
});
|
|
5096
|
+
if (opts?.augment === false)
|
|
5097
|
+
return results;
|
|
4790
5098
|
return yield* augmentRetrieval(results);
|
|
4791
5099
|
});
|
|
4792
|
-
const supersede =
|
|
5100
|
+
const supersede = Effect27.fn("Store.supersede")(function* (key, namespace) {
|
|
4793
5101
|
const now = Date.now();
|
|
4794
|
-
const result = yield*
|
|
5102
|
+
const result = yield* Effect27.tryPromise({
|
|
4795
5103
|
try: () => db.transaction(async (tx) => {
|
|
4796
5104
|
const active = await tx.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.logicalKey, key), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).limit(1);
|
|
4797
5105
|
if (active.length === 0)
|
|
@@ -4803,7 +5111,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4803
5111
|
if (!result)
|
|
4804
5112
|
return yield* new NotFound({ id: key });
|
|
4805
5113
|
});
|
|
4806
|
-
const decrementTreeCf = (model, membership, x) =>
|
|
5114
|
+
const decrementTreeCf = (model, membership, x) => Effect27.tryPromise({
|
|
4807
5115
|
try: () => db.transaction(async (tx) => {
|
|
4808
5116
|
const leafRows = await tx.select({
|
|
4809
5117
|
topicKey: topics.topicKey,
|
|
@@ -4833,10 +5141,10 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4833
5141
|
}),
|
|
4834
5142
|
catch: (cause) => new StoreError({ message: `Store: tree CF decrement: ${cause}` })
|
|
4835
5143
|
});
|
|
4836
|
-
const forget =
|
|
5144
|
+
const forget = Effect27.fn("Store.forget")(function* (id) {
|
|
4837
5145
|
let membership;
|
|
4838
5146
|
if (topicsEnabled) {
|
|
4839
|
-
const memberRows = yield*
|
|
5147
|
+
const memberRows = yield* Effect27.tryPromise({
|
|
4840
5148
|
try: () => db.select({ topicKey: topicMemberships.topicKey, modelKey: topicMemberships.modelKey }).from(topicMemberships).where(eq8(topicMemberships.memoryId, id)),
|
|
4841
5149
|
catch: (cause) => new StoreError({ message: `Store.forget: load membership: ${cause}` })
|
|
4842
5150
|
});
|
|
@@ -4851,14 +5159,14 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4851
5159
|
}
|
|
4852
5160
|
}
|
|
4853
5161
|
}
|
|
4854
|
-
const rs = yield*
|
|
5162
|
+
const rs = yield* Effect27.tryPromise({
|
|
4855
5163
|
try: () => db.delete(memories2).where(eq8(memories2.id, id)),
|
|
4856
5164
|
catch: (cause) => new StoreError({ message: `Store.forget: ${cause}` })
|
|
4857
5165
|
});
|
|
4858
5166
|
if (rs.rowsAffected === 0)
|
|
4859
5167
|
return yield* new NotFound({ id });
|
|
4860
5168
|
if (topicsEnabled && membership && modelConfig && modelConfig.id === membership.modelKey) {
|
|
4861
|
-
const leafRows = yield*
|
|
5169
|
+
const leafRows = yield* Effect27.tryPromise({
|
|
4862
5170
|
try: () => db.select({ memberCount: topics.memberCount }).from(topics).where(and8(eq8(topics.topicKey, membership.topicKey), eq8(topics.modelKey, membership.modelKey))),
|
|
4863
5171
|
catch: (cause) => new StoreError({ message: `Store.forget: load leaf: ${cause}` })
|
|
4864
5172
|
});
|
|
@@ -4866,18 +5174,18 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4866
5174
|
if (liveCount < (topicsConfig.minLeafItems ?? 32)) {
|
|
4867
5175
|
const enqueueMerge = topicsOptions?.enqueueMerge;
|
|
4868
5176
|
if (enqueueMerge) {
|
|
4869
|
-
yield* enqueueMerge(membership.topicKey, membership.modelKey).pipe(
|
|
5177
|
+
yield* enqueueMerge(membership.topicKey, membership.modelKey).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.forget enqueueMerge: ${cause}` })));
|
|
4870
5178
|
} else {
|
|
4871
|
-
yield* mergeOrBorrow(membership.topicKey, modelConfig, topicsConfig).pipe(
|
|
5179
|
+
yield* mergeOrBorrow(membership.topicKey, modelConfig, topicsConfig).pipe(Effect27.provideService(Service2, backend), Effect27.mapError((cause) => new StoreError({ message: `Store.forget mergeOrBorrow: ${cause}` })));
|
|
4872
5180
|
}
|
|
4873
5181
|
}
|
|
4874
5182
|
}
|
|
4875
5183
|
});
|
|
4876
|
-
const sweep =
|
|
5184
|
+
const sweep = Effect27.fn("Store.sweep")(function* (namespace) {
|
|
4877
5185
|
return yield* backend.sweep(namespace);
|
|
4878
5186
|
});
|
|
4879
|
-
const events =
|
|
4880
|
-
const rows = yield*
|
|
5187
|
+
const events = Effect27.fn("Store.events")(function* (memoryId) {
|
|
5188
|
+
const rows = yield* Effect27.tryPromise({
|
|
4881
5189
|
try: () => db.select().from(memoryEvents2).where(eq8(memoryEvents2.memoryId, memoryId)).orderBy(asc5(memoryEvents2.createdAt)),
|
|
4882
5190
|
catch: (cause) => new StoreError({ message: `Store.events: ${cause}` })
|
|
4883
5191
|
});
|
|
@@ -4892,7 +5200,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4892
5200
|
}));
|
|
4893
5201
|
});
|
|
4894
5202
|
const resolveModel = (modelKey) => modelConfig?.id === modelKey ? modelConfig : undefined;
|
|
4895
|
-
const ensureTopicTree2 =
|
|
5203
|
+
const ensureTopicTree2 = Effect27.fn("Store.ensureTopicTree")(function* (modelKey) {
|
|
4896
5204
|
const model = resolveModel(modelKey);
|
|
4897
5205
|
if (!model) {
|
|
4898
5206
|
return yield* new ModelMismatch({
|
|
@@ -4900,33 +5208,33 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4900
5208
|
actual: modelKey
|
|
4901
5209
|
});
|
|
4902
5210
|
}
|
|
4903
|
-
return yield* ensureTopicTree(model, topicsConfig).pipe(
|
|
5211
|
+
return yield* ensureTopicTree(model, topicsConfig).pipe(Effect27.provideService(Service2, backend));
|
|
4904
5212
|
});
|
|
4905
|
-
const resolveTopics2 =
|
|
5213
|
+
const resolveTopics2 = Effect27.fn("Store.resolveTopics")(function* (queryVector, opts) {
|
|
4906
5214
|
if (!modelConfig) {
|
|
4907
5215
|
return yield* new ModelMismatch({
|
|
4908
5216
|
expected: "a configured embedding model",
|
|
4909
5217
|
actual: "(none)"
|
|
4910
5218
|
});
|
|
4911
5219
|
}
|
|
4912
|
-
return yield* resolveTopics(queryVector, modelConfig, topicsConfig, opts).pipe(
|
|
5220
|
+
return yield* resolveTopics(queryVector, modelConfig, topicsConfig, opts).pipe(Effect27.provideService(Service2, backend));
|
|
4913
5221
|
});
|
|
4914
|
-
const topicMembers =
|
|
4915
|
-
const rows = yield*
|
|
5222
|
+
const topicMembers = Effect27.fn("Store.topicMembers")(function* (topicKey) {
|
|
5223
|
+
const rows = yield* Effect27.tryPromise({
|
|
4916
5224
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(eq8(topicMemberships.topicKey, topicKey)).orderBy(asc5(topicMemberships.memoryId)),
|
|
4917
5225
|
catch: (cause) => new StoreError({ message: `Store.topicMembers: ${cause}` })
|
|
4918
5226
|
});
|
|
4919
5227
|
return rows;
|
|
4920
5228
|
});
|
|
4921
|
-
const listTopics =
|
|
4922
|
-
const rows = yield*
|
|
5229
|
+
const listTopics = Effect27.fn("Store.listTopics")(function* (modelKey) {
|
|
5230
|
+
const rows = yield* Effect27.tryPromise({
|
|
4923
5231
|
try: () => db.select().from(topics).where(eq8(topics.modelKey, modelKey)).orderBy(asc5(topics.topicKey)),
|
|
4924
5232
|
catch: (cause) => new StoreError({ message: `Store.listTopics: ${cause}` })
|
|
4925
5233
|
});
|
|
4926
5234
|
return rows;
|
|
4927
5235
|
});
|
|
4928
|
-
const topicStats =
|
|
4929
|
-
const rows = yield*
|
|
5236
|
+
const topicStats = Effect27.fn("Store.topicStats")(function* (modelKey) {
|
|
5237
|
+
const rows = yield* Effect27.tryPromise({
|
|
4930
5238
|
try: () => db.select({ kind: topics.kind, depth: topics.depth }).from(topics).where(eq8(topics.modelKey, modelKey)),
|
|
4931
5239
|
catch: (cause) => new StoreError({ message: `Store.topicStats: ${cause}` })
|
|
4932
5240
|
});
|
|
@@ -4943,7 +5251,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4943
5251
|
}
|
|
4944
5252
|
return { depth, leafCount, internalCount, totalTopics: rows.length };
|
|
4945
5253
|
});
|
|
4946
|
-
const deleteTree =
|
|
5254
|
+
const deleteTree = Effect27.fn("Store.deleteTree")(function* (modelKey) {
|
|
4947
5255
|
const model = resolveModel(modelKey);
|
|
4948
5256
|
if (!model) {
|
|
4949
5257
|
return yield* new ModelMismatch({
|
|
@@ -4952,7 +5260,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4952
5260
|
});
|
|
4953
5261
|
}
|
|
4954
5262
|
yield* backend.deleteTopicVectors(model);
|
|
4955
|
-
yield*
|
|
5263
|
+
yield* Effect27.tryPromise({
|
|
4956
5264
|
try: () => db.transaction(async (tx) => {
|
|
4957
5265
|
const modelTopicKeys2 = tx.select({ topicKey: topics.topicKey }).from(topics).where(eq8(topics.modelKey, modelKey));
|
|
4958
5266
|
await tx.delete(topicChangeLog).where(inArray4(topicChangeLog.topicKey, modelTopicKeys2));
|
|
@@ -4964,7 +5272,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4964
5272
|
catch: (cause) => new StoreError({ message: `Store.deleteTree: ${cause}` })
|
|
4965
5273
|
});
|
|
4966
5274
|
});
|
|
4967
|
-
const processSplits =
|
|
5275
|
+
const processSplits = Effect27.fn("Store.processSplits")(function* (modelKey) {
|
|
4968
5276
|
const model = resolveModel(modelKey);
|
|
4969
5277
|
if (!model) {
|
|
4970
5278
|
return yield* new ModelMismatch({
|
|
@@ -4974,19 +5282,19 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4974
5282
|
}
|
|
4975
5283
|
let processed = 0;
|
|
4976
5284
|
for (let iteration = 0;iteration < 100; iteration++) {
|
|
4977
|
-
const leaves = yield*
|
|
5285
|
+
const leaves = yield* Effect27.tryPromise({
|
|
4978
5286
|
try: () => db.select({ topicKey: topics.topicKey, memberCount: topics.memberCount }).from(topics).where(and8(eq8(topics.modelKey, modelKey), eq8(topics.kind, "leaf"), ne4(topics.status, "inactive"))).orderBy(asc5(topics.topicKey)),
|
|
4979
5287
|
catch: (cause) => new StoreError({ message: `Store.processSplits: ${cause}` })
|
|
4980
5288
|
});
|
|
4981
5289
|
const candidate = leaves.find((leaf) => shouldSplit(leaf.memberCount, topicsConfig));
|
|
4982
5290
|
if (!candidate)
|
|
4983
5291
|
break;
|
|
4984
|
-
yield* splitTopic(candidate.topicKey, model, topicsConfig).pipe(
|
|
5292
|
+
yield* splitTopic(candidate.topicKey, model, topicsConfig).pipe(Effect27.provideService(Service2, backend), Effect27.mapError((cause) => new StoreError({ message: `Store.processSplits: ${cause}` })));
|
|
4985
5293
|
processed++;
|
|
4986
5294
|
}
|
|
4987
5295
|
return processed;
|
|
4988
5296
|
});
|
|
4989
|
-
const mergeOrBorrow2 =
|
|
5297
|
+
const mergeOrBorrow2 = Effect27.fn("Store.mergeOrBorrow")(function* (modelKey, topicKey) {
|
|
4990
5298
|
const model = resolveModel(modelKey);
|
|
4991
5299
|
if (!model) {
|
|
4992
5300
|
return yield* new ModelMismatch({
|
|
@@ -4994,9 +5302,9 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
4994
5302
|
actual: modelKey
|
|
4995
5303
|
});
|
|
4996
5304
|
}
|
|
4997
|
-
return yield* mergeOrBorrow(topicKey, model, topicsConfig).pipe(
|
|
5305
|
+
return yield* mergeOrBorrow(topicKey, model, topicsConfig).pipe(Effect27.provideService(Service2, backend), Effect27.mapError((cause) => new StoreError({ message: `Store.mergeOrBorrow: ${cause}` })));
|
|
4998
5306
|
});
|
|
4999
|
-
const rebalanceSiblings2 =
|
|
5307
|
+
const rebalanceSiblings2 = Effect27.fn("Store.rebalanceSiblings")(function* (modelKey, topicKey) {
|
|
5000
5308
|
const model = resolveModel(modelKey);
|
|
5001
5309
|
if (!model) {
|
|
5002
5310
|
return yield* new ModelMismatch({
|
|
@@ -5004,15 +5312,15 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5004
5312
|
actual: modelKey
|
|
5005
5313
|
});
|
|
5006
5314
|
}
|
|
5007
|
-
return yield* rebalanceSiblings(topicKey, model, topicsConfig).pipe(
|
|
5315
|
+
return yield* rebalanceSiblings(topicKey, model, topicsConfig).pipe(Effect27.provideService(Service2, backend), Effect27.mapError((cause) => new StoreError({ message: `Store.rebalanceSiblings: ${cause}` })));
|
|
5008
5316
|
});
|
|
5009
|
-
const setActiveModel2 =
|
|
5010
|
-
return yield* setActiveModel(model, topicsConfig).pipe(
|
|
5317
|
+
const setActiveModel2 = Effect27.fn("Store.setActiveModel")(function* (model) {
|
|
5318
|
+
return yield* setActiveModel(model, topicsConfig).pipe(Effect27.provideService(Service2, backend));
|
|
5011
5319
|
});
|
|
5012
|
-
const isActive2 =
|
|
5013
|
-
return yield* isActive(modelKey).pipe(
|
|
5320
|
+
const isActive2 = Effect27.fn("Store.isActive")(function* (modelKey) {
|
|
5321
|
+
return yield* isActive(modelKey).pipe(Effect27.provideService(Service2, backend));
|
|
5014
5322
|
});
|
|
5015
|
-
const topicMetrics2 =
|
|
5323
|
+
const topicMetrics2 = Effect27.fn("Store.topicMetrics")(function* (modelKey) {
|
|
5016
5324
|
const model = resolveModel(modelKey);
|
|
5017
5325
|
if (!model) {
|
|
5018
5326
|
return yield* new ModelMismatch({
|
|
@@ -5020,9 +5328,9 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5020
5328
|
actual: modelKey
|
|
5021
5329
|
});
|
|
5022
5330
|
}
|
|
5023
|
-
return yield* topicMetrics(modelKey, topicsConfig).pipe(
|
|
5331
|
+
return yield* topicMetrics(modelKey, topicsConfig).pipe(Effect27.provideService(Service2, backend));
|
|
5024
5332
|
});
|
|
5025
|
-
const topicChangeLog3 =
|
|
5333
|
+
const topicChangeLog3 = Effect27.fn("Store.topicChangeLog")(function* (modelKey, opts) {
|
|
5026
5334
|
const model = resolveModel(modelKey);
|
|
5027
5335
|
if (!model) {
|
|
5028
5336
|
return yield* new ModelMismatch({
|
|
@@ -5030,9 +5338,9 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5030
5338
|
actual: modelKey
|
|
5031
5339
|
});
|
|
5032
5340
|
}
|
|
5033
|
-
return yield* topicChangeLog2(modelKey, opts).pipe(
|
|
5341
|
+
return yield* topicChangeLog2(modelKey, opts).pipe(Effect27.provideService(Service2, backend));
|
|
5034
5342
|
});
|
|
5035
|
-
const topicHealth2 =
|
|
5343
|
+
const topicHealth2 = Effect27.fn("Store.topicHealth")(function* (modelKey) {
|
|
5036
5344
|
const model = resolveModel(modelKey);
|
|
5037
5345
|
if (!model) {
|
|
5038
5346
|
return yield* new ModelMismatch({
|
|
@@ -5040,11 +5348,11 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5040
5348
|
actual: modelKey
|
|
5041
5349
|
});
|
|
5042
5350
|
}
|
|
5043
|
-
return yield* topicHealth(modelKey, topicsConfig).pipe(
|
|
5351
|
+
return yield* topicHealth(modelKey, topicsConfig).pipe(Effect27.provideService(Service2, backend));
|
|
5044
5352
|
});
|
|
5045
|
-
const updateEmbedding =
|
|
5353
|
+
const updateEmbedding = Effect27.fn("Store.updateEmbedding")(function* (id, embedding, modelName, options) {
|
|
5046
5354
|
const now = Date.now();
|
|
5047
|
-
const affected = yield*
|
|
5355
|
+
const affected = yield* Effect27.tryPromise({
|
|
5048
5356
|
try: () => db.transaction(async (tx) => {
|
|
5049
5357
|
const rs = await tx.update(memories2).set({ embedding: [...embedding], embeddingModel: modelName }).where(eq8(memories2.id, id));
|
|
5050
5358
|
if (rs.rowsAffected === 0)
|
|
@@ -5074,7 +5382,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5074
5382
|
if (modelConfig && modelConfig.id === modelName) {
|
|
5075
5383
|
yield* backend.ensureVectorTable(modelConfig);
|
|
5076
5384
|
if (topicsEnabled) {
|
|
5077
|
-
const memberRows = yield*
|
|
5385
|
+
const memberRows = yield* Effect27.tryPromise({
|
|
5078
5386
|
try: () => db.select({
|
|
5079
5387
|
topicKey: topicMemberships.topicKey,
|
|
5080
5388
|
modelKey: topicMemberships.modelKey
|
|
@@ -5089,7 +5397,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5089
5397
|
yield* decrementTreeCf(modelConfig, mem, x);
|
|
5090
5398
|
yield* updateTopicVectorForget(backend, modelConfig, mem.topicKey, x);
|
|
5091
5399
|
}
|
|
5092
|
-
yield*
|
|
5400
|
+
yield* Effect27.tryPromise({
|
|
5093
5401
|
try: () => db.delete(topicMemberships).where(and8(eq8(topicMemberships.memoryId, id), eq8(topicMemberships.modelKey, modelName))),
|
|
5094
5402
|
catch: (cause) => new StoreError({ message: `Store.updateEmbedding: delete membership: ${cause}` })
|
|
5095
5403
|
});
|
|
@@ -5097,23 +5405,76 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5097
5405
|
}
|
|
5098
5406
|
yield* backend.insertVector(id, embedding, modelConfig);
|
|
5099
5407
|
if (topicsEnabled) {
|
|
5100
|
-
const routed = yield* routeInsert(id, embedding, modelConfig, topicsConfig).pipe(
|
|
5101
|
-
if (options?.scanCandidates !== false) {
|
|
5408
|
+
const routed = yield* routeInsert(id, embedding, modelConfig, topicsConfig).pipe(Effect27.provideService(Service2, backend), Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbedding topic route: ${cause}` })));
|
|
5409
|
+
if (options?.scanCandidates !== false && graphEnabled) {
|
|
5102
5410
|
yield* scanLeafCandidates(id, embedding, routed.topicKey, modelConfig);
|
|
5103
5411
|
}
|
|
5104
5412
|
if (routed.overflow) {
|
|
5105
5413
|
const enqueueSplit = topicsOptions?.enqueueSplit;
|
|
5106
5414
|
if (enqueueSplit) {
|
|
5107
|
-
yield* enqueueSplit(routed.topicKey, modelConfig.id).pipe(
|
|
5415
|
+
yield* enqueueSplit(routed.topicKey, modelConfig.id).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbedding enqueueSplit: ${cause}` })));
|
|
5416
|
+
} else {
|
|
5417
|
+
yield* processSplits(modelConfig.id).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbedding processSplits: ${cause}` })));
|
|
5418
|
+
}
|
|
5419
|
+
}
|
|
5420
|
+
}
|
|
5421
|
+
}
|
|
5422
|
+
});
|
|
5423
|
+
const updateEmbeddingMany = Effect27.fn("Store.updateEmbeddingMany")(function* (entries, modelName, options) {
|
|
5424
|
+
if (entries.length === 0)
|
|
5425
|
+
return;
|
|
5426
|
+
const now = Date.now();
|
|
5427
|
+
yield* Effect27.tryPromise({
|
|
5428
|
+
try: () => db.transaction(async (tx) => {
|
|
5429
|
+
await tx.run(sql7`UPDATE ${memories2} SET embedding = CASE id ${sql7.join(entries.map((entry) => sql7`WHEN ${entry.id} THEN ${new Float32Array(entry.embedding).buffer}`), sql7` `)} END, embedding_model = ${modelName} WHERE id IN (${sql7.join(entries.map((entry) => entry.id), sql7`, `)})`);
|
|
5430
|
+
await tx.insert(memoryEvents2).values(entries.map((entry) => ({
|
|
5431
|
+
memoryId: entry.id,
|
|
5432
|
+
eventType: "embedding_updated",
|
|
5433
|
+
reason: modelName,
|
|
5434
|
+
createdAt: now
|
|
5435
|
+
})));
|
|
5436
|
+
}),
|
|
5437
|
+
catch: (cause) => {
|
|
5438
|
+
let chain = "";
|
|
5439
|
+
let c = cause;
|
|
5440
|
+
let i = 0;
|
|
5441
|
+
while (c && i < 6) {
|
|
5442
|
+
chain += " | " + String(c.message ?? c);
|
|
5443
|
+
c = c.cause;
|
|
5444
|
+
i++;
|
|
5445
|
+
}
|
|
5446
|
+
return new StoreError({ message: `Store.updateEmbeddingMany: ${cause}` + chain });
|
|
5447
|
+
}
|
|
5448
|
+
});
|
|
5449
|
+
if (modelConfig && modelConfig.id === modelName) {
|
|
5450
|
+
yield* backend.ensureVectorTable(modelConfig);
|
|
5451
|
+
yield* backend.insertVectors(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig);
|
|
5452
|
+
if (topicsEnabled) {
|
|
5453
|
+
const routed = yield* routeInsertMany(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig, topicsConfig).pipe(Effect27.provideService(Service2, backend), Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany route: ${cause}` })));
|
|
5454
|
+
if (options?.scanCandidates === true && graphEnabled) {
|
|
5455
|
+
for (const routedEntry of routed) {
|
|
5456
|
+
const entry = entries.find((e) => e.id === routedEntry.memoryId);
|
|
5457
|
+
if (entry) {
|
|
5458
|
+
yield* scanLeafCandidates(routedEntry.memoryId, entry.embedding, routedEntry.topicKey, modelConfig);
|
|
5459
|
+
}
|
|
5460
|
+
}
|
|
5461
|
+
}
|
|
5462
|
+
const overflowed = [...new Set(routed.filter((r) => r.overflow).map((r) => r.topicKey))];
|
|
5463
|
+
if (overflowed.length > 0) {
|
|
5464
|
+
const enqueueSplit = topicsOptions?.enqueueSplit;
|
|
5465
|
+
if (enqueueSplit) {
|
|
5466
|
+
for (const topicKey of overflowed) {
|
|
5467
|
+
yield* enqueueSplit(topicKey, modelConfig.id).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany enqueueSplit: ${cause}` })));
|
|
5468
|
+
}
|
|
5108
5469
|
} else {
|
|
5109
|
-
yield* processSplits(modelConfig.id).pipe(
|
|
5470
|
+
yield* processSplits(modelConfig.id).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany processSplits: ${cause}` })));
|
|
5110
5471
|
}
|
|
5111
5472
|
}
|
|
5112
5473
|
}
|
|
5113
5474
|
}
|
|
5114
5475
|
});
|
|
5115
|
-
const findStale =
|
|
5116
|
-
const rows = yield*
|
|
5476
|
+
const findStale = Effect27.fn("Store.findStale")(function* (model) {
|
|
5477
|
+
const rows = yield* Effect27.tryPromise({
|
|
5117
5478
|
try: () => db.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.state, "active"), sql7`(embedding IS NULL OR embedding_model IS NULL OR embedding_model != ${model})`)),
|
|
5118
5479
|
catch: (cause) => new StoreError({ message: `Store.findStale: ${cause}` })
|
|
5119
5480
|
});
|
|
@@ -5133,14 +5494,14 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5133
5494
|
});
|
|
5134
5495
|
return memoryId;
|
|
5135
5496
|
};
|
|
5136
|
-
const reconcile =
|
|
5497
|
+
const reconcile = Effect27.fn("Store.reconcile")(function* (namespace, keyPrefix, keyRoot, walkedRoots, keptKeys) {
|
|
5137
5498
|
const inDomain = (key) => {
|
|
5138
5499
|
const rel = key.slice(keyPrefix.length).split("#")[0];
|
|
5139
5500
|
const filePath = keyRoot === "/" ? `/${rel}` : `${keyRoot}/${rel}`;
|
|
5140
5501
|
return walkedRoots.some((root) => filePath === root || filePath.startsWith(`${root}/`));
|
|
5141
5502
|
};
|
|
5142
5503
|
const owned = (key) => keptKeys.includes(key);
|
|
5143
|
-
const rows = yield*
|
|
5504
|
+
const rows = yield* Effect27.tryPromise({
|
|
5144
5505
|
try: () => db.select({
|
|
5145
5506
|
id: memories2.id,
|
|
5146
5507
|
logicalKey: memories2.logicalKey,
|
|
@@ -5151,7 +5512,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5151
5512
|
const vanished = rows.filter((r) => inDomain(r.logicalKey) && !owned(r.logicalKey));
|
|
5152
5513
|
if (vanished.length > 0) {
|
|
5153
5514
|
const now = Date.now();
|
|
5154
|
-
yield*
|
|
5515
|
+
yield* Effect27.tryPromise({
|
|
5155
5516
|
try: () => db.transaction(async (tx) => {
|
|
5156
5517
|
for (const row of vanished) {
|
|
5157
5518
|
await flipToSupersededById(tx, row.id, now, "source_removed");
|
|
@@ -5162,7 +5523,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5162
5523
|
}
|
|
5163
5524
|
return vanished;
|
|
5164
5525
|
});
|
|
5165
|
-
const writeSuperseding =
|
|
5526
|
+
const writeSuperseding = Effect27.fn("Store.writeSuperseding")(function* (input) {
|
|
5166
5527
|
if (input.content.includes("\x00")) {
|
|
5167
5528
|
return yield* new StoreError({
|
|
5168
5529
|
message: "Store.writeSuperseding: content contains a NUL byte (libSQL would silently truncate it)"
|
|
@@ -5173,7 +5534,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5173
5534
|
const metadataBlob = input.typedMetadata !== undefined ? mergeMetadataBlob(input.metadata, input.typedMetadata) : input.metadata ?? null;
|
|
5174
5535
|
const namespace = input.namespace ?? "default";
|
|
5175
5536
|
let supersededId;
|
|
5176
|
-
yield*
|
|
5537
|
+
yield* Effect27.tryPromise({
|
|
5177
5538
|
try: () => db.transaction(async (tx) => {
|
|
5178
5539
|
const existing = await tx.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.logicalKey, input.logicalKey), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).limit(1);
|
|
5179
5540
|
const old = existing[0];
|
|
@@ -5228,7 +5589,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5228
5589
|
...supersededId !== undefined ? { supersededId } : {}
|
|
5229
5590
|
};
|
|
5230
5591
|
});
|
|
5231
|
-
const addEdge =
|
|
5592
|
+
const addEdge = Effect27.fn("Store.addEdge")(function* (input) {
|
|
5232
5593
|
if (input.fromMemoryId === input.toMemoryId) {
|
|
5233
5594
|
return yield* new StoreError({ message: "Store.addEdge: self-loops are not allowed" });
|
|
5234
5595
|
}
|
|
@@ -5254,7 +5615,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5254
5615
|
message: `Store.addEdge: confidence must be a finite number in [0,1] (got ${String(confidence)})`
|
|
5255
5616
|
});
|
|
5256
5617
|
}
|
|
5257
|
-
const endpoints = yield*
|
|
5618
|
+
const endpoints = yield* Effect27.tryPromise({
|
|
5258
5619
|
try: () => db.select({ id: memories2.id }).from(memories2).where(inArray4(memories2.id, [input.fromMemoryId, input.toMemoryId])),
|
|
5259
5620
|
catch: (cause) => new StoreError({ message: `Store.addEdge: endpoints: ${cause}` })
|
|
5260
5621
|
});
|
|
@@ -5279,7 +5640,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5279
5640
|
createdAt: now,
|
|
5280
5641
|
updatedAt: now
|
|
5281
5642
|
};
|
|
5282
|
-
yield*
|
|
5643
|
+
yield* Effect27.tryPromise({
|
|
5283
5644
|
try: () => db.transaction(async (tx) => {
|
|
5284
5645
|
await tx.insert(artifactEdges).values(row);
|
|
5285
5646
|
if (input.confirmed && input.edgeKind === "memory_supersedes") {
|
|
@@ -5299,17 +5660,17 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5299
5660
|
});
|
|
5300
5661
|
return row;
|
|
5301
5662
|
});
|
|
5302
|
-
const expireEdge =
|
|
5303
|
-
const rs = yield*
|
|
5663
|
+
const expireEdge = Effect27.fn("Store.expireEdge")(function* (id) {
|
|
5664
|
+
const rs = yield* Effect27.tryPromise({
|
|
5304
5665
|
try: () => db.update(artifactEdges).set({ expiredAt: Date.now(), updatedAt: Date.now() }).where(eq8(artifactEdges.id, id)),
|
|
5305
5666
|
catch: (cause) => new StoreError({ message: `Store.expireEdge: ${cause}` })
|
|
5306
5667
|
});
|
|
5307
5668
|
if (rs.rowsAffected === 0)
|
|
5308
5669
|
return yield* new EdgeNotFound({ id });
|
|
5309
5670
|
});
|
|
5310
|
-
const confirmEdge =
|
|
5671
|
+
const confirmEdge = Effect27.fn("Store.confirmEdge")(function* (id) {
|
|
5311
5672
|
const now = Date.now();
|
|
5312
|
-
const found = yield*
|
|
5673
|
+
const found = yield* Effect27.tryPromise({
|
|
5313
5674
|
try: () => db.transaction(async (tx) => {
|
|
5314
5675
|
const rows = await tx.select().from(artifactEdges).where(eq8(artifactEdges.id, id)).limit(1);
|
|
5315
5676
|
const edge = rows[0];
|
|
@@ -5326,15 +5687,15 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5326
5687
|
if (!found)
|
|
5327
5688
|
return yield* new EdgeNotFound({ id });
|
|
5328
5689
|
});
|
|
5329
|
-
const rejectEdge =
|
|
5330
|
-
const rs = yield*
|
|
5690
|
+
const rejectEdge = Effect27.fn("Store.rejectEdge")(function* (id) {
|
|
5691
|
+
const rs = yield* Effect27.tryPromise({
|
|
5331
5692
|
try: () => db.update(artifactEdges).set({ status: "rejected", updatedAt: Date.now() }).where(eq8(artifactEdges.id, id)),
|
|
5332
5693
|
catch: (cause) => new StoreError({ message: `Store.rejectEdge: ${cause}` })
|
|
5333
5694
|
});
|
|
5334
5695
|
if (rs.rowsAffected === 0)
|
|
5335
5696
|
return yield* new EdgeNotFound({ id });
|
|
5336
5697
|
});
|
|
5337
|
-
const listEdges =
|
|
5698
|
+
const listEdges = Effect27.fn("Store.listEdges")(function* (opts) {
|
|
5338
5699
|
const conditions = [];
|
|
5339
5700
|
if (!opts?.includeExpired)
|
|
5340
5701
|
conditions.push(isNull2(artifactEdges.expiredAt));
|
|
@@ -5345,7 +5706,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5345
5706
|
conditions.push(eq8(artifactEdges.edgeKind, opts.edgeKind));
|
|
5346
5707
|
if (opts?.status)
|
|
5347
5708
|
conditions.push(eq8(artifactEdges.status, opts.status));
|
|
5348
|
-
const rows = yield*
|
|
5709
|
+
const rows = yield* Effect27.tryPromise({
|
|
5349
5710
|
try: () => {
|
|
5350
5711
|
const base = db.select().from(artifactEdges);
|
|
5351
5712
|
return (conditions.length === 0 ? base : base.where(and8(...conditions))).orderBy(desc2(artifactEdges.createdAt));
|
|
@@ -5354,7 +5715,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5354
5715
|
});
|
|
5355
5716
|
return rows;
|
|
5356
5717
|
});
|
|
5357
|
-
const explain =
|
|
5718
|
+
const explain = Effect27.fn("Store.explain")(function* (node, opts) {
|
|
5358
5719
|
if (node.kind !== "memory" && node.kind !== "topic" && node.kind !== "concept") {
|
|
5359
5720
|
return yield* new StoreError({
|
|
5360
5721
|
message: `Store.explain: unknown kind "${String(node.kind)}"`
|
|
@@ -5398,37 +5759,41 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5398
5759
|
pendingConflicts: pending
|
|
5399
5760
|
};
|
|
5400
5761
|
});
|
|
5401
|
-
const
|
|
5762
|
+
const writeCandidateEdges = (candidates, now) => Effect27.tryPromise({
|
|
5402
5763
|
try: () => db.transaction(async (tx) => {
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
|
|
5408
|
-
|
|
5409
|
-
|
|
5410
|
-
|
|
5411
|
-
|
|
5412
|
-
|
|
5413
|
-
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5764
|
+
let inserted = 0;
|
|
5765
|
+
for (const candidate of candidates) {
|
|
5766
|
+
const existing = await tx.select({ id: artifactEdges.id }).from(artifactEdges).where(and8(eq8(artifactEdges.fromMemoryId, candidate.fromMemoryId), eq8(artifactEdges.toMemoryId, candidate.toMemoryId), eq8(artifactEdges.edgeKind, candidate.edgeKind))).limit(1);
|
|
5767
|
+
if (existing.length > 0)
|
|
5768
|
+
continue;
|
|
5769
|
+
await tx.insert(artifactEdges).values({
|
|
5770
|
+
id: crypto.randomUUID(),
|
|
5771
|
+
fromMemoryId: candidate.fromMemoryId,
|
|
5772
|
+
toMemoryId: candidate.toMemoryId,
|
|
5773
|
+
edgeKind: candidate.edgeKind,
|
|
5774
|
+
weight: 1,
|
|
5775
|
+
status: candidate.status,
|
|
5776
|
+
source: candidate.source,
|
|
5777
|
+
confidence: candidate.confidence,
|
|
5778
|
+
validAt: candidate.status === "confirmed" ? now : null,
|
|
5779
|
+
expiredAt: null,
|
|
5780
|
+
createdAt: now,
|
|
5781
|
+
updatedAt: now
|
|
5782
|
+
});
|
|
5783
|
+
if (candidate.status === "confirmed") {
|
|
5784
|
+
await flipToSupersededById(tx, candidate.fromMemoryId, now);
|
|
5785
|
+
}
|
|
5786
|
+
inserted += 1;
|
|
5422
5787
|
}
|
|
5423
|
-
return
|
|
5788
|
+
return inserted;
|
|
5424
5789
|
}),
|
|
5425
|
-
catch: (cause) => new StoreError({ message: `Store.
|
|
5790
|
+
catch: (cause) => new StoreError({ message: `Store.writeCandidateEdges: ${cause}` })
|
|
5426
5791
|
});
|
|
5427
|
-
const reconcileRenameCandidates =
|
|
5792
|
+
const reconcileRenameCandidates = Effect27.fn("Store.reconcileRenameCandidates")(function* (vanished, newUnitIds) {
|
|
5428
5793
|
const withVectors = vanished.filter((v) => v.embedding !== null);
|
|
5429
5794
|
if (withVectors.length === 0 || newUnitIds.length === 0)
|
|
5430
5795
|
return 0;
|
|
5431
|
-
const newRows = yield*
|
|
5796
|
+
const newRows = yield* Effect27.tryPromise({
|
|
5432
5797
|
try: () => db.select({ id: memories2.id, embedding: memories2.embedding }).from(memories2).where(and8(inArray4(memories2.id, newUnitIds))),
|
|
5433
5798
|
catch: (cause) => new StoreError({ message: `Store.reconcileRenameCandidates: ${cause}` })
|
|
5434
5799
|
});
|
|
@@ -5437,6 +5802,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5437
5802
|
return 0;
|
|
5438
5803
|
const now = Date.now();
|
|
5439
5804
|
let inserted = 0;
|
|
5805
|
+
const batch = [];
|
|
5440
5806
|
for (const v of withVectors) {
|
|
5441
5807
|
const vVec = Float32Array.from(v.embedding);
|
|
5442
5808
|
for (const n of newWithVectors) {
|
|
@@ -5444,7 +5810,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5444
5810
|
continue;
|
|
5445
5811
|
const cosine = cosineSimilarity(vVec, Float32Array.from(n.embedding));
|
|
5446
5812
|
if (cosine >= GRAPH_CONFLICT_BANDS.nearDuplicate) {
|
|
5447
|
-
|
|
5813
|
+
batch.push({
|
|
5448
5814
|
fromMemoryId: v.id,
|
|
5449
5815
|
toMemoryId: n.id,
|
|
5450
5816
|
edgeKind: "memory_supersedes",
|
|
@@ -5452,24 +5818,23 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5452
5818
|
status: "pending",
|
|
5453
5819
|
confidence: cosine,
|
|
5454
5820
|
reason: "rename"
|
|
5455
|
-
}
|
|
5456
|
-
if (ok)
|
|
5457
|
-
inserted += 1;
|
|
5821
|
+
});
|
|
5458
5822
|
}
|
|
5459
5823
|
}
|
|
5460
5824
|
}
|
|
5825
|
+
inserted = yield* writeCandidateEdges(batch, now);
|
|
5461
5826
|
return inserted;
|
|
5462
5827
|
});
|
|
5463
|
-
const scanLeafCandidates = (memoryId, vector, topicKey, model) =>
|
|
5464
|
-
const memberRows = yield*
|
|
5828
|
+
const scanLeafCandidates = (memoryId, vector, topicKey, model) => Effect27.gen(function* () {
|
|
5829
|
+
const memberRows = yield* Effect27.tryPromise({
|
|
5465
5830
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and8(eq8(topicMemberships.topicKey, topicKey), eq8(topicMemberships.modelKey, model.id))).orderBy(asc5(topicMemberships.memoryId)),
|
|
5466
5831
|
catch: (cause) => new StoreError({ message: `Store.updateEmbedding candidates: members: ${cause}` })
|
|
5467
5832
|
});
|
|
5468
5833
|
const memberIds = memberRows.map((r) => r.memoryId);
|
|
5469
5834
|
if (memberIds.length === 0)
|
|
5470
5835
|
return;
|
|
5471
|
-
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(
|
|
5472
|
-
const memoryRows = yield*
|
|
5836
|
+
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.updateEmbedding candidates: vectors: ${cause}` })));
|
|
5837
|
+
const memoryRows = yield* Effect27.tryPromise({
|
|
5473
5838
|
try: () => db.select({
|
|
5474
5839
|
id: memories2.id,
|
|
5475
5840
|
logicalKey: memories2.logicalKey,
|
|
@@ -5478,7 +5843,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5478
5843
|
}).from(memories2).where(and8(inArray4(memories2.id, memberIds), eq8(memories2.state, "active"))),
|
|
5479
5844
|
catch: (cause) => new StoreError({ message: `Store.updateEmbedding candidates: memories: ${cause}` })
|
|
5480
5845
|
});
|
|
5481
|
-
const selfRows = yield*
|
|
5846
|
+
const selfRows = yield* Effect27.tryPromise({
|
|
5482
5847
|
try: () => db.select({
|
|
5483
5848
|
logicalKey: memories2.logicalKey,
|
|
5484
5849
|
createdAt: memories2.createdAt,
|
|
@@ -5504,13 +5869,11 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5504
5869
|
});
|
|
5505
5870
|
const candidates = leafLocalCandidates(memoryId, new Float32Array(vector), self.createdAt, self.logicalKey, self.metadata, members, GRAPH_CONFLICT_BANDS);
|
|
5506
5871
|
const now = Date.now();
|
|
5507
|
-
|
|
5508
|
-
yield* writeCandidateEdge(candidate, now);
|
|
5509
|
-
}
|
|
5872
|
+
yield* writeCandidateEdges(candidates, now);
|
|
5510
5873
|
});
|
|
5511
|
-
const graphCandidates =
|
|
5874
|
+
const graphCandidates = Effect27.fn("Store.graphCandidates")(function* (opts) {
|
|
5512
5875
|
const limit = Math.max(1, Math.min(opts?.limit ?? 50, GRAPH_CANDIDATE_LIMIT_MAX));
|
|
5513
|
-
const rows = yield*
|
|
5876
|
+
const rows = yield* Effect27.tryPromise({
|
|
5514
5877
|
try: () => db.select({
|
|
5515
5878
|
id: artifactEdges.id,
|
|
5516
5879
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
@@ -5522,7 +5885,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5522
5885
|
}).from(artifactEdges).where(and8(eq8(artifactEdges.status, "pending"), isNull2(artifactEdges.expiredAt))).orderBy(desc2(artifactEdges.createdAt), asc5(artifactEdges.id)).limit(limit),
|
|
5523
5886
|
catch: (cause) => new StoreError({ message: `Store.graphCandidates: pending: ${cause}` })
|
|
5524
5887
|
});
|
|
5525
|
-
const isolatedRows = yield* backend.listIsolatedMemories(0).pipe(
|
|
5888
|
+
const isolatedRows = yield* backend.listIsolatedMemories(0).pipe(Effect27.mapError((cause) => new StoreError({ message: `Store.graphCandidates: isolated: ${cause}` })));
|
|
5526
5889
|
const isolated = isolatedRows.map((row) => ({
|
|
5527
5890
|
id: String(row.id),
|
|
5528
5891
|
createdAt: Number(row.createdAt)
|
|
@@ -5541,13 +5904,13 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5541
5904
|
oldestIsolated: isolated.slice(0, GRAPH_ISOLATED_EXAMPLES)
|
|
5542
5905
|
};
|
|
5543
5906
|
});
|
|
5544
|
-
const graphCoverage2 =
|
|
5545
|
-
return yield* graphCoverage(modelKey).pipe(
|
|
5907
|
+
const graphCoverage2 = Effect27.fn("Store.graphCoverage")(function* (modelKey) {
|
|
5908
|
+
return yield* graphCoverage(modelKey).pipe(Effect27.provideService(Service2, backend));
|
|
5546
5909
|
});
|
|
5547
|
-
const createConcept =
|
|
5910
|
+
const createConcept = Effect27.fn("Store.createConcept")(function* (name, embedding) {
|
|
5548
5911
|
const id = crypto.randomUUID();
|
|
5549
5912
|
const now = Date.now();
|
|
5550
|
-
yield*
|
|
5913
|
+
yield* Effect27.tryPromise({
|
|
5551
5914
|
try: () => db.insert(concepts2).values({
|
|
5552
5915
|
id,
|
|
5553
5916
|
name,
|
|
@@ -5567,8 +5930,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5567
5930
|
lastSeen: now
|
|
5568
5931
|
};
|
|
5569
5932
|
});
|
|
5570
|
-
const getConcept =
|
|
5571
|
-
const rows = yield*
|
|
5933
|
+
const getConcept = Effect27.fn("Store.getConcept")(function* (id) {
|
|
5934
|
+
const rows = yield* Effect27.tryPromise({
|
|
5572
5935
|
try: () => db.select().from(concepts2).where(eq8(concepts2.id, id)).limit(1),
|
|
5573
5936
|
catch: (cause) => new StoreError({ message: `Store.getConcept: ${cause}` })
|
|
5574
5937
|
});
|
|
@@ -5584,8 +5947,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5584
5947
|
lastSeen: row.lastSeen
|
|
5585
5948
|
};
|
|
5586
5949
|
});
|
|
5587
|
-
const listConcepts =
|
|
5588
|
-
const rows = yield*
|
|
5950
|
+
const listConcepts = Effect27.fn("Store.listConcepts")(function* () {
|
|
5951
|
+
const rows = yield* Effect27.tryPromise({
|
|
5589
5952
|
try: () => db.select().from(concepts2),
|
|
5590
5953
|
catch: (cause) => new StoreError({ message: `Store.listConcepts: ${cause}` })
|
|
5591
5954
|
});
|
|
@@ -5598,8 +5961,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5598
5961
|
lastSeen: row.lastSeen
|
|
5599
5962
|
}));
|
|
5600
5963
|
});
|
|
5601
|
-
const annotateMemory =
|
|
5602
|
-
yield*
|
|
5964
|
+
const annotateMemory = Effect27.fn("Store.annotateMemory")(function* (memoryId, conceptId, confidence) {
|
|
5965
|
+
yield* Effect27.tryPromise({
|
|
5603
5966
|
try: () => db.insert(memoryConcepts2).values({
|
|
5604
5967
|
conceptId,
|
|
5605
5968
|
memoryId,
|
|
@@ -5609,14 +5972,14 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5609
5972
|
catch: (cause) => new StoreError({ message: `Store.annotateMemory: ${cause}` })
|
|
5610
5973
|
});
|
|
5611
5974
|
});
|
|
5612
|
-
const getMemoryAnnotations =
|
|
5613
|
-
const rows = yield*
|
|
5975
|
+
const getMemoryAnnotations = Effect27.fn("Store.getMemoryAnnotations")(function* (memoryId) {
|
|
5976
|
+
const rows = yield* Effect27.tryPromise({
|
|
5614
5977
|
try: () => db.select({ conceptId: memoryConcepts2.conceptId }).from(memoryConcepts2).where(eq8(memoryConcepts2.memoryId, memoryId)),
|
|
5615
5978
|
catch: (cause) => new StoreError({ message: `Store.getMemoryAnnotations: ${cause}` })
|
|
5616
5979
|
});
|
|
5617
5980
|
return rows.map((r) => r.conceptId);
|
|
5618
5981
|
});
|
|
5619
|
-
const queryByMetadata =
|
|
5982
|
+
const queryByMetadata = Effect27.fn("Store.queryByMetadata")(function* (filter) {
|
|
5620
5983
|
const rows = yield* backend.queryByMetadata(filter);
|
|
5621
5984
|
const typed = rows;
|
|
5622
5985
|
return typed.map((row) => fromMemoryRow({
|
|
@@ -5633,8 +5996,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5633
5996
|
embeddingModel: row.embedding_model
|
|
5634
5997
|
}));
|
|
5635
5998
|
});
|
|
5636
|
-
const getMemoryMetadata =
|
|
5637
|
-
const rows = yield*
|
|
5999
|
+
const getMemoryMetadata = Effect27.fn("Store.getMemoryMetadata")(function* (id) {
|
|
6000
|
+
const rows = yield* Effect27.tryPromise({
|
|
5638
6001
|
try: () => db.select({ metadata: memories2.metadata }).from(memories2).where(eq8(memories2.id, id)).limit(1),
|
|
5639
6002
|
catch: (cause) => new StoreError({ message: `Store.getMemoryMetadata: ${cause}` })
|
|
5640
6003
|
});
|
|
@@ -5643,8 +6006,8 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5643
6006
|
return yield* new NotFound({ id });
|
|
5644
6007
|
return parseMetadata(row.metadata);
|
|
5645
6008
|
});
|
|
5646
|
-
const setMemoryMetadata =
|
|
5647
|
-
const updated = yield*
|
|
6009
|
+
const setMemoryMetadata = Effect27.fn("Store.setMemoryMetadata")(function* (id, metadata) {
|
|
6010
|
+
const updated = yield* Effect27.tryPromise({
|
|
5648
6011
|
try: () => db.transaction(async (tx) => {
|
|
5649
6012
|
const rows = await tx.select({ metadata: memories2.metadata }).from(memories2).where(eq8(memories2.id, id)).limit(1);
|
|
5650
6013
|
const row = rows[0];
|
|
@@ -5669,6 +6032,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5669
6032
|
recency,
|
|
5670
6033
|
searchFts5,
|
|
5671
6034
|
searchVector,
|
|
6035
|
+
augmentSearchResults,
|
|
5672
6036
|
logAudit,
|
|
5673
6037
|
listSchema: () => backend.listSchema(),
|
|
5674
6038
|
supersede,
|
|
@@ -5676,6 +6040,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5676
6040
|
sweep,
|
|
5677
6041
|
events,
|
|
5678
6042
|
updateEmbedding,
|
|
6043
|
+
updateEmbeddingMany,
|
|
5679
6044
|
findStale,
|
|
5680
6045
|
createConcept,
|
|
5681
6046
|
getConcept,
|
|
@@ -5710,7 +6075,7 @@ var makeStore2 = (backend, modelConfig, topicsOptions, graphConfig, auditOptions
|
|
|
5710
6075
|
};
|
|
5711
6076
|
};
|
|
5712
6077
|
var layer8 = (url, options) => {
|
|
5713
|
-
const base = Layer10.effect(Service5,
|
|
6078
|
+
const base = Layer10.effect(Service5, Effect27.gen(function* () {
|
|
5714
6079
|
const backend = yield* Service2;
|
|
5715
6080
|
return Service5.of(makeStore2(backend, options?.embeddingModelConfig, options?.topics, options?.graphConfig, options?.enqueueAudit !== undefined ? { enqueueAudit: options.enqueueAudit } : undefined, options?.defaultTtlSecs));
|
|
5716
6081
|
})).pipe(Layer10.provideMerge(layer3(url, options)));
|
|
@@ -5723,7 +6088,7 @@ var DISABLED_LOGGING = {
|
|
|
5723
6088
|
format: "jsonl",
|
|
5724
6089
|
ringSize: 4096
|
|
5725
6090
|
};
|
|
5726
|
-
var loggingFromConfig = Layer10.unwrap(
|
|
6091
|
+
var loggingFromConfig = Layer10.unwrap(Effect27.map(Effect27.service(Service4), (cfg) => layer(cfg.config.logging ?? DISABLED_LOGGING)));
|
|
5727
6092
|
// ../store/src/topics/index.ts
|
|
5728
6093
|
var exports_topics = {};
|
|
5729
6094
|
__export(exports_topics, {
|
|
@@ -5744,6 +6109,7 @@ __export(exports_topics, {
|
|
|
5744
6109
|
shouldSplit: () => shouldSplit,
|
|
5745
6110
|
setActiveModel: () => setActiveModel,
|
|
5746
6111
|
scoreChildren: () => scoreChildren,
|
|
6112
|
+
routeInsertMany: () => routeInsertMany,
|
|
5747
6113
|
routeInsert: () => routeInsert,
|
|
5748
6114
|
resolveTopics: () => resolveTopics,
|
|
5749
6115
|
rebalanceSiblings: () => rebalanceSiblings,
|
|
@@ -5781,15 +6147,15 @@ __export(exports_topics, {
|
|
|
5781
6147
|
});
|
|
5782
6148
|
|
|
5783
6149
|
// ../store/src/audit.ts
|
|
5784
|
-
import { Effect as
|
|
6150
|
+
import { Effect as Effect28 } from "effect";
|
|
5785
6151
|
function auditWriteHandler(deps = {}) {
|
|
5786
|
-
const run = (payload, ctx) =>
|
|
6152
|
+
const run = (payload, ctx) => Effect28.gen(function* () {
|
|
5787
6153
|
const envelope = payload;
|
|
5788
6154
|
const backend = yield* Service2;
|
|
5789
|
-
yield* backend.writeAuditRows(envelope.rows).pipe(
|
|
6155
|
+
yield* backend.writeAuditRows(envelope.rows).pipe(Effect28.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `audit.write: ${error}` })));
|
|
5790
6156
|
return;
|
|
5791
6157
|
});
|
|
5792
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6158
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect28.provideService(Service2, deps.backend)) : run;
|
|
5793
6159
|
}
|
|
5794
6160
|
// ../store/src/graph/index.ts
|
|
5795
6161
|
var exports_graph = {};
|
|
@@ -5824,7 +6190,7 @@ __export(exports_graph, {
|
|
|
5824
6190
|
});
|
|
5825
6191
|
|
|
5826
6192
|
// ../store/src/graph/concepts.ts
|
|
5827
|
-
import { Clock as Clock6, Effect as
|
|
6193
|
+
import { Clock as Clock6, Effect as Effect29 } from "effect";
|
|
5828
6194
|
import { and as and9, asc as asc6, eq as eq9, isNotNull, sql as sql8 } from "drizzle-orm";
|
|
5829
6195
|
var DEFAULT_CLUSTER_K = 8;
|
|
5830
6196
|
var DEFAULT_CLUSTER_MIN_SIZE = 3;
|
|
@@ -5947,11 +6313,11 @@ var conceptConfidence = (vector, centroid) => {
|
|
|
5947
6313
|
return 0;
|
|
5948
6314
|
return Math.min(1, Math.max(0, dot3(vector, centroid) / Math.sqrt(norm)));
|
|
5949
6315
|
};
|
|
5950
|
-
var clusterLeaf =
|
|
6316
|
+
var clusterLeaf = Effect29.fn("ConceptCluster.clusterLeaf")(function* (topicKey, model, config) {
|
|
5951
6317
|
const backend = yield* Service2;
|
|
5952
6318
|
const db = backend.db;
|
|
5953
6319
|
const now = yield* Clock6.currentTimeMillis;
|
|
5954
|
-
const memberRows = yield*
|
|
6320
|
+
const memberRows = yield* Effect29.tryPromise({
|
|
5955
6321
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and9(eq9(topicMemberships.topicKey, topicKey), eq9(topicMemberships.modelKey, model.id))).orderBy(asc6(topicMemberships.memoryId)),
|
|
5956
6322
|
catch: (cause) => new StoreError({ message: `clusterLeaf: load members: ${cause}` })
|
|
5957
6323
|
});
|
|
@@ -5970,11 +6336,11 @@ var clusterLeaf = Effect28.fn("ConceptCluster.clusterLeaf")(function* (topicKey,
|
|
|
5970
6336
|
return { concepts: 0, annotated: 0 };
|
|
5971
6337
|
return yield* persistClusters(db, members, config, now);
|
|
5972
6338
|
});
|
|
5973
|
-
var clusterCorpus =
|
|
6339
|
+
var clusterCorpus = Effect29.fn("ConceptCluster.clusterCorpus")(function* (model, config) {
|
|
5974
6340
|
const backend = yield* Service2;
|
|
5975
6341
|
const db = backend.db;
|
|
5976
6342
|
const now = yield* Clock6.currentTimeMillis;
|
|
5977
|
-
const rows = yield*
|
|
6343
|
+
const rows = yield* Effect29.tryPromise({
|
|
5978
6344
|
try: () => db.select({ id: memories.id, embedding: memories.embedding }).from(memories).where(and9(eq9(memories.state, "active"), isNotNull(memories.embedding))).orderBy(asc6(memories.id)),
|
|
5979
6345
|
catch: (cause) => new StoreError({ message: `clusterCorpus: load memories: ${cause}` })
|
|
5980
6346
|
});
|
|
@@ -5991,7 +6357,7 @@ var clusterCorpus = Effect28.fn("ConceptCluster.clusterCorpus")(function* (model
|
|
|
5991
6357
|
return { concepts: 0, annotated: 0 };
|
|
5992
6358
|
return yield* persistClusters(db, members, config, now);
|
|
5993
6359
|
});
|
|
5994
|
-
var persistClusters = (db, members, config, now) =>
|
|
6360
|
+
var persistClusters = (db, members, config, now) => Effect29.tryPromise({
|
|
5995
6361
|
try: () => db.transaction(async (tx) => {
|
|
5996
6362
|
const { k, minClusterSize } = resolvedClusterConfig(config);
|
|
5997
6363
|
const vectors = members.map((m) => m.vector);
|
|
@@ -6022,18 +6388,18 @@ var persistClusters = (db, members, config, now) => Effect28.tryPromise({
|
|
|
6022
6388
|
lastSeen: now
|
|
6023
6389
|
}
|
|
6024
6390
|
});
|
|
6025
|
-
|
|
6391
|
+
await tx.insert(memoryConcepts).values(memberIndices.map((i) => {
|
|
6026
6392
|
const member = members[i];
|
|
6027
|
-
|
|
6393
|
+
return {
|
|
6028
6394
|
conceptId: id,
|
|
6029
6395
|
memoryId: member.id,
|
|
6030
6396
|
confidence: conceptConfidence(member.vector, centroid),
|
|
6031
6397
|
assignedAt: now
|
|
6032
|
-
}
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
}
|
|
6036
|
-
}
|
|
6398
|
+
};
|
|
6399
|
+
})).onConflictDoUpdate({
|
|
6400
|
+
target: [memoryConcepts.conceptId, memoryConcepts.memoryId],
|
|
6401
|
+
set: { confidence: sql8`excluded.confidence`, assignedAt: now }
|
|
6402
|
+
});
|
|
6037
6403
|
conceptCount++;
|
|
6038
6404
|
annotated += memberIndices.length;
|
|
6039
6405
|
}
|
|
@@ -6042,21 +6408,21 @@ var persistClusters = (db, members, config, now) => Effect28.tryPromise({
|
|
|
6042
6408
|
catch: (cause) => new StoreError({ message: `concept cluster: persist: ${cause}` })
|
|
6043
6409
|
});
|
|
6044
6410
|
function conceptClusterHandler(deps) {
|
|
6045
|
-
const run = (payload, ctx) =>
|
|
6411
|
+
const run = (payload, ctx) => Effect29.gen(function* () {
|
|
6046
6412
|
const envelope = payload;
|
|
6047
6413
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
6048
6414
|
if (!model) {
|
|
6049
|
-
return yield*
|
|
6415
|
+
return yield* Effect29.fail(new JobFailure({
|
|
6050
6416
|
jobId: ctx.jobId,
|
|
6051
6417
|
message: `concept.cluster: unknown model "${envelope.modelKey}"`
|
|
6052
6418
|
}));
|
|
6053
6419
|
}
|
|
6054
6420
|
if (envelope.topicKey !== undefined) {
|
|
6055
|
-
yield* clusterLeaf(envelope.topicKey, model, deps.config).pipe(
|
|
6421
|
+
yield* clusterLeaf(envelope.topicKey, model, deps.config).pipe(Effect29.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `concept.cluster: ${error}` })));
|
|
6056
6422
|
} else {
|
|
6057
|
-
yield* clusterCorpus(model, deps.config).pipe(
|
|
6423
|
+
yield* clusterCorpus(model, deps.config).pipe(Effect29.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `concept.cluster: ${error}` })));
|
|
6058
6424
|
}
|
|
6059
|
-
yield*
|
|
6425
|
+
yield* Effect29.logInfo("concepts.clustered").pipe(Effect29.annotateLogs({
|
|
6060
6426
|
scope: "@dimple/store",
|
|
6061
6427
|
event: "concepts.clustered",
|
|
6062
6428
|
modelKey: envelope.modelKey,
|
|
@@ -6064,10 +6430,10 @@ function conceptClusterHandler(deps) {
|
|
|
6064
6430
|
}));
|
|
6065
6431
|
return;
|
|
6066
6432
|
});
|
|
6067
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6433
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect29.provideService(Service2, deps.backend)) : run;
|
|
6068
6434
|
}
|
|
6069
6435
|
// ../store/src/graph/dream.ts
|
|
6070
|
-
import { Clock as Clock7, Effect as
|
|
6436
|
+
import { Clock as Clock7, Effect as Effect30 } from "effect";
|
|
6071
6437
|
import { and as and10, asc as asc7, eq as eq10, gt, inArray as inArray5, isNull as isNull3, sql as sql9 } from "drizzle-orm";
|
|
6072
6438
|
var dreamStateDefaults = {
|
|
6073
6439
|
minHours: 24,
|
|
@@ -6097,110 +6463,99 @@ var flipToSuperseded = async (tx, memoryId, now) => {
|
|
|
6097
6463
|
});
|
|
6098
6464
|
return true;
|
|
6099
6465
|
};
|
|
6100
|
-
var
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
if (row.status !== "pending")
|
|
6107
|
-
return { inserted: false, flipped: false };
|
|
6108
|
-
if (confirmed) {
|
|
6109
|
-
await tx.update(artifactEdges).set({
|
|
6110
|
-
status: "confirmed",
|
|
6111
|
-
validAt: now,
|
|
6112
|
-
confidence: relation.confidence,
|
|
6113
|
-
updatedAt: now
|
|
6114
|
-
}).where(eq10(artifactEdges.id, row.id));
|
|
6115
|
-
const flipped2 = await flipToSuperseded(tx, relation.fromMemoryId, now);
|
|
6116
|
-
return { inserted: false, flipped: flipped2 };
|
|
6117
|
-
}
|
|
6466
|
+
var writeRelationEdgeTx = async (tx, relation, now) => {
|
|
6467
|
+
const existing = await tx.select({ id: artifactEdges.id, status: artifactEdges.status }).from(artifactEdges).where(and10(eq10(artifactEdges.fromMemoryId, relation.fromMemoryId), eq10(artifactEdges.toMemoryId, relation.toMemoryId), eq10(artifactEdges.edgeKind, relation.edgeKind))).limit(1);
|
|
6468
|
+
const confirmed = relation.edgeKind === "memory_supersedes" && relation.confidence >= SUPERSEDE_CONFIRM_BAR;
|
|
6469
|
+
if (existing.length > 0) {
|
|
6470
|
+
const row = existing[0];
|
|
6471
|
+
if (row.status !== "pending")
|
|
6118
6472
|
return { inserted: false, flipped: false };
|
|
6119
|
-
|
|
6473
|
+
if (confirmed) {
|
|
6474
|
+
await tx.update(artifactEdges).set({
|
|
6475
|
+
status: "confirmed",
|
|
6476
|
+
validAt: now,
|
|
6477
|
+
confidence: relation.confidence,
|
|
6478
|
+
updatedAt: now
|
|
6479
|
+
}).where(eq10(artifactEdges.id, row.id));
|
|
6480
|
+
const flipped2 = await flipToSuperseded(tx, relation.fromMemoryId, now);
|
|
6481
|
+
return { inserted: false, flipped: flipped2 };
|
|
6482
|
+
}
|
|
6483
|
+
return { inserted: false, flipped: false };
|
|
6484
|
+
}
|
|
6485
|
+
await tx.insert(artifactEdges).values({
|
|
6486
|
+
id: crypto.randomUUID(),
|
|
6487
|
+
fromMemoryId: relation.fromMemoryId,
|
|
6488
|
+
toMemoryId: relation.toMemoryId,
|
|
6489
|
+
edgeKind: relation.edgeKind,
|
|
6490
|
+
weight: 1,
|
|
6491
|
+
status: confirmed ? "confirmed" : "pending",
|
|
6492
|
+
source: "llm",
|
|
6493
|
+
confidence: relation.confidence,
|
|
6494
|
+
validAt: confirmed ? now : null,
|
|
6495
|
+
expiredAt: null,
|
|
6496
|
+
createdAt: now,
|
|
6497
|
+
updatedAt: now
|
|
6498
|
+
});
|
|
6499
|
+
const flipped = confirmed ? await flipToSuperseded(tx, relation.fromMemoryId, now) : false;
|
|
6500
|
+
return { inserted: true, flipped };
|
|
6501
|
+
};
|
|
6502
|
+
var writeDerivedMemoryTx = async (tx, content, sources, now) => {
|
|
6503
|
+
const logicalKey = derivedKey(content);
|
|
6504
|
+
const existing = await tx.select({ id: memories.id }).from(memories).where(and10(eq10(memories.logicalKey, logicalKey), eq10(memories.namespace, DERIVED_NAMESPACE), eq10(memories.state, "active"))).limit(1);
|
|
6505
|
+
const memoryId = existing[0]?.id ?? crypto.randomUUID();
|
|
6506
|
+
let inserted = false;
|
|
6507
|
+
if (existing.length === 0) {
|
|
6508
|
+
await tx.insert(memories).values({
|
|
6509
|
+
id: memoryId,
|
|
6510
|
+
logicalKey,
|
|
6511
|
+
content,
|
|
6512
|
+
contentType: "text/plain",
|
|
6513
|
+
state: "active",
|
|
6514
|
+
createdAt: now,
|
|
6515
|
+
ttlSecs: DERIVED_TTL_SECS,
|
|
6516
|
+
namespace: DERIVED_NAMESPACE,
|
|
6517
|
+
metadata: JSON.stringify({ sources })
|
|
6518
|
+
});
|
|
6519
|
+
await tx.insert(memoryEvents).values({
|
|
6520
|
+
memoryId,
|
|
6521
|
+
eventType: "created",
|
|
6522
|
+
createdAt: now
|
|
6523
|
+
});
|
|
6524
|
+
inserted = true;
|
|
6525
|
+
}
|
|
6526
|
+
for (const sourceId of sources) {
|
|
6527
|
+
const edge = await tx.select({ id: artifactEdges.id }).from(artifactEdges).where(and10(eq10(artifactEdges.fromMemoryId, sourceId), eq10(artifactEdges.toMemoryId, memoryId), eq10(artifactEdges.edgeKind, "memory_supports"))).limit(1);
|
|
6528
|
+
if (edge.length > 0)
|
|
6529
|
+
continue;
|
|
6120
6530
|
await tx.insert(artifactEdges).values({
|
|
6121
6531
|
id: crypto.randomUUID(),
|
|
6122
|
-
fromMemoryId:
|
|
6123
|
-
toMemoryId:
|
|
6124
|
-
edgeKind:
|
|
6532
|
+
fromMemoryId: sourceId,
|
|
6533
|
+
toMemoryId: memoryId,
|
|
6534
|
+
edgeKind: "memory_supports",
|
|
6125
6535
|
weight: 1,
|
|
6126
|
-
status:
|
|
6536
|
+
status: "pending",
|
|
6127
6537
|
source: "llm",
|
|
6128
|
-
confidence:
|
|
6129
|
-
validAt:
|
|
6538
|
+
confidence: null,
|
|
6539
|
+
validAt: null,
|
|
6130
6540
|
expiredAt: null,
|
|
6131
6541
|
createdAt: now,
|
|
6132
6542
|
updatedAt: now
|
|
6133
6543
|
});
|
|
6134
|
-
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
});
|
|
6139
|
-
var writeDerivedMemory = (db, content, sources, now) => Effect29.tryPromise({
|
|
6140
|
-
try: () => db.transaction(async (tx) => {
|
|
6141
|
-
const logicalKey = derivedKey(content);
|
|
6142
|
-
const existing = await tx.select({ id: memories.id }).from(memories).where(and10(eq10(memories.logicalKey, logicalKey), eq10(memories.namespace, DERIVED_NAMESPACE), eq10(memories.state, "active"))).limit(1);
|
|
6143
|
-
const memoryId = existing[0]?.id ?? crypto.randomUUID();
|
|
6144
|
-
let inserted = false;
|
|
6145
|
-
if (existing.length === 0) {
|
|
6146
|
-
await tx.insert(memories).values({
|
|
6147
|
-
id: memoryId,
|
|
6148
|
-
logicalKey,
|
|
6149
|
-
content,
|
|
6150
|
-
contentType: "text/plain",
|
|
6151
|
-
state: "active",
|
|
6152
|
-
createdAt: now,
|
|
6153
|
-
ttlSecs: DERIVED_TTL_SECS,
|
|
6154
|
-
namespace: DERIVED_NAMESPACE,
|
|
6155
|
-
metadata: JSON.stringify({ sources })
|
|
6156
|
-
});
|
|
6157
|
-
await tx.insert(memoryEvents).values({
|
|
6158
|
-
memoryId,
|
|
6159
|
-
eventType: "created",
|
|
6160
|
-
createdAt: now
|
|
6161
|
-
});
|
|
6162
|
-
inserted = true;
|
|
6163
|
-
}
|
|
6164
|
-
for (const sourceId of sources) {
|
|
6165
|
-
const edge = await tx.select({ id: artifactEdges.id }).from(artifactEdges).where(and10(eq10(artifactEdges.fromMemoryId, sourceId), eq10(artifactEdges.toMemoryId, memoryId), eq10(artifactEdges.edgeKind, "memory_supports"))).limit(1);
|
|
6166
|
-
if (edge.length > 0)
|
|
6167
|
-
continue;
|
|
6168
|
-
await tx.insert(artifactEdges).values({
|
|
6169
|
-
id: crypto.randomUUID(),
|
|
6170
|
-
fromMemoryId: sourceId,
|
|
6171
|
-
toMemoryId: memoryId,
|
|
6172
|
-
edgeKind: "memory_supports",
|
|
6173
|
-
weight: 1,
|
|
6174
|
-
status: "pending",
|
|
6175
|
-
source: "llm",
|
|
6176
|
-
confidence: null,
|
|
6177
|
-
validAt: null,
|
|
6178
|
-
expiredAt: null,
|
|
6179
|
-
createdAt: now,
|
|
6180
|
-
updatedAt: now
|
|
6181
|
-
});
|
|
6182
|
-
}
|
|
6183
|
-
return { inserted };
|
|
6184
|
-
}),
|
|
6185
|
-
catch: (cause) => new StoreError({ message: `memory.dream: write derived: ${cause}` })
|
|
6186
|
-
});
|
|
6187
|
-
var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
6544
|
+
}
|
|
6545
|
+
return { inserted };
|
|
6546
|
+
};
|
|
6547
|
+
var runDream = Effect30.fn("MemoryDream.run")(function* (model, classify) {
|
|
6188
6548
|
const backend = yield* Service2;
|
|
6189
6549
|
const db = backend.db;
|
|
6190
6550
|
const now = yield* Clock7.currentTimeMillis;
|
|
6191
6551
|
const { minHours, minNewMemories, maxPairs, maxDerived } = dreamStateDefaults;
|
|
6192
|
-
const stateRows = yield*
|
|
6552
|
+
const stateRows = yield* Effect30.tryPromise({
|
|
6193
6553
|
try: () => db.select().from(dreamState).where(eq10(dreamState.modelKey, model.id)),
|
|
6194
6554
|
catch: (cause) => new StoreError({ message: `memory.dream: load state: ${cause}` })
|
|
6195
6555
|
});
|
|
6196
6556
|
const lastDreamAt = stateRows[0]?.lastDreamAt ?? 0;
|
|
6197
|
-
const sinceCountRows = yield* Effect29.tryPromise({
|
|
6198
|
-
try: () => db.select({ count: sql9`count(*)` }).from(memories).where(gt(memories.createdAt, lastDreamAt)),
|
|
6199
|
-
catch: (cause) => new StoreError({ message: `memory.dream: count new: ${cause}` })
|
|
6200
|
-
});
|
|
6201
|
-
const memoriesSinceWatermark = Number(sinceCountRows[0]?.count ?? 0);
|
|
6202
6557
|
if (now - lastDreamAt < minHours * 3600000) {
|
|
6203
|
-
yield*
|
|
6558
|
+
yield* Effect30.logDebug("dream.skipped").pipe(Effect30.annotateLogs({ scope: "@dimple/store", event: "dream.skipped", reason: "cooldown" }));
|
|
6204
6559
|
return {
|
|
6205
6560
|
skipped: true,
|
|
6206
6561
|
reason: "cooldown",
|
|
@@ -6209,8 +6564,13 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6209
6564
|
superseded: 0
|
|
6210
6565
|
};
|
|
6211
6566
|
}
|
|
6567
|
+
const sinceCountRows = yield* Effect30.tryPromise({
|
|
6568
|
+
try: () => db.select({ count: sql9`count(*)` }).from(memories).where(gt(memories.createdAt, lastDreamAt)),
|
|
6569
|
+
catch: (cause) => new StoreError({ message: `memory.dream: count new: ${cause}` })
|
|
6570
|
+
});
|
|
6571
|
+
const memoriesSinceWatermark = Number(sinceCountRows[0]?.count ?? 0);
|
|
6212
6572
|
if (memoriesSinceWatermark < minNewMemories) {
|
|
6213
|
-
yield*
|
|
6573
|
+
yield* Effect30.logDebug("dream.skipped").pipe(Effect30.annotateLogs({
|
|
6214
6574
|
scope: "@dimple/store",
|
|
6215
6575
|
event: "dream.skipped",
|
|
6216
6576
|
reason: "insufficient-new-memories",
|
|
@@ -6224,7 +6584,7 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6224
6584
|
superseded: 0
|
|
6225
6585
|
};
|
|
6226
6586
|
}
|
|
6227
|
-
const pendingRows = yield*
|
|
6587
|
+
const pendingRows = yield* Effect30.tryPromise({
|
|
6228
6588
|
try: () => db.select({
|
|
6229
6589
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
6230
6590
|
toMemoryId: artifactEdges.toMemoryId
|
|
@@ -6237,7 +6597,7 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6237
6597
|
};
|
|
6238
6598
|
for (const row of pendingRows)
|
|
6239
6599
|
addPair(row.fromMemoryId, row.toMemoryId);
|
|
6240
|
-
const sinceRows = yield*
|
|
6600
|
+
const sinceRows = yield* Effect30.tryPromise({
|
|
6241
6601
|
try: () => db.select({
|
|
6242
6602
|
id: memories.id,
|
|
6243
6603
|
logicalKey: memories.logicalKey,
|
|
@@ -6248,19 +6608,19 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6248
6608
|
});
|
|
6249
6609
|
if (sinceRows.length > 0) {
|
|
6250
6610
|
const sinceIds = sinceRows.map((r) => r.id);
|
|
6251
|
-
const membershipRows = yield*
|
|
6611
|
+
const membershipRows = yield* Effect30.tryPromise({
|
|
6252
6612
|
try: () => db.select({ memoryId: topicMemberships.memoryId, topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and10(eq10(topicMemberships.modelKey, model.id), inArray5(topicMemberships.memoryId, sinceIds))),
|
|
6253
6613
|
catch: (cause) => new StoreError({ message: `memory.dream: scan memberships: ${cause}` })
|
|
6254
6614
|
});
|
|
6255
6615
|
const leafKeys = [...new Set(membershipRows.map((m) => m.topicKey))];
|
|
6256
6616
|
if (leafKeys.length > 0) {
|
|
6257
|
-
const mateRows = yield*
|
|
6617
|
+
const mateRows = yield* Effect30.tryPromise({
|
|
6258
6618
|
try: () => db.select({ memoryId: topicMemberships.memoryId, topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and10(eq10(topicMemberships.modelKey, model.id), inArray5(topicMemberships.topicKey, leafKeys))),
|
|
6259
6619
|
catch: (cause) => new StoreError({ message: `memory.dream: scan mates: ${cause}` })
|
|
6260
6620
|
});
|
|
6261
6621
|
const memberIds = [...new Set([...sinceIds, ...mateRows.map((m) => m.memoryId)])];
|
|
6262
|
-
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(
|
|
6263
|
-
const memoryRows = yield*
|
|
6622
|
+
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(Effect30.mapError((cause) => new StoreError({ message: `memory.dream: scan vectors: ${cause}` })));
|
|
6623
|
+
const memoryRows = yield* Effect30.tryPromise({
|
|
6264
6624
|
try: () => db.select({
|
|
6265
6625
|
id: memories.id,
|
|
6266
6626
|
logicalKey: memories.logicalKey,
|
|
@@ -6317,7 +6677,7 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6317
6677
|
}
|
|
6318
6678
|
const orderedPairs = [...pairSet.values()].sort((a, b) => cmp3(a.toMemoryId, b.toMemoryId) || cmp3(a.fromMemoryId, b.fromMemoryId)).slice(0, maxPairs);
|
|
6319
6679
|
const endpointIds = [...new Set(orderedPairs.flatMap((p) => [p.fromMemoryId, p.toMemoryId]))];
|
|
6320
|
-
const contentRows = endpointIds.length > 0 ? yield*
|
|
6680
|
+
const contentRows = endpointIds.length > 0 ? yield* Effect30.tryPromise({
|
|
6321
6681
|
try: () => db.select({ id: memories.id, content: memories.content }).from(memories).where(inArray5(memories.id, endpointIds)),
|
|
6322
6682
|
catch: (cause) => new StoreError({ message: `memory.dream: pair contents: ${cause}` })
|
|
6323
6683
|
}) : [];
|
|
@@ -6337,17 +6697,17 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6337
6697
|
}
|
|
6338
6698
|
let classification = { relations: [], derived: [] };
|
|
6339
6699
|
if (dreamPairs.length > 0) {
|
|
6340
|
-
classification = yield* classify.classify(dreamPairs, { maxDerived }).pipe(
|
|
6700
|
+
classification = yield* classify.classify(dreamPairs, { maxDerived }).pipe(Effect30.mapError((error) => new StoreError({ message: `memory.dream: classify: ${error}` })));
|
|
6341
6701
|
}
|
|
6342
6702
|
const endpointIdsOf = [
|
|
6343
6703
|
...new Set(classification.relations.flatMap((r) => [r.fromMemoryId, r.toMemoryId]))
|
|
6344
6704
|
];
|
|
6345
|
-
const endpointRows = endpointIdsOf.length > 0 ? yield*
|
|
6705
|
+
const endpointRows = endpointIdsOf.length > 0 ? yield* Effect30.tryPromise({
|
|
6346
6706
|
try: () => db.select({ id: memories.id, metadata: memories.metadata }).from(memories).where(inArray5(memories.id, endpointIdsOf)),
|
|
6347
6707
|
catch: (cause) => new StoreError({ message: `memory.dream: endpoints: ${cause}` })
|
|
6348
6708
|
}) : [];
|
|
6349
6709
|
const metadataById = new Map(endpointRows.map((r) => [r.id, r.metadata]));
|
|
6350
|
-
|
|
6710
|
+
const acceptedRelations = [];
|
|
6351
6711
|
for (const relation of classification.relations) {
|
|
6352
6712
|
const fromMetadata = metadataById.get(relation.fromMemoryId);
|
|
6353
6713
|
const toMetadata = metadataById.get(relation.toMemoryId);
|
|
@@ -6356,13 +6716,11 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6356
6716
|
if (parseMetadata(fromMetadata)?.pinned === true || parseMetadata(toMetadata)?.pinned === true) {
|
|
6357
6717
|
continue;
|
|
6358
6718
|
}
|
|
6359
|
-
|
|
6360
|
-
if (result.flipped)
|
|
6361
|
-
superseded++;
|
|
6719
|
+
acceptedRelations.push(relation);
|
|
6362
6720
|
}
|
|
6363
|
-
|
|
6721
|
+
const acceptedDerived = [];
|
|
6364
6722
|
for (const derived of classification.derived) {
|
|
6365
|
-
const sourceRows = derived.sources.length > 0 ? yield*
|
|
6723
|
+
const sourceRows = derived.sources.length > 0 ? yield* Effect30.tryPromise({
|
|
6366
6724
|
try: () => db.select({ id: memories.id }).from(memories).where(inArray5(memories.id, derived.sources)),
|
|
6367
6725
|
catch: (cause) => new StoreError({ message: `memory.dream: derived sources: ${cause}` })
|
|
6368
6726
|
}) : [];
|
|
@@ -6370,27 +6728,36 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6370
6728
|
const survivingSources = derived.sources.filter((id) => survivingIds.has(id));
|
|
6371
6729
|
if (survivingSources.length === 0)
|
|
6372
6730
|
continue;
|
|
6373
|
-
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
try: () => db.
|
|
6379
|
-
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
|
|
6383
|
-
|
|
6384
|
-
|
|
6385
|
-
|
|
6386
|
-
|
|
6387
|
-
|
|
6388
|
-
|
|
6389
|
-
|
|
6731
|
+
acceptedDerived.push({ content: derived.content, sources: survivingSources });
|
|
6732
|
+
}
|
|
6733
|
+
let superseded = 0;
|
|
6734
|
+
let created = 0;
|
|
6735
|
+
yield* Effect30.tryPromise({
|
|
6736
|
+
try: () => db.transaction(async (tx) => {
|
|
6737
|
+
for (const relation of acceptedRelations) {
|
|
6738
|
+
const result = await writeRelationEdgeTx(tx, relation, now);
|
|
6739
|
+
if (result.flipped)
|
|
6740
|
+
superseded++;
|
|
6741
|
+
}
|
|
6742
|
+
for (const derived of acceptedDerived) {
|
|
6743
|
+
const outcome = await writeDerivedMemoryTx(tx, derived.content, derived.sources, now);
|
|
6744
|
+
if (outcome.inserted)
|
|
6745
|
+
created++;
|
|
6746
|
+
}
|
|
6747
|
+
const countRows = await tx.select({ count: sql9`count(*)` }).from(memories);
|
|
6748
|
+
const totalMemories = Number(countRows[0]?.count ?? 0);
|
|
6749
|
+
await tx.insert(dreamState).values({
|
|
6750
|
+
modelKey: model.id,
|
|
6751
|
+
lastDreamAt: now,
|
|
6752
|
+
lastDreamMemoryCount: totalMemories
|
|
6753
|
+
}).onConflictDoUpdate({
|
|
6754
|
+
target: dreamState.modelKey,
|
|
6755
|
+
set: { lastDreamAt: now, lastDreamMemoryCount: totalMemories }
|
|
6756
|
+
});
|
|
6390
6757
|
}),
|
|
6391
|
-
catch: (cause) => new StoreError({ message: `memory.dream:
|
|
6758
|
+
catch: (cause) => new StoreError({ message: `memory.dream: persist: ${cause}` })
|
|
6392
6759
|
});
|
|
6393
|
-
yield*
|
|
6760
|
+
yield* Effect30.logInfo("dream.completed").pipe(Effect30.annotateLogs({
|
|
6394
6761
|
scope: "@dimple/store",
|
|
6395
6762
|
event: "dream.completed",
|
|
6396
6763
|
modelKey: model.id,
|
|
@@ -6401,24 +6768,24 @@ var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6401
6768
|
return { skipped: false, classified: dreamPairs.length, created, superseded };
|
|
6402
6769
|
});
|
|
6403
6770
|
function graphDreamHandler(deps) {
|
|
6404
|
-
const run = (payload, ctx) =>
|
|
6771
|
+
const run = (payload, ctx) => Effect30.gen(function* () {
|
|
6405
6772
|
const envelope = payload;
|
|
6406
6773
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
6407
6774
|
if (!model) {
|
|
6408
|
-
return yield*
|
|
6775
|
+
return yield* Effect30.fail(new JobFailure({
|
|
6409
6776
|
jobId: ctx.jobId,
|
|
6410
6777
|
message: `memory.dream: unknown model "${envelope.modelKey}"`
|
|
6411
6778
|
}));
|
|
6412
6779
|
}
|
|
6413
|
-
return yield* runDream(model, deps.classify).pipe(
|
|
6780
|
+
return yield* runDream(model, deps.classify).pipe(Effect30.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `memory.dream: ${error}` })));
|
|
6414
6781
|
});
|
|
6415
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6782
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect30.provideService(Service2, deps.backend)) : run;
|
|
6416
6783
|
}
|
|
6417
6784
|
// ../store/src/graph/repair.ts
|
|
6418
|
-
import { Clock as Clock8, Effect as
|
|
6785
|
+
import { Clock as Clock8, Effect as Effect31 } from "effect";
|
|
6419
6786
|
import { and as and11, eq as eq11, isNotNull as isNotNull2, lt, or as or2, sql as sql10 } from "drizzle-orm";
|
|
6420
6787
|
var DEFAULT_EXPIRED_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
6421
|
-
var runRepair = (db, includePurgeExpired, expiredGraceMs, now) =>
|
|
6788
|
+
var runRepair = (db, includePurgeExpired, expiredGraceMs, now) => Effect31.tryPromise({
|
|
6422
6789
|
try: () => db.transaction(async (tx) => {
|
|
6423
6790
|
let orphanedEdges = 0;
|
|
6424
6791
|
let staleAnnotations = 0;
|
|
@@ -6512,10 +6879,10 @@ var runRepair = (db, includePurgeExpired, expiredGraceMs, now) => Effect30.tryPr
|
|
|
6512
6879
|
catch: (cause) => new StoreError({ message: `graph.repair: ${cause}` })
|
|
6513
6880
|
});
|
|
6514
6881
|
function graphRepairHandler(deps) {
|
|
6515
|
-
const run = (payload, ctx) =>
|
|
6882
|
+
const run = (payload, ctx) => Effect31.gen(function* () {
|
|
6516
6883
|
const envelope = payload;
|
|
6517
6884
|
if (!deps.getModelConfig(envelope.modelKey)) {
|
|
6518
|
-
return yield*
|
|
6885
|
+
return yield* Effect31.fail(new JobFailure({
|
|
6519
6886
|
jobId: ctx.jobId,
|
|
6520
6887
|
message: `graph.repair: unknown model "${envelope.modelKey}"`
|
|
6521
6888
|
}));
|
|
@@ -6523,8 +6890,8 @@ function graphRepairHandler(deps) {
|
|
|
6523
6890
|
const backend = yield* Service2;
|
|
6524
6891
|
const db = backend.db;
|
|
6525
6892
|
const now = yield* Clock8.currentTimeMillis;
|
|
6526
|
-
const result = yield* runRepair(db, deps.includePurgeExpired ?? false, deps.expiredGraceMs ?? DEFAULT_EXPIRED_GRACE_MS, now).pipe(
|
|
6527
|
-
yield*
|
|
6893
|
+
const result = yield* runRepair(db, deps.includePurgeExpired ?? false, deps.expiredGraceMs ?? DEFAULT_EXPIRED_GRACE_MS, now).pipe(Effect31.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `graph.repair: ${error}` })));
|
|
6894
|
+
yield* Effect31.logInfo("repair.completed").pipe(Effect31.annotateLogs({
|
|
6528
6895
|
scope: "@dimple/store",
|
|
6529
6896
|
event: "repair.completed",
|
|
6530
6897
|
modelKey: envelope.modelKey,
|
|
@@ -6534,7 +6901,7 @@ function graphRepairHandler(deps) {
|
|
|
6534
6901
|
}));
|
|
6535
6902
|
return result;
|
|
6536
6903
|
});
|
|
6537
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6904
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect31.provideService(Service2, deps.backend)) : run;
|
|
6538
6905
|
}
|
|
6539
6906
|
// ../store/src/models.ts
|
|
6540
6907
|
import { Schema as Schema20 } from "effect";
|
|
@@ -6621,7 +6988,7 @@ class EmbeddingError extends Schema21.TaggedErrorClass()("EmbeddingProvider.Embe
|
|
|
6621
6988
|
class Service9 extends Context9.Service()("@dimple/embedder/EmbeddingModelRegistry") {
|
|
6622
6989
|
}
|
|
6623
6990
|
// ../embedder/src/transformers.ts
|
|
6624
|
-
import { Config as Config5, Effect as
|
|
6991
|
+
import { Config as Config5, Effect as Effect33, Layer as Layer11 } from "effect";
|
|
6625
6992
|
import { existsSync as existsSync2, readFileSync } from "node:fs";
|
|
6626
6993
|
import { join as join2 } from "node:path";
|
|
6627
6994
|
import { pathToFileURL } from "node:url";
|
|
@@ -6768,47 +7135,59 @@ var defaults2 = {
|
|
|
6768
7135
|
pooling: "mean",
|
|
6769
7136
|
normalize: true
|
|
6770
7137
|
};
|
|
7138
|
+
var outputVectors = (result) => {
|
|
7139
|
+
const { data, dims } = result;
|
|
7140
|
+
const stride = dims.length > 0 ? dims[dims.length - 1] : data.length;
|
|
7141
|
+
if (stride > 0 && data.length % stride === 0) {
|
|
7142
|
+
const vectors = [];
|
|
7143
|
+
for (let offset = 0;offset < data.length; offset += stride) {
|
|
7144
|
+
vectors.push(data.subarray(offset, offset + stride));
|
|
7145
|
+
}
|
|
7146
|
+
return vectors;
|
|
7147
|
+
}
|
|
7148
|
+
return result.tolist().map((v) => new Float32Array(v));
|
|
7149
|
+
};
|
|
6771
7150
|
var make5 = (options) => {
|
|
6772
7151
|
let pipe = null;
|
|
6773
|
-
const getPipeline =
|
|
7152
|
+
const getPipeline = Effect33.fnUntraced(function* () {
|
|
6774
7153
|
if (pipe)
|
|
6775
7154
|
return pipe;
|
|
6776
|
-
const loadModule = () =>
|
|
6777
|
-
let transformers = yield* loadModule().pipe(
|
|
7155
|
+
const loadModule = () => Effect33.tryPromise(() => import("@huggingface/transformers"));
|
|
7156
|
+
let transformers = yield* loadModule().pipe(Effect33.catch(() => {
|
|
6778
7157
|
if (options.autoInstall === false) {
|
|
6779
|
-
return
|
|
7158
|
+
return Effect33.fail(new EmbeddingError({
|
|
6780
7159
|
message: "@huggingface/transformers is not installed — run `bun add @huggingface/transformers` " + "to use local embeddings (or configure a hosted embedding provider)"
|
|
6781
7160
|
}));
|
|
6782
7161
|
}
|
|
6783
|
-
return
|
|
6784
|
-
yield*
|
|
6785
|
-
yield*
|
|
7162
|
+
return Effect33.gen(function* () {
|
|
7163
|
+
yield* Effect33.logInfo("auto-installing @huggingface/transformers (first local-embed use — scripts skipped)").pipe(Effect33.annotateLogs({ scope: "@dimple/embedder", event: "embedder.auto-install" }));
|
|
7164
|
+
yield* Effect33.tryPromise(() => installTransformers(process.cwd())).pipe(Effect33.catch((cause) => {
|
|
6786
7165
|
const inner = cause.cause;
|
|
6787
|
-
return
|
|
7166
|
+
return Effect33.fail(inner instanceof EmbeddingError ? inner : new EmbeddingError({
|
|
6788
7167
|
message: `auto-install failed: ${inner instanceof Error ? inner.message : String(cause)}`
|
|
6789
7168
|
}));
|
|
6790
7169
|
}));
|
|
6791
|
-
return yield*
|
|
7170
|
+
return yield* Effect33.tryPromise(importFromDisk).pipe(Effect33.mapError(() => new EmbeddingError({
|
|
6792
7171
|
message: "@huggingface/transformers still unavailable after auto-install — run `bun add " + "@huggingface/transformers` manually (or set `autoInstall: false` and configure a hosted embedding provider)"
|
|
6793
7172
|
})));
|
|
6794
7173
|
});
|
|
6795
7174
|
}));
|
|
6796
7175
|
if (options.cacheDir)
|
|
6797
7176
|
transformers.env.cacheDir = options.cacheDir;
|
|
6798
|
-
pipe = yield*
|
|
7177
|
+
pipe = yield* Effect33.tryPromise(() => transformers.pipeline("feature-extraction", options.modelId ?? defaults2.modelId, {
|
|
6799
7178
|
device: options.device ?? defaults2.device,
|
|
6800
7179
|
dtype: options.dtype ?? defaults2.dtype
|
|
6801
|
-
})).pipe(
|
|
7180
|
+
})).pipe(Effect33.mapError((cause) => new EmbeddingError({ message: `load model: ${cause}` })));
|
|
6802
7181
|
return pipe;
|
|
6803
7182
|
});
|
|
6804
|
-
const embed =
|
|
7183
|
+
const embed = Effect33.fn("TransformersEmbedding.embed")(function* (texts) {
|
|
6805
7184
|
const started = Date.now();
|
|
6806
7185
|
const p = yield* getPipeline();
|
|
6807
|
-
const result = yield*
|
|
7186
|
+
const result = yield* Effect33.promise(() => p(texts, {
|
|
6808
7187
|
pooling: options.pooling ?? defaults2.pooling,
|
|
6809
7188
|
normalize: options.normalize ?? defaults2.normalize
|
|
6810
|
-
})).pipe(
|
|
6811
|
-
const vectors = result
|
|
7189
|
+
})).pipe(Effect33.mapError((cause) => new EmbeddingError({ message: `embed: ${cause}` })));
|
|
7190
|
+
const vectors = outputVectors(result);
|
|
6812
7191
|
for (const vector of vectors) {
|
|
6813
7192
|
if (options.dimensions !== undefined && vector.length !== options.dimensions) {
|
|
6814
7193
|
return yield* new EmbeddingError({
|
|
@@ -6816,7 +7195,7 @@ var make5 = (options) => {
|
|
|
6816
7195
|
});
|
|
6817
7196
|
}
|
|
6818
7197
|
}
|
|
6819
|
-
yield*
|
|
7198
|
+
yield* Effect33.logDebug("embedder.embed").pipe(Effect33.annotateLogs({
|
|
6820
7199
|
scope: "@dimple/embedder",
|
|
6821
7200
|
event: "embedder.embed",
|
|
6822
7201
|
count: texts.length,
|
|
@@ -6824,13 +7203,13 @@ var make5 = (options) => {
|
|
|
6824
7203
|
}));
|
|
6825
7204
|
return vectors;
|
|
6826
7205
|
});
|
|
6827
|
-
const embedSingle =
|
|
7206
|
+
const embedSingle = Effect33.fn("TransformersEmbedding.embedSingle")(function* (text4) {
|
|
6828
7207
|
const p = yield* getPipeline();
|
|
6829
|
-
const result = yield*
|
|
7208
|
+
const result = yield* Effect33.promise(() => p(text4, {
|
|
6830
7209
|
pooling: options.pooling ?? defaults2.pooling,
|
|
6831
7210
|
normalize: options.normalize ?? defaults2.normalize
|
|
6832
|
-
})).pipe(
|
|
6833
|
-
const vector =
|
|
7211
|
+
})).pipe(Effect33.mapError((cause) => new EmbeddingError({ message: `embedSingle: ${cause}` })));
|
|
7212
|
+
const vector = outputVectors(result)[0] ?? new Float32Array(0);
|
|
6834
7213
|
if (options.dimensions !== undefined && vector.length !== options.dimensions) {
|
|
6835
7214
|
return yield* new EmbeddingError({
|
|
6836
7215
|
message: `embedder dimension mismatch: model outputs ${vector.length} dims but config declares ${String(options.dimensions)}`
|
|
@@ -6851,14 +7230,14 @@ var makeTransformersModel = (options = {}) => {
|
|
|
6851
7230
|
};
|
|
6852
7231
|
};
|
|
6853
7232
|
// ../embedder/src/registry.ts
|
|
6854
|
-
import { Effect as
|
|
7233
|
+
import { Effect as Effect34, Layer as Layer12 } from "effect";
|
|
6855
7234
|
// ../core/src/hosted-embedding.ts
|
|
6856
7235
|
import { embedMany } from "ai";
|
|
6857
7236
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
6858
7237
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
6859
7238
|
import { createCohere } from "@ai-sdk/cohere";
|
|
6860
7239
|
import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
|
|
6861
|
-
import { Effect as
|
|
7240
|
+
import { Effect as Effect35 } from "effect";
|
|
6862
7241
|
var makeHostedEmbeddingModel = (options) => {
|
|
6863
7242
|
const provider = options.provider ?? "openai";
|
|
6864
7243
|
const model = provider === "google" ? createGoogleGenerativeAI({
|
|
@@ -6889,8 +7268,8 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
6889
7268
|
return { openaiCompatible: { dimensions: options.dimensions } };
|
|
6890
7269
|
}
|
|
6891
7270
|
})();
|
|
6892
|
-
const embed =
|
|
6893
|
-
const result = yield*
|
|
7271
|
+
const embed = Effect35.fn("HostedEmbedding.embed")(function* (texts) {
|
|
7272
|
+
const result = yield* Effect35.tryPromise({
|
|
6894
7273
|
try: () => embedMany({ model, values: [...texts], providerOptions }),
|
|
6895
7274
|
catch: (cause) => new EmbeddingError({ message: `hosted embedMany: ${cause}` })
|
|
6896
7275
|
});
|
|
@@ -6903,7 +7282,7 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
6903
7282
|
}
|
|
6904
7283
|
return result.embeddings.map((e) => new Float32Array(e));
|
|
6905
7284
|
});
|
|
6906
|
-
const embedSingle =
|
|
7285
|
+
const embedSingle = Effect35.fn("HostedEmbedding.embedSingle")(function* (text4) {
|
|
6907
7286
|
const results = yield* embed([text4]);
|
|
6908
7287
|
return results[0];
|
|
6909
7288
|
});
|
|
@@ -6917,7 +7296,7 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
6917
7296
|
};
|
|
6918
7297
|
|
|
6919
7298
|
// ../retriever/src/hybrid.ts
|
|
6920
|
-
import { Context as Context10, Effect as
|
|
7299
|
+
import { Context as Context10, Effect as Effect36, Layer as Layer13, Schema as Schema22 } from "effect";
|
|
6921
7300
|
|
|
6922
7301
|
// ../retriever/src/rrf.ts
|
|
6923
7302
|
function fuse(denseRanks, fts5Ranks, k, limit) {
|
|
@@ -6960,15 +7339,15 @@ class HybridError extends Schema22.TaggedErrorClass()("HybridRetrieval.HybridErr
|
|
|
6960
7339
|
|
|
6961
7340
|
class Service10 extends Context10.Service()("@dimple/retriever/HybridRetrieval") {
|
|
6962
7341
|
}
|
|
6963
|
-
var wrapError = (tag) => (effect) => effect.pipe(
|
|
6964
|
-
var layer11 = (options) => Layer13.effect(Service10,
|
|
7342
|
+
var wrapError = (tag) => (effect) => effect.pipe(Effect36.mapError((cause) => new HybridError({ message: `${tag}: ${cause}` })));
|
|
7343
|
+
var layer11 = (options) => Layer13.effect(Service10, Effect36.gen(function* () {
|
|
6965
7344
|
const store = yield* Service5;
|
|
6966
7345
|
const registry = yield* Service9;
|
|
6967
7346
|
const defaultId = yield* registry.defaultModelId.pipe(wrapError("defaultModelId"));
|
|
6968
7347
|
const defaultModel = yield* registry.getModel(defaultId).pipe(wrapError("getModel"));
|
|
6969
7348
|
const defaultK = options?.defaultK ?? 60;
|
|
6970
7349
|
const defaultLimit = options?.defaultLimit ?? 10;
|
|
6971
|
-
const hybrid =
|
|
7350
|
+
const hybrid = Effect36.fn("HybridRetrieval.hybrid")(function* (query2, options2) {
|
|
6972
7351
|
const namespace = options2?.namespace ?? "default";
|
|
6973
7352
|
const limit = options2?.limit ?? defaultLimit;
|
|
6974
7353
|
const k = options2?.k ?? defaultK;
|
|
@@ -6985,12 +7364,16 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
6985
7364
|
const vectorResults = yield* store.searchVector([...embedding], namespace, limit * fetchMultiplier, {
|
|
6986
7365
|
...vectorOpts,
|
|
6987
7366
|
...options2?.contentType !== undefined ? { contentType: options2.contentType } : {},
|
|
6988
|
-
audit: false
|
|
7367
|
+
audit: false,
|
|
7368
|
+
augment: false,
|
|
7369
|
+
includeEmbedding: options2?.includeEmbedding === true
|
|
6989
7370
|
}).pipe(wrapError("searchVector"));
|
|
6990
7371
|
const fts5Results = yield* store.searchFts5(query2, namespace, limit * fetchMultiplier, {
|
|
6991
7372
|
...topicKeys !== undefined ? { topicKeys } : {},
|
|
6992
7373
|
...options2?.contentType !== undefined ? { contentType: options2.contentType } : {},
|
|
6993
|
-
audit: false
|
|
7374
|
+
audit: false,
|
|
7375
|
+
augment: false,
|
|
7376
|
+
includeEmbedding: options2?.includeEmbedding === true
|
|
6994
7377
|
}).pipe(wrapError("searchFts5"));
|
|
6995
7378
|
const vWeight = profile.modeWeights?.vector ?? 1;
|
|
6996
7379
|
const fWeight = profile.modeWeights?.fts ?? 1;
|
|
@@ -7026,25 +7409,15 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7026
7409
|
const now = Date.now();
|
|
7027
7410
|
const DAY_MS = 86400000;
|
|
7028
7411
|
const recencyBoost = profile.recencyBoost ?? 0;
|
|
7029
|
-
const augmentedById = new Map;
|
|
7030
|
-
for (const sm of vectorResults)
|
|
7031
|
-
augmentedById.set(sm.memory.id, sm);
|
|
7032
|
-
for (const sm of fts5Results)
|
|
7033
|
-
if (!augmentedById.has(sm.memory.id))
|
|
7034
|
-
augmentedById.set(sm.memory.id, sm);
|
|
7035
7412
|
const boosted = fused.map((r) => {
|
|
7036
7413
|
const memory = memoryMap.get(r.memoryId);
|
|
7037
7414
|
const ageDays = (now - memory.createdAt) / DAY_MS;
|
|
7038
7415
|
const recencyFactor = 1 / (1 + ageDays);
|
|
7039
7416
|
const score = r.rrfScore * (1 + recencyBoost * recencyFactor);
|
|
7040
|
-
const aug = augmentedById.get(r.memoryId);
|
|
7041
7417
|
return {
|
|
7042
7418
|
memory,
|
|
7043
7419
|
score,
|
|
7044
|
-
source: "hybrid"
|
|
7045
|
-
...aug?.supersededBy !== undefined ? { supersededBy: aug.supersededBy } : {},
|
|
7046
|
-
...aug?.conflicts !== undefined ? { conflicts: aug.conflicts } : {},
|
|
7047
|
-
...aug?.supportedBy !== undefined ? { supportedBy: aug.supportedBy } : {}
|
|
7420
|
+
source: "hybrid"
|
|
7048
7421
|
};
|
|
7049
7422
|
});
|
|
7050
7423
|
boosted.sort((a, b) => b.score - a.score);
|
|
@@ -7077,7 +7450,7 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7077
7450
|
fts: fts5Results.map((s) => s.memory.id)
|
|
7078
7451
|
})
|
|
7079
7452
|
}).pipe(wrapError("logAudit"));
|
|
7080
|
-
yield*
|
|
7453
|
+
yield* Effect36.logInfo("hybrid.search").pipe(Effect36.annotateLogs({
|
|
7081
7454
|
scope: "@dimple/retriever",
|
|
7082
7455
|
event: "hybrid.search",
|
|
7083
7456
|
mode,
|
|
@@ -7091,7 +7464,7 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7091
7464
|
}));
|
|
7092
7465
|
var hybridRetrievalLayer = layer11();
|
|
7093
7466
|
// ../llm/src/llm.ts
|
|
7094
|
-
import { Cause as Cause4, Effect as
|
|
7467
|
+
import { Cause as Cause4, Effect as Effect37, Schema as Schema23 } from "effect";
|
|
7095
7468
|
import {
|
|
7096
7469
|
generateText as generateTextCall,
|
|
7097
7470
|
NoSuchToolError,
|
|
@@ -7145,9 +7518,9 @@ var makeLLM = (model, defaults3 = {}) => {
|
|
|
7145
7518
|
...reasoningEffort !== undefined ? { providerOptions: { openaiCompatible: { reasoningEffort } } } : {}
|
|
7146
7519
|
};
|
|
7147
7520
|
};
|
|
7148
|
-
const generateText =
|
|
7521
|
+
const generateText = Effect37.fn("LLM.generateText")(function* (input) {
|
|
7149
7522
|
const started = Date.now();
|
|
7150
|
-
const text4 = yield*
|
|
7523
|
+
const text4 = yield* Effect37.tryPromise({
|
|
7151
7524
|
try: () => generateTextCall({
|
|
7152
7525
|
model,
|
|
7153
7526
|
prompt: input.prompt,
|
|
@@ -7156,16 +7529,16 @@ var makeLLM = (model, defaults3 = {}) => {
|
|
|
7156
7529
|
}).then((result) => result.text),
|
|
7157
7530
|
catch: (cause) => new LLMError({ message: "LLM text generation failed", cause: Cause4.die(cause) })
|
|
7158
7531
|
});
|
|
7159
|
-
yield*
|
|
7532
|
+
yield* Effect37.logDebug("llm.generate-text").pipe(Effect37.annotateLogs({
|
|
7160
7533
|
scope: "@dimple/llm",
|
|
7161
7534
|
event: "llm.generate-text",
|
|
7162
7535
|
latencyMs: Date.now() - started
|
|
7163
7536
|
}));
|
|
7164
7537
|
return text4;
|
|
7165
7538
|
});
|
|
7166
|
-
const generateObject =
|
|
7539
|
+
const generateObject = Effect37.fn("LLM.generateObject")(function* (input) {
|
|
7167
7540
|
const started = Date.now();
|
|
7168
|
-
const result = yield*
|
|
7541
|
+
const result = yield* Effect37.tryPromise({
|
|
7169
7542
|
try: async () => {
|
|
7170
7543
|
try {
|
|
7171
7544
|
const result3 = await generateTextCall({
|
|
@@ -7208,7 +7581,7 @@ ${schemaJson}`,
|
|
|
7208
7581
|
},
|
|
7209
7582
|
catch: (cause) => new LLMError({ message: "LLM structured generation failed", cause: Cause4.die(cause) })
|
|
7210
7583
|
});
|
|
7211
|
-
yield*
|
|
7584
|
+
yield* Effect37.logDebug("llm.generate-object").pipe(Effect37.annotateLogs({
|
|
7212
7585
|
scope: "@dimple/llm",
|
|
7213
7586
|
event: "llm.generate-object",
|
|
7214
7587
|
latencyMs: Date.now() - started
|
|
@@ -7228,7 +7601,7 @@ var openAICompatible = (options) => {
|
|
|
7228
7601
|
return provider.languageModel(options.modelId ?? "default");
|
|
7229
7602
|
};
|
|
7230
7603
|
// ../llm/src/dream-classifier.ts
|
|
7231
|
-
import { Effect as
|
|
7604
|
+
import { Effect as Effect38 } from "effect";
|
|
7232
7605
|
import { z as z2 } from "zod";
|
|
7233
7606
|
var EDGE_KINDS = ["memory_supersedes", "memory_contradicts", "memory_supports"];
|
|
7234
7607
|
var EDGE_KIND_SET = new Set(EDGE_KINDS);
|
|
@@ -7275,7 +7648,7 @@ var buildPrompt = (pairs, maxDerived) => [
|
|
|
7275
7648
|
].join(`
|
|
7276
7649
|
`);
|
|
7277
7650
|
var makeDreamClassifier = (deps) => {
|
|
7278
|
-
const classify =
|
|
7651
|
+
const classify = Effect38.fn("DreamClassifier.classify")(function* (pairs, opts) {
|
|
7279
7652
|
const system = deps.modelId === undefined ? SYSTEM_PROMPT : `${SYSTEM_PROMPT}
|
|
7280
7653
|
Judging for model: ${deps.modelId}`;
|
|
7281
7654
|
const judged = yield* deps.llm.generateObject({
|
|
@@ -7305,7 +7678,7 @@ Judging for model: ${deps.modelId}`;
|
|
|
7305
7678
|
return { classify };
|
|
7306
7679
|
};
|
|
7307
7680
|
// ../llm/src/service.ts
|
|
7308
|
-
import { Config as Config6, Context as Context11, Effect as
|
|
7681
|
+
import { Config as Config6, Context as Context11, Effect as Effect39, Layer as Layer14 } from "effect";
|
|
7309
7682
|
class Service11 extends Context11.Service()("@dimple/LLM") {
|
|
7310
7683
|
}
|
|
7311
7684
|
// ../core/src/layer.ts
|
|
@@ -7315,7 +7688,7 @@ var DISABLED_LOGGING2 = {
|
|
|
7315
7688
|
format: "jsonl",
|
|
7316
7689
|
ringSize: 4096
|
|
7317
7690
|
};
|
|
7318
|
-
var embedderFromConfig = (model) => model.provider === "transformers" ?
|
|
7691
|
+
var embedderFromConfig = (model) => model.provider === "transformers" ? Effect40.succeed(makeTransformersModel({
|
|
7319
7692
|
modelId: model.model,
|
|
7320
7693
|
dimensions: model.dimensions,
|
|
7321
7694
|
...model.device !== undefined ? { device: model.device } : {},
|
|
@@ -7324,14 +7697,14 @@ var embedderFromConfig = (model) => model.provider === "transformers" ? Effect39
|
|
|
7324
7697
|
...model.normalize !== undefined ? { normalize: model.normalize } : {},
|
|
7325
7698
|
...model.cacheDir !== undefined ? { cacheDir: model.cacheDir } : {},
|
|
7326
7699
|
...model.autoInstall !== undefined ? { autoInstall: model.autoInstall } : {}
|
|
7327
|
-
})) : model.provider === "openai" ? model.apiKey === undefined ?
|
|
7700
|
+
})) : model.provider === "openai" ? model.apiKey === undefined ? Effect40.fail(new EmbeddingError({ message: `OpenAI model ${model.id} requires an apiKey` })) : Effect40.succeed(makeHostedEmbeddingModel({
|
|
7328
7701
|
modelId: model.id,
|
|
7329
7702
|
model: model.model,
|
|
7330
7703
|
dimensions: model.dimensions,
|
|
7331
7704
|
apiKey: model.apiKey,
|
|
7332
7705
|
...model.provider !== "openai" ? { provider: model.provider } : {},
|
|
7333
7706
|
...model.baseURL !== undefined ? { baseURL: model.baseURL } : {}
|
|
7334
|
-
})) :
|
|
7707
|
+
})) : Effect40.fail(new EmbeddingError({ message: `Unknown provider: ${model.provider}` }));
|
|
7335
7708
|
var layer13 = (options) => {
|
|
7336
7709
|
const topicConfig = Schema24.decodeUnknownSync(TopicConfig)(options.topics?.config ?? {});
|
|
7337
7710
|
const graphConfig = Schema24.decodeUnknownSync(exports_graph.GraphConfig)(options.graph ?? {});
|
|
@@ -7339,11 +7712,31 @@ var layer13 = (options) => {
|
|
|
7339
7712
|
const defaultModel = models2.get(options.defaultModelId);
|
|
7340
7713
|
const loggingConfig = options.logging ?? DISABLED_LOGGING2;
|
|
7341
7714
|
const getModelConfig = (key) => models2.get(key);
|
|
7342
|
-
const storeWithSeams = Layer15.unwrap(
|
|
7715
|
+
const storeWithSeams = Layer15.unwrap(Effect40.gen(function* () {
|
|
7343
7716
|
const rt = yield* Service7;
|
|
7344
|
-
const enqueueSplit = (topicKey, modelKey) => rt.enqueue(TopicSplit, { topicId: topicKey, modelKey }).pipe(
|
|
7345
|
-
const enqueueMerge = (topicKey, modelKey) => rt.enqueue(TopicMerge, { topicId: topicKey, modelKey }).pipe(
|
|
7346
|
-
const
|
|
7717
|
+
const enqueueSplit = (topicKey, modelKey) => rt.enqueue(TopicSplit, { topicId: topicKey, modelKey }).pipe(Effect40.asVoid);
|
|
7718
|
+
const enqueueMerge = (topicKey, modelKey) => rt.enqueue(TopicMerge, { topicId: topicKey, modelKey }).pipe(Effect40.asVoid);
|
|
7719
|
+
const auditBuffer = [];
|
|
7720
|
+
let auditTimer;
|
|
7721
|
+
const flushAudit = () => Effect40.gen(function* () {
|
|
7722
|
+
auditTimer = undefined;
|
|
7723
|
+
if (auditBuffer.length === 0)
|
|
7724
|
+
return;
|
|
7725
|
+
const rows = auditBuffer.splice(0, auditBuffer.length);
|
|
7726
|
+
yield* rt.enqueue(AuditWrite, { rows }).pipe(Effect40.asVoid);
|
|
7727
|
+
});
|
|
7728
|
+
const enqueueAudit = (row) => Effect40.gen(function* () {
|
|
7729
|
+
auditBuffer.push(row);
|
|
7730
|
+
if (auditTimer === undefined) {
|
|
7731
|
+
auditTimer = setTimeout(() => {
|
|
7732
|
+
Effect40.runPromise(flushAudit()).catch(() => {});
|
|
7733
|
+
}, 1000);
|
|
7734
|
+
auditTimer.unref?.();
|
|
7735
|
+
}
|
|
7736
|
+
if (auditBuffer.length >= 50) {
|
|
7737
|
+
yield* flushAudit();
|
|
7738
|
+
}
|
|
7739
|
+
});
|
|
7347
7740
|
return layer8(options.storeUrl, {
|
|
7348
7741
|
...options.defaultTtlSecs !== undefined ? { defaultTtlSecs: options.defaultTtlSecs } : {},
|
|
7349
7742
|
...options.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : {},
|
|
@@ -7362,15 +7755,15 @@ var layer13 = (options) => {
|
|
|
7362
7755
|
...options.audit === false ? {} : { enqueueAudit }
|
|
7363
7756
|
});
|
|
7364
7757
|
}));
|
|
7365
|
-
const embedderRegistry = Layer15.effect(Service9,
|
|
7758
|
+
const embedderRegistry = Layer15.effect(Service9, Effect40.sync(() => Service9.of({
|
|
7366
7759
|
getModel: (id) => {
|
|
7367
7760
|
const model = models2.get(id);
|
|
7368
|
-
return model !== undefined ? options.embedderFor(model) :
|
|
7761
|
+
return model !== undefined ? options.embedderFor(model) : Effect40.fail(new EmbeddingError({ message: `Unknown model: ${id}` }));
|
|
7369
7762
|
},
|
|
7370
|
-
defaultModelId:
|
|
7371
|
-
registerModel: () =>
|
|
7763
|
+
defaultModelId: Effect40.succeed(options.defaultModelId),
|
|
7764
|
+
registerModel: () => Effect40.void
|
|
7372
7765
|
})));
|
|
7373
|
-
const coreService = Layer15.effect(Service,
|
|
7766
|
+
const coreService = Layer15.effect(Service, Effect40.gen(function* () {
|
|
7374
7767
|
if (defaultModel === undefined) {
|
|
7375
7768
|
return yield* new CoreConfigError({
|
|
7376
7769
|
message: `@dimple/core: unknown defaultModelId "${options.defaultModelId}"`
|
|
@@ -7383,20 +7776,20 @@ var layer13 = (options) => {
|
|
|
7383
7776
|
const hybrid = yield* Service10;
|
|
7384
7777
|
const logging = yield* Service3;
|
|
7385
7778
|
const extraByName = new Map((options.extraHandlers ?? []).map((h) => [h.name, h.handler]));
|
|
7386
|
-
const register = (name, handler) => extraByName.has(name) ?
|
|
7779
|
+
const register = (name, handler) => extraByName.has(name) ? Effect40.void : registry.register(name, handler);
|
|
7387
7780
|
yield* register("topic.split", exports_topics.topicSplitHandler({
|
|
7388
7781
|
getModelConfig,
|
|
7389
7782
|
config: topicConfig,
|
|
7390
7783
|
backend,
|
|
7391
|
-
onUnevenSplit: (childKey, modelKey) => rt.enqueue(TopicRebalance, { topicId: childKey, modelKey }).pipe(
|
|
7392
|
-
onChildOverflow: (childKey, modelKey) => rt.enqueue(TopicSplit, { topicId: childKey, modelKey }).pipe(
|
|
7784
|
+
onUnevenSplit: (childKey, modelKey) => rt.enqueue(TopicRebalance, { topicId: childKey, modelKey }).pipe(Effect40.asVoid),
|
|
7785
|
+
onChildOverflow: (childKey, modelKey) => rt.enqueue(TopicSplit, { topicId: childKey, modelKey }).pipe(Effect40.asVoid)
|
|
7393
7786
|
}));
|
|
7394
7787
|
yield* register("topic.merge", exports_topics.topicMergeHandler({ getModelConfig, config: topicConfig, backend }));
|
|
7395
7788
|
yield* register("topic.rebalance", exports_topics.topicRebalanceHandler({ getModelConfig, config: topicConfig, backend }));
|
|
7396
7789
|
yield* register("memory.embed", exports_topics.memoryEmbedHandler({
|
|
7397
7790
|
getModelConfig,
|
|
7398
|
-
embedOne: (text4, model) => options.embedderFor(model).pipe(
|
|
7399
|
-
getMemoryContent: (memoryId) => store.get(memoryId).pipe(
|
|
7791
|
+
embedOne: (text4, model) => options.embedderFor(model).pipe(Effect40.flatMap((m) => m.embedSingle(text4))),
|
|
7792
|
+
getMemoryContent: (memoryId) => store.get(memoryId).pipe(Effect40.map((m) => m.content)),
|
|
7400
7793
|
config: topicConfig,
|
|
7401
7794
|
backend
|
|
7402
7795
|
}));
|
|
@@ -7411,12 +7804,12 @@ var layer13 = (options) => {
|
|
|
7411
7804
|
yield* register("memory.dream", (payload, ctx) => {
|
|
7412
7805
|
const envelope = payload;
|
|
7413
7806
|
if (getModelConfig(envelope.modelKey) === undefined) {
|
|
7414
|
-
return
|
|
7807
|
+
return Effect40.fail(new JobFailure({
|
|
7415
7808
|
jobId: ctx.jobId,
|
|
7416
7809
|
message: `memory.dream: unknown model "${envelope.modelKey}"`
|
|
7417
7810
|
}));
|
|
7418
7811
|
}
|
|
7419
|
-
return
|
|
7812
|
+
return Effect40.fail(new JobFailure({
|
|
7420
7813
|
jobId: ctx.jobId,
|
|
7421
7814
|
message: "memory.dream: LLM not configured — provide `llm` to @dimple/core"
|
|
7422
7815
|
}));
|
|
@@ -7451,7 +7844,7 @@ var layer13 = (options) => {
|
|
|
7451
7844
|
...options.defaultLimit !== undefined ? { defaultLimit: options.defaultLimit } : {}
|
|
7452
7845
|
})), Layer15.provideMerge(embedderRegistry), Layer15.provideMerge(storeWithSeams), Layer15.provideMerge(Layer15.mergeAll(jobStack, layer7, layer(loggingConfig))));
|
|
7453
7846
|
};
|
|
7454
|
-
var layerConfig7 = (options) => Layer15.unwrap(
|
|
7847
|
+
var layerConfig7 = (options) => Layer15.unwrap(Effect40.gen(function* () {
|
|
7455
7848
|
const cfg = yield* Service4;
|
|
7456
7849
|
const embedding = cfg.config.embedding;
|
|
7457
7850
|
if (embedding === undefined || embedding.models.length === 0) {
|
|
@@ -7485,7 +7878,7 @@ var layerConfig7 = (options) => Layer15.unwrap(Effect39.gen(function* () {
|
|
|
7485
7878
|
});
|
|
7486
7879
|
}));
|
|
7487
7880
|
// ../extractor/src/code-extractor.ts
|
|
7488
|
-
import { Context as Context12, Effect as
|
|
7881
|
+
import { Context as Context12, Effect as Effect41, Layer as Layer16 } from "effect";
|
|
7489
7882
|
|
|
7490
7883
|
// ../extractor/src/models/extraction.ts
|
|
7491
7884
|
import { Schema as Schema25 } from "effect";
|
|
@@ -7931,9 +8324,9 @@ function extract3(filePath, tree2) {
|
|
|
7931
8324
|
// ../extractor/src/code-extractor.ts
|
|
7932
8325
|
class Service12 extends Context12.Service()("@dimple/extractor/CodeExtractor") {
|
|
7933
8326
|
}
|
|
7934
|
-
var layer14 = Layer16.effect(Service12,
|
|
7935
|
-
extract:
|
|
7936
|
-
const result = yield*
|
|
8327
|
+
var layer14 = Layer16.effect(Service12, Effect41.succeed(Service12.of({
|
|
8328
|
+
extract: Effect41.fn("CodeExtractor.extract")(function* (request) {
|
|
8329
|
+
const result = yield* Effect41.try({
|
|
7937
8330
|
try: () => extract3(request.filePath, request.tree),
|
|
7938
8331
|
catch: (cause) => new ExtractionError({ message: `Extraction failed for ${request.filePath}`, cause })
|
|
7939
8332
|
});
|
|
@@ -7966,14 +8359,14 @@ var layer15 = Layer17.succeed(Service13, Service13.of({
|
|
|
7966
8359
|
extractText: (content) => content
|
|
7967
8360
|
}));
|
|
7968
8361
|
// ../parser/src/code-parser.ts
|
|
7969
|
-
import { Context as Context19, Effect as
|
|
8362
|
+
import { Context as Context19, Effect as Effect48, Layer as Layer24 } from "effect";
|
|
7970
8363
|
import { Parser as Parser2 } from "web-tree-sitter";
|
|
7971
8364
|
|
|
7972
8365
|
// ../parser/src/language-store.ts
|
|
7973
|
-
import { Cache, Context as Context18, Effect as
|
|
8366
|
+
import { Cache, Context as Context18, Effect as Effect47, Layer as Layer23 } from "effect";
|
|
7974
8367
|
|
|
7975
8368
|
// ../wasm/src/wasm-registry.ts
|
|
7976
|
-
import { Context as Context14, Effect as
|
|
8369
|
+
import { Context as Context14, Effect as Effect42, Layer as Layer18, Schema as Schema30 } from "effect";
|
|
7977
8370
|
|
|
7978
8371
|
// ../shared/src/errors/wasm.ts
|
|
7979
8372
|
import { Schema as Schema27 } from "effect";
|
|
@@ -8075,7 +8468,7 @@ for (const grammar of grammars.grammars) {
|
|
|
8075
8468
|
|
|
8076
8469
|
class Service14 extends Context14.Service()("@dimple/wasm/TreeSitterRegistry") {
|
|
8077
8470
|
static layer = Layer18.succeed(Service14, Service14.of({
|
|
8078
|
-
resolve:
|
|
8471
|
+
resolve: Effect42.fn("TreeSitterRegistry.resolve")(function* (lang) {
|
|
8079
8472
|
const found = registryMap.get(lang);
|
|
8080
8473
|
if (found === undefined) {
|
|
8081
8474
|
return yield* new RegistryError2({ message: `Language not found in registry: ${lang}` });
|
|
@@ -8085,11 +8478,11 @@ class Service14 extends Context14.Service()("@dimple/wasm/TreeSitterRegistry") {
|
|
|
8085
8478
|
}));
|
|
8086
8479
|
}
|
|
8087
8480
|
// ../wasm/src/wasm-downloader.ts
|
|
8088
|
-
import { Context as Context15, Effect as
|
|
8481
|
+
import { Context as Context15, Effect as Effect43, Layer as Layer19 } from "effect";
|
|
8089
8482
|
class Service15 extends Context15.Service()("@dimple/wasm/WasmDownloader") {
|
|
8090
|
-
static layer = Layer19.effect(Service15,
|
|
8091
|
-
download:
|
|
8092
|
-
const response = yield*
|
|
8483
|
+
static layer = Layer19.effect(Service15, Effect43.succeed(Service15.of({
|
|
8484
|
+
download: Effect43.fn("WasmDownloader.download")(function* (url) {
|
|
8485
|
+
const response = yield* Effect43.tryPromise({
|
|
8093
8486
|
try: () => fetch(url),
|
|
8094
8487
|
catch: (cause) => new WasmDownloadError({
|
|
8095
8488
|
message: `Failed to fetch from ${url}`,
|
|
@@ -8103,7 +8496,7 @@ class Service15 extends Context15.Service()("@dimple/wasm/WasmDownloader") {
|
|
|
8103
8496
|
url
|
|
8104
8497
|
});
|
|
8105
8498
|
}
|
|
8106
|
-
const arrayBuffer = yield*
|
|
8499
|
+
const arrayBuffer = yield* Effect43.tryPromise({
|
|
8107
8500
|
try: () => response.arrayBuffer(),
|
|
8108
8501
|
catch: (cause) => new WasmDownloadError({
|
|
8109
8502
|
message: `Failed to read response from ${url}`,
|
|
@@ -8116,7 +8509,7 @@ class Service15 extends Context15.Service()("@dimple/wasm/WasmDownloader") {
|
|
|
8116
8509
|
})));
|
|
8117
8510
|
}
|
|
8118
8511
|
// ../wasm/src/wasm-cache.ts
|
|
8119
|
-
import { Context as Context16, Effect as
|
|
8512
|
+
import { Context as Context16, Effect as Effect44, Layer as Layer20 } from "effect";
|
|
8120
8513
|
import { existsSync as existsSync3, mkdirSync, renameSync, writeFileSync } from "node:fs";
|
|
8121
8514
|
import { join as join3 } from "node:path";
|
|
8122
8515
|
|
|
@@ -8213,35 +8606,35 @@ var paths = envPaths("dimple");
|
|
|
8213
8606
|
|
|
8214
8607
|
class Service16 extends Context16.Service()("@dimple/wasm/WasmCache") {
|
|
8215
8608
|
}
|
|
8216
|
-
var layer16 = Layer20.effect(Service16,
|
|
8217
|
-
ensure:
|
|
8609
|
+
var layer16 = Layer20.effect(Service16, Effect44.succeed(Service16.of({
|
|
8610
|
+
ensure: Effect44.fn("WasmCache.ensure")(function* (lang, version, download) {
|
|
8218
8611
|
const versionDir = join3(paths.cache, "tree-sitter", lang, version);
|
|
8219
8612
|
const finalPath = join3(versionDir, "parser.wasm");
|
|
8220
|
-
const exists = yield*
|
|
8613
|
+
const exists = yield* Effect44.try({
|
|
8221
8614
|
try: () => existsSync3(finalPath),
|
|
8222
8615
|
catch: (cause) => new WasmCacheError({ message: `Failed to stat ${finalPath}`, cause })
|
|
8223
8616
|
});
|
|
8224
8617
|
if (exists)
|
|
8225
8618
|
return finalPath;
|
|
8226
|
-
yield*
|
|
8619
|
+
yield* Effect44.try({
|
|
8227
8620
|
try: () => mkdirSync(versionDir, { recursive: true }),
|
|
8228
8621
|
catch: (cause) => new WasmCacheError({ message: `Failed to create ${versionDir}`, cause })
|
|
8229
8622
|
});
|
|
8230
8623
|
const tempPath = join3(versionDir, `.tmp-${process.pid}`);
|
|
8231
8624
|
const wasm = yield* download;
|
|
8232
|
-
yield*
|
|
8625
|
+
yield* Effect44.try({
|
|
8233
8626
|
try: () => writeFileSync(tempPath, wasm),
|
|
8234
8627
|
catch: (cause) => new WasmCacheError({ message: `Failed to write ${tempPath}`, cause })
|
|
8235
8628
|
});
|
|
8236
|
-
const renamed = yield*
|
|
8629
|
+
const renamed = yield* Effect44.try({
|
|
8237
8630
|
try: () => {
|
|
8238
8631
|
renameSync(tempPath, finalPath);
|
|
8239
8632
|
return true;
|
|
8240
8633
|
},
|
|
8241
8634
|
catch: (cause) => new WasmCacheError({ message: `Failed to rename ${tempPath}`, cause })
|
|
8242
|
-
}).pipe(
|
|
8635
|
+
}).pipe(Effect44.catchCause(() => Effect44.succeed(false)));
|
|
8243
8636
|
if (!renamed) {
|
|
8244
|
-
const existsAfter = yield*
|
|
8637
|
+
const existsAfter = yield* Effect44.try({
|
|
8245
8638
|
try: () => existsSync3(finalPath),
|
|
8246
8639
|
catch: (cause) => new WasmCacheError({ message: `Failed to stat ${finalPath}`, cause })
|
|
8247
8640
|
});
|
|
@@ -8255,25 +8648,25 @@ var layer16 = Layer20.effect(Service16, Effect43.succeed(Service16.of({
|
|
|
8255
8648
|
})
|
|
8256
8649
|
})));
|
|
8257
8650
|
// ../wasm/src/grammar-store.ts
|
|
8258
|
-
import { Context as Context17, Effect as
|
|
8651
|
+
import { Context as Context17, Effect as Effect45, Layer as Layer21 } from "effect";
|
|
8259
8652
|
class Service17 extends Context17.Service()("@dimple/wasm/GrammarStore") {
|
|
8260
8653
|
}
|
|
8261
|
-
var layer17 = Layer21.effect(Service17,
|
|
8654
|
+
var layer17 = Layer21.effect(Service17, Effect45.gen(function* () {
|
|
8262
8655
|
const registry = yield* Service14;
|
|
8263
8656
|
const downloader = yield* Service15;
|
|
8264
8657
|
const cache = yield* Service16;
|
|
8265
|
-
const ensure =
|
|
8266
|
-
const metadata = yield* registry.resolve(lang).pipe(
|
|
8658
|
+
const ensure = Effect45.fn("GrammarStore.ensure")(function* (lang) {
|
|
8659
|
+
const metadata = yield* registry.resolve(lang).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
8267
8660
|
language: lang,
|
|
8268
8661
|
message: `Failed to resolve grammar metadata for: ${lang}`,
|
|
8269
8662
|
cause
|
|
8270
8663
|
})));
|
|
8271
|
-
const wasmBytes = yield* downloader.download(metadata.url).pipe(
|
|
8664
|
+
const wasmBytes = yield* downloader.download(metadata.url).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
8272
8665
|
language: lang,
|
|
8273
8666
|
message: `Failed to download grammar for: ${lang}`,
|
|
8274
8667
|
cause
|
|
8275
8668
|
})));
|
|
8276
|
-
const path2 = yield* cache.ensure(lang, metadata.version,
|
|
8669
|
+
const path2 = yield* cache.ensure(lang, metadata.version, Effect45.succeed(wasmBytes)).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
8277
8670
|
language: lang,
|
|
8278
8671
|
message: `Failed to provision grammar for: ${lang}`,
|
|
8279
8672
|
cause
|
|
@@ -8287,9 +8680,9 @@ var allLive = layer17.pipe(Layer21.provideMerge(Layer21.mergeAll(Service14.layer
|
|
|
8287
8680
|
import { Language } from "web-tree-sitter";
|
|
8288
8681
|
|
|
8289
8682
|
// ../parser/src/tree-sitter-engine.ts
|
|
8290
|
-
import { Effect as
|
|
8683
|
+
import { Effect as Effect46, Layer as Layer22 } from "effect";
|
|
8291
8684
|
import { Parser } from "web-tree-sitter";
|
|
8292
|
-
var layer18 = Layer22.effectDiscard(
|
|
8685
|
+
var layer18 = Layer22.effectDiscard(Effect46.tryPromise({
|
|
8293
8686
|
try: () => Parser.init(),
|
|
8294
8687
|
catch: (cause) => new ParserInitError({ message: "Failed to initialize tree sitter", cause })
|
|
8295
8688
|
}));
|
|
@@ -8297,20 +8690,20 @@ var layer18 = Layer22.effectDiscard(Effect45.tryPromise({
|
|
|
8297
8690
|
// ../parser/src/language-store.ts
|
|
8298
8691
|
class Service18 extends Context18.Service()("@dimple/parser/LanguageStore") {
|
|
8299
8692
|
}
|
|
8300
|
-
var layer19 = Layer23.effect(Service18,
|
|
8693
|
+
var layer19 = Layer23.effect(Service18, Effect47.gen(function* () {
|
|
8301
8694
|
const grammarStore = yield* Service17;
|
|
8302
8695
|
const resolveLang = (lang) => lang === "jsx" ? "javascript" : lang;
|
|
8303
8696
|
const cache = yield* Cache.make({
|
|
8304
8697
|
capacity: 100,
|
|
8305
8698
|
timeToLive: Infinity,
|
|
8306
|
-
lookup: (lang) =>
|
|
8699
|
+
lookup: (lang) => Effect47.gen(function* () {
|
|
8307
8700
|
const grammarLang = resolveLang(lang);
|
|
8308
|
-
const wasmPath = yield* grammarStore.ensure(grammarLang).pipe(
|
|
8701
|
+
const wasmPath = yield* grammarStore.ensure(grammarLang).pipe(Effect47.mapError((e) => new LanguageLoadError({
|
|
8309
8702
|
lang,
|
|
8310
8703
|
message: `Failed to provision grammar for ${lang}`,
|
|
8311
8704
|
cause: e
|
|
8312
8705
|
})));
|
|
8313
|
-
return yield*
|
|
8706
|
+
return yield* Effect47.tryPromise({
|
|
8314
8707
|
try: () => Language.load(wasmPath),
|
|
8315
8708
|
catch: (cause) => new LanguageLoadError({ lang, message: `Failed to load language: ${lang}`, cause })
|
|
8316
8709
|
});
|
|
@@ -8333,15 +8726,20 @@ var TreeHandle = Schema31.Struct({
|
|
|
8333
8726
|
// ../parser/src/code-parser.ts
|
|
8334
8727
|
class Service19 extends Context19.Service()("@dimple/parser/CodeParser") {
|
|
8335
8728
|
}
|
|
8336
|
-
var layer20 = Layer24.effect(Service19,
|
|
8729
|
+
var layer20 = Layer24.effect(Service19, Effect48.gen(function* () {
|
|
8337
8730
|
const languageStore = yield* Service18;
|
|
8338
|
-
const
|
|
8339
|
-
|
|
8731
|
+
const parserByLanguage = new Map;
|
|
8732
|
+
const parse2 = Effect48.fn("CodeParser.parse")(function* (request) {
|
|
8733
|
+
const language = yield* languageStore.get(request.lang).pipe(Effect48.mapError((cause) => new ParseError({
|
|
8340
8734
|
message: `Failed to load language '${request.lang}' for parsing`,
|
|
8341
8735
|
cause
|
|
8342
8736
|
})));
|
|
8343
|
-
|
|
8344
|
-
parser
|
|
8737
|
+
let parser = parserByLanguage.get(request.lang);
|
|
8738
|
+
if (parser === undefined) {
|
|
8739
|
+
parser = new Parser2;
|
|
8740
|
+
parser.setLanguage(language);
|
|
8741
|
+
parserByLanguage.set(request.lang, parser);
|
|
8742
|
+
}
|
|
8345
8743
|
const tree2 = parser.parse(request.source);
|
|
8346
8744
|
if (!tree2) {
|
|
8347
8745
|
return yield* new ParseError({ message: "Failed to parse source" });
|
|
@@ -8357,7 +8755,7 @@ var ParseRequest = Schema32.Struct({
|
|
|
8357
8755
|
source: Schema32.String
|
|
8358
8756
|
});
|
|
8359
8757
|
// src/config.ts
|
|
8360
|
-
import { Effect as
|
|
8758
|
+
import { Effect as Effect49, Layer as Layer25 } from "effect";
|
|
8361
8759
|
|
|
8362
8760
|
// src/errors.ts
|
|
8363
8761
|
import { Schema as Schema33 } from "effect";
|
|
@@ -8481,13 +8879,13 @@ var remapError = (error) => {
|
|
|
8481
8879
|
};
|
|
8482
8880
|
|
|
8483
8881
|
// src/config.ts
|
|
8484
|
-
var resolveConfigInput =
|
|
8485
|
-
return yield* (typeof input === "string" ? loadConfigFromFile(input) : loadConfig(input)).pipe(
|
|
8882
|
+
var resolveConfigInput = Effect49.fn("Sdk.resolveConfigInput")(function* (input) {
|
|
8883
|
+
return yield* (typeof input === "string" ? loadConfigFromFile(input) : loadConfig(input)).pipe(Effect49.mapError((error) => new ConfigError({ message: error.message })));
|
|
8486
8884
|
});
|
|
8487
|
-
var configLayerFromInput = (input) => Layer25.effect(Service4, resolveConfigInput(input).pipe(
|
|
8885
|
+
var configLayerFromInput = (input) => Layer25.effect(Service4, resolveConfigInput(input).pipe(Effect49.map((config) => Service4.of({ config }))));
|
|
8488
8886
|
|
|
8489
8887
|
// src/dimple.ts
|
|
8490
|
-
import { Cause as Cause5, Effect as
|
|
8888
|
+
import { Cause as Cause5, Effect as Effect50, Exit as Exit7 } from "effect";
|
|
8491
8889
|
|
|
8492
8890
|
// src/indexer.ts
|
|
8493
8891
|
import { execFileSync } from "node:child_process";
|
|
@@ -8727,7 +9125,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8727
9125
|
}
|
|
8728
9126
|
return;
|
|
8729
9127
|
};
|
|
8730
|
-
const runQuery = (text4, options) => run(
|
|
9128
|
+
const runQuery = (text4, options) => run(Effect50.gen(function* () {
|
|
8731
9129
|
if (typeof text4 !== "string") {
|
|
8732
9130
|
return yield* new InputError({ message: "query: text must be a string" });
|
|
8733
9131
|
}
|
|
@@ -8740,7 +9138,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8740
9138
|
let rows = yield* core().hybrid.hybrid(text4, {
|
|
8741
9139
|
...rest,
|
|
8742
9140
|
...capLimit !== undefined ? { limit: capLimit } : {}
|
|
8743
|
-
}).pipe(
|
|
9141
|
+
}).pipe(Effect50.map((r) => [...r]));
|
|
8744
9142
|
let truncated = false;
|
|
8745
9143
|
if (budget?.maxChars !== undefined && budget.maxChars > 0) {
|
|
8746
9144
|
let used = 0;
|
|
@@ -8830,7 +9228,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8830
9228
|
}
|
|
8831
9229
|
return;
|
|
8832
9230
|
};
|
|
8833
|
-
const writeUnit = (input, embed) =>
|
|
9231
|
+
const writeUnit = (input, embed) => Effect50.gen(function* () {
|
|
8834
9232
|
const mem = yield* core().store.write(input);
|
|
8835
9233
|
if (embed) {
|
|
8836
9234
|
yield* core().store.ensureTopicTree(core().defaultModel.id);
|
|
@@ -8840,8 +9238,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8840
9238
|
}
|
|
8841
9239
|
return mem;
|
|
8842
9240
|
});
|
|
8843
|
-
const upsertUnit = (input, embed) =>
|
|
8844
|
-
const existing = yield* core().store.getByKey(input.logicalKey, input.namespace ?? "default").pipe(
|
|
9241
|
+
const upsertUnit = (input, embed) => Effect50.gen(function* () {
|
|
9242
|
+
const existing = yield* core().store.getByKey(input.logicalKey, input.namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
8845
9243
|
if (existing !== undefined && existing.content === input.content) {
|
|
8846
9244
|
const stamp = codeVersionOf(input.metadata);
|
|
8847
9245
|
if (stamp !== undefined && codeVersionOf(existing.metadata) === undefined) {
|
|
@@ -8906,7 +9304,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8906
9304
|
return;
|
|
8907
9305
|
};
|
|
8908
9306
|
const messageOf3 = (cause) => cause instanceof Error ? cause.message : String(cause);
|
|
8909
|
-
const resolveMemory = (idOrKey) => core().store.get(idOrKey).pipe(
|
|
9307
|
+
const resolveMemory = (idOrKey) => core().store.get(idOrKey).pipe(Effect50.catchTag("Store.NotFound", () => core().store.getByKey(idOrKey, "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.fail(new NotFoundError({ id: idOrKey, message: `Memory not found: ${idOrKey}` }))))));
|
|
8910
9308
|
const codeVersionOf = (blob) => {
|
|
8911
9309
|
if (blob === undefined || blob === null)
|
|
8912
9310
|
return;
|
|
@@ -8923,13 +9321,13 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8923
9321
|
get defaultModelId() {
|
|
8924
9322
|
return core().defaultModel.id;
|
|
8925
9323
|
},
|
|
8926
|
-
write: (input, options) => run(
|
|
9324
|
+
write: (input, options) => run(Effect50.gen(function* () {
|
|
8927
9325
|
const invalid = validateWriteInput(input);
|
|
8928
9326
|
if (invalid !== undefined)
|
|
8929
9327
|
return yield* invalid;
|
|
8930
9328
|
return yield* writeUnit(input, options?.embed !== false);
|
|
8931
9329
|
})),
|
|
8932
|
-
writeMany: (units, options) => run(
|
|
9330
|
+
writeMany: (units, options) => run(Effect50.gen(function* () {
|
|
8933
9331
|
if (!Array.isArray(units)) {
|
|
8934
9332
|
return yield* new InputError({ message: "writeMany: units must be an array" });
|
|
8935
9333
|
}
|
|
@@ -8941,13 +9339,15 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8941
9339
|
const mems = yield* core().store.writeMany(units);
|
|
8942
9340
|
if (options?.embed !== false) {
|
|
8943
9341
|
yield* core().store.ensureTopicTree(core().defaultModel.id);
|
|
8944
|
-
const
|
|
8945
|
-
|
|
8946
|
-
|
|
8947
|
-
|
|
8948
|
-
|
|
9342
|
+
const EMBED_CHUNK = 64;
|
|
9343
|
+
const vectors = [];
|
|
9344
|
+
for (let i = 0;i < mems.length; i += EMBED_CHUNK) {
|
|
9345
|
+
const chunk = mems.slice(i, i + EMBED_CHUNK).map((m) => m.content);
|
|
9346
|
+
const vs = yield* core().defaultEmbedder.embed(chunk);
|
|
9347
|
+
vectors.push(...vs);
|
|
8949
9348
|
}
|
|
8950
|
-
|
|
9349
|
+
yield* core().store.updateEmbeddingMany(mems.map((mem, i) => ({ id: mem.id, embedding: [...vectors[i]] })), core().defaultModel.id);
|
|
9350
|
+
return yield* Effect50.forEach(mems, (mem) => core().store.get(mem.id), {
|
|
8951
9351
|
discard: false
|
|
8952
9352
|
});
|
|
8953
9353
|
}
|
|
@@ -8955,8 +9355,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8955
9355
|
})),
|
|
8956
9356
|
get: (id) => run(resolveMemory(id)),
|
|
8957
9357
|
query: runQuery,
|
|
8958
|
-
forget: (id) => run(resolveMemory(id).pipe(
|
|
8959
|
-
index: (options) => run(
|
|
9358
|
+
forget: (id) => run(resolveMemory(id).pipe(Effect50.flatMap((memory) => core().store.forget(memory.id)))),
|
|
9359
|
+
index: (options) => run(Effect50.gen(function* () {
|
|
8960
9360
|
const invalid = validateIndexOptions(options);
|
|
8961
9361
|
if (invalid !== undefined)
|
|
8962
9362
|
return yield* invalid;
|
|
@@ -8966,7 +9366,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8966
9366
|
const maxFileBytes = options.maxFileBytes ?? 64 * 1024;
|
|
8967
9367
|
const maxChunkBytes = options.maxChunkBytes ?? 8 * 1024;
|
|
8968
9368
|
const extensions = [...options.extensions ?? INDEXABLE_EXTENSIONS];
|
|
8969
|
-
const walk = yield*
|
|
9369
|
+
const walk = yield* Effect50.try({
|
|
8970
9370
|
try: () => walkFiles([...options.paths], extensions),
|
|
8971
9371
|
catch: (cause) => new InputError({
|
|
8972
9372
|
message: `index: failed to walk paths: ${messageOf3(cause)}`
|
|
@@ -8996,8 +9396,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
8996
9396
|
if (language === undefined)
|
|
8997
9397
|
continue;
|
|
8998
9398
|
const chunkKeyOf = (chunkId) => `${keyPrefix}${relative}#${stableChunkId(chunkId)}`;
|
|
8999
|
-
const exit = yield*
|
|
9000
|
-
const source = yield*
|
|
9399
|
+
const exit = yield* Effect50.exit(Effect50.gen(function* () {
|
|
9400
|
+
const source = yield* Effect50.try({
|
|
9001
9401
|
try: () => readCapped(filePath, maxFileBytes),
|
|
9002
9402
|
catch: (cause) => new InputError({ message: `read failed: ${messageOf3(cause)}` })
|
|
9003
9403
|
});
|
|
@@ -9079,13 +9479,14 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9079
9479
|
}
|
|
9080
9480
|
}
|
|
9081
9481
|
}
|
|
9082
|
-
|
|
9482
|
+
const importTargets = resolveImports(filePath, (extracted.imports ?? []).map((imp) => imp.source), extensions);
|
|
9483
|
+
for (const target of importTargets) {
|
|
9083
9484
|
pendingEdges.push({
|
|
9084
9485
|
from: `${keyPrefix}${keyForPath(walk.root, target)}`,
|
|
9085
9486
|
to: fileKey
|
|
9086
9487
|
});
|
|
9087
9488
|
}
|
|
9088
|
-
const importRels =
|
|
9489
|
+
const importRels = importTargets.map((target) => keyForPath(walk.root, target));
|
|
9089
9490
|
for (const chunk of extracted.chunks) {
|
|
9090
9491
|
const callerKey = chunkKeyOf(chunk.id);
|
|
9091
9492
|
if (!unitIds.has(callerKey))
|
|
@@ -9103,7 +9504,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9103
9504
|
}
|
|
9104
9505
|
return { units, upd, dups };
|
|
9105
9506
|
}));
|
|
9106
|
-
if (
|
|
9507
|
+
if (Exit7.isFailure(exit)) {
|
|
9107
9508
|
failed.push({ filePath, message: Cause5.pretty(exit.cause) });
|
|
9108
9509
|
} else {
|
|
9109
9510
|
indexed += exit.value.units;
|
|
@@ -9117,14 +9518,14 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9117
9518
|
const resolveKey = (key) => {
|
|
9118
9519
|
const fromMap = unitIds.get(key);
|
|
9119
9520
|
if (fromMap !== undefined)
|
|
9120
|
-
return
|
|
9121
|
-
return core().store.getByKey(key, namespace ?? "default").pipe(
|
|
9521
|
+
return Effect50.succeed(fromMap);
|
|
9522
|
+
return core().store.getByKey(key, namespace ?? "default").pipe(Effect50.map((m) => m.id), Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9122
9523
|
};
|
|
9123
|
-
const edgeEffect = (edge) =>
|
|
9124
|
-
const [from, to] = yield*
|
|
9524
|
+
const edgeEffect = (edge) => Effect50.gen(function* () {
|
|
9525
|
+
const [from, to] = yield* Effect50.all([resolveKey(edge.from), resolveKey(edge.to)]);
|
|
9125
9526
|
if (from === undefined || to === undefined || from === to)
|
|
9126
9527
|
return;
|
|
9127
|
-
yield*
|
|
9528
|
+
yield* Effect50.exit(core().store.addEdge({
|
|
9128
9529
|
fromMemoryId: from,
|
|
9129
9530
|
toMemoryId: to,
|
|
9130
9531
|
edgeKind: edge.edgeKind,
|
|
@@ -9134,10 +9535,10 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9134
9535
|
}));
|
|
9135
9536
|
});
|
|
9136
9537
|
for (const edge of pendingEdges) {
|
|
9137
|
-
yield*
|
|
9538
|
+
yield* Effect50.exit(edgeEffect({ ...edge, edgeKind: "memory_supports" }));
|
|
9138
9539
|
}
|
|
9139
9540
|
for (const edge of pendingCallEdges) {
|
|
9140
|
-
yield*
|
|
9541
|
+
yield* Effect50.exit(edgeEffect({ ...edge, edgeKind: "code_calls" }));
|
|
9141
9542
|
}
|
|
9142
9543
|
return {
|
|
9143
9544
|
files: walk.files.length,
|
|
@@ -9150,7 +9551,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9150
9551
|
failed
|
|
9151
9552
|
};
|
|
9152
9553
|
})),
|
|
9153
|
-
updateEmbedding: (id, embedding, modelKey) => run(
|
|
9554
|
+
updateEmbedding: (id, embedding, modelKey) => run(Effect50.gen(function* () {
|
|
9154
9555
|
if (!Array.isArray(embedding) || embedding.length === 0 || embedding.some((v) => typeof v !== "number" || !Number.isFinite(v))) {
|
|
9155
9556
|
return yield* new InputError({
|
|
9156
9557
|
message: "updateEmbedding: embedding must be a non-empty array of finite numbers"
|
|
@@ -9161,10 +9562,10 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9161
9562
|
yield* core().store.ensureTopicTree(key);
|
|
9162
9563
|
yield* core().store.updateEmbedding(memory.id, [...embedding], key);
|
|
9163
9564
|
})),
|
|
9164
|
-
embed: (text4) => run(core().defaultEmbedder.embedSingle(text4).pipe(
|
|
9165
|
-
dream: (modelKey) => run(core().runtime.enqueue(MemoryDream, { modelKey, runId: crypto.randomUUID() }).pipe(
|
|
9565
|
+
embed: (text4) => run(core().defaultEmbedder.embedSingle(text4).pipe(Effect50.map((vector) => [...vector]))),
|
|
9566
|
+
dream: (modelKey) => run(core().runtime.enqueue(MemoryDream, { modelKey, runId: crypto.randomUUID() }).pipe(Effect50.flatMap((jobId) => core().runtime.wait(jobId)), Effect50.asVoid)),
|
|
9166
9567
|
dbSchema: () => run(core().store.listSchema()),
|
|
9167
|
-
repair: (modelKey) => run(core().runtime.enqueue(GraphRepair, { modelKey, runId: crypto.randomUUID() }).pipe(
|
|
9568
|
+
repair: (modelKey) => run(core().runtime.enqueue(GraphRepair, { modelKey, runId: crypto.randomUUID() }).pipe(Effect50.flatMap((jobId) => core().runtime.wait(jobId)), Effect50.asVoid)),
|
|
9168
9569
|
explain: (input, options) => run(core().store.explain(input, {
|
|
9169
9570
|
...options?.edgeKinds !== undefined ? { edgeKinds: options.edgeKinds } : {},
|
|
9170
9571
|
...options?.maxDepth !== undefined ? { maxDepth: options.maxDepth } : {},
|
|
@@ -9174,7 +9575,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9174
9575
|
})),
|
|
9175
9576
|
health: (modelKey) => run(core().store.topicHealth(modelKey)),
|
|
9176
9577
|
graph: {
|
|
9177
|
-
addEdge: (input) => run(
|
|
9578
|
+
addEdge: (input) => run(Effect50.gen(function* () {
|
|
9178
9579
|
const invalid = validateAddEdgeInput(input);
|
|
9179
9580
|
if (invalid !== undefined)
|
|
9180
9581
|
return yield* invalid;
|
|
@@ -9216,24 +9617,24 @@ var diffConfig = (oldC, newC) => {
|
|
|
9216
9617
|
};
|
|
9217
9618
|
var RELOAD_DRAIN_MS = 30000;
|
|
9218
9619
|
var bootStack = async (options) => {
|
|
9219
|
-
const scope = await
|
|
9620
|
+
const scope = await Effect51.runPromise(Scope5.make());
|
|
9220
9621
|
let runtime;
|
|
9221
9622
|
try {
|
|
9222
9623
|
const layer21 = ("core" in options ? layer13(options.core) : layerConfig7(options.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : {}).pipe(Layer26.provide(configLayerFromInput(options.config)))).pipe(Layer26.provide(Layer26.succeed(Scope5.Scope, scope)));
|
|
9223
9624
|
const appLayer = Layer26.mergeAll(layer21, layer20, layer14);
|
|
9224
9625
|
const rt = ManagedRuntime.make(appLayer);
|
|
9225
9626
|
runtime = rt;
|
|
9226
|
-
const [core, parser, extractor] = await rt.runPromise(
|
|
9627
|
+
const [core, parser, extractor] = await rt.runPromise(Effect51.all([Service, Service19, Service12]));
|
|
9227
9628
|
return {
|
|
9228
9629
|
runPromise: (effect) => rt.runPromise(effect),
|
|
9229
9630
|
disposeEffect: rt.disposeEffect,
|
|
9230
9631
|
core,
|
|
9231
9632
|
parser,
|
|
9232
9633
|
extractor,
|
|
9233
|
-
dispose: () =>
|
|
9634
|
+
dispose: () => Effect51.runPromise(Scope5.close(scope, Exit8.void).pipe(Effect51.andThen(rt.disposeEffect)))
|
|
9234
9635
|
};
|
|
9235
9636
|
} catch (cause) {
|
|
9236
|
-
await
|
|
9637
|
+
await Effect51.runPromise(Scope5.close(scope, Exit8.void).pipe(Effect51.andThen(runtime?.disposeEffect ?? Effect51.void))).catch(() => {
|
|
9237
9638
|
return;
|
|
9238
9639
|
});
|
|
9239
9640
|
return Promise.reject(cause);
|
|
@@ -9244,7 +9645,7 @@ var makeDimple = async (options) => {
|
|
|
9244
9645
|
const migrationFolder = "config" in options ? options.migrationFolder : undefined;
|
|
9245
9646
|
let currentConfig;
|
|
9246
9647
|
if (configSource !== undefined) {
|
|
9247
|
-
currentConfig = await
|
|
9648
|
+
currentConfig = await Effect51.runPromise(resolveConfigInput(configSource)).catch((cause) => Promise.reject(remapError(cause)));
|
|
9248
9649
|
}
|
|
9249
9650
|
let current = await bootStack(configSource !== undefined && currentConfig !== undefined ? {
|
|
9250
9651
|
config: currentConfig,
|
|
@@ -9266,7 +9667,7 @@ var makeDimple = async (options) => {
|
|
|
9266
9667
|
}
|
|
9267
9668
|
const stack = current;
|
|
9268
9669
|
inflight2 += 1;
|
|
9269
|
-
return stack.runPromise(effect.pipe(
|
|
9670
|
+
return stack.runPromise(effect.pipe(Effect51.mapError(remapError))).catch((cause) => {
|
|
9270
9671
|
return Promise.reject(remapError(cause));
|
|
9271
9672
|
}).finally(() => {
|
|
9272
9673
|
inflight2 -= 1;
|
|
@@ -9322,7 +9723,7 @@ var makeDimple = async (options) => {
|
|
|
9322
9723
|
}));
|
|
9323
9724
|
}
|
|
9324
9725
|
const nextInput = input ?? configSource;
|
|
9325
|
-
const resolved = await
|
|
9726
|
+
const resolved = await Effect51.runPromise(resolveConfigInput(nextInput));
|
|
9326
9727
|
if (currentConfig !== undefined && stableKey(resolved) === stableKey(currentConfig)) {
|
|
9327
9728
|
return {
|
|
9328
9729
|
config: resolved,
|
|
@@ -9345,10 +9746,10 @@ var makeDimple = async (options) => {
|
|
|
9345
9746
|
currentConfig = resolved;
|
|
9346
9747
|
await old.dispose();
|
|
9347
9748
|
if (changes.storeUrl) {
|
|
9348
|
-
await
|
|
9749
|
+
await Effect51.runPromise(Effect51.logWarning(`reload: store database switched "${oldStoreUrl}" → "${resolved.storeUrl}" — ` + "the old db is preserved but no longer served (nothing was migrated)").pipe(Effect51.annotateLogs({ scope: "@dimple/sdk", event: "reload.store.switch" })));
|
|
9349
9750
|
}
|
|
9350
9751
|
if (changes.jobsUrl) {
|
|
9351
|
-
await
|
|
9752
|
+
await Effect51.runPromise(Effect51.logWarning(`reload: jobs database switched "${oldJobsUrl}" → "${resolved.jobs?.dbUrl}" — ` + "pending jobs remain durable in the old db").pipe(Effect51.annotateLogs({ scope: "@dimple/sdk", event: "reload.jobs.switch" })));
|
|
9352
9753
|
}
|
|
9353
9754
|
return { config: resolved, changed: changes, reloaded: true };
|
|
9354
9755
|
} finally {
|