@ponraaj/dimple-sdk 0.0.0-alpha.13 → 0.0.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dimple.d.ts +23 -2
- package/dist/errors.js +505 -481
- package/dist/index.d.ts +1 -1
- package/dist/index.js +624 -590
- package/dist/internal/core/core.d.ts +4 -1
- package/dist/internal/core/layer.d.ts +3 -3
- package/dist/internal/store/topics/routing.d.ts +1 -1
- package/dist/internal/workflow/engine.d.ts +3 -2
- package/dist/internal/workflow/index.d.ts +1 -0
- package/dist/internal/workflow/runtime.d.ts +4 -3
- package/dist/internal/workflow/stats.d.ts +44 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -29,10 +29,10 @@ class CoreConfigError extends Schema.TaggedError()("core.ConfigError", {
|
|
|
29
29
|
class Service extends Context.Service()("@dimple/core/Core") {
|
|
30
30
|
}
|
|
31
31
|
// ../core/src/layer.ts
|
|
32
|
-
import { Effect as
|
|
32
|
+
import { Effect as Effect40, Layer as Layer16, Schedule as Schedule2, Schema as Schema29 } from "effect";
|
|
33
33
|
|
|
34
34
|
// ../store/src/store.ts
|
|
35
|
-
import { Config as Config4, Effect as
|
|
35
|
+
import { Config as Config4, Effect as Effect26, Layer as Layer11, Schema as Schema23 } from "effect";
|
|
36
36
|
import { eq as eq8, and as and8, desc as desc2, asc as asc5, inArray as inArray4, ne as ne4, or, isNull as isNull2, like, sql as sql8 } from "drizzle-orm";
|
|
37
37
|
|
|
38
38
|
// ../store/src/db-backend.ts
|
|
@@ -2611,7 +2611,7 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2611
2611
|
children.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
2612
2612
|
}
|
|
2613
2613
|
const membershipRows = [];
|
|
2614
|
-
const
|
|
2614
|
+
const topicFinal = new Map;
|
|
2615
2615
|
const outcomes = [];
|
|
2616
2616
|
for (const entry of entries) {
|
|
2617
2617
|
const x = new Float32Array(entry.vector);
|
|
@@ -2634,12 +2634,12 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2634
2634
|
modelKey: model.id,
|
|
2635
2635
|
updatedAt: now
|
|
2636
2636
|
});
|
|
2637
|
-
const acc =
|
|
2637
|
+
const acc = topicFinal.get(topicKey);
|
|
2638
2638
|
if (acc) {
|
|
2639
|
-
acc.
|
|
2639
|
+
acc.memberCount = leafCf.count;
|
|
2640
2640
|
acc.ss = leafCf.ss;
|
|
2641
2641
|
} else {
|
|
2642
|
-
|
|
2642
|
+
topicFinal.set(topicKey, { memberCount: leafCf.count, ss: leafCf.ss });
|
|
2643
2643
|
}
|
|
2644
2644
|
let ancestorKey = parentOf.get(topicKey);
|
|
2645
2645
|
while (ancestorKey !== undefined) {
|
|
@@ -2650,11 +2650,11 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2650
2650
|
row.memberCount += 1;
|
|
2651
2651
|
const ancestorSum = addToSum(sumOf(ancestorKey), x);
|
|
2652
2652
|
sumByTopic.set(ancestorKey, ancestorSum);
|
|
2653
|
-
const accA =
|
|
2653
|
+
const accA = topicFinal.get(ancestorKey);
|
|
2654
2654
|
if (accA)
|
|
2655
|
-
accA.
|
|
2655
|
+
accA.memberCount = row.memberCount;
|
|
2656
2656
|
else
|
|
2657
|
-
|
|
2657
|
+
topicFinal.set(ancestorKey, { memberCount: row.memberCount, ss: row.sumSquares });
|
|
2658
2658
|
ancestorKey = parentOf.get(ancestorKey);
|
|
2659
2659
|
}
|
|
2660
2660
|
outcomes.push({
|
|
@@ -2664,14 +2664,14 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2664
2664
|
});
|
|
2665
2665
|
}
|
|
2666
2666
|
await tx.insert(topicMemberships).values(membershipRows);
|
|
2667
|
-
for (const [key,
|
|
2668
|
-
await tx.update(topics).set({ memberCount:
|
|
2667
|
+
for (const [key, final] of topicFinal) {
|
|
2668
|
+
await tx.update(topics).set({ memberCount: final.memberCount, sumSquares: final.ss, updatedAt: now }).where(and(eq(topics.topicKey, key), eq(topics.modelKey, model.id)));
|
|
2669
2669
|
}
|
|
2670
|
-
return { outcomes,
|
|
2670
|
+
return { outcomes, topicFinal };
|
|
2671
2671
|
}),
|
|
2672
2672
|
catch: (cause) => cause
|
|
2673
2673
|
}).pipe(routingFailure("topics.routeInsertMany"));
|
|
2674
|
-
yield* backend.upsertTopicVectors(Array.from(result.
|
|
2674
|
+
yield* backend.upsertTopicVectors(Array.from(result.topicFinal, ([topicKey]) => ({
|
|
2675
2675
|
topicKey,
|
|
2676
2676
|
sumVector: Array.from(sumByTopic.get(topicKey) ?? zeros)
|
|
2677
2677
|
})), model).pipe(routingFailure("topics.routeInsertMany"));
|
|
@@ -2772,7 +2772,7 @@ var resolveTopics = Effect11.fn("topics.resolveTopics")(function* (queryVector,
|
|
|
2772
2772
|
});
|
|
2773
2773
|
|
|
2774
2774
|
// ../store/src/topics/lifecycle.ts
|
|
2775
|
-
import { Clock as Clock3, Cause as Cause3, Effect as
|
|
2775
|
+
import { Clock as Clock3, Cause as Cause3, Effect as Effect21, Exit as Exit6 } from "effect";
|
|
2776
2776
|
import { and as and3, eq as eq3, inArray, ne, sql as sql3 } from "drizzle-orm";
|
|
2777
2777
|
|
|
2778
2778
|
// ../workflow/src/queue.ts
|
|
@@ -2935,12 +2935,12 @@ var queues = [
|
|
|
2935
2935
|
import {
|
|
2936
2936
|
Cause,
|
|
2937
2937
|
Clock as Clock2,
|
|
2938
|
-
Context as
|
|
2938
|
+
Context as Context9,
|
|
2939
2939
|
Deferred as Deferred2,
|
|
2940
2940
|
Duration,
|
|
2941
|
-
Effect as
|
|
2941
|
+
Effect as Effect18,
|
|
2942
2942
|
Exit as Exit5,
|
|
2943
|
-
Layer as
|
|
2943
|
+
Layer as Layer8,
|
|
2944
2944
|
Option as Option8,
|
|
2945
2945
|
Schema as Schema18
|
|
2946
2946
|
} from "effect";
|
|
@@ -3085,7 +3085,7 @@ var notifyExecutionDone = (executionId) => Effect14.gen(function* () {
|
|
|
3085
3085
|
});
|
|
3086
3086
|
|
|
3087
3087
|
// ../workflow/src/engine.ts
|
|
3088
|
-
import { Effect as
|
|
3088
|
+
import { Effect as Effect17, Exit as Exit4, Fiber, FiberMap, Layer as Layer7, Option as Option7, Schema as Schema17 } from "effect";
|
|
3089
3089
|
import { Workflow as Workflow3, WorkflowEngine as WorkflowEngine2 } from "effect/workflow";
|
|
3090
3090
|
import { Persistence as PersistenceNamespace2, PersistedQueue } from "effect/persistence";
|
|
3091
3091
|
|
|
@@ -3242,20 +3242,42 @@ var make5 = (options) => Effect15.gen(function* () {
|
|
|
3242
3242
|
var layerConfig2 = (config) => Layer5.effectContext(Config3.unwrap(config).pipe(Effect15.flatMap(make5), Effect15.map((client) => Context7.make(LibsqlClient, client).pipe(Context7.add(Client.SqlClient, client))))).pipe(Layer5.provide(Reactivity.layer));
|
|
3243
3243
|
var layer5 = (config) => Layer5.effectContext(Effect15.map(make5(config), (client) => Context7.make(LibsqlClient, client).pipe(Context7.add(Client.SqlClient, client)))).pipe(Layer5.provide(Reactivity.layer));
|
|
3244
3244
|
// ../workflow/src/engine.ts
|
|
3245
|
-
import { SqlError as
|
|
3245
|
+
import { SqlError as SqlErrorNamespace2 } from "effect/sql";
|
|
3246
|
+
|
|
3247
|
+
// ../workflow/src/stats.ts
|
|
3248
|
+
import { Context as Context8, Effect as Effect16, Layer as Layer6 } from "effect";
|
|
3249
|
+
import { SqlClient as SqlClient2 } from "effect/sql";
|
|
3250
|
+
class Service7 extends Context8.Service()("@dimple/workflow/QueueStats") {
|
|
3251
|
+
}
|
|
3252
|
+
var layer6 = Layer6.effect(Service7, Effect16.gen(function* () {
|
|
3253
|
+
const sql3 = yield* SqlClient2.SqlClient;
|
|
3254
|
+
const pending = Effect16.fn("QueueStats.pending")(function* () {
|
|
3255
|
+
const rows = yield* sql3`
|
|
3256
|
+
SELECT queue_name AS queue, COUNT(*) AS count
|
|
3257
|
+
FROM effect_queue
|
|
3258
|
+
WHERE state = 'pending' AND queue_name != ${RESUME_QUEUE}
|
|
3259
|
+
GROUP BY queue_name
|
|
3260
|
+
ORDER BY queue_name
|
|
3261
|
+
`;
|
|
3262
|
+
return rows.map((row) => ({ queue: row.queue, pending: Number(row.count) }));
|
|
3263
|
+
});
|
|
3264
|
+
return Service7.of({ pending });
|
|
3265
|
+
}));
|
|
3266
|
+
|
|
3267
|
+
// ../workflow/src/engine.ts
|
|
3246
3268
|
var Persistence = PersistenceNamespace2.Persistence;
|
|
3247
3269
|
var PersistedQueueFactory = PersistedQueue.PersistedQueueFactory;
|
|
3248
3270
|
var PayloadCodec = Schema17.Record(Schema17.String, Schema17.Json);
|
|
3249
|
-
var layerDurable =
|
|
3271
|
+
var layerDurable = Layer7.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(function* () {
|
|
3250
3272
|
const persistence = yield* Persistence;
|
|
3251
3273
|
const queueFactory = yield* PersistedQueueFactory;
|
|
3252
|
-
const scope = yield*
|
|
3274
|
+
const scope = yield* Effect17.scope;
|
|
3253
3275
|
const store = yield* persistence.make({ storeId: STORE_ID });
|
|
3254
3276
|
const resumeQueue = yield* queueFactory.make({ name: RESUME_QUEUE, schema: ResumeSignal });
|
|
3255
3277
|
const workflows = new Map;
|
|
3256
3278
|
const executions = new Map;
|
|
3257
3279
|
const clocks = yield* FiberMap.make();
|
|
3258
|
-
const resume = (state) =>
|
|
3280
|
+
const resume = (state) => Effect17.gen(function* () {
|
|
3259
3281
|
const exit = state.fiber?.pollUnsafe();
|
|
3260
3282
|
if (exit && exit._tag === "Success" && exit.value._tag === "Complete")
|
|
3261
3283
|
return;
|
|
@@ -3267,32 +3289,32 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3267
3289
|
const instance = WorkflowEngine2.WorkflowInstance.initial(state.instance.workflow, state.instance.executionId);
|
|
3268
3290
|
instance.interrupted = state.instance.interrupted;
|
|
3269
3291
|
state.instance = instance;
|
|
3270
|
-
state.fiber = yield* state.execute(state.payload, state.instance.executionId).pipe(
|
|
3292
|
+
state.fiber = yield* state.execute(state.payload, state.instance.executionId).pipe(Effect17.onExit(() => {
|
|
3271
3293
|
if (!instance.interrupted)
|
|
3272
|
-
return
|
|
3294
|
+
return Effect17.void;
|
|
3273
3295
|
instance.suspended = false;
|
|
3274
|
-
return
|
|
3275
|
-
}), Workflow3.intoResult,
|
|
3296
|
+
return Effect17.withFiber((fiber) => Effect17.interruptible(Fiber.interrupt(fiber)));
|
|
3297
|
+
}), Workflow3.intoResult, Effect17.provideService(WorkflowEngine2.WorkflowInstance, instance), Effect17.provideService(WorkflowEngine2.WorkflowEngine, engine), Effect17.tap((result) => {
|
|
3276
3298
|
if (result._tag !== "Complete")
|
|
3277
|
-
return
|
|
3278
|
-
return
|
|
3299
|
+
return Effect17.void;
|
|
3300
|
+
return Effect17.gen(function* () {
|
|
3279
3301
|
const entry2 = workflows.get(state.instance.workflow._tag);
|
|
3280
3302
|
if (!entry2)
|
|
3281
3303
|
return;
|
|
3282
|
-
const encoded = yield*
|
|
3304
|
+
const encoded = yield* Effect17.orDie(Schema17.encodeEffect(resultCodecFor(entry2.workflow))(result));
|
|
3283
3305
|
yield* setEncodedExit(store, resultKey(state.instance.executionId), Exit4.succeed(encoded));
|
|
3284
3306
|
yield* notifyExecutionDone(state.instance.executionId);
|
|
3285
3307
|
});
|
|
3286
|
-
}),
|
|
3308
|
+
}), Effect17.tap((result) => {
|
|
3287
3309
|
if (!state.parent || result._tag !== "Complete")
|
|
3288
|
-
return
|
|
3310
|
+
return Effect17.void;
|
|
3289
3311
|
const parent = executions.get(state.parent);
|
|
3290
3312
|
if (parent)
|
|
3291
|
-
return
|
|
3292
|
-
return
|
|
3293
|
-
}),
|
|
3313
|
+
return Effect17.forkIn(resume(parent), scope);
|
|
3314
|
+
return Effect17.void;
|
|
3315
|
+
}), Effect17.forkIn(entry.scope));
|
|
3294
3316
|
});
|
|
3295
|
-
const resumeExecution = (executionId) =>
|
|
3317
|
+
const resumeExecution = (executionId) => Effect17.gen(function* () {
|
|
3296
3318
|
const state = executions.get(executionId);
|
|
3297
3319
|
if (state) {
|
|
3298
3320
|
yield* resume(state);
|
|
@@ -3315,22 +3337,22 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3315
3337
|
}
|
|
3316
3338
|
});
|
|
3317
3339
|
const engine = WorkflowEngine2.makeUnsafe({
|
|
3318
|
-
register:
|
|
3340
|
+
register: Effect17.fnUntraced(function* (workflow, execute) {
|
|
3319
3341
|
workflows.set(workflow._tag, {
|
|
3320
3342
|
workflow,
|
|
3321
3343
|
execute,
|
|
3322
|
-
scope: yield*
|
|
3344
|
+
scope: yield* Effect17.scope
|
|
3323
3345
|
});
|
|
3324
3346
|
}),
|
|
3325
3347
|
execute: executeEncoded(workflows, executions, store, resume),
|
|
3326
|
-
interrupt:
|
|
3348
|
+
interrupt: Effect17.fnUntraced(function* (_workflow, executionId) {
|
|
3327
3349
|
const state = executions.get(executionId);
|
|
3328
3350
|
if (!state)
|
|
3329
3351
|
return;
|
|
3330
3352
|
state.instance.interrupted = true;
|
|
3331
3353
|
yield* resume(state);
|
|
3332
3354
|
}),
|
|
3333
|
-
interruptUnsafe:
|
|
3355
|
+
interruptUnsafe: Effect17.fnUntraced(function* (_workflow, executionId) {
|
|
3334
3356
|
const state = executions.get(executionId);
|
|
3335
3357
|
if (!state)
|
|
3336
3358
|
return;
|
|
@@ -3340,36 +3362,36 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3340
3362
|
}
|
|
3341
3363
|
}),
|
|
3342
3364
|
resume: (_workflow, executionId) => resumeExecution(executionId),
|
|
3343
|
-
activityExecute:
|
|
3365
|
+
activityExecute: Effect17.fnUntraced(function* (activity, attempt) {
|
|
3344
3366
|
const instance = yield* WorkflowEngine2.WorkflowInstance;
|
|
3345
3367
|
const key = activityKey(instance.executionId, activity.name, attempt);
|
|
3346
3368
|
const persisted = yield* getEncodedExit(store, key);
|
|
3347
3369
|
if (persisted && Exit4.isSuccess(persisted)) {
|
|
3348
3370
|
const entry = workflows.get(instance.workflow._tag);
|
|
3349
3371
|
if (!entry)
|
|
3350
|
-
return yield*
|
|
3351
|
-
return yield*
|
|
3372
|
+
return yield* Effect17.die(`activity replay: unknown workflow ${instance.workflow._tag}`);
|
|
3373
|
+
return yield* Effect17.orDie(Schema17.decodeUnknownEffect(resultCodecFor(entry.workflow))(persisted.value));
|
|
3352
3374
|
}
|
|
3353
3375
|
const activityInstance = WorkflowEngine2.WorkflowInstance.initial(instance.workflow, instance.executionId);
|
|
3354
3376
|
activityInstance.interrupted = instance.interrupted;
|
|
3355
|
-
const result = yield* activity.executeEncoded.pipe(Workflow3.intoResult,
|
|
3377
|
+
const result = yield* activity.executeEncoded.pipe(Workflow3.intoResult, Effect17.provideService(WorkflowEngine2.WorkflowInstance, activityInstance), Effect17.exit);
|
|
3356
3378
|
if (result._tag === "Success" && result.value._tag !== "Suspended") {
|
|
3357
3379
|
const entry = workflows.get(instance.workflow._tag);
|
|
3358
3380
|
if (entry) {
|
|
3359
|
-
const encoded = yield*
|
|
3381
|
+
const encoded = yield* Effect17.orDie(Schema17.encodeEffect(resultCodecFor(entry.workflow))(result.value));
|
|
3360
3382
|
yield* setEncodedExit(store, key, Exit4.succeed(encoded));
|
|
3361
3383
|
}
|
|
3362
3384
|
}
|
|
3363
3385
|
return yield* result;
|
|
3364
3386
|
}),
|
|
3365
|
-
poll: (_workflow, executionId) =>
|
|
3387
|
+
poll: (_workflow, executionId) => Effect17.gen(function* () {
|
|
3366
3388
|
const state = executions.get(executionId);
|
|
3367
3389
|
if (!state) {
|
|
3368
3390
|
const persisted = yield* getEncodedExit(store, resultKey(executionId));
|
|
3369
3391
|
if (persisted && Exit4.isSuccess(persisted)) {
|
|
3370
3392
|
const workflow = yield* workflowTagFor(workflows, store, executionId);
|
|
3371
3393
|
if (workflow) {
|
|
3372
|
-
return Option7.some(yield*
|
|
3394
|
+
return Option7.some(yield* Effect17.orDie(Schema17.decodeUnknownEffect(resultCodecFor(workflow))(persisted.value)));
|
|
3373
3395
|
}
|
|
3374
3396
|
}
|
|
3375
3397
|
return Option7.none();
|
|
@@ -3379,9 +3401,9 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3379
3401
|
return Option7.none();
|
|
3380
3402
|
if (exit._tag === "Success")
|
|
3381
3403
|
return Option7.some(exit.value);
|
|
3382
|
-
return yield*
|
|
3404
|
+
return yield* Effect17.die(exit.cause);
|
|
3383
3405
|
}),
|
|
3384
|
-
deferredResult:
|
|
3406
|
+
deferredResult: Effect17.fnUntraced(function* (deferred) {
|
|
3385
3407
|
const instance = yield* WorkflowEngine2.WorkflowInstance;
|
|
3386
3408
|
const persisted = yield* getEncodedExit(store, deferredKey(instance.executionId, deferred.name));
|
|
3387
3409
|
if (persisted) {
|
|
@@ -3389,9 +3411,9 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3389
3411
|
}
|
|
3390
3412
|
return Option7.none();
|
|
3391
3413
|
}),
|
|
3392
|
-
deferredDone: (options) =>
|
|
3414
|
+
deferredDone: (options) => Effect17.gen(function* () {
|
|
3393
3415
|
yield* setEncodedExit(store, deferredKey(options.executionId, options.deferredName), options.exit);
|
|
3394
|
-
yield* resumeQueue.offer({ executionId: options.executionId }, { id: options.executionId }).pipe(
|
|
3416
|
+
yield* resumeQueue.offer({ executionId: options.executionId }, { id: options.executionId }).pipe(Effect17.orDie);
|
|
3395
3417
|
yield* resumeExecution(options.executionId);
|
|
3396
3418
|
}),
|
|
3397
3419
|
scheduleClock: (workflow, options) => engine.deferredDone(options.clock.deferred, {
|
|
@@ -3399,17 +3421,17 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3399
3421
|
executionId: options.executionId,
|
|
3400
3422
|
deferredName: options.clock.deferred.name,
|
|
3401
3423
|
exit: Exit4.void
|
|
3402
|
-
}).pipe(
|
|
3424
|
+
}).pipe(Effect17.delay(options.clock.duration), FiberMap.run(clocks, `${options.executionId}/${options.clock.name}`, {
|
|
3403
3425
|
onlyIfMissing: true
|
|
3404
|
-
}),
|
|
3426
|
+
}), Effect17.asVoid)
|
|
3405
3427
|
});
|
|
3406
3428
|
return engine;
|
|
3407
3429
|
}));
|
|
3408
|
-
var executeEncoded = (workflows, executions, store, resume) =>
|
|
3409
|
-
const payload = yield*
|
|
3430
|
+
var executeEncoded = (workflows, executions, store, resume) => Effect17.fnUntraced(function* (workflow, options) {
|
|
3431
|
+
const payload = yield* Effect17.orDie(Schema17.decodeUnknownEffect(PayloadCodec)(options.payload));
|
|
3410
3432
|
const entry = workflows.get(workflow._tag);
|
|
3411
3433
|
if (!entry) {
|
|
3412
|
-
return yield*
|
|
3434
|
+
return yield* Effect17.orDie(Effect17.fail(`Workflow ${workflow._tag} is not registered`));
|
|
3413
3435
|
}
|
|
3414
3436
|
let state = executions.get(options.executionId);
|
|
3415
3437
|
if (!state) {
|
|
@@ -3425,43 +3447,43 @@ var executeEncoded = (workflows, executions, store, resume) => Effect16.fnUntrac
|
|
|
3425
3447
|
yield* resume(state);
|
|
3426
3448
|
}
|
|
3427
3449
|
if (options.discard)
|
|
3428
|
-
return yield*
|
|
3450
|
+
return yield* Effect17.void;
|
|
3429
3451
|
if (!state.fiber) {
|
|
3430
|
-
return yield*
|
|
3452
|
+
return yield* Effect17.orDie(Effect17.fail(`Workflow ${workflow._tag}: execution ${options.executionId} has no fiber`));
|
|
3431
3453
|
}
|
|
3432
3454
|
return yield* Fiber.join(state.fiber);
|
|
3433
3455
|
});
|
|
3434
|
-
var loadRecord = (store, executionId) =>
|
|
3456
|
+
var loadRecord = (store, executionId) => Effect17.gen(function* () {
|
|
3435
3457
|
const persisted = yield* getEncodedExit(store, recordKey(executionId));
|
|
3436
3458
|
if (persisted && Exit4.isSuccess(persisted)) {
|
|
3437
|
-
return yield*
|
|
3459
|
+
return yield* Effect17.orDie(Schema17.decodeUnknownEffect(ExecutionRecord)(persisted.value));
|
|
3438
3460
|
}
|
|
3439
3461
|
return;
|
|
3440
3462
|
});
|
|
3441
|
-
var workflowTagFor = (workflows, store, executionId) =>
|
|
3463
|
+
var workflowTagFor = (workflows, store, executionId) => Effect17.gen(function* () {
|
|
3442
3464
|
const record = yield* loadRecord(store, executionId);
|
|
3443
3465
|
if (!record)
|
|
3444
3466
|
return;
|
|
3445
3467
|
return workflows.get(record.workflowTag)?.workflow;
|
|
3446
3468
|
});
|
|
3447
|
-
var libsqlStack = (options) =>
|
|
3448
|
-
yield*
|
|
3469
|
+
var libsqlStack = (options) => Layer7.unwrap(Effect17.gen(function* () {
|
|
3470
|
+
yield* Effect17.tryPromise({
|
|
3449
3471
|
try: () => prepareFileDatabase(options.url),
|
|
3450
3472
|
catch: (cause) => new Error(String(cause))
|
|
3451
|
-
}).pipe(
|
|
3473
|
+
}).pipe(Effect17.catch(() => Effect17.void));
|
|
3452
3474
|
const storeLayer = PersistedQueue.layerStoreSql();
|
|
3453
|
-
const inner =
|
|
3454
|
-
const ctx = yield*
|
|
3475
|
+
const inner = Layer7.mergeAll(PersistenceNamespace2.layerSql, PersistedQueue.layer.pipe(Layer7.provide(storeLayer)), PersistedQueue.layerCleanup().pipe(Layer7.provide(storeLayer)), layer6).pipe(Layer7.provide(exports_LibsqlClient.layer(options)));
|
|
3476
|
+
const ctx = yield* Effect17.acquireUseRelease(Effect17.tryPromise({
|
|
3455
3477
|
try: () => acquireMigrationLock(options.url),
|
|
3456
|
-
catch: (cause) => new
|
|
3457
|
-
reason: new
|
|
3478
|
+
catch: (cause) => new SqlErrorNamespace2.SqlError({
|
|
3479
|
+
reason: new SqlErrorNamespace2.LockTimeoutError({
|
|
3458
3480
|
cause,
|
|
3459
3481
|
message: "timed out acquiring the jobs-database migration lock",
|
|
3460
3482
|
operation: "libsqlStack"
|
|
3461
3483
|
})
|
|
3462
3484
|
})
|
|
3463
|
-
}), () =>
|
|
3464
|
-
return
|
|
3485
|
+
}), () => Layer7.build(inner), (release) => Effect17.sync(() => release?.()));
|
|
3486
|
+
return Layer7.succeedContext(ctx);
|
|
3465
3487
|
}));
|
|
3466
3488
|
|
|
3467
3489
|
// ../workflow/src/runtime.ts
|
|
@@ -3473,14 +3495,14 @@ var DEFAULT_TIMEOUT = Duration.minutes(5);
|
|
|
3473
3495
|
var DEFAULT_POLL_INTERVAL = Duration.millis(250);
|
|
3474
3496
|
var PROCESS_WINDOW = Duration.seconds(30);
|
|
3475
3497
|
|
|
3476
|
-
class
|
|
3498
|
+
class Service8 extends Context9.Service()("@dimple/workflow/JobRuntime") {
|
|
3477
3499
|
}
|
|
3478
3500
|
var jobsError = (jobId, cause) => {
|
|
3479
3501
|
if (cause instanceof JobFailed || cause instanceof JobTimeout || cause instanceof JobCancelled || cause instanceof QueueError)
|
|
3480
3502
|
return cause;
|
|
3481
3503
|
return new QueueError({ message: cause instanceof Error ? cause.message : String(cause) });
|
|
3482
3504
|
};
|
|
3483
|
-
var
|
|
3505
|
+
var layer7 = (defs) => Layer8.effect(Service8, Effect18.gen(function* () {
|
|
3484
3506
|
const engine = yield* WorkflowEngine3.WorkflowEngine;
|
|
3485
3507
|
const persistence = yield* Persistence2;
|
|
3486
3508
|
const queueFactory = yield* PersistedQueueFactory2;
|
|
@@ -3488,24 +3510,24 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3488
3510
|
for (const def of defs) {
|
|
3489
3511
|
yield* def.register(engine);
|
|
3490
3512
|
}
|
|
3491
|
-
const enqueue =
|
|
3492
|
-
const decoded = yield* Schema18.decodeUnknownEffect(def.payloadSchema)(payload).pipe(
|
|
3513
|
+
const enqueue = Effect18.fn("JobRuntime.enqueue")(function* (def, payload) {
|
|
3514
|
+
const decoded = yield* Schema18.decodeUnknownEffect(def.payloadSchema)(payload).pipe(Effect18.mapError((e) => new QueueError({ message: `invalid payload: ${e.message}` })));
|
|
3493
3515
|
const token = def.idempotencyKey(decoded);
|
|
3494
3516
|
const envelope = { token, payload: decoded, traceId: "" };
|
|
3495
|
-
const jobId = yield* def.workflow.executionId(envelope).pipe(
|
|
3496
|
-
yield* def.workflow.execute(envelope, { discard: true }).pipe(
|
|
3517
|
+
const jobId = yield* def.workflow.executionId(envelope).pipe(Effect18.provideService(WorkflowEngine3.WorkflowEngine, engine), Effect18.mapError((cause) => new QueueError({ message: cause instanceof Error ? cause.message : String(cause) })));
|
|
3518
|
+
yield* def.workflow.execute(envelope, { discard: true }).pipe(Effect18.provideService(WorkflowEngine3.WorkflowEngine, engine), Effect18.mapError((cause) => jobsError(jobId, cause)));
|
|
3497
3519
|
return jobId;
|
|
3498
3520
|
});
|
|
3499
|
-
const wait =
|
|
3521
|
+
const wait = Effect18.fn("JobRuntime.wait")(function* (jobId, options) {
|
|
3500
3522
|
const timeout = Duration.fromInput(options?.timeout ?? DEFAULT_TIMEOUT).pipe(Option8.getOrElse(() => Duration.minutes(5)));
|
|
3501
3523
|
const pollInterval = Duration.fromInput(options?.pollInterval ?? DEFAULT_POLL_INTERVAL).pipe(Option8.getOrElse(() => Duration.millis(250)));
|
|
3502
|
-
const deadline = yield* Clock2.currentTimeMillis.pipe(
|
|
3524
|
+
const deadline = yield* Clock2.currentTimeMillis.pipe(Effect18.map((now) => now + Duration.toMillis(timeout)));
|
|
3503
3525
|
const completionSignal = yield* Deferred2.make();
|
|
3504
3526
|
onExecutionDone(jobId, completionSignal);
|
|
3505
3527
|
try {
|
|
3506
3528
|
while (true) {
|
|
3507
3529
|
if (yield* readMarker(store, jobId, CANCELLED)) {
|
|
3508
|
-
return yield*
|
|
3530
|
+
return yield* Effect18.fail(new JobCancelled({ jobId }));
|
|
3509
3531
|
}
|
|
3510
3532
|
const def = yield* findDef(defs, jobId, store);
|
|
3511
3533
|
if (def) {
|
|
@@ -3516,7 +3538,7 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3516
3538
|
if (Exit5.isSuccess(r.exit))
|
|
3517
3539
|
return r.exit.value;
|
|
3518
3540
|
const failure = Exit5.getCause(r.exit).pipe(Option8.flatMap(Cause.findErrorOption), Option8.map((error) => Option8.getOrElse(Schema18.decodeUnknownOption(Schema18.String)(error), () => String(error))));
|
|
3519
|
-
return yield*
|
|
3541
|
+
return yield* Effect18.fail(new JobFailed({
|
|
3520
3542
|
jobId,
|
|
3521
3543
|
message: Option8.getOrElse(failure, () => "workflow failed")
|
|
3522
3544
|
}));
|
|
@@ -3526,19 +3548,19 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3526
3548
|
const now = yield* Clock2.currentTimeMillis;
|
|
3527
3549
|
if (now >= deadline) {
|
|
3528
3550
|
yield* writeMarker(store, jobId, TIMED_OUT);
|
|
3529
|
-
return yield*
|
|
3551
|
+
return yield* Effect18.fail(new JobTimeout({ jobId, timeout }));
|
|
3530
3552
|
}
|
|
3531
|
-
yield*
|
|
3553
|
+
yield* Effect18.race(Deferred2.await(completionSignal).pipe(Effect18.asVoid), Effect18.sleep(pollInterval));
|
|
3532
3554
|
}
|
|
3533
3555
|
} finally {
|
|
3534
3556
|
offExecutionDone(jobId, completionSignal);
|
|
3535
3557
|
}
|
|
3536
3558
|
});
|
|
3537
|
-
const submit =
|
|
3559
|
+
const submit = Effect18.fn("JobRuntime.submit")(function* (def, payload) {
|
|
3538
3560
|
const jobId = yield* enqueue(def, payload);
|
|
3539
3561
|
return yield* wait(jobId);
|
|
3540
3562
|
});
|
|
3541
|
-
const status =
|
|
3563
|
+
const status = Effect18.fn("JobRuntime.status")(function* (jobId) {
|
|
3542
3564
|
if (yield* readMarker(store, jobId, CANCELLED))
|
|
3543
3565
|
return "cancelled";
|
|
3544
3566
|
if (yield* readMarker(store, jobId, TIMED_OUT))
|
|
@@ -3555,66 +3577,66 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3555
3577
|
}
|
|
3556
3578
|
return "pending";
|
|
3557
3579
|
});
|
|
3558
|
-
const cancel =
|
|
3580
|
+
const cancel = Effect18.fn("JobRuntime.cancel")(function* (jobId) {
|
|
3559
3581
|
yield* writeMarker(store, jobId, CANCELLED);
|
|
3560
3582
|
return true;
|
|
3561
3583
|
});
|
|
3562
|
-
const processPending =
|
|
3584
|
+
const processPending = Effect18.fn("JobRuntime.processPending")(function* (options) {
|
|
3563
3585
|
const resumeQueue = yield* queueFactory.make({
|
|
3564
3586
|
name: RESUME_QUEUE,
|
|
3565
3587
|
schema: ResumeSignal,
|
|
3566
3588
|
maxAttempts: 1
|
|
3567
3589
|
});
|
|
3568
3590
|
const window = Duration.toMillis(Duration.fromInput(options?.window ?? PROCESS_WINDOW).pipe(Option8.getOrElse(() => Duration.seconds(30))));
|
|
3569
|
-
const deadline = yield* Clock2.currentTimeMillis.pipe(
|
|
3591
|
+
const deadline = yield* Clock2.currentTimeMillis.pipe(Effect18.map((now) => now + window));
|
|
3570
3592
|
let processed = 0;
|
|
3571
3593
|
while (true) {
|
|
3572
3594
|
const now = yield* Clock2.currentTimeMillis;
|
|
3573
3595
|
if (now >= deadline)
|
|
3574
3596
|
break;
|
|
3575
|
-
const signal = yield* resumeQueue.take((value2) =>
|
|
3597
|
+
const signal = yield* resumeQueue.take((value2) => Effect18.succeed(Option8.some(value2))).pipe(Effect18.mapError((e) => new QueueError({ message: e instanceof Error ? e.message : String(e) })), Effect18.timeoutOrElse({
|
|
3576
3598
|
duration: Duration.millis(100),
|
|
3577
|
-
orElse: () =>
|
|
3599
|
+
orElse: () => Effect18.succeed(Option8.none())
|
|
3578
3600
|
}));
|
|
3579
3601
|
if (Option8.isNone(signal))
|
|
3580
3602
|
continue;
|
|
3581
3603
|
const executionId = signal.value.executionId;
|
|
3582
3604
|
const def = yield* findDef(defs, executionId, store);
|
|
3583
3605
|
if (def) {
|
|
3584
|
-
yield* engine.resume(def.workflow, executionId).pipe(
|
|
3606
|
+
yield* engine.resume(def.workflow, executionId).pipe(Effect18.orDie);
|
|
3585
3607
|
processed++;
|
|
3586
3608
|
}
|
|
3587
3609
|
}
|
|
3588
3610
|
return processed;
|
|
3589
3611
|
});
|
|
3590
|
-
return
|
|
3612
|
+
return Service8.of({ enqueue, wait, submit, status, cancel, processPending });
|
|
3591
3613
|
}));
|
|
3592
|
-
var findDef = (defs, executionId, store) =>
|
|
3614
|
+
var findDef = (defs, executionId, store) => Effect18.gen(function* () {
|
|
3593
3615
|
const persisted = yield* getEncodedExit(store, `${executionId}/record`);
|
|
3594
3616
|
if (!persisted || !Exit5.isSuccess(persisted))
|
|
3595
3617
|
return;
|
|
3596
|
-
const record = yield*
|
|
3618
|
+
const record = yield* Effect18.orDie(Schema18.decodeUnknownEffect(ExecutionRecord)(persisted.value));
|
|
3597
3619
|
return defs.find((d) => d.workflow._tag === record.workflowTag);
|
|
3598
3620
|
});
|
|
3599
3621
|
var layerLibsql = (defs, options) => {
|
|
3600
3622
|
const stack = libsqlStack(options);
|
|
3601
|
-
return
|
|
3623
|
+
return layer7(defs).pipe(Layer8.provideMerge(Layer8.mergeAll(layerDurable.pipe(Layer8.provide(stack)), stack)));
|
|
3602
3624
|
};
|
|
3603
|
-
var configured3 = (defs, options) => make({ name: "JobStack", layer:
|
|
3625
|
+
var configured3 = (defs, options) => make({ name: "JobStack", layer: Layer8.suspend(() => layerLibsql(defs, options)), deps: [] });
|
|
3604
3626
|
var node3 = configured3(queues, { url: JobsConfigDefaults.dbUrl });
|
|
3605
3627
|
// ../workflow/src/registry.ts
|
|
3606
|
-
import { Context as
|
|
3628
|
+
import { Context as Context10, Effect as Effect19, Layer as Layer9, Option as Option9, Ref, Schema as Schema19 } from "effect";
|
|
3607
3629
|
class JobFailure extends Schema19.TaggedError()("JobFailure", {
|
|
3608
3630
|
jobId: Schema19.String,
|
|
3609
3631
|
message: Schema19.String
|
|
3610
3632
|
}) {
|
|
3611
3633
|
}
|
|
3612
3634
|
|
|
3613
|
-
class
|
|
3635
|
+
class Service9 extends Context10.Service()("@dimple/workflow/JobRegistry") {
|
|
3614
3636
|
}
|
|
3615
|
-
var
|
|
3637
|
+
var layer8 = Layer9.effect(Service9, Effect19.gen(function* () {
|
|
3616
3638
|
const handlers = yield* Ref.make(new Map);
|
|
3617
|
-
const register =
|
|
3639
|
+
const register = Effect19.fn("JobRegistry.register")(function* (type, handler) {
|
|
3618
3640
|
const existed = yield* Ref.modify(handlers, (map2) => {
|
|
3619
3641
|
if (map2.has(type)) {
|
|
3620
3642
|
return [true, map2];
|
|
@@ -3622,48 +3644,48 @@ var layer7 = Layer8.effect(Service8, Effect18.gen(function* () {
|
|
|
3622
3644
|
return [false, new Map(map2).set(type, handler)];
|
|
3623
3645
|
});
|
|
3624
3646
|
if (existed) {
|
|
3625
|
-
return yield*
|
|
3647
|
+
return yield* Effect19.fail(new RegistryError({ type, message: `Job type already registered: ${type}` }));
|
|
3626
3648
|
}
|
|
3627
3649
|
});
|
|
3628
|
-
const resolve =
|
|
3629
|
-
const handler = yield* Ref.get(handlers).pipe(
|
|
3650
|
+
const resolve = Effect19.fn("JobRegistry.resolve")(function* (type) {
|
|
3651
|
+
const handler = yield* Ref.get(handlers).pipe(Effect19.map((map2) => Option9.fromNullishOr(map2.get(type))));
|
|
3630
3652
|
if (Option9.isNone(handler)) {
|
|
3631
|
-
return yield*
|
|
3653
|
+
return yield* Effect19.fail(new UnknownJobType({ type }));
|
|
3632
3654
|
}
|
|
3633
3655
|
return handler.value;
|
|
3634
3656
|
});
|
|
3635
|
-
return
|
|
3657
|
+
return Service9.of({ register, resolve });
|
|
3636
3658
|
}));
|
|
3637
|
-
var configured4 = () => make({ name: "JobRegistry", layer:
|
|
3659
|
+
var configured4 = () => make({ name: "JobRegistry", layer: layer8, deps: [] });
|
|
3638
3660
|
var node4 = configured4();
|
|
3639
3661
|
// ../workflow/src/executor.ts
|
|
3640
|
-
import { Cause as Cause2, Duration as Duration2, Effect as
|
|
3662
|
+
import { Cause as Cause2, Duration as Duration2, Effect as Effect20, Layer as Layer10, Schedule } from "effect";
|
|
3641
3663
|
import { DurableQueue as DurableQueue2, WorkflowEngine as WorkflowEngine4 } from "effect/workflow";
|
|
3642
|
-
var runHandler = (def, payload, jobId, maxAttempts) =>
|
|
3643
|
-
const registry = yield*
|
|
3664
|
+
var runHandler = (def, payload, jobId, maxAttempts) => Effect20.gen(function* () {
|
|
3665
|
+
const registry = yield* Service9;
|
|
3644
3666
|
const handler = yield* registry.resolve(def.name);
|
|
3645
|
-
yield*
|
|
3667
|
+
yield* Effect20.logInfo("job.started").pipe(Effect20.annotateLogs({
|
|
3646
3668
|
scope: "@dimple/workflow",
|
|
3647
3669
|
event: "job.started",
|
|
3648
3670
|
job: def.name,
|
|
3649
3671
|
idempotencyKey: jobId
|
|
3650
3672
|
}));
|
|
3651
3673
|
const run = () => handler(payload, { jobId });
|
|
3652
|
-
const outcome = yield* (maxAttempts > 1 ? run().pipe(
|
|
3674
|
+
const outcome = yield* (maxAttempts > 1 ? run().pipe(Effect20.retry({
|
|
3653
3675
|
times: maxAttempts - 1,
|
|
3654
3676
|
schedule: Schedule.spaced(Duration2.millis(250)).pipe(Schedule.jittered)
|
|
3655
|
-
})) : run()).pipe(
|
|
3677
|
+
})) : run()).pipe(Effect20.exit);
|
|
3656
3678
|
if (outcome._tag === "Failure") {
|
|
3657
|
-
yield*
|
|
3679
|
+
yield* Effect20.logError("job.failed").pipe(Effect20.annotateLogs({
|
|
3658
3680
|
scope: "@dimple/workflow",
|
|
3659
3681
|
event: "job.failed",
|
|
3660
3682
|
job: def.name,
|
|
3661
3683
|
idempotencyKey: jobId,
|
|
3662
3684
|
error: Cause2.pretty(outcome.cause)
|
|
3663
3685
|
}));
|
|
3664
|
-
return yield*
|
|
3686
|
+
return yield* Effect20.failCause(outcome.cause).pipe(Effect20.mapError((e) => e.message));
|
|
3665
3687
|
}
|
|
3666
|
-
yield*
|
|
3688
|
+
yield* Effect20.logInfo("job.completed").pipe(Effect20.annotateLogs({
|
|
3667
3689
|
scope: "@dimple/workflow",
|
|
3668
3690
|
event: "job.completed",
|
|
3669
3691
|
job: def.name,
|
|
@@ -3678,23 +3700,23 @@ var workerLayer = (defs, options = {}) => {
|
|
|
3678
3700
|
const layers = defs.map((def) => workerFor(def, options.concurrency ?? 1, maxAttempts));
|
|
3679
3701
|
const [first, ...rest] = layers;
|
|
3680
3702
|
if (!first)
|
|
3681
|
-
return
|
|
3682
|
-
const workers = rest.reduce((acc,
|
|
3683
|
-
return options.logging === undefined ? workers :
|
|
3703
|
+
return Layer10.empty;
|
|
3704
|
+
const workers = rest.reduce((acc, layer9) => Layer10.mergeAll(acc, layer9), first);
|
|
3705
|
+
return options.logging === undefined ? workers : Layer10.provideMerge(layer(options.logging))(workers);
|
|
3684
3706
|
};
|
|
3685
|
-
var configured5 = (defs, options, deps) => make({ name: "Workers", layer:
|
|
3707
|
+
var configured5 = (defs, options, deps) => make({ name: "Workers", layer: Layer10.suspend(() => workerLayer(defs, options)), deps });
|
|
3686
3708
|
// ../store/src/topics/lifecycle.ts
|
|
3687
|
-
var routingFailure3 = (tag) => (effect) => effect.pipe(
|
|
3688
|
-
var isActive =
|
|
3709
|
+
var routingFailure3 = (tag) => (effect) => effect.pipe(Effect21.mapError((cause) => new RoutingFailure({ cause: { operation: tag, cause } })));
|
|
3710
|
+
var isActive = Effect21.fn("topics.isActive")(function* (modelKey) {
|
|
3689
3711
|
const backend = yield* Service2;
|
|
3690
3712
|
const db = backend.db;
|
|
3691
|
-
const rows = yield*
|
|
3713
|
+
const rows = yield* Effect21.tryPromise({
|
|
3692
3714
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and3(eq3(topics.modelKey, modelKey), eq3(topics.status, "active"))).limit(1),
|
|
3693
3715
|
catch: (cause) => cause
|
|
3694
3716
|
}).pipe(routingFailure3("topics.isActive"));
|
|
3695
3717
|
return rows.length > 0;
|
|
3696
3718
|
});
|
|
3697
|
-
var setActiveModel =
|
|
3719
|
+
var setActiveModel = Effect21.fn("topics.setActiveModel")(function* (model, config) {
|
|
3698
3720
|
const backend = yield* Service2;
|
|
3699
3721
|
const db = backend.db;
|
|
3700
3722
|
yield* ensureTopicTree(model, config).pipe(routingFailure3("topics.setActiveModel.seed"));
|
|
@@ -3702,14 +3724,14 @@ var setActiveModel = Effect20.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3702
3724
|
let routed = 0;
|
|
3703
3725
|
let alreadyInTree = 0;
|
|
3704
3726
|
if (memoryIds.length > 0) {
|
|
3705
|
-
const topicRows = yield*
|
|
3727
|
+
const topicRows = yield* Effect21.tryPromise({
|
|
3706
3728
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(eq3(topics.modelKey, model.id)),
|
|
3707
3729
|
catch: (cause) => cause
|
|
3708
3730
|
}).pipe(routingFailure3("topics.setActiveModel.topics"));
|
|
3709
3731
|
const modelTopicKeys = topicRows.map((r) => r.topicKey);
|
|
3710
3732
|
if (modelTopicKeys.length > 0) {
|
|
3711
3733
|
for (const memoryId of memoryIds) {
|
|
3712
|
-
const existing = yield*
|
|
3734
|
+
const existing = yield* Effect21.tryPromise({
|
|
3713
3735
|
try: () => db.select({ topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and3(eq3(topicMemberships.memoryId, memoryId), eq3(topicMemberships.modelKey, model.id), inArray(topicMemberships.topicKey, modelTopicKeys))).limit(1),
|
|
3714
3736
|
catch: (cause) => cause
|
|
3715
3737
|
}).pipe(routingFailure3("topics.setActiveModel.membership"));
|
|
@@ -3727,7 +3749,7 @@ var setActiveModel = Effect20.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3727
3749
|
}
|
|
3728
3750
|
}
|
|
3729
3751
|
const auditTimestamp = yield* Clock3.currentTimeMillis;
|
|
3730
|
-
const deactivated = yield*
|
|
3752
|
+
const deactivated = yield* Effect21.tryPromise({
|
|
3731
3753
|
try: () => db.transaction(async (tx) => {
|
|
3732
3754
|
const others = await tx.selectDistinct({ modelKey: topics.modelKey }).from(topics).where(and3(ne(topics.modelKey, model.id), ne(topics.status, "inactive")));
|
|
3733
3755
|
const affected = others.map((r) => r.modelKey).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
@@ -3750,18 +3772,18 @@ var setActiveModel = Effect20.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3750
3772
|
}).pipe(routingFailure3("topics.setActiveModel.deactivate"));
|
|
3751
3773
|
return { routed, alreadyInTree, deactivated };
|
|
3752
3774
|
});
|
|
3753
|
-
var embedOneMemory = (deps, payload, ctx) =>
|
|
3775
|
+
var embedOneMemory = (deps, payload, ctx) => Effect21.gen(function* () {
|
|
3754
3776
|
const envelope = payload;
|
|
3755
3777
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
3756
3778
|
if (!model) {
|
|
3757
|
-
return yield*
|
|
3779
|
+
return yield* Effect21.fail(new JobFailure({
|
|
3758
3780
|
jobId: ctx.jobId,
|
|
3759
3781
|
message: `memory.embed: unknown model "${envelope.modelKey}"`
|
|
3760
3782
|
}));
|
|
3761
3783
|
}
|
|
3762
|
-
const content = yield* deps.getMemoryContent(envelope.memoryId).pipe(
|
|
3784
|
+
const content = yield* deps.getMemoryContent(envelope.memoryId).pipe(Effect21.catchTag("Store.NotFound", () => Effect21.succeed(undefined)), Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: load content: ${e}` })));
|
|
3763
3785
|
if (content === undefined) {
|
|
3764
|
-
yield*
|
|
3786
|
+
yield* Effect21.logDebug("embed.stale-reoffer").pipe(Effect21.annotateLogs({
|
|
3765
3787
|
scope: "@dimple/store",
|
|
3766
3788
|
event: "embed.stale-reoffer",
|
|
3767
3789
|
memoryId: envelope.memoryId,
|
|
@@ -3769,24 +3791,24 @@ var embedOneMemory = (deps, payload, ctx) => Effect20.gen(function* () {
|
|
|
3769
3791
|
}));
|
|
3770
3792
|
return;
|
|
3771
3793
|
}
|
|
3772
|
-
const vector = yield* deps.embedOne(content, model).pipe(
|
|
3794
|
+
const vector = yield* deps.embedOne(content, model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: embed: ${e}` })));
|
|
3773
3795
|
const backend = yield* Service2;
|
|
3774
3796
|
const db = backend.db;
|
|
3775
|
-
const existing = yield*
|
|
3797
|
+
const existing = yield* Effect21.tryPromise({
|
|
3776
3798
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and3(eq3(topicMemberships.memoryId, envelope.memoryId), eq3(topicMemberships.modelKey, model.id))),
|
|
3777
3799
|
catch: (cause) => new StoreError({ message: `memory.embed: membership check: ${cause}` })
|
|
3778
|
-
}).pipe(
|
|
3779
|
-
yield* ensureTopicTree(model, deps.config).pipe(
|
|
3780
|
-
yield* backend.ensureVectorTable(model).pipe(
|
|
3781
|
-
yield* backend.insertVector(envelope.memoryId, [...vector], model).pipe(
|
|
3782
|
-
yield*
|
|
3800
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: membership check: ${e}` })));
|
|
3801
|
+
yield* ensureTopicTree(model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: seed tree: ${e}` })));
|
|
3802
|
+
yield* backend.ensureVectorTable(model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: vector table: ${e}` })));
|
|
3803
|
+
yield* backend.insertVector(envelope.memoryId, [...vector], model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: insert vector: ${e}` })));
|
|
3804
|
+
yield* Effect21.tryPromise({
|
|
3783
3805
|
try: () => db.update(memories).set({ embedding: [...vector], embeddingModel: model.id }).where(eq3(memories.id, envelope.memoryId)),
|
|
3784
3806
|
catch: (cause) => new StoreError({ message: `memory.embed: stamp embedding: ${cause}` })
|
|
3785
|
-
}).pipe(
|
|
3807
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: stamp embedding: ${e}` })));
|
|
3786
3808
|
if (existing.length === 0) {
|
|
3787
|
-
yield* routeInsert(envelope.memoryId, [...vector], model, deps.config).pipe(
|
|
3809
|
+
yield* routeInsert(envelope.memoryId, [...vector], model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: route: ${e}` })));
|
|
3788
3810
|
}
|
|
3789
|
-
yield*
|
|
3811
|
+
yield* Effect21.logInfo("embed.completed").pipe(Effect21.annotateLogs({
|
|
3790
3812
|
scope: "@dimple/store",
|
|
3791
3813
|
event: "embed.completed",
|
|
3792
3814
|
memoryId: envelope.memoryId,
|
|
@@ -3797,9 +3819,9 @@ var embedOneMemory = (deps, payload, ctx) => Effect20.gen(function* () {
|
|
|
3797
3819
|
});
|
|
3798
3820
|
function memoryEmbedHandler(deps) {
|
|
3799
3821
|
const run = (payload, ctx) => embedOneMemory(deps, payload, ctx);
|
|
3800
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
3822
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect21.provideService(Service2, deps.backend)) : run;
|
|
3801
3823
|
}
|
|
3802
|
-
var logBatchCompleted = (modelKey, embedded, failed, ctx) =>
|
|
3824
|
+
var logBatchCompleted = (modelKey, embedded, failed, ctx) => Effect21.logInfo("embed.batch.completed").pipe(Effect21.annotateLogs({
|
|
3803
3825
|
scope: "@dimple/store",
|
|
3804
3826
|
event: "embed.batch.completed",
|
|
3805
3827
|
jobId: ctx.jobId,
|
|
@@ -3807,15 +3829,15 @@ var logBatchCompleted = (modelKey, embedded, failed, ctx) => Effect20.logInfo("e
|
|
|
3807
3829
|
embedded,
|
|
3808
3830
|
failed
|
|
3809
3831
|
}));
|
|
3810
|
-
var batchFailure = (ctx, failed, total, first) =>
|
|
3832
|
+
var batchFailure = (ctx, failed, total, first) => Effect21.fail(new JobFailure({
|
|
3811
3833
|
jobId: ctx.jobId,
|
|
3812
3834
|
message: `memory.embed.batch: ${failed}/${total} failed — ${first}`
|
|
3813
3835
|
}));
|
|
3814
3836
|
function memoryEmbedBatchHandler(deps) {
|
|
3815
|
-
const run = (payload, ctx) =>
|
|
3837
|
+
const run = (payload, ctx) => Effect21.gen(function* () {
|
|
3816
3838
|
const model = deps.getModelConfig(payload.modelKey);
|
|
3817
3839
|
if (!model) {
|
|
3818
|
-
return yield*
|
|
3840
|
+
return yield* Effect21.fail(new JobFailure({
|
|
3819
3841
|
jobId: ctx.jobId,
|
|
3820
3842
|
message: `memory.embed: unknown model "${payload.modelKey}"`
|
|
3821
3843
|
}));
|
|
@@ -3826,23 +3848,23 @@ function memoryEmbedBatchHandler(deps) {
|
|
|
3826
3848
|
let first;
|
|
3827
3849
|
const embedded = [];
|
|
3828
3850
|
const stale = [];
|
|
3829
|
-
yield*
|
|
3830
|
-
const content = yield* deps.getMemoryContent(memoryId).pipe(
|
|
3851
|
+
yield* Effect21.forEach(payload.memoryIds, (memoryId) => Effect21.gen(function* () {
|
|
3852
|
+
const content = yield* deps.getMemoryContent(memoryId).pipe(Effect21.catchTag("Store.NotFound", () => Effect21.succeed(undefined)), Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: load content: ${e}` })));
|
|
3831
3853
|
if (content === undefined) {
|
|
3832
3854
|
stale.push(memoryId);
|
|
3833
3855
|
return;
|
|
3834
3856
|
}
|
|
3835
|
-
const vector = yield* deps.embedOne(content, model).pipe(
|
|
3857
|
+
const vector = yield* deps.embedOne(content, model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: embed: ${e}` })));
|
|
3836
3858
|
embedded.push({ memoryId, vector });
|
|
3837
|
-
}).pipe(
|
|
3859
|
+
}).pipe(Effect21.exit, Effect21.tap((exit) => {
|
|
3838
3860
|
if (Exit6.isSuccess(exit))
|
|
3839
|
-
return
|
|
3861
|
+
return Effect21.void;
|
|
3840
3862
|
failed += 1;
|
|
3841
3863
|
first ??= Cause3.pretty(exit.cause);
|
|
3842
|
-
return
|
|
3864
|
+
return Effect21.void;
|
|
3843
3865
|
})), { concurrency: 1, discard: true });
|
|
3844
3866
|
for (const memoryId of stale) {
|
|
3845
|
-
yield*
|
|
3867
|
+
yield* Effect21.logDebug("embed.stale-reoffer").pipe(Effect21.annotateLogs({
|
|
3846
3868
|
scope: "@dimple/store",
|
|
3847
3869
|
event: "embed.stale-reoffer",
|
|
3848
3870
|
memoryId,
|
|
@@ -3856,25 +3878,25 @@ function memoryEmbedBatchHandler(deps) {
|
|
|
3856
3878
|
}
|
|
3857
3879
|
return;
|
|
3858
3880
|
}
|
|
3859
|
-
yield* ensureTopicTree(model, deps.config).pipe(
|
|
3860
|
-
yield* backend.ensureVectorTable(model).pipe(
|
|
3881
|
+
yield* ensureTopicTree(model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: seed tree: ${e}` })));
|
|
3882
|
+
yield* backend.ensureVectorTable(model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: vector table: ${e}` })));
|
|
3861
3883
|
const ids = embedded.map((e) => e.memoryId);
|
|
3862
|
-
const members = yield*
|
|
3884
|
+
const members = yield* Effect21.tryPromise({
|
|
3863
3885
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and3(eq3(topicMemberships.modelKey, model.id), inArray(topicMemberships.memoryId, ids))),
|
|
3864
3886
|
catch: (cause) => new StoreError({ message: `memory.embed: membership check: ${cause}` })
|
|
3865
|
-
}).pipe(
|
|
3887
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: membership check: ${e}` })));
|
|
3866
3888
|
const memberSet = new Set(members.map((r) => r.memoryId));
|
|
3867
3889
|
const entries = embedded.map((e) => ({ memoryId: e.memoryId, vector: [...e.vector] }));
|
|
3868
|
-
yield* backend.insertVectors(entries, model).pipe(
|
|
3869
|
-
yield*
|
|
3890
|
+
yield* backend.insertVectors(entries, model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: insert vectors: ${e}` })));
|
|
3891
|
+
yield* Effect21.tryPromise({
|
|
3870
3892
|
try: () => db.transaction(async (tx) => {
|
|
3871
3893
|
await tx.run(sql3`UPDATE ${memories} SET embedding = CASE id ${sql3.join(entries.map((entry) => sql3`WHEN ${entry.memoryId} THEN ${new Float32Array(entry.vector).buffer}`), sql3` `)} END, embedding_model = ${model.id} WHERE id IN (${sql3.join(entries.map((entry) => sql3`${entry.memoryId}`), sql3`, `)})`);
|
|
3872
3894
|
}),
|
|
3873
3895
|
catch: (cause) => new StoreError({ message: `memory.embed: stamp embeddings: ${cause}` })
|
|
3874
|
-
}).pipe(
|
|
3896
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: stamp embeddings: ${e}` })));
|
|
3875
3897
|
const fresh = entries.filter((entry) => !memberSet.has(entry.memoryId));
|
|
3876
3898
|
if (fresh.length > 0) {
|
|
3877
|
-
yield* routeInsertMany(fresh, model, deps.config).pipe(
|
|
3899
|
+
yield* routeInsertMany(fresh, model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: route: ${e}` })));
|
|
3878
3900
|
}
|
|
3879
3901
|
yield* logBatchCompleted(payload.modelKey, embedded.length, failed, ctx);
|
|
3880
3902
|
if (failed > 0) {
|
|
@@ -3882,11 +3904,11 @@ function memoryEmbedBatchHandler(deps) {
|
|
|
3882
3904
|
}
|
|
3883
3905
|
return;
|
|
3884
3906
|
});
|
|
3885
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
3907
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect21.provideService(Service2, deps.backend)) : run;
|
|
3886
3908
|
}
|
|
3887
3909
|
|
|
3888
3910
|
// ../store/src/topics/jobs.ts
|
|
3889
|
-
import { Clock as Clock4, Effect as
|
|
3911
|
+
import { Clock as Clock4, Effect as Effect22 } from "effect";
|
|
3890
3912
|
import { and as and4, asc as asc3, eq as eq4, inArray as inArray2, sql as sql5 } from "drizzle-orm";
|
|
3891
3913
|
|
|
3892
3914
|
// ../store/src/topics/topic-vectors.ts
|
|
@@ -3925,13 +3947,13 @@ var childBaseKey = (parent, vectors) => {
|
|
|
3925
3947
|
}
|
|
3926
3948
|
return `t_${(hash >>> 0).toString(16).padStart(8, "0")}`;
|
|
3927
3949
|
};
|
|
3928
|
-
var splitTopic =
|
|
3950
|
+
var splitTopic = Effect22.fn("TopicSplit.splitTopic")(function* (topicKey, model, config) {
|
|
3929
3951
|
const backend = yield* Service2;
|
|
3930
3952
|
const db = backend.db;
|
|
3931
3953
|
yield* backend.ensureTopicVectorTable(model);
|
|
3932
3954
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS;
|
|
3933
3955
|
const hardLimit = config.splitDeferredHardLimit ?? DEFAULT_SPLIT_DEFERRED_HARD_LIMIT;
|
|
3934
|
-
const leafRows = yield*
|
|
3956
|
+
const leafRows = yield* Effect22.tryPromise({
|
|
3935
3957
|
try: () => db.select().from(topics).where(and4(eq4(topics.topicKey, topicKey), eq4(topics.modelKey, model.id))),
|
|
3936
3958
|
catch: (cause) => new StoreError({ message: `splitTopic: load topic: ${cause}` })
|
|
3937
3959
|
});
|
|
@@ -3942,13 +3964,13 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3942
3964
|
if (leaf.modelKey !== model.id) {
|
|
3943
3965
|
return yield* new ModelMismatch({ expected: model.id, actual: leaf.modelKey });
|
|
3944
3966
|
}
|
|
3945
|
-
const memberRows = yield*
|
|
3967
|
+
const memberRows = yield* Effect22.tryPromise({
|
|
3946
3968
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and4(eq4(topicMemberships.topicKey, topicKey), eq4(topicMemberships.modelKey, model.id))).orderBy(asc3(topicMemberships.memoryId)),
|
|
3947
3969
|
catch: (cause) => new StoreError({ message: `splitTopic: load members: ${cause}` })
|
|
3948
3970
|
});
|
|
3949
3971
|
const memberIds = memberRows.map((r) => r.memoryId);
|
|
3950
3972
|
const memberCount = memberIds.length;
|
|
3951
|
-
const growthRows = yield*
|
|
3973
|
+
const growthRows = yield* Effect22.tryPromise({
|
|
3952
3974
|
try: () => db.select().from(topicGrowthState).where(and4(eq4(topicGrowthState.topicKey, topicKey), eq4(topicGrowthState.modelKey, model.id))),
|
|
3953
3975
|
catch: (cause) => new StoreError({ message: `splitTopic: load growth state: ${cause}` })
|
|
3954
3976
|
});
|
|
@@ -3980,7 +4002,7 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3980
4002
|
const forced = memberCount > hardLimit;
|
|
3981
4003
|
if (!plan.accepted && !forced) {
|
|
3982
4004
|
const now2 = yield* Clock4.currentTimeMillis;
|
|
3983
|
-
yield*
|
|
4005
|
+
yield* Effect22.tryPromise({
|
|
3984
4006
|
try: () => db.transaction(async (tx) => {
|
|
3985
4007
|
await tx.insert(topicGrowthState).values({
|
|
3986
4008
|
topicKey,
|
|
@@ -4013,7 +4035,7 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
4013
4035
|
};
|
|
4014
4036
|
}
|
|
4015
4037
|
const now = yield* Clock4.currentTimeMillis;
|
|
4016
|
-
const children = yield*
|
|
4038
|
+
const children = yield* Effect22.tryPromise({
|
|
4017
4039
|
try: () => db.transaction(async (tx) => {
|
|
4018
4040
|
const child0Cf = cfFromVectors(groupVectors(0));
|
|
4019
4041
|
const child1Cf = cfFromVectors(groupVectors(1));
|
|
@@ -4134,34 +4156,34 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
4134
4156
|
});
|
|
4135
4157
|
return { accepted: true, children: children.keys, childCounts: children.counts };
|
|
4136
4158
|
});
|
|
4137
|
-
var staleSplitNoOp = (notFound, model) =>
|
|
4159
|
+
var staleSplitNoOp = (notFound, model) => Effect22.gen(function* () {
|
|
4138
4160
|
const backend = yield* Service2;
|
|
4139
4161
|
const db = backend.db;
|
|
4140
|
-
const rows = yield*
|
|
4162
|
+
const rows = yield* Effect22.tryPromise({
|
|
4141
4163
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and4(eq4(topics.topicKey, notFound.topicKey), eq4(topics.modelKey, model.id))),
|
|
4142
4164
|
catch: (cause) => new StoreError({ message: `topic.split: stale check: ${cause}` })
|
|
4143
4165
|
});
|
|
4144
4166
|
if (rows.length > 0) {
|
|
4145
4167
|
return { accepted: false, reason: "no-improvement" };
|
|
4146
4168
|
}
|
|
4147
|
-
return yield*
|
|
4169
|
+
return yield* Effect22.fail(notFound);
|
|
4148
4170
|
});
|
|
4149
4171
|
function topicSplitHandler(deps) {
|
|
4150
|
-
const run = (payload, ctx) =>
|
|
4172
|
+
const run = (payload, ctx) => Effect22.gen(function* () {
|
|
4151
4173
|
const envelope = payload;
|
|
4152
4174
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4153
4175
|
if (!model) {
|
|
4154
|
-
return yield*
|
|
4176
|
+
return yield* Effect22.fail(new JobFailure({
|
|
4155
4177
|
jobId: ctx.jobId,
|
|
4156
4178
|
message: `topic.split: unknown model "${envelope.modelKey}"`
|
|
4157
4179
|
}));
|
|
4158
4180
|
}
|
|
4159
|
-
const outcome = yield* splitTopic(envelope.topicId, model, deps.config).pipe(
|
|
4181
|
+
const outcome = yield* splitTopic(envelope.topicId, model, deps.config).pipe(Effect22.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleSplitNoOp(notFound, model)), Effect22.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4160
4182
|
if (outcome.accepted && deps.onUnevenSplit) {
|
|
4161
4183
|
const [n0, n1] = outcome.childCounts;
|
|
4162
4184
|
const [k0] = outcome.children;
|
|
4163
4185
|
if (Math.max(n0, n1) > 2 * Math.min(n0, n1)) {
|
|
4164
|
-
yield* deps.onUnevenSplit(k0, envelope.modelKey).pipe(
|
|
4186
|
+
yield* deps.onUnevenSplit(k0, envelope.modelKey).pipe(Effect22.mapError((error) => new JobFailure({
|
|
4165
4187
|
jobId: ctx.jobId,
|
|
4166
4188
|
message: `topic.split: enqueue rebalance: ${error}`
|
|
4167
4189
|
})));
|
|
@@ -4171,14 +4193,14 @@ function topicSplitHandler(deps) {
|
|
|
4171
4193
|
const maxLeafItems = deps.config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS;
|
|
4172
4194
|
for (let i = 0;i < outcome.children.length; i++) {
|
|
4173
4195
|
if (outcome.childCounts[i] > maxLeafItems) {
|
|
4174
|
-
yield* deps.onChildOverflow(outcome.children[i], envelope.modelKey).pipe(
|
|
4196
|
+
yield* deps.onChildOverflow(outcome.children[i], envelope.modelKey).pipe(Effect22.mapError((error) => new JobFailure({
|
|
4175
4197
|
jobId: ctx.jobId,
|
|
4176
4198
|
message: `topic.split: enqueue child split: ${error}`
|
|
4177
4199
|
})));
|
|
4178
4200
|
}
|
|
4179
4201
|
}
|
|
4180
4202
|
}
|
|
4181
|
-
yield*
|
|
4203
|
+
yield* Effect22.logInfo("split.completed").pipe(Effect22.annotateLogs({
|
|
4182
4204
|
scope: "@dimple/store",
|
|
4183
4205
|
event: "split.completed",
|
|
4184
4206
|
topicId: envelope.topicId,
|
|
@@ -4188,16 +4210,16 @@ function topicSplitHandler(deps) {
|
|
|
4188
4210
|
}));
|
|
4189
4211
|
return;
|
|
4190
4212
|
});
|
|
4191
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4213
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect22.provideService(Service2, deps.backend)) : run;
|
|
4192
4214
|
}
|
|
4193
4215
|
|
|
4194
4216
|
// ../store/src/topics/maintenance.ts
|
|
4195
|
-
import { Clock as Clock5, Effect as
|
|
4217
|
+
import { Clock as Clock5, Effect as Effect23 } from "effect";
|
|
4196
4218
|
import { and as and5, asc as asc4, eq as eq5, ne as ne2 } from "drizzle-orm";
|
|
4197
4219
|
var DEFAULT_MIN_LEAF_ITEMS = 32;
|
|
4198
4220
|
var DEFAULT_MAX_LEAF_ITEMS2 = 128;
|
|
4199
4221
|
var DEFAULT_REBALANCE_COOLDOWN_MS = 60000;
|
|
4200
|
-
var maintenanceError = (tag) => (effect) => effect.pipe(
|
|
4222
|
+
var maintenanceError = (tag) => (effect) => effect.pipe(Effect23.mapError((cause) => new StoreError({ message: `${tag}: ${cause}` })));
|
|
4201
4223
|
var dot2 = (a, b) => {
|
|
4202
4224
|
let sum = 0;
|
|
4203
4225
|
for (let i = 0;i < a.length; i++)
|
|
@@ -4215,12 +4237,12 @@ var resolvedConfig2 = (config) => ({
|
|
|
4215
4237
|
maxLeafItems: config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS2,
|
|
4216
4238
|
rebalanceCooldownMs: config.rebalanceCooldownMs ?? DEFAULT_REBALANCE_COOLDOWN_MS
|
|
4217
4239
|
});
|
|
4218
|
-
var mergeOrBorrow =
|
|
4240
|
+
var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, model, config) {
|
|
4219
4241
|
const backend = yield* Service2;
|
|
4220
4242
|
const db = backend.db;
|
|
4221
4243
|
const { minLeafItems, maxLeafItems } = resolvedConfig2(config);
|
|
4222
4244
|
const now = yield* Clock5.currentTimeMillis;
|
|
4223
|
-
const leafRows = yield*
|
|
4245
|
+
const leafRows = yield* Effect23.tryPromise({
|
|
4224
4246
|
try: () => db.select().from(topics).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id))),
|
|
4225
4247
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load topic: ${cause}` })
|
|
4226
4248
|
});
|
|
@@ -4233,7 +4255,7 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4233
4255
|
}
|
|
4234
4256
|
if (leaf.memberCount >= minLeafItems)
|
|
4235
4257
|
return { action: "none" };
|
|
4236
|
-
const parentRows = yield*
|
|
4258
|
+
const parentRows = yield* Effect23.tryPromise({
|
|
4237
4259
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.parentKey)),
|
|
4238
4260
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load parent: ${cause}` })
|
|
4239
4261
|
});
|
|
@@ -4241,16 +4263,16 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4241
4263
|
return yield* logUnderfilled(db, topicKey, model, now, "no-borrowable");
|
|
4242
4264
|
}
|
|
4243
4265
|
const parentKey = parentRows[0].parentKey;
|
|
4244
|
-
const parentRowRows = yield*
|
|
4266
|
+
const parentRowRows = yield* Effect23.tryPromise({
|
|
4245
4267
|
try: () => db.select({ childCount: topics.childCount }).from(topics).where(and5(eq5(topics.topicKey, parentKey), eq5(topics.modelKey, model.id))),
|
|
4246
4268
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load parent row: ${cause}` })
|
|
4247
4269
|
});
|
|
4248
4270
|
const parentRow = parentRowRows[0];
|
|
4249
|
-
const siblingRows = yield*
|
|
4271
|
+
const siblingRows = yield* Effect23.tryPromise({
|
|
4250
4272
|
try: () => db.select().from(topics).where(and5(eq5(topics.modelKey, model.id), eq5(topics.kind, "leaf"), ne2(topics.status, "inactive"))).orderBy(asc4(topics.topicKey)),
|
|
4251
4273
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load siblings: ${cause}` })
|
|
4252
4274
|
});
|
|
4253
|
-
const relationRows = yield*
|
|
4275
|
+
const relationRows = yield* Effect23.tryPromise({
|
|
4254
4276
|
try: () => db.select({ childKey: topicRelations.childKey }).from(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.childKey)),
|
|
4255
4277
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load relations: ${cause}` })
|
|
4256
4278
|
});
|
|
@@ -4272,7 +4294,7 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4272
4294
|
const membersByTopic = new Map;
|
|
4273
4295
|
const vecById = new Map;
|
|
4274
4296
|
for (const sibling of siblingsOf) {
|
|
4275
|
-
const members = yield*
|
|
4297
|
+
const members = yield* Effect23.tryPromise({
|
|
4276
4298
|
try: () => db.select({
|
|
4277
4299
|
memoryId: topicMemberships.memoryId,
|
|
4278
4300
|
manualLock: topicMemberships.manualLock
|
|
@@ -4285,7 +4307,7 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4285
4307
|
vecById.set(row.memoryId, new Float32Array(row.vector));
|
|
4286
4308
|
}
|
|
4287
4309
|
const movedIds = new Set;
|
|
4288
|
-
const outcome = yield*
|
|
4310
|
+
const outcome = yield* Effect23.tryPromise({
|
|
4289
4311
|
try: () => db.transaction(async (tx) => {
|
|
4290
4312
|
const leafCf0 = cfOf(leaf);
|
|
4291
4313
|
let leafCf = leafCf0;
|
|
@@ -4454,12 +4476,12 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4454
4476
|
});
|
|
4455
4477
|
return outcome;
|
|
4456
4478
|
});
|
|
4457
|
-
var rebalanceSiblings =
|
|
4479
|
+
var rebalanceSiblings = Effect23.fn("topics.rebalanceSiblings")(function* (topicKey, model, config) {
|
|
4458
4480
|
const backend = yield* Service2;
|
|
4459
4481
|
const db = backend.db;
|
|
4460
4482
|
const { rebalanceCooldownMs } = resolvedConfig2(config);
|
|
4461
4483
|
const now = yield* Clock5.currentTimeMillis;
|
|
4462
|
-
const rows = yield*
|
|
4484
|
+
const rows = yield* Effect23.tryPromise({
|
|
4463
4485
|
try: () => db.select().from(topics).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id))),
|
|
4464
4486
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load topic: ${cause}` })
|
|
4465
4487
|
});
|
|
@@ -4468,19 +4490,19 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4468
4490
|
return yield* new TopicNotFound({ topicKey });
|
|
4469
4491
|
if (row.modelKey !== model.id)
|
|
4470
4492
|
return yield* new ModelMismatch({ expected: model.id, actual: row.modelKey });
|
|
4471
|
-
const parentRows = yield*
|
|
4493
|
+
const parentRows = yield* Effect23.tryPromise({
|
|
4472
4494
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.parentKey)),
|
|
4473
4495
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load parent: ${cause}` })
|
|
4474
4496
|
});
|
|
4475
4497
|
if (parentRows.length === 0)
|
|
4476
4498
|
return { action: "none", reason: "balanced" };
|
|
4477
4499
|
const parentKey = parentRows[0].parentKey;
|
|
4478
|
-
const relationRows = yield*
|
|
4500
|
+
const relationRows = yield* Effect23.tryPromise({
|
|
4479
4501
|
try: () => db.select({ childKey: topicRelations.childKey }).from(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.childKey)),
|
|
4480
4502
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load relations: ${cause}` })
|
|
4481
4503
|
});
|
|
4482
4504
|
const siblingKeys = relationRows.map((r) => r.childKey);
|
|
4483
|
-
const topicRows = yield*
|
|
4505
|
+
const topicRows = yield* Effect23.tryPromise({
|
|
4484
4506
|
try: () => db.select().from(topics).where(and5(eq5(topics.modelKey, model.id), eq5(topics.kind, "leaf"), ne2(topics.status, "inactive"))),
|
|
4485
4507
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load siblings: ${cause}` })
|
|
4486
4508
|
});
|
|
@@ -4488,7 +4510,7 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4488
4510
|
const siblings = siblingKeys.map((k) => map2.get(k)).filter(Boolean);
|
|
4489
4511
|
if (siblings.length < 2)
|
|
4490
4512
|
return { action: "none", reason: "balanced" };
|
|
4491
|
-
const growthRows = yield*
|
|
4513
|
+
const growthRows = yield* Effect23.tryPromise({
|
|
4492
4514
|
try: () => db.select().from(topicGrowthState).where(and5(eq5(topicGrowthState.topicKey, topicKey), eq5(topicGrowthState.modelKey, model.id))),
|
|
4493
4515
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load growth: ${cause}` })
|
|
4494
4516
|
});
|
|
@@ -4513,7 +4535,7 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4513
4535
|
const membersByTopic = new Map;
|
|
4514
4536
|
const vecById = new Map;
|
|
4515
4537
|
for (const sibling of siblings) {
|
|
4516
|
-
const members = yield*
|
|
4538
|
+
const members = yield* Effect23.tryPromise({
|
|
4517
4539
|
try: () => db.select({
|
|
4518
4540
|
memoryId: topicMemberships.memoryId,
|
|
4519
4541
|
manualLock: topicMemberships.manualLock
|
|
@@ -4526,7 +4548,7 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4526
4548
|
vecById.set(row2.memoryId, new Float32Array(row2.vector));
|
|
4527
4549
|
}
|
|
4528
4550
|
const movedIds = new Set;
|
|
4529
|
-
const outcome = yield*
|
|
4551
|
+
const outcome = yield* Effect23.tryPromise({
|
|
4530
4552
|
try: () => db.transaction(async (tx) => {
|
|
4531
4553
|
const liveCfs = new Map(siblings.map((s) => [s.topicKey, cfOf(s)]));
|
|
4532
4554
|
const liveList = () => [...liveCfs.entries()].map(([topicKey2, cf]) => ({ topicKey: topicKey2, cf })).sort((a, b) => b.cf.count - a.cf.count || (a.topicKey < b.topicKey ? -1 : 1));
|
|
@@ -4613,7 +4635,7 @@ var loadMembers = async (tx, topicKey, modelKey) => {
|
|
|
4613
4635
|
const rows = await tx.select({ memoryId: topicMemberships.memoryId, manualLock: topicMemberships.manualLock }).from(topicMemberships).where(and5(eq5(topicMemberships.topicKey, topicKey), eq5(topicMemberships.modelKey, modelKey))).orderBy(asc4(topicMemberships.memoryId));
|
|
4614
4636
|
return rows;
|
|
4615
4637
|
};
|
|
4616
|
-
var logUnderfilled = (db, topicKey, model, now, reason) =>
|
|
4638
|
+
var logUnderfilled = (db, topicKey, model, now, reason) => Effect23.tryPromise({
|
|
4617
4639
|
try: () => db.insert(topicChangeLog).values({
|
|
4618
4640
|
topicKey,
|
|
4619
4641
|
operation: "merge",
|
|
@@ -4623,7 +4645,7 @@ var logUnderfilled = (db, topicKey, model, now, reason) => Effect22.tryPromise({
|
|
|
4623
4645
|
createdAt: now
|
|
4624
4646
|
}),
|
|
4625
4647
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: log underfilled: ${cause}` })
|
|
4626
|
-
}).pipe(
|
|
4648
|
+
}).pipe(Effect23.map(() => ({ action: "underfilled", reason })));
|
|
4627
4649
|
var logUnderfilledTx = async (tx, topicKey, model, now, reason) => {
|
|
4628
4650
|
await tx.insert(topicChangeLog).values({
|
|
4629
4651
|
topicKey,
|
|
@@ -4636,61 +4658,61 @@ var logUnderfilledTx = async (tx, topicKey, model, now, reason) => {
|
|
|
4636
4658
|
return { action: "underfilled", reason };
|
|
4637
4659
|
};
|
|
4638
4660
|
function topicMergeHandler(deps) {
|
|
4639
|
-
const run = (payload, ctx) =>
|
|
4661
|
+
const run = (payload, ctx) => Effect23.gen(function* () {
|
|
4640
4662
|
const envelope = payload;
|
|
4641
4663
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4642
4664
|
if (!model) {
|
|
4643
|
-
return yield*
|
|
4665
|
+
return yield* Effect23.fail(new JobFailure({
|
|
4644
4666
|
jobId: ctx.jobId,
|
|
4645
4667
|
message: `topic.merge: unknown model "${envelope.modelKey}"`
|
|
4646
4668
|
}));
|
|
4647
4669
|
}
|
|
4648
|
-
yield* mergeOrBorrow(envelope.topicId, model, deps.config).pipe(
|
|
4670
|
+
yield* mergeOrBorrow(envelope.topicId, model, deps.config).pipe(Effect23.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleNoOp(notFound, model)), Effect23.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4649
4671
|
return;
|
|
4650
4672
|
});
|
|
4651
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4673
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect23.provideService(Service2, deps.backend)) : run;
|
|
4652
4674
|
}
|
|
4653
4675
|
function topicRebalanceHandler(deps) {
|
|
4654
|
-
const run = (payload, ctx) =>
|
|
4676
|
+
const run = (payload, ctx) => Effect23.gen(function* () {
|
|
4655
4677
|
const envelope = payload;
|
|
4656
4678
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4657
4679
|
if (!model) {
|
|
4658
|
-
return yield*
|
|
4680
|
+
return yield* Effect23.fail(new JobFailure({
|
|
4659
4681
|
jobId: ctx.jobId,
|
|
4660
4682
|
message: `topic.rebalance: unknown model "${envelope.modelKey}"`
|
|
4661
4683
|
}));
|
|
4662
4684
|
}
|
|
4663
|
-
yield* rebalanceSiblings(envelope.topicId, model, deps.config).pipe(
|
|
4685
|
+
yield* rebalanceSiblings(envelope.topicId, model, deps.config).pipe(Effect23.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleNoOp(notFound, model)), Effect23.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4664
4686
|
return;
|
|
4665
4687
|
});
|
|
4666
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4688
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect23.provideService(Service2, deps.backend)) : run;
|
|
4667
4689
|
}
|
|
4668
|
-
var staleNoOp = (notFound, model) =>
|
|
4690
|
+
var staleNoOp = (notFound, model) => Effect23.gen(function* () {
|
|
4669
4691
|
const backend = yield* Service2;
|
|
4670
4692
|
const db = backend.db;
|
|
4671
|
-
const rows = yield*
|
|
4693
|
+
const rows = yield* Effect23.tryPromise({
|
|
4672
4694
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and5(eq5(topics.topicKey, notFound.topicKey), eq5(topics.modelKey, model.id))),
|
|
4673
4695
|
catch: (cause) => new StoreError({ message: `maintenance: stale check: ${cause}` })
|
|
4674
4696
|
});
|
|
4675
4697
|
if (rows.length > 0) {
|
|
4676
4698
|
return;
|
|
4677
4699
|
}
|
|
4678
|
-
return yield*
|
|
4679
|
-
}).pipe(
|
|
4700
|
+
return yield* Effect23.fail(new JobFailure({ jobId: notFound.topicKey, message: `topic missing: ${notFound.topicKey}` }));
|
|
4701
|
+
}).pipe(Effect23.mapError((error) => error instanceof JobFailure ? error : new JobFailure({ jobId: notFound.topicKey, message: `${error}` })));
|
|
4680
4702
|
|
|
4681
4703
|
// ../store/src/topics/observability.ts
|
|
4682
|
-
import { Effect as
|
|
4704
|
+
import { Effect as Effect24 } from "effect";
|
|
4683
4705
|
import { and as and6, desc, eq as eq6, inArray as inArray3, ne as ne3, sql as sql6 } from "drizzle-orm";
|
|
4684
4706
|
var DEFAULT_MAX_LEAF_ITEMS3 = 128;
|
|
4685
4707
|
var DEFAULT_SPLIT_DEFERRED_HARD_LIMIT2 = 512;
|
|
4686
4708
|
var DEFAULT_CHANGE_LOG_LIMIT = 50;
|
|
4687
|
-
var routingFailure4 = (tag) => (effect) => effect.pipe(
|
|
4709
|
+
var routingFailure4 = (tag) => (effect) => effect.pipe(Effect24.mapError((cause) => new RoutingFailure({ cause: { operation: tag, cause } })));
|
|
4688
4710
|
var modelTopicKeys = (db, modelKey) => db.select({ topicKey: topics.topicKey }).from(topics).where(eq6(topics.modelKey, modelKey));
|
|
4689
|
-
var topicMetrics =
|
|
4711
|
+
var topicMetrics = Effect24.fn("topics.topicMetrics")(function* (modelKey, config) {
|
|
4690
4712
|
const backend = yield* Service2;
|
|
4691
4713
|
const db = backend.db;
|
|
4692
4714
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS3;
|
|
4693
|
-
const rows = yield*
|
|
4715
|
+
const rows = yield* Effect24.tryPromise({
|
|
4694
4716
|
try: () => db.select({
|
|
4695
4717
|
topicKey: topics.topicKey,
|
|
4696
4718
|
kind: topics.kind,
|
|
@@ -4723,12 +4745,12 @@ var topicMetrics = Effect23.fn("topics.topicMetrics")(function* (modelKey, confi
|
|
|
4723
4745
|
deferredLeaves++;
|
|
4724
4746
|
}
|
|
4725
4747
|
leafMemberCounts.sort((a, b) => a - b);
|
|
4726
|
-
const growth = yield*
|
|
4748
|
+
const growth = yield* Effect24.tryPromise({
|
|
4727
4749
|
try: () => db.select({ total: sql6`coalesce(sum(${topicGrowthState.splitRejectedCount}), 0)` }).from(topicGrowthState).where(eq6(topicGrowthState.modelKey, modelKey)),
|
|
4728
4750
|
catch: (cause) => cause
|
|
4729
4751
|
}).pipe(routingFailure4("topics.topicMetrics.growth"));
|
|
4730
4752
|
const splitRejectedTotal = Number(growth[0]?.total ?? 0);
|
|
4731
|
-
const lastLog = yield*
|
|
4753
|
+
const lastLog = yield* Effect24.tryPromise({
|
|
4732
4754
|
try: () => db.select({
|
|
4733
4755
|
operation: topicChangeLog.operation,
|
|
4734
4756
|
createdAt: topicChangeLog.createdAt
|
|
@@ -4748,11 +4770,11 @@ var topicMetrics = Effect23.fn("topics.topicMetrics")(function* (modelKey, confi
|
|
|
4748
4770
|
lastOperation
|
|
4749
4771
|
};
|
|
4750
4772
|
});
|
|
4751
|
-
var topicChangeLog2 =
|
|
4773
|
+
var topicChangeLog2 = Effect24.fn("topics.topicChangeLog")(function* (modelKey, opts) {
|
|
4752
4774
|
const backend = yield* Service2;
|
|
4753
4775
|
const db = backend.db;
|
|
4754
4776
|
const limit = opts?.limit ?? DEFAULT_CHANGE_LOG_LIMIT;
|
|
4755
|
-
const rows = yield*
|
|
4777
|
+
const rows = yield* Effect24.tryPromise({
|
|
4756
4778
|
try: () => {
|
|
4757
4779
|
const scoped = inArray3(topicChangeLog.topicKey, modelTopicKeys(db, modelKey));
|
|
4758
4780
|
const filtered = opts?.operation ? and6(scoped, eq6(topicChangeLog.operation, opts.operation)) : scoped;
|
|
@@ -4762,14 +4784,14 @@ var topicChangeLog2 = Effect23.fn("topics.topicChangeLog")(function* (modelKey,
|
|
|
4762
4784
|
}).pipe(routingFailure4("topics.topicChangeLog"));
|
|
4763
4785
|
return rows;
|
|
4764
4786
|
});
|
|
4765
|
-
var topicHealth =
|
|
4787
|
+
var topicHealth = Effect24.fn("topics.topicHealth")(function* (modelKey, config) {
|
|
4766
4788
|
const backend = yield* Service2;
|
|
4767
4789
|
const db = backend.db;
|
|
4768
4790
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS3;
|
|
4769
4791
|
const hardLimit = config.splitDeferredHardLimit ?? DEFAULT_SPLIT_DEFERRED_HARD_LIMIT2;
|
|
4770
4792
|
const slack = Math.max(2, Math.ceil(hardLimit / 2) - maxLeafItems);
|
|
4771
4793
|
const leafLimit = maxLeafItems + slack;
|
|
4772
|
-
const rows = yield*
|
|
4794
|
+
const rows = yield* Effect24.tryPromise({
|
|
4773
4795
|
try: () => db.select({
|
|
4774
4796
|
topicKey: topics.topicKey,
|
|
4775
4797
|
kind: topics.kind,
|
|
@@ -4779,7 +4801,7 @@ var topicHealth = Effect23.fn("topics.topicHealth")(function* (modelKey, config)
|
|
|
4779
4801
|
}).from(topics).where(and6(eq6(topics.modelKey, modelKey), ne3(topics.status, "inactive"))),
|
|
4780
4802
|
catch: (cause) => cause
|
|
4781
4803
|
}).pipe(routingFailure4("topics.topicHealth.topics"));
|
|
4782
|
-
const countRows = yield*
|
|
4804
|
+
const countRows = yield* Effect24.tryPromise({
|
|
4783
4805
|
try: () => db.select({ topicKey: topicMemberships.topicKey, count: sql6`count(*)` }).from(topicMemberships).where(eq6(topicMemberships.modelKey, modelKey)).groupBy(topicMemberships.topicKey),
|
|
4784
4806
|
catch: (cause) => cause
|
|
4785
4807
|
}).pipe(routingFailure4("topics.topicHealth.memberships"));
|
|
@@ -4804,30 +4826,30 @@ var topicHealth = Effect23.fn("topics.topicHealth")(function* (modelKey, config)
|
|
|
4804
4826
|
});
|
|
4805
4827
|
|
|
4806
4828
|
// ../store/src/graph/observability.ts
|
|
4807
|
-
import { Effect as
|
|
4829
|
+
import { Effect as Effect25 } from "effect";
|
|
4808
4830
|
import { and as and7, eq as eq7, isNull, sql as sql7 } from "drizzle-orm";
|
|
4809
|
-
var graphCoverage =
|
|
4831
|
+
var graphCoverage = Effect25.fn("GraphCoverage.run")(function* (modelKey) {
|
|
4810
4832
|
const backend = yield* Service2;
|
|
4811
4833
|
const db = backend.db;
|
|
4812
|
-
const totalRows = yield*
|
|
4834
|
+
const totalRows = yield* Effect25.tryPromise({
|
|
4813
4835
|
try: () => db.select({ id: memories.id }).from(memories).where(and7(eq7(memories.state, "active"), eq7(memories.embeddingModel, modelKey))),
|
|
4814
4836
|
catch: (cause) => new StoreError({ message: `graphCoverage: total: ${cause}` })
|
|
4815
4837
|
});
|
|
4816
4838
|
const totalMemories = totalRows.length;
|
|
4817
|
-
const isolatedRows = yield* backend.listIsolatedMemories(0, modelKey).pipe(
|
|
4839
|
+
const isolatedRows = yield* backend.listIsolatedMemories(0, modelKey).pipe(Effect25.mapError((cause) => new StoreError({ message: `graphCoverage: isolated: ${cause}` })));
|
|
4818
4840
|
const isolatedMemories = isolatedRows.length;
|
|
4819
4841
|
const coveredMemories = totalMemories - isolatedMemories;
|
|
4820
4842
|
const coveragePercent = totalMemories === 0 ? 0 : Math.round(coveredMemories / totalMemories * 1000) / 10;
|
|
4821
|
-
const pendingRows = yield*
|
|
4843
|
+
const pendingRows = yield* Effect25.tryPromise({
|
|
4822
4844
|
try: () => db.select({ count: sql7`count(*)` }).from(artifactEdges).where(and7(eq7(artifactEdges.status, "pending"), isNull(artifactEdges.expiredAt))),
|
|
4823
4845
|
catch: (cause) => new StoreError({ message: `graphCoverage: pending: ${cause}` })
|
|
4824
4846
|
});
|
|
4825
4847
|
const pendingCandidates = totalMemories === 0 ? 0 : Number(pendingRows[0]?.count ?? 0);
|
|
4826
|
-
const conceptCount = totalMemories === 0 ? 0 : (yield*
|
|
4848
|
+
const conceptCount = totalMemories === 0 ? 0 : (yield* Effect25.tryPromise({
|
|
4827
4849
|
try: () => db.select({ id: concepts.id }).from(concepts),
|
|
4828
4850
|
catch: (cause) => new StoreError({ message: `graphCoverage: concepts: ${cause}` })
|
|
4829
4851
|
})).length;
|
|
4830
|
-
const annotatedMemories = totalMemories === 0 ? 0 : new Set((yield*
|
|
4852
|
+
const annotatedMemories = totalMemories === 0 ? 0 : new Set((yield* Effect25.tryPromise({
|
|
4831
4853
|
try: () => db.select({ memoryId: memoryConcepts.memoryId }).from(memoryConcepts),
|
|
4832
4854
|
catch: (cause) => new StoreError({ message: `graphCoverage: annotations: ${cause}` })
|
|
4833
4855
|
})).map((row) => row.memoryId)).size;
|
|
@@ -5062,25 +5084,25 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5062
5084
|
const db = backend.db;
|
|
5063
5085
|
const topicsConfig = topicsOptions?.config ?? Schema23.decodeUnknownSync(TopicConfig)({});
|
|
5064
5086
|
const topicsEnabled = topicsOptions?.enabled ?? false;
|
|
5065
|
-
const updateTopicVectorForget = (backendImpl, model, topicKey, x) =>
|
|
5066
|
-
const current = yield* backendImpl.fetchTopicVector(topicKey, model).pipe(
|
|
5087
|
+
const updateTopicVectorForget = (backendImpl, model, topicKey, x) => Effect26.gen(function* () {
|
|
5088
|
+
const current = yield* backendImpl.fetchTopicVector(topicKey, model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: load vector: ${cause}` })));
|
|
5067
5089
|
const ls = current ? new Float32Array(current.sumVector) : new Float32Array(model.dimensions);
|
|
5068
5090
|
for (let i = 0;i < ls.length; i++)
|
|
5069
5091
|
ls[i] = ls[i] - x[i];
|
|
5070
|
-
yield* backendImpl.upsertTopicVector(topicKey, Array.from(ls), model).pipe(
|
|
5071
|
-
const relRows = yield*
|
|
5092
|
+
yield* backendImpl.upsertTopicVector(topicKey, Array.from(ls), model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: write vector: ${cause}` })));
|
|
5093
|
+
const relRows = yield* Effect26.tryPromise({
|
|
5072
5094
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and8(eq8(topicRelations.childKey, topicKey), eq8(topicRelations.modelKey, model.id), eq8(topicRelations.relationKind, "parent_child"))),
|
|
5073
5095
|
catch: storeError("Store.forget: load ancestor")
|
|
5074
5096
|
});
|
|
5075
5097
|
let ancestorKey = relRows[0]?.parentKey;
|
|
5076
5098
|
while (ancestorKey !== undefined) {
|
|
5077
5099
|
const key = ancestorKey;
|
|
5078
|
-
const a = yield* backendImpl.fetchTopicVector(key, model).pipe(
|
|
5100
|
+
const a = yield* backendImpl.fetchTopicVector(key, model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: load vector: ${cause}` })));
|
|
5079
5101
|
const als = a ? new Float32Array(a.sumVector) : new Float32Array(model.dimensions);
|
|
5080
5102
|
for (let i = 0;i < als.length; i++)
|
|
5081
5103
|
als[i] = als[i] - x[i];
|
|
5082
|
-
yield* backendImpl.upsertTopicVector(key, Array.from(als), model).pipe(
|
|
5083
|
-
const up = yield*
|
|
5104
|
+
yield* backendImpl.upsertTopicVector(key, Array.from(als), model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: write vector: ${cause}` })));
|
|
5105
|
+
const up = yield* Effect26.tryPromise({
|
|
5084
5106
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and8(eq8(topicRelations.childKey, key), eq8(topicRelations.modelKey, model.id), eq8(topicRelations.relationKind, "parent_child"))),
|
|
5085
5107
|
catch: storeError("Store.forget: load ancestor")
|
|
5086
5108
|
});
|
|
@@ -5130,14 +5152,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5130
5152
|
};
|
|
5131
5153
|
const seam = auditOptions?.enqueueAudit;
|
|
5132
5154
|
if (seam !== undefined) {
|
|
5133
|
-
return propagateSeam ? seam(row).pipe(
|
|
5155
|
+
return propagateSeam ? seam(row).pipe(Effect26.mapError((cause) => new StoreError({ message: `audit.enqueue-failed: ${cause}` }))) : seam(row).pipe(Effect26.catch((cause) => Effect26.logError("audit.enqueue-failed").pipe(Effect26.annotateLogs({
|
|
5134
5156
|
scope: "@dimple/store",
|
|
5135
5157
|
event: "audit.enqueue-failed",
|
|
5136
5158
|
mode,
|
|
5137
5159
|
cause: String(cause)
|
|
5138
5160
|
}))));
|
|
5139
5161
|
}
|
|
5140
|
-
return
|
|
5162
|
+
return Effect26.sync(() => {
|
|
5141
5163
|
db.insert(retrievalLog2).values({
|
|
5142
5164
|
id: row.id,
|
|
5143
5165
|
queryHash: row.queryHash,
|
|
@@ -5152,7 +5174,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5152
5174
|
}).catch(() => {});
|
|
5153
5175
|
});
|
|
5154
5176
|
};
|
|
5155
|
-
const logAudit =
|
|
5177
|
+
const logAudit = Effect26.fn("Store.logAudit")(function* (input) {
|
|
5156
5178
|
return yield* logRetrieval(input.query, input.namespace, input.mode, input.resultCount, input.startTime, input.queryProfile, input.sources, input.hashSeed, true);
|
|
5157
5179
|
});
|
|
5158
5180
|
const GRAPH_SUPERSEDE_DEMOTION = graphConfig?.supersedeDemotion ?? 0.5;
|
|
@@ -5163,7 +5185,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5163
5185
|
const graphEnabled = graphConfig?.enabled !== false;
|
|
5164
5186
|
const GRAPH_CANDIDATE_LIMIT_MAX = 200;
|
|
5165
5187
|
const GRAPH_ISOLATED_EXAMPLES = 10;
|
|
5166
|
-
const fetchResultEdges = (ids) =>
|
|
5188
|
+
const fetchResultEdges = (ids) => Effect26.tryPromise({
|
|
5167
5189
|
try: () => db.select({
|
|
5168
5190
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
5169
5191
|
toMemoryId: artifactEdges.toMemoryId,
|
|
@@ -5172,11 +5194,11 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5172
5194
|
}).from(artifactEdges).where(and8(or(inArray4(artifactEdges.fromMemoryId, ids), inArray4(artifactEdges.toMemoryId, ids)), ne4(artifactEdges.status, "rejected"), isNull2(artifactEdges.expiredAt))),
|
|
5173
5195
|
catch: storeError("Store.retrieval post-pass: edges")
|
|
5174
5196
|
});
|
|
5175
|
-
const fetchResultConcepts = (ids) =>
|
|
5197
|
+
const fetchResultConcepts = (ids) => Effect26.tryPromise({
|
|
5176
5198
|
try: () => db.select({ memoryId: memoryConcepts2.memoryId, conceptId: memoryConcepts2.conceptId }).from(memoryConcepts2).where(inArray4(memoryConcepts2.memoryId, ids)),
|
|
5177
5199
|
catch: storeError("Store.retrieval post-pass: concepts")
|
|
5178
5200
|
});
|
|
5179
|
-
const augmentRetrieval = (results) =>
|
|
5201
|
+
const augmentRetrieval = (results) => Effect26.gen(function* () {
|
|
5180
5202
|
if (results.length === 0)
|
|
5181
5203
|
return results;
|
|
5182
5204
|
const ids = results.map((r) => r.memory.id);
|
|
@@ -5186,7 +5208,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5186
5208
|
supersedeDemotion: GRAPH_SUPERSEDE_DEMOTION
|
|
5187
5209
|
});
|
|
5188
5210
|
});
|
|
5189
|
-
const augmentSearchResults =
|
|
5211
|
+
const augmentSearchResults = Effect26.fn("Store.augmentSearchResults")(function* (results) {
|
|
5190
5212
|
return yield* augmentRetrieval(results);
|
|
5191
5213
|
});
|
|
5192
5214
|
const EDGE_KINDS = [
|
|
@@ -5196,7 +5218,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5196
5218
|
"code_calls"
|
|
5197
5219
|
];
|
|
5198
5220
|
const EDGE_SOURCES = ["rule", "llm", "agent"];
|
|
5199
|
-
const write =
|
|
5221
|
+
const write = Effect26.fn("Store.write")(function* (input) {
|
|
5200
5222
|
if (input.content.includes("\x00")) {
|
|
5201
5223
|
return yield* new StoreError({
|
|
5202
5224
|
message: "Store.write: content contains a NUL byte (libSQL would silently truncate it)"
|
|
@@ -5205,7 +5227,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5205
5227
|
const id = crypto.randomUUID();
|
|
5206
5228
|
const now = Date.now();
|
|
5207
5229
|
const metadataBlob = input.typedMetadata !== undefined ? mergeMetadataBlob(input.metadata, input.typedMetadata) : input.metadata ?? null;
|
|
5208
|
-
yield*
|
|
5230
|
+
yield* Effect26.tryPromise({
|
|
5209
5231
|
try: () => db.transaction(async (tx) => {
|
|
5210
5232
|
await tx.insert(memories2).values({
|
|
5211
5233
|
id,
|
|
@@ -5238,7 +5260,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5238
5260
|
embeddingModel: undefined
|
|
5239
5261
|
};
|
|
5240
5262
|
});
|
|
5241
|
-
const writeMany =
|
|
5263
|
+
const writeMany = Effect26.fn("Store.writeMany")(function* (inputs) {
|
|
5242
5264
|
if (inputs.length === 0)
|
|
5243
5265
|
return [];
|
|
5244
5266
|
for (const input of inputs) {
|
|
@@ -5262,7 +5284,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5262
5284
|
metadata: metadataBlob
|
|
5263
5285
|
};
|
|
5264
5286
|
});
|
|
5265
|
-
yield*
|
|
5287
|
+
yield* Effect26.tryPromise({
|
|
5266
5288
|
try: () => db.transaction(async (tx) => {
|
|
5267
5289
|
await tx.insert(memories2).values(rows);
|
|
5268
5290
|
await tx.insert(memoryEvents2).values(rows.map((row) => ({
|
|
@@ -5290,8 +5312,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5290
5312
|
embeddingModel: undefined
|
|
5291
5313
|
}));
|
|
5292
5314
|
});
|
|
5293
|
-
const get =
|
|
5294
|
-
const rows = yield*
|
|
5315
|
+
const get = Effect26.fn("Store.get")(function* (id) {
|
|
5316
|
+
const rows = yield* Effect26.tryPromise({
|
|
5295
5317
|
try: () => db.select(memoryFields).from(memories2).where(eq8(memories2.id, id)).limit(1),
|
|
5296
5318
|
catch: storeError("Store.get")
|
|
5297
5319
|
});
|
|
@@ -5300,8 +5322,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5300
5322
|
return yield* new NotFound({ id });
|
|
5301
5323
|
return fromMemoryRow(row);
|
|
5302
5324
|
});
|
|
5303
|
-
const getByKey =
|
|
5304
|
-
const rows = yield*
|
|
5325
|
+
const getByKey = Effect26.fn("Store.getByKey")(function* (key, namespace) {
|
|
5326
|
+
const rows = yield* Effect26.tryPromise({
|
|
5305
5327
|
try: () => db.select(memoryFields).from(memories2).where(and8(eq8(memories2.logicalKey, key), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).orderBy(desc2(memories2.createdAt)).limit(1),
|
|
5306
5328
|
catch: storeError("Store.getByKey")
|
|
5307
5329
|
});
|
|
@@ -5310,9 +5332,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5310
5332
|
return yield* new NotFound({ id: key });
|
|
5311
5333
|
return fromMemoryRow(row);
|
|
5312
5334
|
});
|
|
5313
|
-
const recency =
|
|
5335
|
+
const recency = Effect26.fn("Store.recency")(function* (namespace, limit) {
|
|
5314
5336
|
const start = Date.now();
|
|
5315
|
-
const rows = yield*
|
|
5337
|
+
const rows = yield* Effect26.tryPromise({
|
|
5316
5338
|
try: () => db.select(memoryFields).from(memories2).where(and8(eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).orderBy(desc2(memories2.createdAt)).limit(limit),
|
|
5317
5339
|
catch: storeError("Store.recency")
|
|
5318
5340
|
});
|
|
@@ -5325,7 +5347,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5325
5347
|
yield* logRetrieval(namespace, namespace, "recency", results.length, start);
|
|
5326
5348
|
return yield* augmentRetrieval(results);
|
|
5327
5349
|
});
|
|
5328
|
-
const searchFts5 =
|
|
5350
|
+
const searchFts5 = Effect26.fn("Store.searchFts5")(function* (query, namespace, limit, opts) {
|
|
5329
5351
|
const start = Date.now();
|
|
5330
5352
|
const rows = yield* backend.ftsSearch(query, namespace, limit, opts?.topicKeys, modelConfig?.id, opts?.contentType, opts?.includeEmbedding === true);
|
|
5331
5353
|
if (opts?.audit !== false) {
|
|
@@ -5356,7 +5378,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5356
5378
|
return results;
|
|
5357
5379
|
return yield* augmentRetrieval(results);
|
|
5358
5380
|
});
|
|
5359
|
-
const searchVector =
|
|
5381
|
+
const searchVector = Effect26.fn("Store.searchVector")(function* (query, namespace, limit, opts) {
|
|
5360
5382
|
if (!modelConfig) {
|
|
5361
5383
|
return yield* new StoreError({
|
|
5362
5384
|
message: "Store.searchVector: no embedding model configured"
|
|
@@ -5366,7 +5388,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5366
5388
|
let allowedMemoryIds;
|
|
5367
5389
|
const conceptIds = opts?.conceptIds;
|
|
5368
5390
|
if (conceptIds !== undefined && conceptIds.length > 0) {
|
|
5369
|
-
const conceptRows = yield*
|
|
5391
|
+
const conceptRows = yield* Effect26.tryPromise({
|
|
5370
5392
|
try: () => db.select({ memoryId: memoryConcepts2.memoryId }).from(memoryConcepts2).where(inArray4(memoryConcepts2.conceptId, conceptIds)),
|
|
5371
5393
|
catch: storeError("Store.searchVector: concepts")
|
|
5372
5394
|
});
|
|
@@ -5403,9 +5425,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5403
5425
|
return results;
|
|
5404
5426
|
return yield* augmentRetrieval(results);
|
|
5405
5427
|
});
|
|
5406
|
-
const supersede =
|
|
5428
|
+
const supersede = Effect26.fn("Store.supersede")(function* (key, namespace) {
|
|
5407
5429
|
const now = Date.now();
|
|
5408
|
-
const result = yield*
|
|
5430
|
+
const result = yield* Effect26.tryPromise({
|
|
5409
5431
|
try: () => db.transaction(async (tx) => {
|
|
5410
5432
|
const active = await tx.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.logicalKey, key), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).limit(1);
|
|
5411
5433
|
if (active.length === 0)
|
|
@@ -5417,7 +5439,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5417
5439
|
if (!result)
|
|
5418
5440
|
return yield* new NotFound({ id: key });
|
|
5419
5441
|
});
|
|
5420
|
-
const decrementTreeCf = (model, membership, x) =>
|
|
5442
|
+
const decrementTreeCf = (model, membership, x) => Effect26.tryPromise({
|
|
5421
5443
|
try: () => db.transaction(async (tx) => {
|
|
5422
5444
|
const leafRows = await tx.select({
|
|
5423
5445
|
topicKey: topics.topicKey,
|
|
@@ -5447,10 +5469,10 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5447
5469
|
}),
|
|
5448
5470
|
catch: storeError("Store: tree CF decrement")
|
|
5449
5471
|
});
|
|
5450
|
-
const forget =
|
|
5472
|
+
const forget = Effect26.fn("Store.forget")(function* (id) {
|
|
5451
5473
|
let membership;
|
|
5452
5474
|
if (topicsEnabled) {
|
|
5453
|
-
const memberRows = yield*
|
|
5475
|
+
const memberRows = yield* Effect26.tryPromise({
|
|
5454
5476
|
try: () => db.select({ topicKey: topicMemberships.topicKey, modelKey: topicMemberships.modelKey }).from(topicMemberships).where(eq8(topicMemberships.memoryId, id)),
|
|
5455
5477
|
catch: storeError("Store.forget: load membership")
|
|
5456
5478
|
});
|
|
@@ -5465,14 +5487,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5465
5487
|
}
|
|
5466
5488
|
}
|
|
5467
5489
|
}
|
|
5468
|
-
const rs = yield*
|
|
5490
|
+
const rs = yield* Effect26.tryPromise({
|
|
5469
5491
|
try: () => db.delete(memories2).where(eq8(memories2.id, id)),
|
|
5470
5492
|
catch: storeError("Store.forget")
|
|
5471
5493
|
});
|
|
5472
5494
|
if (rs.rowsAffected === 0)
|
|
5473
5495
|
return yield* new NotFound({ id });
|
|
5474
5496
|
if (topicsEnabled && membership && modelConfig && modelConfig.id === membership.modelKey) {
|
|
5475
|
-
const leafRows = yield*
|
|
5497
|
+
const leafRows = yield* Effect26.tryPromise({
|
|
5476
5498
|
try: () => db.select({ memberCount: topics.memberCount }).from(topics).where(and8(eq8(topics.topicKey, membership.topicKey), eq8(topics.modelKey, membership.modelKey))),
|
|
5477
5499
|
catch: storeError("Store.forget: load leaf")
|
|
5478
5500
|
});
|
|
@@ -5480,18 +5502,18 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5480
5502
|
if (liveCount < (topicsConfig.minLeafItems ?? 32)) {
|
|
5481
5503
|
const enqueueMerge = topicsOptions?.enqueueMerge;
|
|
5482
5504
|
if (enqueueMerge) {
|
|
5483
|
-
yield* enqueueMerge(membership.topicKey, membership.modelKey).pipe(
|
|
5505
|
+
yield* enqueueMerge(membership.topicKey, membership.modelKey).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget enqueueMerge: ${cause}` })));
|
|
5484
5506
|
} else {
|
|
5485
|
-
yield* mergeOrBorrow(membership.topicKey, modelConfig, topicsConfig).pipe(
|
|
5507
|
+
yield* mergeOrBorrow(membership.topicKey, modelConfig, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.forget mergeOrBorrow: ${cause}` })));
|
|
5486
5508
|
}
|
|
5487
5509
|
}
|
|
5488
5510
|
}
|
|
5489
5511
|
});
|
|
5490
|
-
const sweep =
|
|
5512
|
+
const sweep = Effect26.fn("Store.sweep")(function* (namespace) {
|
|
5491
5513
|
return yield* backend.sweep(namespace);
|
|
5492
5514
|
});
|
|
5493
|
-
const events =
|
|
5494
|
-
const rows = yield*
|
|
5515
|
+
const events = Effect26.fn("Store.events")(function* (memoryId) {
|
|
5516
|
+
const rows = yield* Effect26.tryPromise({
|
|
5495
5517
|
try: () => db.select().from(memoryEvents2).where(eq8(memoryEvents2.memoryId, memoryId)).orderBy(asc5(memoryEvents2.createdAt)),
|
|
5496
5518
|
catch: storeError("Store.events")
|
|
5497
5519
|
});
|
|
@@ -5508,7 +5530,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5508
5530
|
});
|
|
5509
5531
|
});
|
|
5510
5532
|
const resolveModel = (modelKey) => modelConfig?.id === modelKey ? modelConfig : undefined;
|
|
5511
|
-
const ensureTopicTree2 =
|
|
5533
|
+
const ensureTopicTree2 = Effect26.fn("Store.ensureTopicTree")(function* (modelKey) {
|
|
5512
5534
|
const model = resolveModel(modelKey);
|
|
5513
5535
|
if (!model) {
|
|
5514
5536
|
return yield* new ModelMismatch({
|
|
@@ -5516,33 +5538,33 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5516
5538
|
actual: modelKey
|
|
5517
5539
|
});
|
|
5518
5540
|
}
|
|
5519
|
-
return yield* ensureTopicTree(model, topicsConfig).pipe(
|
|
5541
|
+
return yield* ensureTopicTree(model, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5520
5542
|
});
|
|
5521
|
-
const resolveTopics2 =
|
|
5543
|
+
const resolveTopics2 = Effect26.fn("Store.resolveTopics")(function* (queryVector, opts) {
|
|
5522
5544
|
if (!modelConfig) {
|
|
5523
5545
|
return yield* new ModelMismatch({
|
|
5524
5546
|
expected: "a configured embedding model",
|
|
5525
5547
|
actual: "(none)"
|
|
5526
5548
|
});
|
|
5527
5549
|
}
|
|
5528
|
-
return yield* resolveTopics(queryVector, modelConfig, topicsConfig, opts).pipe(
|
|
5550
|
+
return yield* resolveTopics(queryVector, modelConfig, topicsConfig, opts).pipe(Effect26.provideService(Service2, backend));
|
|
5529
5551
|
});
|
|
5530
|
-
const topicMembers =
|
|
5531
|
-
const rows = yield*
|
|
5552
|
+
const topicMembers = Effect26.fn("Store.topicMembers")(function* (topicKey) {
|
|
5553
|
+
const rows = yield* Effect26.tryPromise({
|
|
5532
5554
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(eq8(topicMemberships.topicKey, topicKey)).orderBy(asc5(topicMemberships.memoryId)),
|
|
5533
5555
|
catch: storeError("Store.topicMembers")
|
|
5534
5556
|
});
|
|
5535
5557
|
return rows;
|
|
5536
5558
|
});
|
|
5537
|
-
const listTopics =
|
|
5538
|
-
const rows = yield*
|
|
5559
|
+
const listTopics = Effect26.fn("Store.listTopics")(function* (modelKey) {
|
|
5560
|
+
const rows = yield* Effect26.tryPromise({
|
|
5539
5561
|
try: () => db.select().from(topics).where(eq8(topics.modelKey, modelKey)).orderBy(asc5(topics.topicKey)),
|
|
5540
5562
|
catch: storeError("Store.listTopics")
|
|
5541
5563
|
});
|
|
5542
5564
|
return rows;
|
|
5543
5565
|
});
|
|
5544
|
-
const topicStats =
|
|
5545
|
-
const rows = yield*
|
|
5566
|
+
const topicStats = Effect26.fn("Store.topicStats")(function* (modelKey) {
|
|
5567
|
+
const rows = yield* Effect26.tryPromise({
|
|
5546
5568
|
try: () => db.select({ kind: topics.kind, depth: topics.depth }).from(topics).where(eq8(topics.modelKey, modelKey)),
|
|
5547
5569
|
catch: storeError("Store.topicStats")
|
|
5548
5570
|
});
|
|
@@ -5559,7 +5581,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5559
5581
|
}
|
|
5560
5582
|
return { depth, leafCount, internalCount, totalTopics: rows.length };
|
|
5561
5583
|
});
|
|
5562
|
-
const deleteTree =
|
|
5584
|
+
const deleteTree = Effect26.fn("Store.deleteTree")(function* (modelKey) {
|
|
5563
5585
|
const model = resolveModel(modelKey);
|
|
5564
5586
|
if (!model) {
|
|
5565
5587
|
return yield* new ModelMismatch({
|
|
@@ -5568,7 +5590,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5568
5590
|
});
|
|
5569
5591
|
}
|
|
5570
5592
|
yield* backend.deleteTopicVectors(model);
|
|
5571
|
-
yield*
|
|
5593
|
+
yield* Effect26.tryPromise({
|
|
5572
5594
|
try: () => db.transaction(async (tx) => {
|
|
5573
5595
|
const modelTopicKeys2 = tx.select({ topicKey: topics.topicKey }).from(topics).where(eq8(topics.modelKey, modelKey));
|
|
5574
5596
|
await tx.delete(topicChangeLog).where(inArray4(topicChangeLog.topicKey, modelTopicKeys2));
|
|
@@ -5580,7 +5602,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5580
5602
|
catch: storeError("Store.deleteTree")
|
|
5581
5603
|
});
|
|
5582
5604
|
});
|
|
5583
|
-
const processSplits =
|
|
5605
|
+
const processSplits = Effect26.fn("Store.processSplits")(function* (modelKey) {
|
|
5584
5606
|
const model = resolveModel(modelKey);
|
|
5585
5607
|
if (!model) {
|
|
5586
5608
|
return yield* new ModelMismatch({
|
|
@@ -5590,19 +5612,19 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5590
5612
|
}
|
|
5591
5613
|
let processed = 0;
|
|
5592
5614
|
for (let iteration = 0;iteration < 100; iteration++) {
|
|
5593
|
-
const leaves = yield*
|
|
5615
|
+
const leaves = yield* Effect26.tryPromise({
|
|
5594
5616
|
try: () => db.select({ topicKey: topics.topicKey, memberCount: topics.memberCount }).from(topics).where(and8(eq8(topics.modelKey, modelKey), eq8(topics.kind, "leaf"), ne4(topics.status, "inactive"))).orderBy(asc5(topics.topicKey)),
|
|
5595
5617
|
catch: storeError("Store.processSplits")
|
|
5596
5618
|
});
|
|
5597
5619
|
const candidate = leaves.find((leaf) => shouldSplit(leaf.memberCount, topicsConfig));
|
|
5598
5620
|
if (!candidate)
|
|
5599
5621
|
break;
|
|
5600
|
-
yield* splitTopic(candidate.topicKey, model, topicsConfig).pipe(
|
|
5622
|
+
yield* splitTopic(candidate.topicKey, model, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.processSplits: ${cause}` })));
|
|
5601
5623
|
processed++;
|
|
5602
5624
|
}
|
|
5603
5625
|
return processed;
|
|
5604
5626
|
});
|
|
5605
|
-
const mergeOrBorrow2 =
|
|
5627
|
+
const mergeOrBorrow2 = Effect26.fn("Store.mergeOrBorrow")(function* (modelKey, topicKey) {
|
|
5606
5628
|
const model = resolveModel(modelKey);
|
|
5607
5629
|
if (!model) {
|
|
5608
5630
|
return yield* new ModelMismatch({
|
|
@@ -5610,9 +5632,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5610
5632
|
actual: modelKey
|
|
5611
5633
|
});
|
|
5612
5634
|
}
|
|
5613
|
-
return yield* mergeOrBorrow(topicKey, model, topicsConfig).pipe(
|
|
5635
|
+
return yield* mergeOrBorrow(topicKey, model, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.mergeOrBorrow: ${cause}` })));
|
|
5614
5636
|
});
|
|
5615
|
-
const rebalanceSiblings2 =
|
|
5637
|
+
const rebalanceSiblings2 = Effect26.fn("Store.rebalanceSiblings")(function* (modelKey, topicKey) {
|
|
5616
5638
|
const model = resolveModel(modelKey);
|
|
5617
5639
|
if (!model) {
|
|
5618
5640
|
return yield* new ModelMismatch({
|
|
@@ -5620,15 +5642,15 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5620
5642
|
actual: modelKey
|
|
5621
5643
|
});
|
|
5622
5644
|
}
|
|
5623
|
-
return yield* rebalanceSiblings(topicKey, model, topicsConfig).pipe(
|
|
5645
|
+
return yield* rebalanceSiblings(topicKey, model, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.rebalanceSiblings: ${cause}` })));
|
|
5624
5646
|
});
|
|
5625
|
-
const setActiveModel2 =
|
|
5626
|
-
return yield* setActiveModel(model, topicsConfig).pipe(
|
|
5647
|
+
const setActiveModel2 = Effect26.fn("Store.setActiveModel")(function* (model) {
|
|
5648
|
+
return yield* setActiveModel(model, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5627
5649
|
});
|
|
5628
|
-
const isActive2 =
|
|
5629
|
-
return yield* isActive(modelKey).pipe(
|
|
5650
|
+
const isActive2 = Effect26.fn("Store.isActive")(function* (modelKey) {
|
|
5651
|
+
return yield* isActive(modelKey).pipe(Effect26.provideService(Service2, backend));
|
|
5630
5652
|
});
|
|
5631
|
-
const topicMetrics2 =
|
|
5653
|
+
const topicMetrics2 = Effect26.fn("Store.topicMetrics")(function* (modelKey) {
|
|
5632
5654
|
const model = resolveModel(modelKey);
|
|
5633
5655
|
if (!model) {
|
|
5634
5656
|
return yield* new ModelMismatch({
|
|
@@ -5636,9 +5658,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5636
5658
|
actual: modelKey
|
|
5637
5659
|
});
|
|
5638
5660
|
}
|
|
5639
|
-
return yield* topicMetrics(modelKey, topicsConfig).pipe(
|
|
5661
|
+
return yield* topicMetrics(modelKey, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5640
5662
|
});
|
|
5641
|
-
const topicChangeLog3 =
|
|
5663
|
+
const topicChangeLog3 = Effect26.fn("Store.topicChangeLog")(function* (modelKey, opts) {
|
|
5642
5664
|
const model = resolveModel(modelKey);
|
|
5643
5665
|
if (!model) {
|
|
5644
5666
|
return yield* new ModelMismatch({
|
|
@@ -5646,9 +5668,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5646
5668
|
actual: modelKey
|
|
5647
5669
|
});
|
|
5648
5670
|
}
|
|
5649
|
-
return yield* topicChangeLog2(modelKey, opts).pipe(
|
|
5671
|
+
return yield* topicChangeLog2(modelKey, opts).pipe(Effect26.provideService(Service2, backend));
|
|
5650
5672
|
});
|
|
5651
|
-
const topicHealth2 =
|
|
5673
|
+
const topicHealth2 = Effect26.fn("Store.topicHealth")(function* (modelKey) {
|
|
5652
5674
|
const model = resolveModel(modelKey);
|
|
5653
5675
|
if (!model) {
|
|
5654
5676
|
return yield* new ModelMismatch({
|
|
@@ -5656,11 +5678,11 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5656
5678
|
actual: modelKey
|
|
5657
5679
|
});
|
|
5658
5680
|
}
|
|
5659
|
-
return yield* topicHealth(modelKey, topicsConfig).pipe(
|
|
5681
|
+
return yield* topicHealth(modelKey, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5660
5682
|
});
|
|
5661
|
-
const updateEmbedding =
|
|
5683
|
+
const updateEmbedding = Effect26.fn("Store.updateEmbedding")(function* (id, embedding, modelName, options) {
|
|
5662
5684
|
const now = Date.now();
|
|
5663
|
-
const affected = yield*
|
|
5685
|
+
const affected = yield* Effect26.tryPromise({
|
|
5664
5686
|
try: () => db.transaction(async (tx) => {
|
|
5665
5687
|
const rs = await tx.update(memories2).set({ embedding: [...embedding], embeddingModel: modelName }).where(eq8(memories2.id, id));
|
|
5666
5688
|
if (rs.rowsAffected === 0)
|
|
@@ -5690,7 +5712,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5690
5712
|
if (modelConfig && modelConfig.id === modelName) {
|
|
5691
5713
|
yield* backend.ensureVectorTable(modelConfig);
|
|
5692
5714
|
if (topicsEnabled) {
|
|
5693
|
-
const memberRows = yield*
|
|
5715
|
+
const memberRows = yield* Effect26.tryPromise({
|
|
5694
5716
|
try: () => db.select({
|
|
5695
5717
|
topicKey: topicMemberships.topicKey,
|
|
5696
5718
|
modelKey: topicMemberships.modelKey
|
|
@@ -5705,7 +5727,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5705
5727
|
yield* decrementTreeCf(modelConfig, mem, x);
|
|
5706
5728
|
yield* updateTopicVectorForget(backend, modelConfig, mem.topicKey, x);
|
|
5707
5729
|
}
|
|
5708
|
-
yield*
|
|
5730
|
+
yield* Effect26.tryPromise({
|
|
5709
5731
|
try: () => db.delete(topicMemberships).where(and8(eq8(topicMemberships.memoryId, id), eq8(topicMemberships.modelKey, modelName))),
|
|
5710
5732
|
catch: storeError("Store.updateEmbedding: delete membership")
|
|
5711
5733
|
});
|
|
@@ -5713,26 +5735,26 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5713
5735
|
}
|
|
5714
5736
|
yield* backend.insertVector(id, embedding, modelConfig);
|
|
5715
5737
|
if (topicsEnabled) {
|
|
5716
|
-
const routed = yield* routeInsert(id, embedding, modelConfig, topicsConfig).pipe(
|
|
5738
|
+
const routed = yield* routeInsert(id, embedding, modelConfig, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding topic route: ${cause}` })));
|
|
5717
5739
|
if (options?.scanCandidates !== false && graphEnabled) {
|
|
5718
5740
|
yield* scanLeafCandidates(id, embedding, routed.topicKey, modelConfig);
|
|
5719
5741
|
}
|
|
5720
5742
|
if (routed.overflow) {
|
|
5721
5743
|
const enqueueSplit = topicsOptions?.enqueueSplit;
|
|
5722
5744
|
if (enqueueSplit) {
|
|
5723
|
-
yield* enqueueSplit(routed.topicKey, modelConfig.id).pipe(
|
|
5745
|
+
yield* enqueueSplit(routed.topicKey, modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding enqueueSplit: ${cause}` })));
|
|
5724
5746
|
} else {
|
|
5725
|
-
yield* processSplits(modelConfig.id).pipe(
|
|
5747
|
+
yield* processSplits(modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding processSplits: ${cause}` })));
|
|
5726
5748
|
}
|
|
5727
5749
|
}
|
|
5728
5750
|
}
|
|
5729
5751
|
}
|
|
5730
5752
|
});
|
|
5731
|
-
const updateEmbeddingMany =
|
|
5753
|
+
const updateEmbeddingMany = Effect26.fn("Store.updateEmbeddingMany")(function* (entries, modelName, options) {
|
|
5732
5754
|
if (entries.length === 0)
|
|
5733
5755
|
return;
|
|
5734
5756
|
const now = Date.now();
|
|
5735
|
-
yield*
|
|
5757
|
+
yield* Effect26.tryPromise({
|
|
5736
5758
|
try: () => db.transaction(async (tx) => {
|
|
5737
5759
|
await tx.run(sql8`UPDATE ${memories2} SET embedding = CASE id ${sql8.join(entries.map((entry) => sql8`WHEN ${entry.id} THEN ${new Float32Array(entry.embedding).buffer}`), sql8` `)} END, embedding_model = ${modelName} WHERE id IN (${sql8.join(entries.map((entry) => entry.id), sql8`, `)})`);
|
|
5738
5760
|
await tx.insert(memoryEvents2).values(entries.map((entry) => ({
|
|
@@ -5758,7 +5780,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5758
5780
|
yield* backend.ensureVectorTable(modelConfig);
|
|
5759
5781
|
yield* backend.insertVectors(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig);
|
|
5760
5782
|
if (topicsEnabled) {
|
|
5761
|
-
const routed = yield* routeInsertMany(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig, topicsConfig).pipe(
|
|
5783
|
+
const routed = yield* routeInsertMany(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany route: ${cause}` })));
|
|
5762
5784
|
if (options?.scanCandidates === true && graphEnabled) {
|
|
5763
5785
|
for (const routedEntry of routed) {
|
|
5764
5786
|
const entry = entries.find((e) => e.id === routedEntry.memoryId);
|
|
@@ -5772,17 +5794,17 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5772
5794
|
const enqueueSplit = topicsOptions?.enqueueSplit;
|
|
5773
5795
|
if (enqueueSplit) {
|
|
5774
5796
|
for (const topicKey of overflowed) {
|
|
5775
|
-
yield* enqueueSplit(topicKey, modelConfig.id).pipe(
|
|
5797
|
+
yield* enqueueSplit(topicKey, modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany enqueueSplit: ${cause}` })));
|
|
5776
5798
|
}
|
|
5777
5799
|
} else {
|
|
5778
|
-
yield* processSplits(modelConfig.id).pipe(
|
|
5800
|
+
yield* processSplits(modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany processSplits: ${cause}` })));
|
|
5779
5801
|
}
|
|
5780
5802
|
}
|
|
5781
5803
|
}
|
|
5782
5804
|
}
|
|
5783
5805
|
});
|
|
5784
|
-
const findStale =
|
|
5785
|
-
const rows = yield*
|
|
5806
|
+
const findStale = Effect26.fn("Store.findStale")(function* (model) {
|
|
5807
|
+
const rows = yield* Effect26.tryPromise({
|
|
5786
5808
|
try: () => db.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.state, "active"), sql8`(embedding IS NULL OR embedding_model IS NULL OR embedding_model != ${model})`)),
|
|
5787
5809
|
catch: storeError("Store.findStale")
|
|
5788
5810
|
});
|
|
@@ -5802,14 +5824,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5802
5824
|
});
|
|
5803
5825
|
return memoryId;
|
|
5804
5826
|
};
|
|
5805
|
-
const reconcile =
|
|
5827
|
+
const reconcile = Effect26.fn("Store.reconcile")(function* (namespace, keyPrefix, keyRoot, walkedRoots, keptKeys) {
|
|
5806
5828
|
const inDomain = (key) => {
|
|
5807
5829
|
const rel = key.slice(keyPrefix.length).split("#")[0];
|
|
5808
5830
|
const filePath = keyRoot === "/" ? `/${rel}` : `${keyRoot}/${rel}`;
|
|
5809
5831
|
return walkedRoots.some((root) => root === "/" ? filePath.startsWith("/") : filePath === root || filePath.startsWith(`${root}/`));
|
|
5810
5832
|
};
|
|
5811
5833
|
const owned = (key) => keptKeys.includes(key);
|
|
5812
|
-
const rows = yield*
|
|
5834
|
+
const rows = yield* Effect26.tryPromise({
|
|
5813
5835
|
try: () => db.select({
|
|
5814
5836
|
id: memories2.id,
|
|
5815
5837
|
logicalKey: memories2.logicalKey,
|
|
@@ -5820,7 +5842,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5820
5842
|
const vanished = rows.filter((r) => inDomain(r.logicalKey) && !owned(r.logicalKey));
|
|
5821
5843
|
if (vanished.length > 0) {
|
|
5822
5844
|
const now = Date.now();
|
|
5823
|
-
yield*
|
|
5845
|
+
yield* Effect26.tryPromise({
|
|
5824
5846
|
try: () => db.transaction(async (tx) => {
|
|
5825
5847
|
for (const row of vanished) {
|
|
5826
5848
|
await flipToSupersededById(tx, row.id, now, "source_removed");
|
|
@@ -5831,7 +5853,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5831
5853
|
}
|
|
5832
5854
|
return vanished;
|
|
5833
5855
|
});
|
|
5834
|
-
const writeSuperseding =
|
|
5856
|
+
const writeSuperseding = Effect26.fn("Store.writeSuperseding")(function* (input) {
|
|
5835
5857
|
if (input.content.includes("\x00")) {
|
|
5836
5858
|
return yield* new StoreError({
|
|
5837
5859
|
message: "Store.writeSuperseding: content contains a NUL byte (libSQL would silently truncate it)"
|
|
@@ -5842,7 +5864,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5842
5864
|
const metadataBlob = input.typedMetadata !== undefined ? mergeMetadataBlob(input.metadata, input.typedMetadata) : input.metadata ?? null;
|
|
5843
5865
|
const namespace = input.namespace ?? "default";
|
|
5844
5866
|
let supersededId;
|
|
5845
|
-
yield*
|
|
5867
|
+
yield* Effect26.tryPromise({
|
|
5846
5868
|
try: () => db.transaction(async (tx) => {
|
|
5847
5869
|
const existing = await tx.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.logicalKey, input.logicalKey), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).limit(1);
|
|
5848
5870
|
const old = existing[0];
|
|
@@ -5897,7 +5919,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5897
5919
|
...supersededId !== undefined ? { supersededId } : undefined
|
|
5898
5920
|
};
|
|
5899
5921
|
});
|
|
5900
|
-
const addEdge =
|
|
5922
|
+
const addEdge = Effect26.fn("Store.addEdge")(function* (input) {
|
|
5901
5923
|
if (input.fromMemoryId === input.toMemoryId) {
|
|
5902
5924
|
return yield* new StoreError({ message: "Store.addEdge: self-loops are not allowed" });
|
|
5903
5925
|
}
|
|
@@ -5923,7 +5945,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5923
5945
|
message: `Store.addEdge: confidence must be a finite number in [0,1] (got ${String(confidence)})`
|
|
5924
5946
|
});
|
|
5925
5947
|
}
|
|
5926
|
-
const endpoints = yield*
|
|
5948
|
+
const endpoints = yield* Effect26.tryPromise({
|
|
5927
5949
|
try: () => db.select({ id: memories2.id }).from(memories2).where(inArray4(memories2.id, [input.fromMemoryId, input.toMemoryId])),
|
|
5928
5950
|
catch: storeError("Store.addEdge: endpoints")
|
|
5929
5951
|
});
|
|
@@ -5948,7 +5970,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5948
5970
|
createdAt: now,
|
|
5949
5971
|
updatedAt: now
|
|
5950
5972
|
};
|
|
5951
|
-
yield*
|
|
5973
|
+
yield* Effect26.tryPromise({
|
|
5952
5974
|
try: () => db.transaction(async (tx) => {
|
|
5953
5975
|
await tx.insert(artifactEdges).values(row);
|
|
5954
5976
|
if (input.confirmed && input.edgeKind === "memory_supersedes") {
|
|
@@ -5968,17 +5990,17 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5968
5990
|
});
|
|
5969
5991
|
return row;
|
|
5970
5992
|
});
|
|
5971
|
-
const expireEdge =
|
|
5972
|
-
const rs = yield*
|
|
5993
|
+
const expireEdge = Effect26.fn("Store.expireEdge")(function* (id) {
|
|
5994
|
+
const rs = yield* Effect26.tryPromise({
|
|
5973
5995
|
try: () => db.update(artifactEdges).set({ expiredAt: Date.now(), updatedAt: Date.now() }).where(eq8(artifactEdges.id, id)),
|
|
5974
5996
|
catch: storeError("Store.expireEdge")
|
|
5975
5997
|
});
|
|
5976
5998
|
if (rs.rowsAffected === 0)
|
|
5977
5999
|
return yield* new EdgeNotFound({ id });
|
|
5978
6000
|
});
|
|
5979
|
-
const confirmEdge =
|
|
6001
|
+
const confirmEdge = Effect26.fn("Store.confirmEdge")(function* (id) {
|
|
5980
6002
|
const now = Date.now();
|
|
5981
|
-
const found = yield*
|
|
6003
|
+
const found = yield* Effect26.tryPromise({
|
|
5982
6004
|
try: () => db.transaction(async (tx) => {
|
|
5983
6005
|
const rows = await tx.select().from(artifactEdges).where(eq8(artifactEdges.id, id)).limit(1);
|
|
5984
6006
|
const edge = rows[0];
|
|
@@ -5995,15 +6017,15 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5995
6017
|
if (!found)
|
|
5996
6018
|
return yield* new EdgeNotFound({ id });
|
|
5997
6019
|
});
|
|
5998
|
-
const rejectEdge =
|
|
5999
|
-
const rs = yield*
|
|
6020
|
+
const rejectEdge = Effect26.fn("Store.rejectEdge")(function* (id) {
|
|
6021
|
+
const rs = yield* Effect26.tryPromise({
|
|
6000
6022
|
try: () => db.update(artifactEdges).set({ status: "rejected", updatedAt: Date.now() }).where(eq8(artifactEdges.id, id)),
|
|
6001
6023
|
catch: storeError("Store.rejectEdge")
|
|
6002
6024
|
});
|
|
6003
6025
|
if (rs.rowsAffected === 0)
|
|
6004
6026
|
return yield* new EdgeNotFound({ id });
|
|
6005
6027
|
});
|
|
6006
|
-
const listEdges =
|
|
6028
|
+
const listEdges = Effect26.fn("Store.listEdges")(function* (opts) {
|
|
6007
6029
|
const conditions = [];
|
|
6008
6030
|
if (!opts?.includeExpired)
|
|
6009
6031
|
conditions.push(isNull2(artifactEdges.expiredAt));
|
|
@@ -6014,7 +6036,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6014
6036
|
conditions.push(eq8(artifactEdges.edgeKind, opts.edgeKind));
|
|
6015
6037
|
if (opts?.status)
|
|
6016
6038
|
conditions.push(eq8(artifactEdges.status, opts.status));
|
|
6017
|
-
const rows = yield*
|
|
6039
|
+
const rows = yield* Effect26.tryPromise({
|
|
6018
6040
|
try: () => {
|
|
6019
6041
|
const base = db.select().from(artifactEdges);
|
|
6020
6042
|
return (conditions.length === 0 ? base : base.where(and8(...conditions))).orderBy(desc2(artifactEdges.createdAt));
|
|
@@ -6023,7 +6045,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6023
6045
|
});
|
|
6024
6046
|
return rows;
|
|
6025
6047
|
});
|
|
6026
|
-
const explain =
|
|
6048
|
+
const explain = Effect26.fn("Store.explain")(function* (node5, opts) {
|
|
6027
6049
|
if (node5.kind !== "memory" && node5.kind !== "topic" && node5.kind !== "concept") {
|
|
6028
6050
|
return yield* new StoreError({
|
|
6029
6051
|
message: `Store.explain: unknown kind "${String(node5.kind)}"`
|
|
@@ -6067,7 +6089,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6067
6089
|
pendingConflicts: pending
|
|
6068
6090
|
};
|
|
6069
6091
|
});
|
|
6070
|
-
const writeCandidateEdges = (candidates, now) =>
|
|
6092
|
+
const writeCandidateEdges = (candidates, now) => Effect26.tryPromise({
|
|
6071
6093
|
try: () => db.transaction(async (tx) => {
|
|
6072
6094
|
let inserted = 0;
|
|
6073
6095
|
for (const candidate of candidates) {
|
|
@@ -6097,11 +6119,11 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6097
6119
|
}),
|
|
6098
6120
|
catch: storeError("Store.writeCandidateEdges")
|
|
6099
6121
|
});
|
|
6100
|
-
const reconcileRenameCandidates =
|
|
6122
|
+
const reconcileRenameCandidates = Effect26.fn("Store.reconcileRenameCandidates")(function* (vanished, newUnitIds) {
|
|
6101
6123
|
const withVectors = vanished.filter((v) => v.embedding !== null);
|
|
6102
6124
|
if (withVectors.length === 0 || newUnitIds.length === 0)
|
|
6103
6125
|
return 0;
|
|
6104
|
-
const newRows = yield*
|
|
6126
|
+
const newRows = yield* Effect26.tryPromise({
|
|
6105
6127
|
try: () => db.select({ id: memories2.id, embedding: memories2.embedding }).from(memories2).where(and8(inArray4(memories2.id, newUnitIds))),
|
|
6106
6128
|
catch: storeError("Store.reconcileRenameCandidates")
|
|
6107
6129
|
});
|
|
@@ -6133,16 +6155,16 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6133
6155
|
inserted = yield* writeCandidateEdges(batch, now);
|
|
6134
6156
|
return inserted;
|
|
6135
6157
|
});
|
|
6136
|
-
const scanLeafCandidates = (memoryId, vector, topicKey, model) =>
|
|
6137
|
-
const memberRows = yield*
|
|
6158
|
+
const scanLeafCandidates = (memoryId, vector, topicKey, model) => Effect26.gen(function* () {
|
|
6159
|
+
const memberRows = yield* Effect26.tryPromise({
|
|
6138
6160
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and8(eq8(topicMemberships.topicKey, topicKey), eq8(topicMemberships.modelKey, model.id))).orderBy(asc5(topicMemberships.memoryId)),
|
|
6139
6161
|
catch: storeError("Store.updateEmbedding candidates: members")
|
|
6140
6162
|
});
|
|
6141
6163
|
const memberIds = memberRows.map((r) => r.memoryId);
|
|
6142
6164
|
if (memberIds.length === 0)
|
|
6143
6165
|
return;
|
|
6144
|
-
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(
|
|
6145
|
-
const memoryRows = yield*
|
|
6166
|
+
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding candidates: vectors: ${cause}` })));
|
|
6167
|
+
const memoryRows = yield* Effect26.tryPromise({
|
|
6146
6168
|
try: () => db.select({
|
|
6147
6169
|
id: memories2.id,
|
|
6148
6170
|
logicalKey: memories2.logicalKey,
|
|
@@ -6151,7 +6173,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6151
6173
|
}).from(memories2).where(and8(inArray4(memories2.id, memberIds), eq8(memories2.state, "active"))),
|
|
6152
6174
|
catch: storeError("Store.updateEmbedding candidates: memories")
|
|
6153
6175
|
});
|
|
6154
|
-
const selfRows = yield*
|
|
6176
|
+
const selfRows = yield* Effect26.tryPromise({
|
|
6155
6177
|
try: () => db.select({
|
|
6156
6178
|
logicalKey: memories2.logicalKey,
|
|
6157
6179
|
createdAt: memories2.createdAt,
|
|
@@ -6179,9 +6201,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6179
6201
|
const now = Date.now();
|
|
6180
6202
|
yield* writeCandidateEdges(candidates, now);
|
|
6181
6203
|
});
|
|
6182
|
-
const graphCandidates =
|
|
6204
|
+
const graphCandidates = Effect26.fn("Store.graphCandidates")(function* (opts) {
|
|
6183
6205
|
const limit = Math.max(1, Math.min(opts?.limit ?? 50, GRAPH_CANDIDATE_LIMIT_MAX));
|
|
6184
|
-
const rows = yield*
|
|
6206
|
+
const rows = yield* Effect26.tryPromise({
|
|
6185
6207
|
try: () => db.select({
|
|
6186
6208
|
id: artifactEdges.id,
|
|
6187
6209
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
@@ -6193,7 +6215,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6193
6215
|
}).from(artifactEdges).where(and8(eq8(artifactEdges.status, "pending"), isNull2(artifactEdges.expiredAt))).orderBy(desc2(artifactEdges.createdAt), asc5(artifactEdges.id)).limit(limit),
|
|
6194
6216
|
catch: storeError("Store.graphCandidates: pending")
|
|
6195
6217
|
});
|
|
6196
|
-
const isolatedRows = yield* backend.listIsolatedMemories(0).pipe(
|
|
6218
|
+
const isolatedRows = yield* backend.listIsolatedMemories(0).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.graphCandidates: isolated: ${cause}` })));
|
|
6197
6219
|
const isolatedRowsDecoded = Schema23.decodeUnknownSync(Schema23.Array(Schema23.Struct({ id: Schema23.String, createdAt: Schema23.Number })))(isolatedRows);
|
|
6198
6220
|
const isolated = isolatedRowsDecoded.map((row) => ({
|
|
6199
6221
|
id: row.id,
|
|
@@ -6213,13 +6235,13 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6213
6235
|
oldestIsolated: isolated.slice(0, GRAPH_ISOLATED_EXAMPLES)
|
|
6214
6236
|
};
|
|
6215
6237
|
});
|
|
6216
|
-
const graphCoverage2 =
|
|
6217
|
-
return yield* graphCoverage(modelKey).pipe(
|
|
6238
|
+
const graphCoverage2 = Effect26.fn("Store.graphCoverage")(function* (modelKey) {
|
|
6239
|
+
return yield* graphCoverage(modelKey).pipe(Effect26.provideService(Service2, backend));
|
|
6218
6240
|
});
|
|
6219
|
-
const createConcept =
|
|
6241
|
+
const createConcept = Effect26.fn("Store.createConcept")(function* (name, embedding) {
|
|
6220
6242
|
const id = crypto.randomUUID();
|
|
6221
6243
|
const now = Date.now();
|
|
6222
|
-
yield*
|
|
6244
|
+
yield* Effect26.tryPromise({
|
|
6223
6245
|
try: () => db.insert(concepts2).values({
|
|
6224
6246
|
id,
|
|
6225
6247
|
name,
|
|
@@ -6239,8 +6261,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6239
6261
|
lastSeen: now
|
|
6240
6262
|
};
|
|
6241
6263
|
});
|
|
6242
|
-
const getConcept =
|
|
6243
|
-
const rows = yield*
|
|
6264
|
+
const getConcept = Effect26.fn("Store.getConcept")(function* (id) {
|
|
6265
|
+
const rows = yield* Effect26.tryPromise({
|
|
6244
6266
|
try: () => db.select().from(concepts2).where(eq8(concepts2.id, id)).limit(1),
|
|
6245
6267
|
catch: storeError("Store.getConcept")
|
|
6246
6268
|
});
|
|
@@ -6256,8 +6278,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6256
6278
|
lastSeen: row.lastSeen
|
|
6257
6279
|
};
|
|
6258
6280
|
});
|
|
6259
|
-
const listConcepts =
|
|
6260
|
-
const rows = yield*
|
|
6281
|
+
const listConcepts = Effect26.fn("Store.listConcepts")(function* () {
|
|
6282
|
+
const rows = yield* Effect26.tryPromise({
|
|
6261
6283
|
try: () => db.select().from(concepts2),
|
|
6262
6284
|
catch: storeError("Store.listConcepts")
|
|
6263
6285
|
});
|
|
@@ -6270,8 +6292,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6270
6292
|
lastSeen: row.lastSeen
|
|
6271
6293
|
}));
|
|
6272
6294
|
});
|
|
6273
|
-
const annotateMemory =
|
|
6274
|
-
yield*
|
|
6295
|
+
const annotateMemory = Effect26.fn("Store.annotateMemory")(function* (memoryId, conceptId, confidence) {
|
|
6296
|
+
yield* Effect26.tryPromise({
|
|
6275
6297
|
try: () => db.insert(memoryConcepts2).values({
|
|
6276
6298
|
conceptId,
|
|
6277
6299
|
memoryId,
|
|
@@ -6281,14 +6303,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6281
6303
|
catch: storeError("Store.annotateMemory")
|
|
6282
6304
|
});
|
|
6283
6305
|
});
|
|
6284
|
-
const getMemoryAnnotations =
|
|
6285
|
-
const rows = yield*
|
|
6306
|
+
const getMemoryAnnotations = Effect26.fn("Store.getMemoryAnnotations")(function* (memoryId) {
|
|
6307
|
+
const rows = yield* Effect26.tryPromise({
|
|
6286
6308
|
try: () => db.select({ conceptId: memoryConcepts2.conceptId }).from(memoryConcepts2).where(eq8(memoryConcepts2.memoryId, memoryId)),
|
|
6287
6309
|
catch: storeError("Store.getMemoryAnnotations")
|
|
6288
6310
|
});
|
|
6289
6311
|
return rows.map((r) => r.conceptId);
|
|
6290
6312
|
});
|
|
6291
|
-
const queryByMetadata =
|
|
6313
|
+
const queryByMetadata = Effect26.fn("Store.queryByMetadata")(function* (filter) {
|
|
6292
6314
|
const rows = yield* backend.queryByMetadata(filter);
|
|
6293
6315
|
const typed = rows;
|
|
6294
6316
|
return typed.map((row) => fromMemoryRow({
|
|
@@ -6305,8 +6327,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6305
6327
|
embeddingModel: row.embedding_model
|
|
6306
6328
|
}));
|
|
6307
6329
|
});
|
|
6308
|
-
const getMemoryMetadata =
|
|
6309
|
-
const rows = yield*
|
|
6330
|
+
const getMemoryMetadata = Effect26.fn("Store.getMemoryMetadata")(function* (id) {
|
|
6331
|
+
const rows = yield* Effect26.tryPromise({
|
|
6310
6332
|
try: () => db.select({ metadata: memories2.metadata }).from(memories2).where(eq8(memories2.id, id)).limit(1),
|
|
6311
6333
|
catch: storeError("Store.getMemoryMetadata")
|
|
6312
6334
|
});
|
|
@@ -6315,8 +6337,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6315
6337
|
return yield* new NotFound({ id });
|
|
6316
6338
|
return parseMetadata(row.metadata);
|
|
6317
6339
|
});
|
|
6318
|
-
const setMemoryMetadata =
|
|
6319
|
-
const updated = yield*
|
|
6340
|
+
const setMemoryMetadata = Effect26.fn("Store.setMemoryMetadata")(function* (id, metadata) {
|
|
6341
|
+
const updated = yield* Effect26.tryPromise({
|
|
6320
6342
|
try: () => db.transaction(async (tx) => {
|
|
6321
6343
|
const rows = await tx.select({ metadata: memories2.metadata }).from(memories2).where(eq8(memories2.id, id)).limit(1);
|
|
6322
6344
|
const row = rows[0];
|
|
@@ -6383,17 +6405,17 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6383
6405
|
graphCoverage: graphCoverage2
|
|
6384
6406
|
};
|
|
6385
6407
|
};
|
|
6386
|
-
var
|
|
6387
|
-
const base =
|
|
6408
|
+
var layer9 = (url, options) => {
|
|
6409
|
+
const base = Layer11.effect(Service5, Effect26.gen(function* () {
|
|
6388
6410
|
const backend = yield* Service2;
|
|
6389
6411
|
return Service5.of(makeStore(backend, options?.embeddingModelConfig, options?.topics, options?.graphConfig, options?.enqueueAudit !== undefined ? { enqueueAudit: options.enqueueAudit } : undefined, options?.defaultTtlSecs));
|
|
6390
|
-
})).pipe(
|
|
6412
|
+
})).pipe(Layer11.provideMerge(layer3(url, options)));
|
|
6391
6413
|
const config = options?.logging ?? silentConfig;
|
|
6392
|
-
return
|
|
6414
|
+
return Layer11.provideMerge(layer(config))(base);
|
|
6393
6415
|
};
|
|
6394
|
-
var configured6 = (url, options) => make({ name: "Store", layer:
|
|
6416
|
+
var configured6 = (url, options) => make({ name: "Store", layer: Layer11.suspend(() => layer9(url, options)), deps: [] });
|
|
6395
6417
|
var node5 = configured6("file:./.dimple/dimple.db");
|
|
6396
|
-
var loggingFromConfig =
|
|
6418
|
+
var loggingFromConfig = Layer11.unwrap(Effect26.map(Effect26.service(Service4), (cfg) => layer(cfg.config.logging ?? silentConfig)));
|
|
6397
6419
|
// ../store/src/topics/index.ts
|
|
6398
6420
|
var exports_topics = {};
|
|
6399
6421
|
__export(exports_topics, {
|
|
@@ -6452,15 +6474,15 @@ __export(exports_topics, {
|
|
|
6452
6474
|
});
|
|
6453
6475
|
|
|
6454
6476
|
// ../store/src/audit.ts
|
|
6455
|
-
import { Effect as
|
|
6477
|
+
import { Effect as Effect27 } from "effect";
|
|
6456
6478
|
function auditWriteHandler(deps = {}) {
|
|
6457
|
-
const run = (payload, ctx) =>
|
|
6479
|
+
const run = (payload, ctx) => Effect27.gen(function* () {
|
|
6458
6480
|
const envelope = payload;
|
|
6459
6481
|
const backend = yield* Service2;
|
|
6460
|
-
yield* backend.writeAuditRows(envelope.rows).pipe(
|
|
6482
|
+
yield* backend.writeAuditRows(envelope.rows).pipe(Effect27.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `audit.write: ${error}` })));
|
|
6461
6483
|
return;
|
|
6462
6484
|
});
|
|
6463
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6485
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect27.provideService(Service2, deps.backend)) : run;
|
|
6464
6486
|
}
|
|
6465
6487
|
// ../store/src/graph/index.ts
|
|
6466
6488
|
var exports_graph = {};
|
|
@@ -6495,7 +6517,7 @@ __export(exports_graph, {
|
|
|
6495
6517
|
});
|
|
6496
6518
|
|
|
6497
6519
|
// ../store/src/graph/concepts.ts
|
|
6498
|
-
import { Clock as Clock6, Effect as
|
|
6520
|
+
import { Clock as Clock6, Effect as Effect28 } from "effect";
|
|
6499
6521
|
import { and as and9, asc as asc6, eq as eq9, isNotNull, sql as sql9 } from "drizzle-orm";
|
|
6500
6522
|
var DEFAULT_CLUSTER_K = 8;
|
|
6501
6523
|
var DEFAULT_CLUSTER_MIN_SIZE = 3;
|
|
@@ -6618,11 +6640,11 @@ var conceptConfidence = (vector, centroid) => {
|
|
|
6618
6640
|
return 0;
|
|
6619
6641
|
return Math.min(1, Math.max(0, dot3(vector, centroid) / Math.sqrt(norm)));
|
|
6620
6642
|
};
|
|
6621
|
-
var clusterLeaf =
|
|
6643
|
+
var clusterLeaf = Effect28.fn("ConceptCluster.clusterLeaf")(function* (topicKey, model, config) {
|
|
6622
6644
|
const backend = yield* Service2;
|
|
6623
6645
|
const db = backend.db;
|
|
6624
6646
|
const now = yield* Clock6.currentTimeMillis;
|
|
6625
|
-
const memberRows = yield*
|
|
6647
|
+
const memberRows = yield* Effect28.tryPromise({
|
|
6626
6648
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and9(eq9(topicMemberships.topicKey, topicKey), eq9(topicMemberships.modelKey, model.id))).orderBy(asc6(topicMemberships.memoryId)),
|
|
6627
6649
|
catch: (cause) => new StoreError({ message: `clusterLeaf: load members: ${cause}` })
|
|
6628
6650
|
});
|
|
@@ -6641,11 +6663,11 @@ var clusterLeaf = Effect27.fn("ConceptCluster.clusterLeaf")(function* (topicKey,
|
|
|
6641
6663
|
return { concepts: 0, annotated: 0 };
|
|
6642
6664
|
return yield* persistClusters(db, members, config, now);
|
|
6643
6665
|
});
|
|
6644
|
-
var clusterCorpus =
|
|
6666
|
+
var clusterCorpus = Effect28.fn("ConceptCluster.clusterCorpus")(function* (model, config) {
|
|
6645
6667
|
const backend = yield* Service2;
|
|
6646
6668
|
const db = backend.db;
|
|
6647
6669
|
const now = yield* Clock6.currentTimeMillis;
|
|
6648
|
-
const rows = yield*
|
|
6670
|
+
const rows = yield* Effect28.tryPromise({
|
|
6649
6671
|
try: () => db.select({ id: memories.id, embedding: memories.embedding }).from(memories).where(and9(eq9(memories.state, "active"), isNotNull(memories.embedding))).orderBy(asc6(memories.id)),
|
|
6650
6672
|
catch: (cause) => new StoreError({ message: `clusterCorpus: load memories: ${cause}` })
|
|
6651
6673
|
});
|
|
@@ -6662,7 +6684,7 @@ var clusterCorpus = Effect27.fn("ConceptCluster.clusterCorpus")(function* (model
|
|
|
6662
6684
|
return { concepts: 0, annotated: 0 };
|
|
6663
6685
|
return yield* persistClusters(db, members, config, now);
|
|
6664
6686
|
});
|
|
6665
|
-
var persistClusters = (db, members, config, now) =>
|
|
6687
|
+
var persistClusters = (db, members, config, now) => Effect28.tryPromise({
|
|
6666
6688
|
try: () => db.transaction(async (tx) => {
|
|
6667
6689
|
const { k, minClusterSize } = resolvedClusterConfig(config);
|
|
6668
6690
|
const vectors = members.map((m) => m.vector);
|
|
@@ -6713,21 +6735,21 @@ var persistClusters = (db, members, config, now) => Effect27.tryPromise({
|
|
|
6713
6735
|
catch: (cause) => new StoreError({ message: `concept cluster: persist: ${cause}` })
|
|
6714
6736
|
});
|
|
6715
6737
|
function conceptClusterHandler(deps) {
|
|
6716
|
-
const run = (payload, ctx) =>
|
|
6738
|
+
const run = (payload, ctx) => Effect28.gen(function* () {
|
|
6717
6739
|
const envelope = payload;
|
|
6718
6740
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
6719
6741
|
if (!model) {
|
|
6720
|
-
return yield*
|
|
6742
|
+
return yield* Effect28.fail(new JobFailure({
|
|
6721
6743
|
jobId: ctx.jobId,
|
|
6722
6744
|
message: `concept.cluster: unknown model "${envelope.modelKey}"`
|
|
6723
6745
|
}));
|
|
6724
6746
|
}
|
|
6725
6747
|
if (envelope.topicKey !== undefined) {
|
|
6726
|
-
yield* clusterLeaf(envelope.topicKey, model, deps.config).pipe(
|
|
6748
|
+
yield* clusterLeaf(envelope.topicKey, model, deps.config).pipe(Effect28.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `concept.cluster: ${error}` })));
|
|
6727
6749
|
} else {
|
|
6728
|
-
yield* clusterCorpus(model, deps.config).pipe(
|
|
6750
|
+
yield* clusterCorpus(model, deps.config).pipe(Effect28.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `concept.cluster: ${error}` })));
|
|
6729
6751
|
}
|
|
6730
|
-
yield*
|
|
6752
|
+
yield* Effect28.logInfo("concepts.clustered").pipe(Effect28.annotateLogs({
|
|
6731
6753
|
scope: "@dimple/store",
|
|
6732
6754
|
event: "concepts.clustered",
|
|
6733
6755
|
modelKey: envelope.modelKey,
|
|
@@ -6735,10 +6757,10 @@ function conceptClusterHandler(deps) {
|
|
|
6735
6757
|
}));
|
|
6736
6758
|
return;
|
|
6737
6759
|
});
|
|
6738
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6760
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect28.provideService(Service2, deps.backend)) : run;
|
|
6739
6761
|
}
|
|
6740
6762
|
// ../store/src/graph/dream.ts
|
|
6741
|
-
import { Clock as Clock7, Effect as
|
|
6763
|
+
import { Clock as Clock7, Effect as Effect29 } from "effect";
|
|
6742
6764
|
import { and as and10, asc as asc7, eq as eq10, gt, inArray as inArray5, isNull as isNull3, sql as sql10 } from "drizzle-orm";
|
|
6743
6765
|
var dreamStateDefaults = {
|
|
6744
6766
|
minHours: 24,
|
|
@@ -6849,18 +6871,18 @@ var writeDerivedMemoryTx = async (tx, content, sources, now) => {
|
|
|
6849
6871
|
}
|
|
6850
6872
|
return { inserted };
|
|
6851
6873
|
};
|
|
6852
|
-
var runDream =
|
|
6874
|
+
var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
6853
6875
|
const backend = yield* Service2;
|
|
6854
6876
|
const db = backend.db;
|
|
6855
6877
|
const now = yield* Clock7.currentTimeMillis;
|
|
6856
6878
|
const { minHours, minNewMemories, maxPairs, maxDerived } = dreamStateDefaults;
|
|
6857
|
-
const stateRows = yield*
|
|
6879
|
+
const stateRows = yield* Effect29.tryPromise({
|
|
6858
6880
|
try: () => db.select().from(dreamState).where(eq10(dreamState.modelKey, model.id)),
|
|
6859
6881
|
catch: (cause) => new StoreError({ message: `memory.dream: load state: ${cause}` })
|
|
6860
6882
|
});
|
|
6861
6883
|
const lastDreamAt = stateRows[0]?.lastDreamAt ?? 0;
|
|
6862
6884
|
if (now - lastDreamAt < minHours * 3600000) {
|
|
6863
|
-
yield*
|
|
6885
|
+
yield* Effect29.logDebug("dream.skipped").pipe(Effect29.annotateLogs({ scope: "@dimple/store", event: "dream.skipped", reason: "cooldown" }));
|
|
6864
6886
|
return {
|
|
6865
6887
|
skipped: true,
|
|
6866
6888
|
reason: "cooldown",
|
|
@@ -6869,13 +6891,13 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6869
6891
|
superseded: 0
|
|
6870
6892
|
};
|
|
6871
6893
|
}
|
|
6872
|
-
const sinceCountRows = yield*
|
|
6894
|
+
const sinceCountRows = yield* Effect29.tryPromise({
|
|
6873
6895
|
try: () => db.select({ count: sql10`count(*)` }).from(memories).where(gt(memories.createdAt, lastDreamAt)),
|
|
6874
6896
|
catch: (cause) => new StoreError({ message: `memory.dream: count new: ${cause}` })
|
|
6875
6897
|
});
|
|
6876
6898
|
const memoriesSinceWatermark = Number(sinceCountRows[0]?.count ?? 0);
|
|
6877
6899
|
if (memoriesSinceWatermark < minNewMemories) {
|
|
6878
|
-
yield*
|
|
6900
|
+
yield* Effect29.logDebug("dream.skipped").pipe(Effect29.annotateLogs({
|
|
6879
6901
|
scope: "@dimple/store",
|
|
6880
6902
|
event: "dream.skipped",
|
|
6881
6903
|
reason: "insufficient-new-memories",
|
|
@@ -6889,7 +6911,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6889
6911
|
superseded: 0
|
|
6890
6912
|
};
|
|
6891
6913
|
}
|
|
6892
|
-
const pendingRows = yield*
|
|
6914
|
+
const pendingRows = yield* Effect29.tryPromise({
|
|
6893
6915
|
try: () => db.select({
|
|
6894
6916
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
6895
6917
|
toMemoryId: artifactEdges.toMemoryId
|
|
@@ -6902,7 +6924,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6902
6924
|
};
|
|
6903
6925
|
for (const row of pendingRows)
|
|
6904
6926
|
addPair(row.fromMemoryId, row.toMemoryId);
|
|
6905
|
-
const sinceRows = yield*
|
|
6927
|
+
const sinceRows = yield* Effect29.tryPromise({
|
|
6906
6928
|
try: () => db.select({
|
|
6907
6929
|
id: memories.id,
|
|
6908
6930
|
logicalKey: memories.logicalKey,
|
|
@@ -6913,19 +6935,19 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6913
6935
|
});
|
|
6914
6936
|
if (sinceRows.length > 0) {
|
|
6915
6937
|
const sinceIds = sinceRows.map((r) => r.id);
|
|
6916
|
-
const membershipRows = yield*
|
|
6938
|
+
const membershipRows = yield* Effect29.tryPromise({
|
|
6917
6939
|
try: () => db.select({ memoryId: topicMemberships.memoryId, topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and10(eq10(topicMemberships.modelKey, model.id), inArray5(topicMemberships.memoryId, sinceIds))),
|
|
6918
6940
|
catch: (cause) => new StoreError({ message: `memory.dream: scan memberships: ${cause}` })
|
|
6919
6941
|
});
|
|
6920
6942
|
const leafKeys = [...new Set(membershipRows.map((m) => m.topicKey))];
|
|
6921
6943
|
if (leafKeys.length > 0) {
|
|
6922
|
-
const mateRows = yield*
|
|
6944
|
+
const mateRows = yield* Effect29.tryPromise({
|
|
6923
6945
|
try: () => db.select({ memoryId: topicMemberships.memoryId, topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and10(eq10(topicMemberships.modelKey, model.id), inArray5(topicMemberships.topicKey, leafKeys))),
|
|
6924
6946
|
catch: (cause) => new StoreError({ message: `memory.dream: scan mates: ${cause}` })
|
|
6925
6947
|
});
|
|
6926
6948
|
const memberIds = [...new Set([...sinceIds, ...mateRows.map((m) => m.memoryId)])];
|
|
6927
|
-
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(
|
|
6928
|
-
const memoryRows = yield*
|
|
6949
|
+
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(Effect29.mapError((cause) => new StoreError({ message: `memory.dream: scan vectors: ${cause}` })));
|
|
6950
|
+
const memoryRows = yield* Effect29.tryPromise({
|
|
6929
6951
|
try: () => db.select({
|
|
6930
6952
|
id: memories.id,
|
|
6931
6953
|
logicalKey: memories.logicalKey,
|
|
@@ -6982,7 +7004,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6982
7004
|
}
|
|
6983
7005
|
const orderedPairs = [...pairSet.values()].sort((a, b) => cmp3(a.toMemoryId, b.toMemoryId) || cmp3(a.fromMemoryId, b.fromMemoryId)).slice(0, maxPairs);
|
|
6984
7006
|
const endpointIds = [...new Set(orderedPairs.flatMap((p) => [p.fromMemoryId, p.toMemoryId]))];
|
|
6985
|
-
const contentRows = endpointIds.length > 0 ? yield*
|
|
7007
|
+
const contentRows = endpointIds.length > 0 ? yield* Effect29.tryPromise({
|
|
6986
7008
|
try: () => db.select({ id: memories.id, content: memories.content }).from(memories).where(inArray5(memories.id, endpointIds)),
|
|
6987
7009
|
catch: (cause) => new StoreError({ message: `memory.dream: pair contents: ${cause}` })
|
|
6988
7010
|
}) : [];
|
|
@@ -7002,12 +7024,12 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7002
7024
|
}
|
|
7003
7025
|
let classification = { relations: [], derived: [] };
|
|
7004
7026
|
if (dreamPairs.length > 0) {
|
|
7005
|
-
classification = yield* classify.classify(dreamPairs, { maxDerived }).pipe(
|
|
7027
|
+
classification = yield* classify.classify(dreamPairs, { maxDerived }).pipe(Effect29.mapError((error) => new StoreError({ message: `memory.dream: classify: ${error}` })));
|
|
7006
7028
|
}
|
|
7007
7029
|
const endpointIdsOf = [
|
|
7008
7030
|
...new Set(classification.relations.flatMap((r) => [r.fromMemoryId, r.toMemoryId]))
|
|
7009
7031
|
];
|
|
7010
|
-
const endpointRows = endpointIdsOf.length > 0 ? yield*
|
|
7032
|
+
const endpointRows = endpointIdsOf.length > 0 ? yield* Effect29.tryPromise({
|
|
7011
7033
|
try: () => db.select({ id: memories.id, metadata: memories.metadata }).from(memories).where(inArray5(memories.id, endpointIdsOf)),
|
|
7012
7034
|
catch: (cause) => new StoreError({ message: `memory.dream: endpoints: ${cause}` })
|
|
7013
7035
|
}) : [];
|
|
@@ -7025,7 +7047,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7025
7047
|
}
|
|
7026
7048
|
const acceptedDerived = [];
|
|
7027
7049
|
for (const derived of classification.derived) {
|
|
7028
|
-
const sourceRows = derived.sources.length > 0 ? yield*
|
|
7050
|
+
const sourceRows = derived.sources.length > 0 ? yield* Effect29.tryPromise({
|
|
7029
7051
|
try: () => db.select({ id: memories.id }).from(memories).where(inArray5(memories.id, derived.sources)),
|
|
7030
7052
|
catch: (cause) => new StoreError({ message: `memory.dream: derived sources: ${cause}` })
|
|
7031
7053
|
}) : [];
|
|
@@ -7037,7 +7059,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7037
7059
|
}
|
|
7038
7060
|
let superseded = 0;
|
|
7039
7061
|
let created = 0;
|
|
7040
|
-
yield*
|
|
7062
|
+
yield* Effect29.tryPromise({
|
|
7041
7063
|
try: () => db.transaction(async (tx) => {
|
|
7042
7064
|
for (const relation of acceptedRelations) {
|
|
7043
7065
|
const result = await writeRelationEdgeTx(tx, relation, now);
|
|
@@ -7062,7 +7084,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7062
7084
|
}),
|
|
7063
7085
|
catch: (cause) => new StoreError({ message: `memory.dream: persist: ${cause}` })
|
|
7064
7086
|
});
|
|
7065
|
-
yield*
|
|
7087
|
+
yield* Effect29.logInfo("dream.completed").pipe(Effect29.annotateLogs({
|
|
7066
7088
|
scope: "@dimple/store",
|
|
7067
7089
|
event: "dream.completed",
|
|
7068
7090
|
modelKey: model.id,
|
|
@@ -7073,24 +7095,24 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7073
7095
|
return { skipped: false, classified: dreamPairs.length, created, superseded };
|
|
7074
7096
|
});
|
|
7075
7097
|
function graphDreamHandler(deps) {
|
|
7076
|
-
const run = (payload, ctx) =>
|
|
7098
|
+
const run = (payload, ctx) => Effect29.gen(function* () {
|
|
7077
7099
|
const envelope = payload;
|
|
7078
7100
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
7079
7101
|
if (!model) {
|
|
7080
|
-
return yield*
|
|
7102
|
+
return yield* Effect29.fail(new JobFailure({
|
|
7081
7103
|
jobId: ctx.jobId,
|
|
7082
7104
|
message: `memory.dream: unknown model "${envelope.modelKey}"`
|
|
7083
7105
|
}));
|
|
7084
7106
|
}
|
|
7085
|
-
return yield* runDream(model, deps.classify).pipe(
|
|
7107
|
+
return yield* runDream(model, deps.classify).pipe(Effect29.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `memory.dream: ${error}` })));
|
|
7086
7108
|
});
|
|
7087
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
7109
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect29.provideService(Service2, deps.backend)) : run;
|
|
7088
7110
|
}
|
|
7089
7111
|
// ../store/src/graph/repair.ts
|
|
7090
|
-
import { Clock as Clock8, Effect as
|
|
7112
|
+
import { Clock as Clock8, Effect as Effect30 } from "effect";
|
|
7091
7113
|
import { and as and11, eq as eq11, isNotNull as isNotNull2, lt, or as or2, sql as sql11 } from "drizzle-orm";
|
|
7092
7114
|
var DEFAULT_EXPIRED_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
7093
|
-
var runRepair = (db, includePurgeExpired, expiredGraceMs, now) =>
|
|
7115
|
+
var runRepair = (db, includePurgeExpired, expiredGraceMs, now) => Effect30.tryPromise({
|
|
7094
7116
|
try: () => db.transaction(async (tx) => {
|
|
7095
7117
|
let orphanedEdges = 0;
|
|
7096
7118
|
let staleAnnotations = 0;
|
|
@@ -7184,10 +7206,10 @@ var runRepair = (db, includePurgeExpired, expiredGraceMs, now) => Effect29.tryPr
|
|
|
7184
7206
|
catch: (cause) => new StoreError({ message: `graph.repair: ${cause}` })
|
|
7185
7207
|
});
|
|
7186
7208
|
function graphRepairHandler(deps) {
|
|
7187
|
-
const run = (payload, ctx) =>
|
|
7209
|
+
const run = (payload, ctx) => Effect30.gen(function* () {
|
|
7188
7210
|
const envelope = payload;
|
|
7189
7211
|
if (!deps.getModelConfig(envelope.modelKey)) {
|
|
7190
|
-
return yield*
|
|
7212
|
+
return yield* Effect30.fail(new JobFailure({
|
|
7191
7213
|
jobId: ctx.jobId,
|
|
7192
7214
|
message: `graph.repair: unknown model "${envelope.modelKey}"`
|
|
7193
7215
|
}));
|
|
@@ -7195,8 +7217,8 @@ function graphRepairHandler(deps) {
|
|
|
7195
7217
|
const backend = yield* Service2;
|
|
7196
7218
|
const db = backend.db;
|
|
7197
7219
|
const now = yield* Clock8.currentTimeMillis;
|
|
7198
|
-
const result = yield* runRepair(db, deps.includePurgeExpired ?? false, deps.expiredGraceMs ?? DEFAULT_EXPIRED_GRACE_MS, now).pipe(
|
|
7199
|
-
yield*
|
|
7220
|
+
const result = yield* runRepair(db, deps.includePurgeExpired ?? false, deps.expiredGraceMs ?? DEFAULT_EXPIRED_GRACE_MS, now).pipe(Effect30.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `graph.repair: ${error}` })));
|
|
7221
|
+
yield* Effect30.logInfo("repair.completed").pipe(Effect30.annotateLogs({
|
|
7200
7222
|
scope: "@dimple/store",
|
|
7201
7223
|
event: "repair.completed",
|
|
7202
7224
|
modelKey: envelope.modelKey,
|
|
@@ -7206,7 +7228,7 @@ function graphRepairHandler(deps) {
|
|
|
7206
7228
|
}));
|
|
7207
7229
|
return result;
|
|
7208
7230
|
});
|
|
7209
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
7231
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect30.provideService(Service2, deps.backend)) : run;
|
|
7210
7232
|
}
|
|
7211
7233
|
// ../store/src/models.ts
|
|
7212
7234
|
import { Schema as Schema24 } from "effect";
|
|
@@ -7285,21 +7307,21 @@ var RetrievalLogEntry = Schema24.Struct({
|
|
|
7285
7307
|
createdAt: Schema24.Number
|
|
7286
7308
|
});
|
|
7287
7309
|
// ../embedder/src/provider.ts
|
|
7288
|
-
import { Context as
|
|
7310
|
+
import { Context as Context11, Schema as Schema25 } from "effect";
|
|
7289
7311
|
|
|
7290
7312
|
class EmbeddingError extends Schema25.TaggedError()("EmbeddingProvider.EmbeddingError", { message: Schema25.String, cause: Schema25.optional(Schema25.Defect()) }) {
|
|
7291
7313
|
}
|
|
7292
7314
|
|
|
7293
|
-
class
|
|
7315
|
+
class Service10 extends Context11.Service()("@dimple/embedder/EmbeddingModelRegistry") {
|
|
7294
7316
|
}
|
|
7295
7317
|
// ../embedder/src/transformers.ts
|
|
7296
|
-
import { Config as Config5, Effect as
|
|
7318
|
+
import { Config as Config5, Effect as Effect33, Layer as Layer12, Option as Option11, Schema as Schema26 } from "effect";
|
|
7297
7319
|
import { existsSync as existsSync2, readFileSync } from "node:fs";
|
|
7298
7320
|
import { join as join2 } from "node:path";
|
|
7299
7321
|
import { pathToFileURL } from "node:url";
|
|
7300
7322
|
|
|
7301
7323
|
// ../embedder/src/auto-install.ts
|
|
7302
|
-
import { Effect as
|
|
7324
|
+
import { Effect as Effect32 } from "effect";
|
|
7303
7325
|
|
|
7304
7326
|
// ../installer/src/index.ts
|
|
7305
7327
|
import { existsSync } from "node:fs";
|
|
@@ -7390,7 +7412,7 @@ var installOptionalPeers = (packages, opts) => {
|
|
|
7390
7412
|
return run;
|
|
7391
7413
|
};
|
|
7392
7414
|
// ../embedder/src/auto-install.ts
|
|
7393
|
-
var installTransformers = (cwd) =>
|
|
7415
|
+
var installTransformers = (cwd) => Effect32.tryPromise({
|
|
7394
7416
|
try: () => installOptionalPeers(["@huggingface/transformers"], { cwd, timeoutMs: 300000 }),
|
|
7395
7417
|
catch: (cause) => new EmbeddingError({
|
|
7396
7418
|
message: cause instanceof InstallerError ? `${cause.message} — or set \`autoInstall: false\` and configure a hosted embedding provider` : `auto-install of @huggingface/transformers failed: ${String(cause)} — run it manually ` + "or set `autoInstall: false` and configure a hosted embedding provider"
|
|
@@ -7506,20 +7528,20 @@ var outputVectors = (result) => {
|
|
|
7506
7528
|
};
|
|
7507
7529
|
var make6 = (options) => {
|
|
7508
7530
|
let pipe = null;
|
|
7509
|
-
const getPipeline =
|
|
7531
|
+
const getPipeline = Effect33.fnUntraced(function* () {
|
|
7510
7532
|
if (pipe)
|
|
7511
7533
|
return pipe;
|
|
7512
|
-
const loadModule = () =>
|
|
7513
|
-
let transformers = yield* loadModule().pipe(
|
|
7534
|
+
const loadModule = () => Effect33.tryPromise(() => import("@huggingface/transformers"));
|
|
7535
|
+
let transformers = yield* loadModule().pipe(Effect33.catch(() => {
|
|
7514
7536
|
if (options.autoInstall === false) {
|
|
7515
|
-
return
|
|
7537
|
+
return Effect33.fail(new EmbeddingError({
|
|
7516
7538
|
message: "@huggingface/transformers is not installed — run `bun add @huggingface/transformers` " + "to use local embeddings (or configure a hosted embedding provider)"
|
|
7517
7539
|
}));
|
|
7518
7540
|
}
|
|
7519
|
-
return
|
|
7520
|
-
yield*
|
|
7541
|
+
return Effect33.gen(function* () {
|
|
7542
|
+
yield* Effect33.logInfo("auto-installing @huggingface/transformers (first local-embed use — scripts skipped)").pipe(Effect33.annotateLogs({ scope: "@dimple/embedder", event: "embedder.auto-install" }));
|
|
7521
7543
|
yield* installTransformers(process.cwd());
|
|
7522
|
-
return yield*
|
|
7544
|
+
return yield* Effect33.tryPromise(importFromDisk).pipe(Effect33.mapError(() => new EmbeddingError({
|
|
7523
7545
|
message: "@huggingface/transformers still unavailable after auto-install — run `bun add " + "@huggingface/transformers` manually (or set `autoInstall: false` and configure a hosted embedding provider)"
|
|
7524
7546
|
})));
|
|
7525
7547
|
});
|
|
@@ -7527,17 +7549,17 @@ var make6 = (options) => {
|
|
|
7527
7549
|
if (options.cacheDir)
|
|
7528
7550
|
transformers.env.cacheDir = options.cacheDir;
|
|
7529
7551
|
applyWasmOptions(transformers.env, options.wasm);
|
|
7530
|
-
pipe = yield*
|
|
7552
|
+
pipe = yield* Effect33.tryPromise(() => transformers.pipeline("feature-extraction", options.modelId ?? defaults2.modelId, {
|
|
7531
7553
|
device: options.device ?? defaults2.device,
|
|
7532
7554
|
dtype: options.dtype ?? defaults2.dtype,
|
|
7533
7555
|
session_options: buildSessionOptions(options.sessionOptions ?? {})
|
|
7534
|
-
})).pipe(
|
|
7556
|
+
})).pipe(Effect33.mapError((cause) => new EmbeddingError({ message: `load model: ${cause}` })));
|
|
7535
7557
|
return pipe;
|
|
7536
7558
|
});
|
|
7537
|
-
const embed =
|
|
7559
|
+
const embed = Effect33.fn("TransformersEmbedding.embed")(function* (texts) {
|
|
7538
7560
|
const started = Date.now();
|
|
7539
7561
|
const p = yield* getPipeline();
|
|
7540
|
-
const result = yield*
|
|
7562
|
+
const result = yield* Effect33.tryPromise({
|
|
7541
7563
|
try: () => p(texts, {
|
|
7542
7564
|
pooling: options.pooling ?? defaults2.pooling,
|
|
7543
7565
|
normalize: options.normalize ?? defaults2.normalize
|
|
@@ -7552,7 +7574,7 @@ var make6 = (options) => {
|
|
|
7552
7574
|
});
|
|
7553
7575
|
}
|
|
7554
7576
|
}
|
|
7555
|
-
yield*
|
|
7577
|
+
yield* Effect33.logDebug("embedder.embed").pipe(Effect33.annotateLogs({
|
|
7556
7578
|
scope: "@dimple/embedder",
|
|
7557
7579
|
event: "embedder.embed",
|
|
7558
7580
|
count: texts.length,
|
|
@@ -7560,9 +7582,9 @@ var make6 = (options) => {
|
|
|
7560
7582
|
}));
|
|
7561
7583
|
return vectors;
|
|
7562
7584
|
});
|
|
7563
|
-
const embedSingle =
|
|
7585
|
+
const embedSingle = Effect33.fn("TransformersEmbedding.embedSingle")(function* (text4) {
|
|
7564
7586
|
const p = yield* getPipeline();
|
|
7565
|
-
const result = yield*
|
|
7587
|
+
const result = yield* Effect33.tryPromise({
|
|
7566
7588
|
try: () => p(text4, {
|
|
7567
7589
|
pooling: options.pooling ?? defaults2.pooling,
|
|
7568
7590
|
normalize: options.normalize ?? defaults2.normalize
|
|
@@ -7590,14 +7612,14 @@ var makeTransformersModel = (options = {}) => {
|
|
|
7590
7612
|
};
|
|
7591
7613
|
};
|
|
7592
7614
|
// ../embedder/src/registry.ts
|
|
7593
|
-
import { Effect as
|
|
7615
|
+
import { Effect as Effect34, Layer as Layer13 } from "effect";
|
|
7594
7616
|
// ../core/src/hosted-embedding.ts
|
|
7595
7617
|
import { embedMany } from "ai";
|
|
7596
7618
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
7597
7619
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
7598
7620
|
import { createCohere } from "@ai-sdk/cohere";
|
|
7599
7621
|
import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
|
|
7600
|
-
import { Effect as
|
|
7622
|
+
import { Effect as Effect35 } from "effect";
|
|
7601
7623
|
var makeHostedEmbeddingModel = (options) => {
|
|
7602
7624
|
const provider = options.provider ?? "openai";
|
|
7603
7625
|
const model = provider === "google" ? createGoogleGenerativeAI({
|
|
@@ -7628,8 +7650,8 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
7628
7650
|
return { openaiCompatible: { dimensions: options.dimensions } };
|
|
7629
7651
|
}
|
|
7630
7652
|
})();
|
|
7631
|
-
const embed =
|
|
7632
|
-
const result = yield*
|
|
7653
|
+
const embed = Effect35.fn("HostedEmbedding.embed")(function* (texts) {
|
|
7654
|
+
const result = yield* Effect35.tryPromise({
|
|
7633
7655
|
try: () => embedMany({ model, values: [...texts], providerOptions }),
|
|
7634
7656
|
catch: (cause) => new EmbeddingError({ message: `hosted embedMany: ${cause}` })
|
|
7635
7657
|
});
|
|
@@ -7642,7 +7664,7 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
7642
7664
|
}
|
|
7643
7665
|
return result.embeddings.map((e) => new Float32Array(e));
|
|
7644
7666
|
});
|
|
7645
|
-
const embedSingle =
|
|
7667
|
+
const embedSingle = Effect35.fn("HostedEmbedding.embedSingle")(function* (text4) {
|
|
7646
7668
|
const results = yield* embed([text4]);
|
|
7647
7669
|
return results[0];
|
|
7648
7670
|
});
|
|
@@ -7656,7 +7678,7 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
7656
7678
|
};
|
|
7657
7679
|
|
|
7658
7680
|
// ../retriever/src/hybrid.ts
|
|
7659
|
-
import { Clock as Clock9, Context as
|
|
7681
|
+
import { Clock as Clock9, Context as Context12, Effect as Effect36, Layer as Layer14, Schema as Schema27 } from "effect";
|
|
7660
7682
|
|
|
7661
7683
|
// ../retriever/src/rrf.ts
|
|
7662
7684
|
function fuse(denseRanks, fts5Ranks, k, limit) {
|
|
@@ -7699,17 +7721,17 @@ class HybridError extends Schema27.TaggedError()("HybridRetrieval.HybridError",
|
|
|
7699
7721
|
}) {
|
|
7700
7722
|
}
|
|
7701
7723
|
|
|
7702
|
-
class
|
|
7724
|
+
class Service11 extends Context12.Service()("@dimple/retriever/HybridRetrieval") {
|
|
7703
7725
|
}
|
|
7704
|
-
var wrapError = (tag) => (effect) => effect.pipe(
|
|
7705
|
-
var
|
|
7726
|
+
var wrapError = (tag) => (effect) => effect.pipe(Effect36.mapError((cause) => new HybridError({ message: `${tag}: ${cause}` })));
|
|
7727
|
+
var layer12 = (options) => Layer14.effect(Service11, Effect36.gen(function* () {
|
|
7706
7728
|
const store = yield* Service5;
|
|
7707
|
-
const registry = yield*
|
|
7729
|
+
const registry = yield* Service10;
|
|
7708
7730
|
const defaultId = yield* registry.defaultModelId.pipe(wrapError("defaultModelId"));
|
|
7709
7731
|
const defaultModel = yield* registry.getModel(defaultId).pipe(wrapError("getModel"));
|
|
7710
7732
|
const defaultK = options?.defaultK ?? 60;
|
|
7711
7733
|
const defaultLimit = options?.defaultLimit ?? 10;
|
|
7712
|
-
const hybrid =
|
|
7734
|
+
const hybrid = Effect36.fn("HybridRetrieval.hybrid")(function* (query2, options2) {
|
|
7713
7735
|
const namespace = options2?.namespace ?? "default";
|
|
7714
7736
|
const limit = options2?.limit ?? defaultLimit;
|
|
7715
7737
|
const k = options2?.k ?? defaultK;
|
|
@@ -7812,7 +7834,7 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7812
7834
|
fts: fts5Results.map((s) => s.memory.id)
|
|
7813
7835
|
})
|
|
7814
7836
|
}).pipe(wrapError("logAudit"));
|
|
7815
|
-
yield*
|
|
7837
|
+
yield* Effect36.logInfo("hybrid.search").pipe(Effect36.annotateLogs({
|
|
7816
7838
|
scope: "@dimple/retriever",
|
|
7817
7839
|
event: "hybrid.search",
|
|
7818
7840
|
mode,
|
|
@@ -7822,16 +7844,16 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7822
7844
|
}));
|
|
7823
7845
|
return results;
|
|
7824
7846
|
});
|
|
7825
|
-
return
|
|
7847
|
+
return Service11.of({ hybrid });
|
|
7826
7848
|
}));
|
|
7827
|
-
var hybridRetrievalLayer =
|
|
7849
|
+
var hybridRetrievalLayer = layer12();
|
|
7828
7850
|
var configured8 = (options, deps) => make({
|
|
7829
7851
|
name: "HybridRetrieval",
|
|
7830
|
-
layer: options === undefined ? hybridRetrievalLayer :
|
|
7852
|
+
layer: options === undefined ? hybridRetrievalLayer : Layer14.suspend(() => layer12(options)),
|
|
7831
7853
|
deps
|
|
7832
7854
|
});
|
|
7833
7855
|
// ../llm/src/llm.ts
|
|
7834
|
-
import { Cause as Cause4, Effect as
|
|
7856
|
+
import { Cause as Cause4, Effect as Effect37, Option as Option12, Schema as Schema28 } from "effect";
|
|
7835
7857
|
import {
|
|
7836
7858
|
generateText as generateTextCall,
|
|
7837
7859
|
NoSuchToolError,
|
|
@@ -7879,9 +7901,9 @@ var makeLLM = (model, defaults3 = {}) => {
|
|
|
7879
7901
|
...reasoningEffort !== undefined ? { providerOptions: { openaiCompatible: { reasoningEffort } } } : undefined
|
|
7880
7902
|
};
|
|
7881
7903
|
};
|
|
7882
|
-
const generateText =
|
|
7904
|
+
const generateText = Effect37.fn("LLM.generateText")(function* (input) {
|
|
7883
7905
|
const started = Date.now();
|
|
7884
|
-
const text4 = yield*
|
|
7906
|
+
const text4 = yield* Effect37.tryPromise({
|
|
7885
7907
|
try: () => generateTextCall({
|
|
7886
7908
|
model,
|
|
7887
7909
|
prompt: input.prompt,
|
|
@@ -7890,16 +7912,16 @@ var makeLLM = (model, defaults3 = {}) => {
|
|
|
7890
7912
|
}).then((result) => result.text),
|
|
7891
7913
|
catch: (cause) => new LLMError({ message: "LLM text generation failed", cause: Cause4.die(cause) })
|
|
7892
7914
|
});
|
|
7893
|
-
yield*
|
|
7915
|
+
yield* Effect37.logDebug("llm.generate-text").pipe(Effect37.annotateLogs({
|
|
7894
7916
|
scope: "@dimple/llm",
|
|
7895
7917
|
event: "llm.generate-text",
|
|
7896
7918
|
latencyMs: Date.now() - started
|
|
7897
7919
|
}));
|
|
7898
7920
|
return text4;
|
|
7899
7921
|
});
|
|
7900
|
-
const generateObject =
|
|
7922
|
+
const generateObject = Effect37.fn("LLM.generateObject")(function* (input) {
|
|
7901
7923
|
const started = Date.now();
|
|
7902
|
-
const result = yield*
|
|
7924
|
+
const result = yield* Effect37.tryPromise({
|
|
7903
7925
|
try: async () => {
|
|
7904
7926
|
try {
|
|
7905
7927
|
const result3 = await generateTextCall({
|
|
@@ -7942,7 +7964,7 @@ ${schemaJson}`,
|
|
|
7942
7964
|
},
|
|
7943
7965
|
catch: (cause) => new LLMError({ message: "LLM structured generation failed", cause: Cause4.die(cause) })
|
|
7944
7966
|
});
|
|
7945
|
-
yield*
|
|
7967
|
+
yield* Effect37.logDebug("llm.generate-object").pipe(Effect37.annotateLogs({
|
|
7946
7968
|
scope: "@dimple/llm",
|
|
7947
7969
|
event: "llm.generate-object",
|
|
7948
7970
|
latencyMs: Date.now() - started
|
|
@@ -7962,7 +7984,7 @@ var openAICompatible = (options) => {
|
|
|
7962
7984
|
return provider.languageModel(options.modelId ?? "default");
|
|
7963
7985
|
};
|
|
7964
7986
|
// ../llm/src/dream-classifier.ts
|
|
7965
|
-
import { Effect as
|
|
7987
|
+
import { Effect as Effect38 } from "effect";
|
|
7966
7988
|
import { z as z2 } from "zod";
|
|
7967
7989
|
var EDGE_KINDS = ["memory_supersedes", "memory_contradicts", "memory_supports"];
|
|
7968
7990
|
var EDGE_KIND_SET = new Set(EDGE_KINDS);
|
|
@@ -8009,7 +8031,7 @@ var buildPrompt = (pairs, maxDerived) => [
|
|
|
8009
8031
|
].join(`
|
|
8010
8032
|
`);
|
|
8011
8033
|
var makeDreamClassifier = (deps) => {
|
|
8012
|
-
const classify =
|
|
8034
|
+
const classify = Effect38.fn("DreamClassifier.classify")(function* (pairs, opts) {
|
|
8013
8035
|
const system = deps.modelId === undefined ? SYSTEM_PROMPT : `${SYSTEM_PROMPT}
|
|
8014
8036
|
Judging for model: ${deps.modelId}`;
|
|
8015
8037
|
const judged = yield* deps.llm.generateObject({
|
|
@@ -8039,11 +8061,11 @@ Judging for model: ${deps.modelId}`;
|
|
|
8039
8061
|
return { classify };
|
|
8040
8062
|
};
|
|
8041
8063
|
// ../llm/src/service.ts
|
|
8042
|
-
import { Config as Config6, Context as
|
|
8043
|
-
class
|
|
8064
|
+
import { Config as Config6, Context as Context13, Effect as Effect39, Layer as Layer15 } from "effect";
|
|
8065
|
+
class Service12 extends Context13.Service()("@dimple/LLM") {
|
|
8044
8066
|
}
|
|
8045
8067
|
// ../core/src/layer.ts
|
|
8046
|
-
var embedderFromConfig = (model) => model.provider === "transformers" ?
|
|
8068
|
+
var embedderFromConfig = (model) => model.provider === "transformers" ? Effect40.succeed(makeTransformersModel({
|
|
8047
8069
|
modelId: model.model,
|
|
8048
8070
|
dimensions: model.dimensions,
|
|
8049
8071
|
...model.device !== undefined ? { device: model.device } : undefined,
|
|
@@ -8054,40 +8076,40 @@ var embedderFromConfig = (model) => model.provider === "transformers" ? Effect39
|
|
|
8054
8076
|
...model.autoInstall !== undefined ? { autoInstall: model.autoInstall } : undefined,
|
|
8055
8077
|
...model.sessionOptions !== undefined ? { sessionOptions: model.sessionOptions } : undefined,
|
|
8056
8078
|
...model.wasm !== undefined ? { wasm: model.wasm } : undefined
|
|
8057
|
-
})) : model.provider === "openai" ? model.apiKey === undefined ?
|
|
8079
|
+
})) : model.provider === "openai" ? model.apiKey === undefined ? Effect40.fail(new EmbeddingError({ message: `OpenAI model ${model.id} requires an apiKey` })) : Effect40.succeed(makeHostedEmbeddingModel({
|
|
8058
8080
|
modelId: model.id,
|
|
8059
8081
|
model: model.model,
|
|
8060
8082
|
dimensions: model.dimensions,
|
|
8061
8083
|
apiKey: model.apiKey,
|
|
8062
8084
|
...model.provider !== "openai" ? { provider: model.provider } : undefined,
|
|
8063
8085
|
...model.baseURL !== undefined ? { baseURL: model.baseURL } : undefined
|
|
8064
|
-
})) :
|
|
8065
|
-
var
|
|
8086
|
+
})) : Effect40.fail(new EmbeddingError({ message: `Unknown provider: ${model.provider}` }));
|
|
8087
|
+
var layer14 = (options) => {
|
|
8066
8088
|
const topicConfig = Schema29.decodeUnknownSync(TopicConfig)(options.topics?.config ?? {});
|
|
8067
8089
|
const graphConfig = Schema29.decodeUnknownSync(exports_graph.GraphConfig)(options.graph ?? {});
|
|
8068
8090
|
const models2 = new Map(options.models.map((m) => [m.id, m]));
|
|
8069
8091
|
const defaultModel = models2.get(options.defaultModelId);
|
|
8070
8092
|
const loggingConfig = options.logging ?? silentConfig;
|
|
8071
8093
|
const getModelConfig = (key) => models2.get(key);
|
|
8072
|
-
const storeWithSeams =
|
|
8073
|
-
const rt = yield*
|
|
8074
|
-
const enqueueSplit = (topicKey, modelKey) => rt.enqueue(TopicSplit, { topicId: topicKey, modelKey }).pipe(
|
|
8075
|
-
const enqueueMerge = (topicKey, modelKey) => rt.enqueue(TopicMerge, { topicId: topicKey, modelKey }).pipe(
|
|
8094
|
+
const storeWithSeams = Layer16.unwrap(Effect40.gen(function* () {
|
|
8095
|
+
const rt = yield* Service8;
|
|
8096
|
+
const enqueueSplit = (topicKey, modelKey) => rt.enqueue(TopicSplit, { topicId: topicKey, modelKey }).pipe(Effect40.asVoid);
|
|
8097
|
+
const enqueueMerge = (topicKey, modelKey) => rt.enqueue(TopicMerge, { topicId: topicKey, modelKey }).pipe(Effect40.asVoid);
|
|
8076
8098
|
const auditBuffer = [];
|
|
8077
|
-
const flushAudit =
|
|
8099
|
+
const flushAudit = Effect40.gen(function* () {
|
|
8078
8100
|
if (auditBuffer.length === 0)
|
|
8079
8101
|
return;
|
|
8080
8102
|
const rows = auditBuffer.splice(0, auditBuffer.length);
|
|
8081
|
-
yield* rt.enqueue(AuditWrite, { rows }).pipe(
|
|
8103
|
+
yield* rt.enqueue(AuditWrite, { rows }).pipe(Effect40.asVoid);
|
|
8082
8104
|
});
|
|
8083
|
-
const enqueueAudit = (row) =>
|
|
8105
|
+
const enqueueAudit = (row) => Effect40.gen(function* () {
|
|
8084
8106
|
auditBuffer.push(row);
|
|
8085
8107
|
if (auditBuffer.length >= 50) {
|
|
8086
8108
|
yield* flushAudit;
|
|
8087
8109
|
}
|
|
8088
8110
|
});
|
|
8089
|
-
yield* flushAudit.pipe(
|
|
8090
|
-
return
|
|
8111
|
+
yield* flushAudit.pipe(Effect40.catchCause((cause) => Effect40.logWarning("audit flush failed", cause)), Effect40.repeat(Schedule2.spaced(1000)), Effect40.forkScoped);
|
|
8112
|
+
return layer9(options.storeUrl, {
|
|
8091
8113
|
...options.defaultTtlSecs !== undefined ? { defaultTtlSecs: options.defaultTtlSecs } : undefined,
|
|
8092
8114
|
...options.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : undefined,
|
|
8093
8115
|
releaseClient: true,
|
|
@@ -8105,48 +8127,49 @@ var layer13 = (options) => {
|
|
|
8105
8127
|
...options.audit !== false ? { enqueueAudit } : undefined
|
|
8106
8128
|
});
|
|
8107
8129
|
}));
|
|
8108
|
-
const embedderRegistry =
|
|
8130
|
+
const embedderRegistry = Layer16.effect(Service10, Effect40.sync(() => Service10.of({
|
|
8109
8131
|
getModel: (id) => {
|
|
8110
8132
|
const model = models2.get(id);
|
|
8111
|
-
return model !== undefined ? options.embedderFor(model) :
|
|
8133
|
+
return model !== undefined ? options.embedderFor(model) : Effect40.fail(new EmbeddingError({ message: `Unknown model: ${id}` }));
|
|
8112
8134
|
},
|
|
8113
|
-
defaultModelId:
|
|
8114
|
-
registerModel: () =>
|
|
8135
|
+
defaultModelId: Effect40.succeed(options.defaultModelId),
|
|
8136
|
+
registerModel: () => Effect40.void
|
|
8115
8137
|
})));
|
|
8116
|
-
const coreService =
|
|
8138
|
+
const coreService = Layer16.effect(Service, Effect40.gen(function* () {
|
|
8117
8139
|
if (defaultModel === undefined) {
|
|
8118
8140
|
return yield* new CoreConfigError({
|
|
8119
8141
|
message: `@dimple/core: unknown defaultModelId "${options.defaultModelId}"`
|
|
8120
8142
|
});
|
|
8121
8143
|
}
|
|
8122
8144
|
const store2 = yield* Service5;
|
|
8123
|
-
const rt = yield*
|
|
8124
|
-
const
|
|
8145
|
+
const rt = yield* Service8;
|
|
8146
|
+
const queueStats = yield* Service7;
|
|
8147
|
+
const registry = yield* Service9;
|
|
8125
8148
|
const backend = yield* Service2;
|
|
8126
|
-
const hybrid = yield*
|
|
8149
|
+
const hybrid = yield* Service11;
|
|
8127
8150
|
const logging = yield* Service3;
|
|
8128
8151
|
const extraByName = new Map((options.extraHandlers ?? []).map((h) => [h.name, h.handler]));
|
|
8129
|
-
const register = (name, handler) => extraByName.has(name) ?
|
|
8152
|
+
const register = (name, handler) => extraByName.has(name) ? Effect40.void : registry.register(name, handler);
|
|
8130
8153
|
yield* register("topic.split", exports_topics.topicSplitHandler({
|
|
8131
8154
|
getModelConfig,
|
|
8132
8155
|
config: topicConfig,
|
|
8133
8156
|
backend,
|
|
8134
|
-
onUnevenSplit: (childKey, modelKey) => rt.enqueue(TopicRebalance, { topicId: childKey, modelKey }).pipe(
|
|
8135
|
-
onChildOverflow: (childKey, modelKey) => rt.enqueue(TopicSplit, { topicId: childKey, modelKey }).pipe(
|
|
8157
|
+
onUnevenSplit: (childKey, modelKey) => rt.enqueue(TopicRebalance, { topicId: childKey, modelKey }).pipe(Effect40.asVoid),
|
|
8158
|
+
onChildOverflow: (childKey, modelKey) => rt.enqueue(TopicSplit, { topicId: childKey, modelKey }).pipe(Effect40.asVoid)
|
|
8136
8159
|
}));
|
|
8137
8160
|
yield* register("topic.merge", exports_topics.topicMergeHandler({ getModelConfig, config: topicConfig, backend }));
|
|
8138
8161
|
yield* register("topic.rebalance", exports_topics.topicRebalanceHandler({ getModelConfig, config: topicConfig, backend }));
|
|
8139
8162
|
yield* register("memory.embed", exports_topics.memoryEmbedHandler({
|
|
8140
8163
|
getModelConfig,
|
|
8141
|
-
embedOne: (text4, model) => options.embedderFor(model).pipe(
|
|
8142
|
-
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(
|
|
8164
|
+
embedOne: (text4, model) => options.embedderFor(model).pipe(Effect40.flatMap((m) => m.embedSingle(text4))),
|
|
8165
|
+
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(Effect40.map((m) => m.content)),
|
|
8143
8166
|
config: topicConfig,
|
|
8144
8167
|
backend
|
|
8145
8168
|
}));
|
|
8146
8169
|
yield* register("memory.embed.batch", exports_topics.memoryEmbedBatchHandler({
|
|
8147
8170
|
getModelConfig,
|
|
8148
|
-
embedOne: (text4, model) => options.embedderFor(model).pipe(
|
|
8149
|
-
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(
|
|
8171
|
+
embedOne: (text4, model) => options.embedderFor(model).pipe(Effect40.flatMap((m) => m.embedSingle(text4))),
|
|
8172
|
+
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(Effect40.map((m) => m.content)),
|
|
8150
8173
|
config: topicConfig,
|
|
8151
8174
|
backend
|
|
8152
8175
|
}));
|
|
@@ -8161,12 +8184,12 @@ var layer13 = (options) => {
|
|
|
8161
8184
|
yield* register("memory.dream", (payload, ctx) => {
|
|
8162
8185
|
const envelope = payload;
|
|
8163
8186
|
if (getModelConfig(envelope.modelKey) === undefined) {
|
|
8164
|
-
return
|
|
8187
|
+
return Effect40.fail(new JobFailure({
|
|
8165
8188
|
jobId: ctx.jobId,
|
|
8166
8189
|
message: `memory.dream: unknown model "${envelope.modelKey}"`
|
|
8167
8190
|
}));
|
|
8168
8191
|
}
|
|
8169
|
-
return
|
|
8192
|
+
return Effect40.fail(new JobFailure({
|
|
8170
8193
|
jobId: ctx.jobId,
|
|
8171
8194
|
message: "memory.dream: LLM not configured — provide `llm` to @dimple/core"
|
|
8172
8195
|
}));
|
|
@@ -8181,6 +8204,7 @@ var layer13 = (options) => {
|
|
|
8181
8204
|
store: store2,
|
|
8182
8205
|
hybrid,
|
|
8183
8206
|
runtime: rt,
|
|
8207
|
+
queueStats,
|
|
8184
8208
|
registry,
|
|
8185
8209
|
models: models2,
|
|
8186
8210
|
defaultModel,
|
|
@@ -8210,7 +8234,7 @@ var layer13 = (options) => {
|
|
|
8210
8234
|
});
|
|
8211
8235
|
return compile(group([jobStack, jobRegistry, store, embedder, retrieval, workers, core]));
|
|
8212
8236
|
};
|
|
8213
|
-
var layerConfig7 = (options) =>
|
|
8237
|
+
var layerConfig7 = (options) => Layer16.unwrap(Effect40.gen(function* () {
|
|
8214
8238
|
const cfg = yield* Service4;
|
|
8215
8239
|
const embedding = cfg.config.embedding;
|
|
8216
8240
|
if (embedding === undefined || embedding.models.length === 0) {
|
|
@@ -8232,7 +8256,7 @@ var layerConfig7 = (options) => Layer15.unwrap(Effect39.gen(function* () {
|
|
|
8232
8256
|
...sessionOptions !== undefined ? { sessionOptions } : undefined
|
|
8233
8257
|
};
|
|
8234
8258
|
});
|
|
8235
|
-
return
|
|
8259
|
+
return layer14({
|
|
8236
8260
|
storeUrl: cfg.config.storeUrl,
|
|
8237
8261
|
...cfg.config.defaultK !== undefined ? { defaultK: cfg.config.defaultK } : undefined,
|
|
8238
8262
|
...cfg.config.defaultLimit !== undefined ? { defaultLimit: cfg.config.defaultLimit } : undefined,
|
|
@@ -8374,10 +8398,10 @@ var remapError = (cause) => {
|
|
|
8374
8398
|
};
|
|
8375
8399
|
|
|
8376
8400
|
// src/make.ts
|
|
8377
|
-
import { Effect as
|
|
8401
|
+
import { Effect as Effect51, Exit as Exit8, Layer as Layer27, ManagedRuntime, Scope as Scope5 } from "effect";
|
|
8378
8402
|
|
|
8379
8403
|
// ../extractor/src/code-extractor.ts
|
|
8380
|
-
import { Context as
|
|
8404
|
+
import { Context as Context14, Effect as Effect41, Layer as Layer17 } from "effect";
|
|
8381
8405
|
|
|
8382
8406
|
// ../extractor/src/models/extraction.ts
|
|
8383
8407
|
import { Schema as Schema31 } from "effect";
|
|
@@ -8821,11 +8845,11 @@ function extract3(filePath, tree2) {
|
|
|
8821
8845
|
}
|
|
8822
8846
|
|
|
8823
8847
|
// ../extractor/src/code-extractor.ts
|
|
8824
|
-
class
|
|
8848
|
+
class Service13 extends Context14.Service()("@dimple/extractor/CodeExtractor") {
|
|
8825
8849
|
}
|
|
8826
|
-
var
|
|
8827
|
-
extract:
|
|
8828
|
-
const result = yield*
|
|
8850
|
+
var layer15 = Layer17.effect(Service13, Effect41.succeed(Service13.of({
|
|
8851
|
+
extract: Effect41.fn("CodeExtractor.extract")(function* (request) {
|
|
8852
|
+
const result = yield* Effect41.try({
|
|
8829
8853
|
try: () => extract3(request.filePath, request.tree),
|
|
8830
8854
|
catch: (cause) => new ExtractionError({ message: `Extraction failed for ${request.filePath}`, cause })
|
|
8831
8855
|
});
|
|
@@ -8841,10 +8865,10 @@ var layer14 = Layer16.effect(Service12, Effect40.succeed(Service12.of({
|
|
|
8841
8865
|
});
|
|
8842
8866
|
})
|
|
8843
8867
|
})));
|
|
8844
|
-
var configured10 = () => make({ name: "CodeExtractor", layer:
|
|
8868
|
+
var configured10 = () => make({ name: "CodeExtractor", layer: layer15, deps: [] });
|
|
8845
8869
|
var node6 = configured10();
|
|
8846
8870
|
// ../extractor/src/content-handler.ts
|
|
8847
|
-
import { Context as
|
|
8871
|
+
import { Context as Context15, Layer as Layer18, Schema as Schema32 } from "effect";
|
|
8848
8872
|
|
|
8849
8873
|
class ContentError extends Schema32.TaggedError()("Content.ContentError", {
|
|
8850
8874
|
contentType: Schema32.String,
|
|
@@ -8852,22 +8876,22 @@ class ContentError extends Schema32.TaggedError()("Content.ContentError", {
|
|
|
8852
8876
|
}) {
|
|
8853
8877
|
}
|
|
8854
8878
|
|
|
8855
|
-
class
|
|
8879
|
+
class Service14 extends Context15.Service()("@dimple/extractor/ContentHandler") {
|
|
8856
8880
|
}
|
|
8857
|
-
var
|
|
8881
|
+
var layer16 = Layer18.succeed(Service14, Service14.of({
|
|
8858
8882
|
contentType: "text/plain",
|
|
8859
8883
|
renderText: (content) => content,
|
|
8860
8884
|
extractText: (content) => content
|
|
8861
8885
|
}));
|
|
8862
8886
|
// ../parser/src/code-parser.ts
|
|
8863
|
-
import { Context as
|
|
8887
|
+
import { Context as Context21, Effect as Effect48, Layer as Layer25 } from "effect";
|
|
8864
8888
|
import { Parser as Parser2 } from "web-tree-sitter";
|
|
8865
8889
|
|
|
8866
8890
|
// ../parser/src/language-store.ts
|
|
8867
|
-
import { Cache, Context as
|
|
8891
|
+
import { Cache, Context as Context20, Effect as Effect47, Layer as Layer24 } from "effect";
|
|
8868
8892
|
|
|
8869
8893
|
// ../wasm/src/wasm-registry.ts
|
|
8870
|
-
import { Context as
|
|
8894
|
+
import { Context as Context16, Effect as Effect42, Layer as Layer19, Schema as Schema36 } from "effect";
|
|
8871
8895
|
|
|
8872
8896
|
// ../shared/src/errors/wasm.ts
|
|
8873
8897
|
import { Schema as Schema33 } from "effect";
|
|
@@ -8967,9 +8991,9 @@ for (const grammar of grammars.grammars) {
|
|
|
8967
8991
|
registryMap.set(grammar.lang, grammar);
|
|
8968
8992
|
}
|
|
8969
8993
|
|
|
8970
|
-
class
|
|
8971
|
-
static layer =
|
|
8972
|
-
resolve:
|
|
8994
|
+
class Service15 extends Context16.Service()("@dimple/wasm/TreeSitterRegistry") {
|
|
8995
|
+
static layer = Layer19.succeed(Service15, Service15.of({
|
|
8996
|
+
resolve: Effect42.fn("TreeSitterRegistry.resolve")(function* (lang) {
|
|
8973
8997
|
const found = registryMap.get(lang);
|
|
8974
8998
|
if (found === undefined) {
|
|
8975
8999
|
return yield* new RegistryError2({ message: `Language not found in registry: ${lang}` });
|
|
@@ -8979,11 +9003,11 @@ class Service14 extends Context15.Service()("@dimple/wasm/TreeSitterRegistry") {
|
|
|
8979
9003
|
}));
|
|
8980
9004
|
}
|
|
8981
9005
|
// ../wasm/src/wasm-downloader.ts
|
|
8982
|
-
import { Context as
|
|
8983
|
-
class
|
|
8984
|
-
static layer =
|
|
8985
|
-
download:
|
|
8986
|
-
const response = yield*
|
|
9006
|
+
import { Context as Context17, Effect as Effect43, Layer as Layer20 } from "effect";
|
|
9007
|
+
class Service16 extends Context17.Service()("@dimple/wasm/WasmDownloader") {
|
|
9008
|
+
static layer = Layer20.effect(Service16, Effect43.succeed(Service16.of({
|
|
9009
|
+
download: Effect43.fn("WasmDownloader.download")(function* (url) {
|
|
9010
|
+
const response = yield* Effect43.tryPromise({
|
|
8987
9011
|
try: () => fetch(url),
|
|
8988
9012
|
catch: (cause) => new WasmDownloadError({
|
|
8989
9013
|
message: `Failed to fetch from ${url}`,
|
|
@@ -8997,7 +9021,7 @@ class Service15 extends Context16.Service()("@dimple/wasm/WasmDownloader") {
|
|
|
8997
9021
|
url
|
|
8998
9022
|
});
|
|
8999
9023
|
}
|
|
9000
|
-
const arrayBuffer = yield*
|
|
9024
|
+
const arrayBuffer = yield* Effect43.tryPromise({
|
|
9001
9025
|
try: () => response.arrayBuffer(),
|
|
9002
9026
|
catch: (cause) => new WasmDownloadError({
|
|
9003
9027
|
message: `Failed to read response from ${url}`,
|
|
@@ -9010,7 +9034,7 @@ class Service15 extends Context16.Service()("@dimple/wasm/WasmDownloader") {
|
|
|
9010
9034
|
})));
|
|
9011
9035
|
}
|
|
9012
9036
|
// ../wasm/src/wasm-cache.ts
|
|
9013
|
-
import { Context as
|
|
9037
|
+
import { Context as Context18, Effect as Effect44, Layer as Layer21 } from "effect";
|
|
9014
9038
|
import { existsSync as existsSync3, mkdirSync, renameSync, writeFileSync } from "node:fs";
|
|
9015
9039
|
import { join as join3 } from "node:path";
|
|
9016
9040
|
|
|
@@ -9105,37 +9129,37 @@ function envPaths(name, { suffix = "nodejs" } = {}) {
|
|
|
9105
9129
|
// ../wasm/src/wasm-cache.ts
|
|
9106
9130
|
var paths = envPaths("dimple");
|
|
9107
9131
|
|
|
9108
|
-
class
|
|
9132
|
+
class Service17 extends Context18.Service()("@dimple/wasm/WasmCache") {
|
|
9109
9133
|
}
|
|
9110
|
-
var
|
|
9111
|
-
ensure:
|
|
9134
|
+
var layer17 = Layer21.effect(Service17, Effect44.succeed(Service17.of({
|
|
9135
|
+
ensure: Effect44.fn("WasmCache.ensure")(function* (lang, version, download) {
|
|
9112
9136
|
const versionDir = join3(paths.cache, "tree-sitter", lang, version);
|
|
9113
9137
|
const finalPath = join3(versionDir, "parser.wasm");
|
|
9114
|
-
const exists = yield*
|
|
9138
|
+
const exists = yield* Effect44.try({
|
|
9115
9139
|
try: () => existsSync3(finalPath),
|
|
9116
9140
|
catch: (cause) => new WasmCacheError({ message: `Failed to stat ${finalPath}`, cause })
|
|
9117
9141
|
});
|
|
9118
9142
|
if (exists)
|
|
9119
9143
|
return finalPath;
|
|
9120
|
-
yield*
|
|
9144
|
+
yield* Effect44.try({
|
|
9121
9145
|
try: () => mkdirSync(versionDir, { recursive: true }),
|
|
9122
9146
|
catch: (cause) => new WasmCacheError({ message: `Failed to create ${versionDir}`, cause })
|
|
9123
9147
|
});
|
|
9124
9148
|
const tempPath = join3(versionDir, `.tmp-${process.pid}`);
|
|
9125
9149
|
const wasm = yield* download;
|
|
9126
|
-
yield*
|
|
9150
|
+
yield* Effect44.try({
|
|
9127
9151
|
try: () => writeFileSync(tempPath, wasm),
|
|
9128
9152
|
catch: (cause) => new WasmCacheError({ message: `Failed to write ${tempPath}`, cause })
|
|
9129
9153
|
});
|
|
9130
|
-
const renamed = yield*
|
|
9154
|
+
const renamed = yield* Effect44.try({
|
|
9131
9155
|
try: () => {
|
|
9132
9156
|
renameSync(tempPath, finalPath);
|
|
9133
9157
|
return true;
|
|
9134
9158
|
},
|
|
9135
9159
|
catch: (cause) => new WasmCacheError({ message: `Failed to rename ${tempPath}`, cause })
|
|
9136
|
-
}).pipe(
|
|
9160
|
+
}).pipe(Effect44.catchCause(() => Effect44.succeed(false)));
|
|
9137
9161
|
if (!renamed) {
|
|
9138
|
-
const existsAfter = yield*
|
|
9162
|
+
const existsAfter = yield* Effect44.try({
|
|
9139
9163
|
try: () => existsSync3(finalPath),
|
|
9140
9164
|
catch: (cause) => new WasmCacheError({ message: `Failed to stat ${finalPath}`, cause })
|
|
9141
9165
|
});
|
|
@@ -9149,72 +9173,72 @@ var layer16 = Layer20.effect(Service16, Effect43.succeed(Service16.of({
|
|
|
9149
9173
|
})
|
|
9150
9174
|
})));
|
|
9151
9175
|
// ../wasm/src/grammar-store.ts
|
|
9152
|
-
import { Context as
|
|
9153
|
-
class
|
|
9176
|
+
import { Context as Context19, Effect as Effect45, Layer as Layer22 } from "effect";
|
|
9177
|
+
class Service18 extends Context19.Service()("@dimple/wasm/GrammarStore") {
|
|
9154
9178
|
}
|
|
9155
|
-
var
|
|
9156
|
-
const registry = yield*
|
|
9157
|
-
const downloader = yield*
|
|
9158
|
-
const cache = yield*
|
|
9159
|
-
const ensure =
|
|
9160
|
-
const metadata = yield* registry.resolve(lang).pipe(
|
|
9179
|
+
var layer18 = Layer22.effect(Service18, Effect45.gen(function* () {
|
|
9180
|
+
const registry = yield* Service15;
|
|
9181
|
+
const downloader = yield* Service16;
|
|
9182
|
+
const cache = yield* Service17;
|
|
9183
|
+
const ensure = Effect45.fn("GrammarStore.ensure")(function* (lang) {
|
|
9184
|
+
const metadata = yield* registry.resolve(lang).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
9161
9185
|
language: lang,
|
|
9162
9186
|
message: `Failed to resolve grammar metadata for: ${lang}`,
|
|
9163
9187
|
cause
|
|
9164
9188
|
})));
|
|
9165
|
-
const wasmBytes = yield* downloader.download(metadata.url).pipe(
|
|
9189
|
+
const wasmBytes = yield* downloader.download(metadata.url).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
9166
9190
|
language: lang,
|
|
9167
9191
|
message: `Failed to download grammar for: ${lang}`,
|
|
9168
9192
|
cause
|
|
9169
9193
|
})));
|
|
9170
|
-
const path2 = yield* cache.ensure(lang, metadata.version,
|
|
9194
|
+
const path2 = yield* cache.ensure(lang, metadata.version, Effect45.succeed(wasmBytes)).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
9171
9195
|
language: lang,
|
|
9172
9196
|
message: `Failed to provision grammar for: ${lang}`,
|
|
9173
9197
|
cause
|
|
9174
9198
|
})));
|
|
9175
9199
|
return path2;
|
|
9176
9200
|
});
|
|
9177
|
-
return
|
|
9201
|
+
return Service18.of({ ensure });
|
|
9178
9202
|
}));
|
|
9179
|
-
var allLive =
|
|
9203
|
+
var allLive = layer18.pipe(Layer22.provideMerge(Layer22.mergeAll(Service15.layer, Service16.layer, layer17)));
|
|
9180
9204
|
var configured11 = () => make({ name: "GrammarStore", layer: allLive, deps: [] });
|
|
9181
9205
|
var node7 = configured11();
|
|
9182
9206
|
// ../parser/src/language-store.ts
|
|
9183
9207
|
import { Language } from "web-tree-sitter";
|
|
9184
9208
|
|
|
9185
9209
|
// ../parser/src/tree-sitter-engine.ts
|
|
9186
|
-
import { Effect as
|
|
9210
|
+
import { Effect as Effect46, Layer as Layer23 } from "effect";
|
|
9187
9211
|
import { Parser } from "web-tree-sitter";
|
|
9188
|
-
var
|
|
9212
|
+
var layer19 = Layer23.effectDiscard(Effect46.tryPromise({
|
|
9189
9213
|
try: () => Parser.init(),
|
|
9190
9214
|
catch: (cause) => new ParserInitError({ message: "Failed to initialize tree sitter", cause })
|
|
9191
9215
|
}));
|
|
9192
9216
|
|
|
9193
9217
|
// ../parser/src/language-store.ts
|
|
9194
|
-
class
|
|
9218
|
+
class Service19 extends Context20.Service()("@dimple/parser/LanguageStore") {
|
|
9195
9219
|
}
|
|
9196
|
-
var
|
|
9197
|
-
const grammarStore = yield*
|
|
9220
|
+
var layer20 = Layer24.effect(Service19, Effect47.gen(function* () {
|
|
9221
|
+
const grammarStore = yield* Service18;
|
|
9198
9222
|
const resolveLang = (lang) => lang === "jsx" ? "javascript" : lang;
|
|
9199
9223
|
const cache = yield* Cache.make({
|
|
9200
9224
|
capacity: 100,
|
|
9201
9225
|
timeToLive: Infinity,
|
|
9202
|
-
lookup: (lang) =>
|
|
9226
|
+
lookup: (lang) => Effect47.gen(function* () {
|
|
9203
9227
|
const grammarLang = resolveLang(lang);
|
|
9204
|
-
const wasmPath = yield* grammarStore.ensure(grammarLang).pipe(
|
|
9228
|
+
const wasmPath = yield* grammarStore.ensure(grammarLang).pipe(Effect47.mapError((e) => new LanguageLoadError({
|
|
9205
9229
|
lang,
|
|
9206
9230
|
message: `Failed to provision grammar for ${lang}`,
|
|
9207
9231
|
cause: e
|
|
9208
9232
|
})));
|
|
9209
|
-
return yield*
|
|
9233
|
+
return yield* Effect47.tryPromise({
|
|
9210
9234
|
try: () => Language.load(wasmPath),
|
|
9211
9235
|
catch: (cause) => new LanguageLoadError({ lang, message: `Failed to load language: ${lang}`, cause })
|
|
9212
9236
|
});
|
|
9213
9237
|
})
|
|
9214
9238
|
});
|
|
9215
9239
|
const get = (lang) => Cache.get(cache, resolveLang(lang));
|
|
9216
|
-
return
|
|
9217
|
-
})).pipe(
|
|
9240
|
+
return Service19.of({ get });
|
|
9241
|
+
})).pipe(Layer24.provide(Layer24.mergeAll(allLive, layer19)));
|
|
9218
9242
|
|
|
9219
9243
|
// ../parser/src/models/treeHandle.ts
|
|
9220
9244
|
import { Schema as Schema37 } from "effect";
|
|
@@ -9227,13 +9251,13 @@ var TreeHandle = Schema37.Struct({
|
|
|
9227
9251
|
});
|
|
9228
9252
|
|
|
9229
9253
|
// ../parser/src/code-parser.ts
|
|
9230
|
-
class
|
|
9254
|
+
class Service20 extends Context21.Service()("@dimple/parser/CodeParser") {
|
|
9231
9255
|
}
|
|
9232
|
-
var
|
|
9233
|
-
const languageStore = yield*
|
|
9256
|
+
var layer21 = Layer25.effect(Service20, Effect48.gen(function* () {
|
|
9257
|
+
const languageStore = yield* Service19;
|
|
9234
9258
|
const parserByLanguage = new Map;
|
|
9235
|
-
const parse2 =
|
|
9236
|
-
const language = yield* languageStore.get(request.lang).pipe(
|
|
9259
|
+
const parse2 = Effect48.fn("CodeParser.parse")(function* (request) {
|
|
9260
|
+
const language = yield* languageStore.get(request.lang).pipe(Effect48.mapError((cause) => new ParseError({
|
|
9237
9261
|
message: `Failed to load language '${request.lang}' for parsing`,
|
|
9238
9262
|
cause
|
|
9239
9263
|
})));
|
|
@@ -9249,9 +9273,9 @@ var layer20 = Layer24.effect(Service19, Effect47.gen(function* () {
|
|
|
9249
9273
|
}
|
|
9250
9274
|
return TreeHandle.make({ tree: tree2, lang: request.lang, source: request.source });
|
|
9251
9275
|
});
|
|
9252
|
-
return
|
|
9253
|
-
})).pipe(
|
|
9254
|
-
var configured12 = () => make({ name: "CodeParser", layer:
|
|
9276
|
+
return Service20.of({ parse: parse2 });
|
|
9277
|
+
})).pipe(Layer25.provide(layer20));
|
|
9278
|
+
var configured12 = () => make({ name: "CodeParser", layer: layer21, deps: [] });
|
|
9255
9279
|
var node8 = configured12();
|
|
9256
9280
|
// ../parser/src/models/parseRequest.ts
|
|
9257
9281
|
import { Schema as Schema38 } from "effect";
|
|
@@ -9260,15 +9284,15 @@ var ParseRequest = Schema38.Struct({
|
|
|
9260
9284
|
source: Schema38.String
|
|
9261
9285
|
});
|
|
9262
9286
|
// src/config.ts
|
|
9263
|
-
import { Effect as
|
|
9287
|
+
import { Effect as Effect49, Layer as Layer26, Option as Option14, Schema as Schema39 } from "effect";
|
|
9264
9288
|
var isString = (value2) => Option14.isSome(Schema39.decodeUnknownOption(Schema39.String)(value2));
|
|
9265
|
-
var resolveConfigInput =
|
|
9266
|
-
return yield* (isString(input) ? loadConfigFromFile(input) : loadConfig(input)).pipe(
|
|
9289
|
+
var resolveConfigInput = Effect49.fn("Sdk.resolveConfigInput")(function* (input) {
|
|
9290
|
+
return yield* (isString(input) ? loadConfigFromFile(input) : loadConfig(input)).pipe(Effect49.mapError((error) => new ConfigError({ message: error.message })));
|
|
9267
9291
|
});
|
|
9268
|
-
var configLayerFromInput = (input) =>
|
|
9292
|
+
var configLayerFromInput = (input) => Layer26.effect(Service4, resolveConfigInput(input).pipe(Effect49.map((config) => Service4.of({ config }))));
|
|
9269
9293
|
|
|
9270
9294
|
// src/dimple.ts
|
|
9271
|
-
import { Cause as Cause5, Effect as
|
|
9295
|
+
import { Cause as Cause5, Effect as Effect50, Exit as Exit7, Option as Option15, Schema as Schema40 } from "effect";
|
|
9272
9296
|
|
|
9273
9297
|
// src/indexer.ts
|
|
9274
9298
|
import { execFileSync } from "node:child_process";
|
|
@@ -9517,7 +9541,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9517
9541
|
}
|
|
9518
9542
|
return;
|
|
9519
9543
|
};
|
|
9520
|
-
const runQuery = (text4, options) => run(
|
|
9544
|
+
const runQuery = (text4, options) => run(Effect50.gen(function* () {
|
|
9521
9545
|
if (!asString(text4)) {
|
|
9522
9546
|
return yield* new InputError({ message: "query: text must be a string" });
|
|
9523
9547
|
}
|
|
@@ -9530,7 +9554,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9530
9554
|
let rows = yield* core().hybrid.hybrid(text4, {
|
|
9531
9555
|
...rest,
|
|
9532
9556
|
...capLimit !== undefined ? { limit: capLimit } : undefined
|
|
9533
|
-
}).pipe(
|
|
9557
|
+
}).pipe(Effect50.map((r) => [...r]));
|
|
9534
9558
|
let truncated = false;
|
|
9535
9559
|
if (budget?.maxChars !== undefined && budget.maxChars > 0) {
|
|
9536
9560
|
let used = 0;
|
|
@@ -9617,7 +9641,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9617
9641
|
}
|
|
9618
9642
|
return;
|
|
9619
9643
|
};
|
|
9620
|
-
const writeUnit = (input, embed) =>
|
|
9644
|
+
const writeUnit = (input, embed) => Effect50.gen(function* () {
|
|
9621
9645
|
const mem = yield* core().store.write(input);
|
|
9622
9646
|
if (embed) {
|
|
9623
9647
|
yield* core().store.ensureTopicTree(core().defaultModel.id);
|
|
@@ -9627,8 +9651,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9627
9651
|
}
|
|
9628
9652
|
return mem;
|
|
9629
9653
|
});
|
|
9630
|
-
const upsertUnit = (input, embed) =>
|
|
9631
|
-
const existing = yield* core().store.getByKey(input.logicalKey, input.namespace ?? "default").pipe(
|
|
9654
|
+
const upsertUnit = (input, embed) => Effect50.gen(function* () {
|
|
9655
|
+
const existing = yield* core().store.getByKey(input.logicalKey, input.namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9632
9656
|
if (existing !== undefined && existing.content === input.content) {
|
|
9633
9657
|
const stamp = codeVersionOf(input.metadata);
|
|
9634
9658
|
if (stamp !== undefined && codeVersionOf(existing.metadata) === undefined) {
|
|
@@ -9693,7 +9717,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9693
9717
|
return;
|
|
9694
9718
|
};
|
|
9695
9719
|
const messageOf3 = (cause) => cause instanceof Error ? cause.message : String(cause);
|
|
9696
|
-
const resolveMemory = (idOrKey) => core().store.get(idOrKey).pipe(
|
|
9720
|
+
const resolveMemory = (idOrKey) => core().store.get(idOrKey).pipe(Effect50.catchTag("Store.NotFound", () => core().store.getByKey(idOrKey, "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.fail(new NotFoundError({ id: idOrKey, message: `Memory not found: ${idOrKey}` }))))));
|
|
9697
9721
|
const codeVersionOf = (blob) => {
|
|
9698
9722
|
if (blob === undefined || blob === null)
|
|
9699
9723
|
return;
|
|
@@ -9713,13 +9737,13 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9713
9737
|
get defaultModelId() {
|
|
9714
9738
|
return core().defaultModel.id;
|
|
9715
9739
|
},
|
|
9716
|
-
write: (input, options) => run(
|
|
9740
|
+
write: (input, options) => run(Effect50.gen(function* () {
|
|
9717
9741
|
const invalid = validateWriteInput(input);
|
|
9718
9742
|
if (invalid !== undefined)
|
|
9719
9743
|
return yield* invalid;
|
|
9720
9744
|
return yield* writeUnit(input, options?.embed !== false);
|
|
9721
9745
|
})),
|
|
9722
|
-
writeMany: (units, options) => run(
|
|
9746
|
+
writeMany: (units, options) => run(Effect50.gen(function* () {
|
|
9723
9747
|
if (!Array.isArray(units)) {
|
|
9724
9748
|
return yield* new InputError({ message: "writeMany: units must be an array" });
|
|
9725
9749
|
}
|
|
@@ -9739,7 +9763,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9739
9763
|
vectors.push(...vs);
|
|
9740
9764
|
}
|
|
9741
9765
|
yield* core().store.updateEmbeddingMany(mems.map((mem, i) => ({ id: mem.id, embedding: [...vectors[i]] })), core().defaultModel.id);
|
|
9742
|
-
return yield*
|
|
9766
|
+
return yield* Effect50.forEach(mems, (mem) => core().store.get(mem.id), {
|
|
9743
9767
|
discard: false
|
|
9744
9768
|
});
|
|
9745
9769
|
}
|
|
@@ -9747,8 +9771,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9747
9771
|
})),
|
|
9748
9772
|
get: (id) => run(resolveMemory(id)),
|
|
9749
9773
|
query: runQuery,
|
|
9750
|
-
forget: (id) => run(resolveMemory(id).pipe(
|
|
9751
|
-
index: (options) => run(
|
|
9774
|
+
forget: (id) => run(resolveMemory(id).pipe(Effect50.flatMap((memory) => core().store.forget(memory.id)))),
|
|
9775
|
+
index: (options) => run(Effect50.gen(function* () {
|
|
9752
9776
|
const invalid = validateIndexOptions(options);
|
|
9753
9777
|
if (invalid !== undefined)
|
|
9754
9778
|
return yield* invalid;
|
|
@@ -9760,7 +9784,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9760
9784
|
const maxFileBytes = options.maxFileBytes ?? 64 * 1024;
|
|
9761
9785
|
const maxChunkBytes = options.maxChunkBytes ?? 8 * 1024;
|
|
9762
9786
|
const extensions = [...options.extensions ?? INDEXABLE_EXTENSIONS];
|
|
9763
|
-
const walk = yield*
|
|
9787
|
+
const walk = yield* Effect50.try({
|
|
9764
9788
|
try: () => walkFiles([...options.paths], extensions),
|
|
9765
9789
|
catch: (cause) => new InputError({
|
|
9766
9790
|
message: `index: failed to walk paths: ${messageOf3(cause)}`
|
|
@@ -9793,8 +9817,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9793
9817
|
if (language === undefined)
|
|
9794
9818
|
continue;
|
|
9795
9819
|
const chunkKeyOf = (chunkId) => `${keyPrefix}${relative}#${stableChunkId(chunkId)}`;
|
|
9796
|
-
const exit = yield*
|
|
9797
|
-
const source = yield*
|
|
9820
|
+
const exit = yield* Effect50.exit(Effect50.gen(function* () {
|
|
9821
|
+
const source = yield* Effect50.try({
|
|
9798
9822
|
try: () => readCapped(filePath, maxFileBytes),
|
|
9799
9823
|
catch: (cause) => new InputError({ message: `read failed: ${messageOf3(cause)}` })
|
|
9800
9824
|
});
|
|
@@ -9873,7 +9897,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9873
9897
|
unitIds.set(key, id);
|
|
9874
9898
|
freshIds.push(id);
|
|
9875
9899
|
};
|
|
9876
|
-
const upsertAll = (candidates2) =>
|
|
9900
|
+
const upsertAll = (candidates2) => Effect50.gen(function* () {
|
|
9877
9901
|
const ids = [];
|
|
9878
9902
|
for (const spec of candidates2) {
|
|
9879
9903
|
const outcome = yield* upsertUnit(spec.input, embedUnits);
|
|
@@ -9885,7 +9909,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9885
9909
|
if (deferEmbed) {
|
|
9886
9910
|
const fresh = [];
|
|
9887
9911
|
for (const spec of specs) {
|
|
9888
|
-
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(
|
|
9912
|
+
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9889
9913
|
if (existing === undefined) {
|
|
9890
9914
|
fresh.push(spec);
|
|
9891
9915
|
continue;
|
|
@@ -9913,7 +9937,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9913
9937
|
} else {
|
|
9914
9938
|
const fresh = [];
|
|
9915
9939
|
for (const spec of specs) {
|
|
9916
|
-
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(
|
|
9940
|
+
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9917
9941
|
if (existing === undefined) {
|
|
9918
9942
|
fresh.push(spec);
|
|
9919
9943
|
continue;
|
|
@@ -9981,14 +10005,14 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9981
10005
|
const resolveKey = (key) => {
|
|
9982
10006
|
const fromMap = unitIds.get(key);
|
|
9983
10007
|
if (fromMap !== undefined)
|
|
9984
|
-
return
|
|
9985
|
-
return core().store.getByKey(key, namespace ?? "default").pipe(
|
|
10008
|
+
return Effect50.succeed(fromMap);
|
|
10009
|
+
return core().store.getByKey(key, namespace ?? "default").pipe(Effect50.map((m) => m.id), Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9986
10010
|
};
|
|
9987
|
-
const edgeEffect = (edge) =>
|
|
9988
|
-
const [from, to] = yield*
|
|
10011
|
+
const edgeEffect = (edge) => Effect50.gen(function* () {
|
|
10012
|
+
const [from, to] = yield* Effect50.all([resolveKey(edge.from), resolveKey(edge.to)]);
|
|
9989
10013
|
if (from === undefined || to === undefined || from === to)
|
|
9990
10014
|
return;
|
|
9991
|
-
yield*
|
|
10015
|
+
yield* Effect50.exit(core().store.addEdge({
|
|
9992
10016
|
fromMemoryId: from,
|
|
9993
10017
|
toMemoryId: to,
|
|
9994
10018
|
edgeKind: edge.edgeKind,
|
|
@@ -9998,10 +10022,10 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9998
10022
|
}));
|
|
9999
10023
|
});
|
|
10000
10024
|
for (const edge of pendingEdges) {
|
|
10001
|
-
yield*
|
|
10025
|
+
yield* Effect50.exit(edgeEffect({ ...edge, edgeKind: "memory_supports" }));
|
|
10002
10026
|
}
|
|
10003
10027
|
for (const edge of pendingCallEdges) {
|
|
10004
|
-
yield*
|
|
10028
|
+
yield* Effect50.exit(edgeEffect({ ...edge, edgeKind: "code_calls" }));
|
|
10005
10029
|
}
|
|
10006
10030
|
const EMBED_BATCH = 512;
|
|
10007
10031
|
if (deferEmbed) {
|
|
@@ -10025,21 +10049,27 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
10025
10049
|
...deferEmbed ? { pendingEmbeddings: pendingEmbedIds.length } : undefined
|
|
10026
10050
|
};
|
|
10027
10051
|
})),
|
|
10028
|
-
updateEmbedding: (id, embedding, modelKey) => run(
|
|
10052
|
+
updateEmbedding: (id, embedding, modelKey) => run(Effect50.gen(function* () {
|
|
10029
10053
|
if (!Array.isArray(embedding) || embedding.length === 0 || embedding.some((v) => !asNumber(v) || !Number.isFinite(v))) {
|
|
10030
10054
|
return yield* new InputError({
|
|
10031
10055
|
message: "updateEmbedding: embedding must be a non-empty array of finite numbers"
|
|
10032
10056
|
});
|
|
10033
10057
|
}
|
|
10034
10058
|
const key = modelKey ?? core().defaultModel.id;
|
|
10059
|
+
const model = core().models.get(key);
|
|
10060
|
+
if (model !== undefined && embedding.length !== model.dimensions) {
|
|
10061
|
+
return yield* new InputError({
|
|
10062
|
+
message: `updateEmbedding: model "${key}" expects ${model.dimensions}-dimensional vectors, got ${embedding.length}`
|
|
10063
|
+
});
|
|
10064
|
+
}
|
|
10035
10065
|
const memory = yield* resolveMemory(id);
|
|
10036
10066
|
yield* core().store.ensureTopicTree(key);
|
|
10037
10067
|
yield* core().store.updateEmbedding(memory.id, [...embedding], key);
|
|
10038
10068
|
})),
|
|
10039
|
-
embed: (text4) => run(core().defaultEmbedder.embedSingle(text4).pipe(
|
|
10040
|
-
dream: (modelKey) => run(core().runtime.enqueue(MemoryDream, { modelKey, runId: crypto.randomUUID() }).pipe(
|
|
10069
|
+
embed: (text4) => run(core().defaultEmbedder.embedSingle(text4).pipe(Effect50.map((vector) => [...vector]))),
|
|
10070
|
+
dream: (modelKey) => run(core().runtime.enqueue(MemoryDream, { modelKey, runId: crypto.randomUUID() }).pipe(Effect50.flatMap((jobId) => core().runtime.wait(jobId)), Effect50.asVoid)),
|
|
10041
10071
|
dbSchema: () => run(core().store.listSchema()),
|
|
10042
|
-
repair: (modelKey) => run(core().runtime.enqueue(GraphRepair, { modelKey, runId: crypto.randomUUID() }).pipe(
|
|
10072
|
+
repair: (modelKey) => run(core().runtime.enqueue(GraphRepair, { modelKey, runId: crypto.randomUUID() }).pipe(Effect50.flatMap((jobId) => core().runtime.wait(jobId)), Effect50.asVoid)),
|
|
10043
10073
|
explain: (input, options) => run(core().store.explain(input, {
|
|
10044
10074
|
...options?.edgeKinds !== undefined ? { edgeKinds: options.edgeKinds } : undefined,
|
|
10045
10075
|
...options?.maxDepth !== undefined ? { maxDepth: options.maxDepth } : undefined,
|
|
@@ -10047,15 +10077,19 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
10047
10077
|
...options?.maxTotal !== undefined ? { maxTotal: options.maxTotal } : undefined,
|
|
10048
10078
|
...options?.includePending !== undefined ? { includePending: options.includePending } : undefined
|
|
10049
10079
|
})),
|
|
10050
|
-
health: (modelKey) => run(
|
|
10080
|
+
health: (modelKey) => run(Effect50.gen(function* () {
|
|
10081
|
+
const topics3 = yield* core().store.topicHealth(modelKey);
|
|
10082
|
+
const jobs = yield* core().queueStats.pending();
|
|
10083
|
+
return { ...topics3, jobs };
|
|
10084
|
+
})),
|
|
10051
10085
|
graph: {
|
|
10052
|
-
addEdge: (input) => run(
|
|
10086
|
+
addEdge: (input) => run(Effect50.gen(function* () {
|
|
10053
10087
|
const invalid = validateAddEdgeInput(input);
|
|
10054
10088
|
if (invalid !== undefined)
|
|
10055
10089
|
return yield* invalid;
|
|
10056
10090
|
return yield* core().store.addEdge(input);
|
|
10057
10091
|
})),
|
|
10058
|
-
candidates: () => run(core().store.graphCandidates()),
|
|
10092
|
+
candidates: (options) => run(core().store.graphCandidates(options)),
|
|
10059
10093
|
coverage: (modelKey) => run(core().store.graphCoverage(modelKey)),
|
|
10060
10094
|
confirm: (id) => run(core().store.confirmEdge(id)),
|
|
10061
10095
|
reject: (id) => run(core().store.rejectEdge(id)),
|
|
@@ -10091,28 +10125,28 @@ var diffConfig = (oldC, newC) => {
|
|
|
10091
10125
|
};
|
|
10092
10126
|
var RELOAD_DRAIN_MS = 30000;
|
|
10093
10127
|
var bootStack = async (options) => {
|
|
10094
|
-
const scope = await
|
|
10128
|
+
const scope = await Effect51.runPromise(Scope5.make());
|
|
10095
10129
|
let runtime;
|
|
10096
10130
|
try {
|
|
10097
10131
|
const coreNode = make({
|
|
10098
10132
|
name: "SdkCore",
|
|
10099
10133
|
deps: [],
|
|
10100
|
-
layer: "core" in options ?
|
|
10134
|
+
layer: "core" in options ? layer14(options.core) : layerConfig7(options.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : {}).pipe(Layer27.provide(configLayerFromInput(options.config)))
|
|
10101
10135
|
});
|
|
10102
|
-
const appLayer = compile(group([coreNode, node8, node6])).pipe(
|
|
10136
|
+
const appLayer = compile(group([coreNode, node8, node6])).pipe(Layer27.provide(Layer27.succeed(Scope5.Scope, scope)));
|
|
10103
10137
|
const rt = ManagedRuntime.make(appLayer);
|
|
10104
10138
|
runtime = rt;
|
|
10105
|
-
const [core, parser, extractor] = await rt.runPromise(
|
|
10139
|
+
const [core, parser, extractor] = await rt.runPromise(Effect51.all([Service, Service20, Service13]));
|
|
10106
10140
|
return {
|
|
10107
10141
|
runPromise: (effect) => rt.runPromise(effect),
|
|
10108
10142
|
disposeEffect: rt.disposeEffect,
|
|
10109
10143
|
core,
|
|
10110
10144
|
parser,
|
|
10111
10145
|
extractor,
|
|
10112
|
-
dispose: () =>
|
|
10146
|
+
dispose: () => Effect51.runPromise(Scope5.close(scope, Exit8.void).pipe(Effect51.andThen(rt.disposeEffect)))
|
|
10113
10147
|
};
|
|
10114
10148
|
} catch (cause) {
|
|
10115
|
-
await
|
|
10149
|
+
await Effect51.runPromise(Scope5.close(scope, Exit8.void).pipe(Effect51.andThen(runtime?.disposeEffect ?? Effect51.void))).catch(() => {
|
|
10116
10150
|
return;
|
|
10117
10151
|
});
|
|
10118
10152
|
return Promise.reject(cause);
|
|
@@ -10123,7 +10157,7 @@ var makeDimple = async (options) => {
|
|
|
10123
10157
|
const migrationFolder = "config" in options ? options.migrationFolder : undefined;
|
|
10124
10158
|
let currentConfig;
|
|
10125
10159
|
if (configSource !== undefined) {
|
|
10126
|
-
currentConfig = await
|
|
10160
|
+
currentConfig = await Effect51.runPromise(resolveConfigInput(configSource)).catch((cause) => Promise.reject(remapError(cause)));
|
|
10127
10161
|
}
|
|
10128
10162
|
let current = await bootStack(configSource !== undefined && currentConfig !== undefined ? {
|
|
10129
10163
|
config: currentConfig,
|
|
@@ -10145,7 +10179,7 @@ var makeDimple = async (options) => {
|
|
|
10145
10179
|
}
|
|
10146
10180
|
const stack = current;
|
|
10147
10181
|
inflight2 += 1;
|
|
10148
|
-
return stack.runPromise(effect.pipe(
|
|
10182
|
+
return stack.runPromise(effect.pipe(Effect51.mapError(remapError))).catch((cause) => {
|
|
10149
10183
|
return Promise.reject(remapError(cause));
|
|
10150
10184
|
}).finally(() => {
|
|
10151
10185
|
inflight2 -= 1;
|
|
@@ -10201,7 +10235,7 @@ var makeDimple = async (options) => {
|
|
|
10201
10235
|
}));
|
|
10202
10236
|
}
|
|
10203
10237
|
const nextInput = input ?? configSource;
|
|
10204
|
-
const resolved = await
|
|
10238
|
+
const resolved = await Effect51.runPromise(resolveConfigInput(nextInput));
|
|
10205
10239
|
if (currentConfig !== undefined && stableKey(resolved) === stableKey(currentConfig)) {
|
|
10206
10240
|
return {
|
|
10207
10241
|
config: resolved,
|
|
@@ -10224,10 +10258,10 @@ var makeDimple = async (options) => {
|
|
|
10224
10258
|
currentConfig = resolved;
|
|
10225
10259
|
await old.dispose();
|
|
10226
10260
|
if (changes.storeUrl) {
|
|
10227
|
-
await
|
|
10261
|
+
await Effect51.runPromise(Effect51.logWarning(`reload: store database switched "${oldStoreUrl}" → "${resolved.storeUrl}" — ` + "the old db is preserved but no longer served (nothing was migrated)").pipe(Effect51.annotateLogs({ scope: "@dimple/sdk", event: "reload.store.switch" })));
|
|
10228
10262
|
}
|
|
10229
10263
|
if (changes.jobsUrl) {
|
|
10230
|
-
await
|
|
10264
|
+
await Effect51.runPromise(Effect51.logWarning(`reload: jobs database switched "${oldJobsUrl}" → "${resolved.jobs?.dbUrl}" — ` + "pending jobs remain durable in the old db").pipe(Effect51.annotateLogs({ scope: "@dimple/sdk", event: "reload.jobs.switch" })));
|
|
10231
10265
|
}
|
|
10232
10266
|
return { config: resolved, changed: changes, reloaded: true };
|
|
10233
10267
|
} finally {
|