@ponraaj/dimple-sdk 0.0.0-alpha.12 → 0.0.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dimple.d.ts +23 -2
- package/dist/errors.js +583 -501
- package/dist/index.d.ts +1 -1
- package/dist/index.js +702 -610
- package/dist/internal/config/index.d.ts +2 -2
- package/dist/internal/config/schema.d.ts +116 -21
- package/dist/internal/core/core.d.ts +4 -1
- package/dist/internal/core/layer.d.ts +3 -3
- package/dist/internal/embedder/index.d.ts +1 -1
- package/dist/internal/embedder/transformers.d.ts +63 -19
- package/dist/internal/store/topics/routing.d.ts +1 -1
- package/dist/internal/workflow/engine.d.ts +3 -2
- package/dist/internal/workflow/index.d.ts +1 -0
- package/dist/internal/workflow/runtime.d.ts +4 -3
- package/dist/internal/workflow/stats.d.ts +44 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -29,10 +29,10 @@ class CoreConfigError extends Schema.TaggedError()("core.ConfigError", {
|
|
|
29
29
|
class Service extends Context.Service()("@dimple/core/Core") {
|
|
30
30
|
}
|
|
31
31
|
// ../core/src/layer.ts
|
|
32
|
-
import { Effect as
|
|
32
|
+
import { Effect as Effect40, Layer as Layer16, Schedule as Schedule2, Schema as Schema29 } from "effect";
|
|
33
33
|
|
|
34
34
|
// ../store/src/store.ts
|
|
35
|
-
import { Config as Config4, Effect as
|
|
35
|
+
import { Config as Config4, Effect as Effect26, Layer as Layer11, Schema as Schema23 } from "effect";
|
|
36
36
|
import { eq as eq8, and as and8, desc as desc2, asc as asc5, inArray as inArray4, ne as ne4, or, isNull as isNull2, like, sql as sql8 } from "drizzle-orm";
|
|
37
37
|
|
|
38
38
|
// ../store/src/db-backend.ts
|
|
@@ -840,6 +840,43 @@ var IndexConfig = Schema8.Struct({
|
|
|
840
840
|
maxNeighbors: Schema8.optional(Schema8.Number),
|
|
841
841
|
searchL: Schema8.optional(Schema8.Number)
|
|
842
842
|
});
|
|
843
|
+
var ThreadCount = Schema8.Number.pipe(Schema8.refine((n) => Number.isInteger(n) && n >= 0 && n <= 1024, {
|
|
844
|
+
identifier: "ThreadCount"
|
|
845
|
+
}));
|
|
846
|
+
var SessionOptions = Schema8.Struct({
|
|
847
|
+
intraOpNumThreads: Schema8.optional(ThreadCount),
|
|
848
|
+
interOpNumThreads: Schema8.optional(ThreadCount),
|
|
849
|
+
executionMode: Schema8.optional(Schema8.Literals(["sequential", "parallel"])),
|
|
850
|
+
graphOptimizationLevel: Schema8.optional(Schema8.Literals(["disabled", "basic", "extended", "all", "layout"])),
|
|
851
|
+
enableCpuMemArena: Schema8.optional(Schema8.Boolean),
|
|
852
|
+
enableMemPattern: Schema8.optional(Schema8.Boolean),
|
|
853
|
+
gpuMemLimit: Schema8.optional(Schema8.Number),
|
|
854
|
+
arenaExtendStrategy: Schema8.optional(Schema8.Literals([0, 1]))
|
|
855
|
+
});
|
|
856
|
+
var mergeSessionOptions = (global, model) => {
|
|
857
|
+
const pick = (key) => model?.[key] ?? global?.[key];
|
|
858
|
+
const intraOpNumThreads = pick("intraOpNumThreads");
|
|
859
|
+
const interOpNumThreads = pick("interOpNumThreads");
|
|
860
|
+
const executionMode = pick("executionMode");
|
|
861
|
+
const graphOptimizationLevel = pick("graphOptimizationLevel");
|
|
862
|
+
const enableCpuMemArena = pick("enableCpuMemArena");
|
|
863
|
+
const enableMemPattern = pick("enableMemPattern");
|
|
864
|
+
const gpuMemLimit = pick("gpuMemLimit");
|
|
865
|
+
const arenaExtendStrategy = pick("arenaExtendStrategy");
|
|
866
|
+
const empty = intraOpNumThreads === undefined && interOpNumThreads === undefined && executionMode === undefined && graphOptimizationLevel === undefined && enableCpuMemArena === undefined && enableMemPattern === undefined && gpuMemLimit === undefined && arenaExtendStrategy === undefined;
|
|
867
|
+
if (empty)
|
|
868
|
+
return;
|
|
869
|
+
return {
|
|
870
|
+
...intraOpNumThreads !== undefined ? { intraOpNumThreads } : undefined,
|
|
871
|
+
...interOpNumThreads !== undefined ? { interOpNumThreads } : undefined,
|
|
872
|
+
...executionMode !== undefined ? { executionMode } : undefined,
|
|
873
|
+
...graphOptimizationLevel !== undefined ? { graphOptimizationLevel } : undefined,
|
|
874
|
+
...enableCpuMemArena !== undefined ? { enableCpuMemArena } : undefined,
|
|
875
|
+
...enableMemPattern !== undefined ? { enableMemPattern } : undefined,
|
|
876
|
+
...gpuMemLimit !== undefined ? { gpuMemLimit } : undefined,
|
|
877
|
+
...arenaExtendStrategy !== undefined ? { arenaExtendStrategy } : undefined
|
|
878
|
+
};
|
|
879
|
+
};
|
|
843
880
|
var EmbeddingModelConfig = Schema8.Struct({
|
|
844
881
|
id: Schema8.String,
|
|
845
882
|
provider: Schema8.String,
|
|
@@ -877,14 +914,13 @@ var EmbeddingModelConfig = Schema8.Struct({
|
|
|
877
914
|
pooling: Schema8.optional(Schema8.Literals(["mean", "cls", "none"])),
|
|
878
915
|
normalize: Schema8.optional(Schema8.Boolean),
|
|
879
916
|
cacheDir: Schema8.optional(Schema8.String),
|
|
880
|
-
sessionOptions: Schema8.optional(
|
|
881
|
-
|
|
882
|
-
arenaExtendStrategy: Schema8.optional(Schema8.Literals([0, 1]))
|
|
883
|
-
})),
|
|
917
|
+
sessionOptions: Schema8.optional(SessionOptions),
|
|
918
|
+
wasm: Schema8.optional(Schema8.Struct({ numThreads: Schema8.optional(ThreadCount) })),
|
|
884
919
|
autoInstall: Schema8.optional(Schema8.Boolean)
|
|
885
920
|
});
|
|
886
921
|
var EmbeddingConfig = Schema8.Struct({
|
|
887
922
|
defaultModel: Schema8.String,
|
|
923
|
+
sessionOptions: Schema8.optional(SessionOptions),
|
|
888
924
|
models: Schema8.Array(EmbeddingModelConfig)
|
|
889
925
|
});
|
|
890
926
|
var MaxAttempts = Schema8.Number.pipe(Schema8.refine((n) => Number.isInteger(n) && n > 0 && n <= 100, {
|
|
@@ -2575,7 +2611,7 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2575
2611
|
children.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
2576
2612
|
}
|
|
2577
2613
|
const membershipRows = [];
|
|
2578
|
-
const
|
|
2614
|
+
const topicFinal = new Map;
|
|
2579
2615
|
const outcomes = [];
|
|
2580
2616
|
for (const entry of entries) {
|
|
2581
2617
|
const x = new Float32Array(entry.vector);
|
|
@@ -2598,12 +2634,12 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2598
2634
|
modelKey: model.id,
|
|
2599
2635
|
updatedAt: now
|
|
2600
2636
|
});
|
|
2601
|
-
const acc =
|
|
2637
|
+
const acc = topicFinal.get(topicKey);
|
|
2602
2638
|
if (acc) {
|
|
2603
|
-
acc.
|
|
2639
|
+
acc.memberCount = leafCf.count;
|
|
2604
2640
|
acc.ss = leafCf.ss;
|
|
2605
2641
|
} else {
|
|
2606
|
-
|
|
2642
|
+
topicFinal.set(topicKey, { memberCount: leafCf.count, ss: leafCf.ss });
|
|
2607
2643
|
}
|
|
2608
2644
|
let ancestorKey = parentOf.get(topicKey);
|
|
2609
2645
|
while (ancestorKey !== undefined) {
|
|
@@ -2614,11 +2650,11 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2614
2650
|
row.memberCount += 1;
|
|
2615
2651
|
const ancestorSum = addToSum(sumOf(ancestorKey), x);
|
|
2616
2652
|
sumByTopic.set(ancestorKey, ancestorSum);
|
|
2617
|
-
const accA =
|
|
2653
|
+
const accA = topicFinal.get(ancestorKey);
|
|
2618
2654
|
if (accA)
|
|
2619
|
-
accA.
|
|
2655
|
+
accA.memberCount = row.memberCount;
|
|
2620
2656
|
else
|
|
2621
|
-
|
|
2657
|
+
topicFinal.set(ancestorKey, { memberCount: row.memberCount, ss: row.sumSquares });
|
|
2622
2658
|
ancestorKey = parentOf.get(ancestorKey);
|
|
2623
2659
|
}
|
|
2624
2660
|
outcomes.push({
|
|
@@ -2628,14 +2664,14 @@ var routeInsertMany = Effect10.fn("topics.routeInsertMany")(function* (entries,
|
|
|
2628
2664
|
});
|
|
2629
2665
|
}
|
|
2630
2666
|
await tx.insert(topicMemberships).values(membershipRows);
|
|
2631
|
-
for (const [key,
|
|
2632
|
-
await tx.update(topics).set({ memberCount:
|
|
2667
|
+
for (const [key, final] of topicFinal) {
|
|
2668
|
+
await tx.update(topics).set({ memberCount: final.memberCount, sumSquares: final.ss, updatedAt: now }).where(and(eq(topics.topicKey, key), eq(topics.modelKey, model.id)));
|
|
2633
2669
|
}
|
|
2634
|
-
return { outcomes,
|
|
2670
|
+
return { outcomes, topicFinal };
|
|
2635
2671
|
}),
|
|
2636
2672
|
catch: (cause) => cause
|
|
2637
2673
|
}).pipe(routingFailure("topics.routeInsertMany"));
|
|
2638
|
-
yield* backend.upsertTopicVectors(Array.from(result.
|
|
2674
|
+
yield* backend.upsertTopicVectors(Array.from(result.topicFinal, ([topicKey]) => ({
|
|
2639
2675
|
topicKey,
|
|
2640
2676
|
sumVector: Array.from(sumByTopic.get(topicKey) ?? zeros)
|
|
2641
2677
|
})), model).pipe(routingFailure("topics.routeInsertMany"));
|
|
@@ -2736,7 +2772,7 @@ var resolveTopics = Effect11.fn("topics.resolveTopics")(function* (queryVector,
|
|
|
2736
2772
|
});
|
|
2737
2773
|
|
|
2738
2774
|
// ../store/src/topics/lifecycle.ts
|
|
2739
|
-
import { Clock as Clock3, Cause as Cause3, Effect as
|
|
2775
|
+
import { Clock as Clock3, Cause as Cause3, Effect as Effect21, Exit as Exit6 } from "effect";
|
|
2740
2776
|
import { and as and3, eq as eq3, inArray, ne, sql as sql3 } from "drizzle-orm";
|
|
2741
2777
|
|
|
2742
2778
|
// ../workflow/src/queue.ts
|
|
@@ -2899,12 +2935,12 @@ var queues = [
|
|
|
2899
2935
|
import {
|
|
2900
2936
|
Cause,
|
|
2901
2937
|
Clock as Clock2,
|
|
2902
|
-
Context as
|
|
2938
|
+
Context as Context9,
|
|
2903
2939
|
Deferred as Deferred2,
|
|
2904
2940
|
Duration,
|
|
2905
|
-
Effect as
|
|
2941
|
+
Effect as Effect18,
|
|
2906
2942
|
Exit as Exit5,
|
|
2907
|
-
Layer as
|
|
2943
|
+
Layer as Layer8,
|
|
2908
2944
|
Option as Option8,
|
|
2909
2945
|
Schema as Schema18
|
|
2910
2946
|
} from "effect";
|
|
@@ -3049,7 +3085,7 @@ var notifyExecutionDone = (executionId) => Effect14.gen(function* () {
|
|
|
3049
3085
|
});
|
|
3050
3086
|
|
|
3051
3087
|
// ../workflow/src/engine.ts
|
|
3052
|
-
import { Effect as
|
|
3088
|
+
import { Effect as Effect17, Exit as Exit4, Fiber, FiberMap, Layer as Layer7, Option as Option7, Schema as Schema17 } from "effect";
|
|
3053
3089
|
import { Workflow as Workflow3, WorkflowEngine as WorkflowEngine2 } from "effect/workflow";
|
|
3054
3090
|
import { Persistence as PersistenceNamespace2, PersistedQueue } from "effect/persistence";
|
|
3055
3091
|
|
|
@@ -3206,20 +3242,42 @@ var make5 = (options) => Effect15.gen(function* () {
|
|
|
3206
3242
|
var layerConfig2 = (config) => Layer5.effectContext(Config3.unwrap(config).pipe(Effect15.flatMap(make5), Effect15.map((client) => Context7.make(LibsqlClient, client).pipe(Context7.add(Client.SqlClient, client))))).pipe(Layer5.provide(Reactivity.layer));
|
|
3207
3243
|
var layer5 = (config) => Layer5.effectContext(Effect15.map(make5(config), (client) => Context7.make(LibsqlClient, client).pipe(Context7.add(Client.SqlClient, client)))).pipe(Layer5.provide(Reactivity.layer));
|
|
3208
3244
|
// ../workflow/src/engine.ts
|
|
3209
|
-
import { SqlError as
|
|
3245
|
+
import { SqlError as SqlErrorNamespace2 } from "effect/sql";
|
|
3246
|
+
|
|
3247
|
+
// ../workflow/src/stats.ts
|
|
3248
|
+
import { Context as Context8, Effect as Effect16, Layer as Layer6 } from "effect";
|
|
3249
|
+
import { SqlClient as SqlClient2 } from "effect/sql";
|
|
3250
|
+
class Service7 extends Context8.Service()("@dimple/workflow/QueueStats") {
|
|
3251
|
+
}
|
|
3252
|
+
var layer6 = Layer6.effect(Service7, Effect16.gen(function* () {
|
|
3253
|
+
const sql3 = yield* SqlClient2.SqlClient;
|
|
3254
|
+
const pending = Effect16.fn("QueueStats.pending")(function* () {
|
|
3255
|
+
const rows = yield* sql3`
|
|
3256
|
+
SELECT queue_name AS queue, COUNT(*) AS count
|
|
3257
|
+
FROM effect_queue
|
|
3258
|
+
WHERE state = 'pending' AND queue_name != ${RESUME_QUEUE}
|
|
3259
|
+
GROUP BY queue_name
|
|
3260
|
+
ORDER BY queue_name
|
|
3261
|
+
`;
|
|
3262
|
+
return rows.map((row) => ({ queue: row.queue, pending: Number(row.count) }));
|
|
3263
|
+
});
|
|
3264
|
+
return Service7.of({ pending });
|
|
3265
|
+
}));
|
|
3266
|
+
|
|
3267
|
+
// ../workflow/src/engine.ts
|
|
3210
3268
|
var Persistence = PersistenceNamespace2.Persistence;
|
|
3211
3269
|
var PersistedQueueFactory = PersistedQueue.PersistedQueueFactory;
|
|
3212
3270
|
var PayloadCodec = Schema17.Record(Schema17.String, Schema17.Json);
|
|
3213
|
-
var layerDurable =
|
|
3271
|
+
var layerDurable = Layer7.effect(WorkflowEngine2.WorkflowEngine)(Effect17.gen(function* () {
|
|
3214
3272
|
const persistence = yield* Persistence;
|
|
3215
3273
|
const queueFactory = yield* PersistedQueueFactory;
|
|
3216
|
-
const scope = yield*
|
|
3274
|
+
const scope = yield* Effect17.scope;
|
|
3217
3275
|
const store = yield* persistence.make({ storeId: STORE_ID });
|
|
3218
3276
|
const resumeQueue = yield* queueFactory.make({ name: RESUME_QUEUE, schema: ResumeSignal });
|
|
3219
3277
|
const workflows = new Map;
|
|
3220
3278
|
const executions = new Map;
|
|
3221
3279
|
const clocks = yield* FiberMap.make();
|
|
3222
|
-
const resume = (state) =>
|
|
3280
|
+
const resume = (state) => Effect17.gen(function* () {
|
|
3223
3281
|
const exit = state.fiber?.pollUnsafe();
|
|
3224
3282
|
if (exit && exit._tag === "Success" && exit.value._tag === "Complete")
|
|
3225
3283
|
return;
|
|
@@ -3231,32 +3289,32 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3231
3289
|
const instance = WorkflowEngine2.WorkflowInstance.initial(state.instance.workflow, state.instance.executionId);
|
|
3232
3290
|
instance.interrupted = state.instance.interrupted;
|
|
3233
3291
|
state.instance = instance;
|
|
3234
|
-
state.fiber = yield* state.execute(state.payload, state.instance.executionId).pipe(
|
|
3292
|
+
state.fiber = yield* state.execute(state.payload, state.instance.executionId).pipe(Effect17.onExit(() => {
|
|
3235
3293
|
if (!instance.interrupted)
|
|
3236
|
-
return
|
|
3294
|
+
return Effect17.void;
|
|
3237
3295
|
instance.suspended = false;
|
|
3238
|
-
return
|
|
3239
|
-
}), Workflow3.intoResult,
|
|
3296
|
+
return Effect17.withFiber((fiber) => Effect17.interruptible(Fiber.interrupt(fiber)));
|
|
3297
|
+
}), Workflow3.intoResult, Effect17.provideService(WorkflowEngine2.WorkflowInstance, instance), Effect17.provideService(WorkflowEngine2.WorkflowEngine, engine), Effect17.tap((result) => {
|
|
3240
3298
|
if (result._tag !== "Complete")
|
|
3241
|
-
return
|
|
3242
|
-
return
|
|
3299
|
+
return Effect17.void;
|
|
3300
|
+
return Effect17.gen(function* () {
|
|
3243
3301
|
const entry2 = workflows.get(state.instance.workflow._tag);
|
|
3244
3302
|
if (!entry2)
|
|
3245
3303
|
return;
|
|
3246
|
-
const encoded = yield*
|
|
3304
|
+
const encoded = yield* Effect17.orDie(Schema17.encodeEffect(resultCodecFor(entry2.workflow))(result));
|
|
3247
3305
|
yield* setEncodedExit(store, resultKey(state.instance.executionId), Exit4.succeed(encoded));
|
|
3248
3306
|
yield* notifyExecutionDone(state.instance.executionId);
|
|
3249
3307
|
});
|
|
3250
|
-
}),
|
|
3308
|
+
}), Effect17.tap((result) => {
|
|
3251
3309
|
if (!state.parent || result._tag !== "Complete")
|
|
3252
|
-
return
|
|
3310
|
+
return Effect17.void;
|
|
3253
3311
|
const parent = executions.get(state.parent);
|
|
3254
3312
|
if (parent)
|
|
3255
|
-
return
|
|
3256
|
-
return
|
|
3257
|
-
}),
|
|
3313
|
+
return Effect17.forkIn(resume(parent), scope);
|
|
3314
|
+
return Effect17.void;
|
|
3315
|
+
}), Effect17.forkIn(entry.scope));
|
|
3258
3316
|
});
|
|
3259
|
-
const resumeExecution = (executionId) =>
|
|
3317
|
+
const resumeExecution = (executionId) => Effect17.gen(function* () {
|
|
3260
3318
|
const state = executions.get(executionId);
|
|
3261
3319
|
if (state) {
|
|
3262
3320
|
yield* resume(state);
|
|
@@ -3279,22 +3337,22 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3279
3337
|
}
|
|
3280
3338
|
});
|
|
3281
3339
|
const engine = WorkflowEngine2.makeUnsafe({
|
|
3282
|
-
register:
|
|
3340
|
+
register: Effect17.fnUntraced(function* (workflow, execute) {
|
|
3283
3341
|
workflows.set(workflow._tag, {
|
|
3284
3342
|
workflow,
|
|
3285
3343
|
execute,
|
|
3286
|
-
scope: yield*
|
|
3344
|
+
scope: yield* Effect17.scope
|
|
3287
3345
|
});
|
|
3288
3346
|
}),
|
|
3289
3347
|
execute: executeEncoded(workflows, executions, store, resume),
|
|
3290
|
-
interrupt:
|
|
3348
|
+
interrupt: Effect17.fnUntraced(function* (_workflow, executionId) {
|
|
3291
3349
|
const state = executions.get(executionId);
|
|
3292
3350
|
if (!state)
|
|
3293
3351
|
return;
|
|
3294
3352
|
state.instance.interrupted = true;
|
|
3295
3353
|
yield* resume(state);
|
|
3296
3354
|
}),
|
|
3297
|
-
interruptUnsafe:
|
|
3355
|
+
interruptUnsafe: Effect17.fnUntraced(function* (_workflow, executionId) {
|
|
3298
3356
|
const state = executions.get(executionId);
|
|
3299
3357
|
if (!state)
|
|
3300
3358
|
return;
|
|
@@ -3304,36 +3362,36 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3304
3362
|
}
|
|
3305
3363
|
}),
|
|
3306
3364
|
resume: (_workflow, executionId) => resumeExecution(executionId),
|
|
3307
|
-
activityExecute:
|
|
3365
|
+
activityExecute: Effect17.fnUntraced(function* (activity, attempt) {
|
|
3308
3366
|
const instance = yield* WorkflowEngine2.WorkflowInstance;
|
|
3309
3367
|
const key = activityKey(instance.executionId, activity.name, attempt);
|
|
3310
3368
|
const persisted = yield* getEncodedExit(store, key);
|
|
3311
3369
|
if (persisted && Exit4.isSuccess(persisted)) {
|
|
3312
3370
|
const entry = workflows.get(instance.workflow._tag);
|
|
3313
3371
|
if (!entry)
|
|
3314
|
-
return yield*
|
|
3315
|
-
return yield*
|
|
3372
|
+
return yield* Effect17.die(`activity replay: unknown workflow ${instance.workflow._tag}`);
|
|
3373
|
+
return yield* Effect17.orDie(Schema17.decodeUnknownEffect(resultCodecFor(entry.workflow))(persisted.value));
|
|
3316
3374
|
}
|
|
3317
3375
|
const activityInstance = WorkflowEngine2.WorkflowInstance.initial(instance.workflow, instance.executionId);
|
|
3318
3376
|
activityInstance.interrupted = instance.interrupted;
|
|
3319
|
-
const result = yield* activity.executeEncoded.pipe(Workflow3.intoResult,
|
|
3377
|
+
const result = yield* activity.executeEncoded.pipe(Workflow3.intoResult, Effect17.provideService(WorkflowEngine2.WorkflowInstance, activityInstance), Effect17.exit);
|
|
3320
3378
|
if (result._tag === "Success" && result.value._tag !== "Suspended") {
|
|
3321
3379
|
const entry = workflows.get(instance.workflow._tag);
|
|
3322
3380
|
if (entry) {
|
|
3323
|
-
const encoded = yield*
|
|
3381
|
+
const encoded = yield* Effect17.orDie(Schema17.encodeEffect(resultCodecFor(entry.workflow))(result.value));
|
|
3324
3382
|
yield* setEncodedExit(store, key, Exit4.succeed(encoded));
|
|
3325
3383
|
}
|
|
3326
3384
|
}
|
|
3327
3385
|
return yield* result;
|
|
3328
3386
|
}),
|
|
3329
|
-
poll: (_workflow, executionId) =>
|
|
3387
|
+
poll: (_workflow, executionId) => Effect17.gen(function* () {
|
|
3330
3388
|
const state = executions.get(executionId);
|
|
3331
3389
|
if (!state) {
|
|
3332
3390
|
const persisted = yield* getEncodedExit(store, resultKey(executionId));
|
|
3333
3391
|
if (persisted && Exit4.isSuccess(persisted)) {
|
|
3334
3392
|
const workflow = yield* workflowTagFor(workflows, store, executionId);
|
|
3335
3393
|
if (workflow) {
|
|
3336
|
-
return Option7.some(yield*
|
|
3394
|
+
return Option7.some(yield* Effect17.orDie(Schema17.decodeUnknownEffect(resultCodecFor(workflow))(persisted.value)));
|
|
3337
3395
|
}
|
|
3338
3396
|
}
|
|
3339
3397
|
return Option7.none();
|
|
@@ -3343,9 +3401,9 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3343
3401
|
return Option7.none();
|
|
3344
3402
|
if (exit._tag === "Success")
|
|
3345
3403
|
return Option7.some(exit.value);
|
|
3346
|
-
return yield*
|
|
3404
|
+
return yield* Effect17.die(exit.cause);
|
|
3347
3405
|
}),
|
|
3348
|
-
deferredResult:
|
|
3406
|
+
deferredResult: Effect17.fnUntraced(function* (deferred) {
|
|
3349
3407
|
const instance = yield* WorkflowEngine2.WorkflowInstance;
|
|
3350
3408
|
const persisted = yield* getEncodedExit(store, deferredKey(instance.executionId, deferred.name));
|
|
3351
3409
|
if (persisted) {
|
|
@@ -3353,9 +3411,9 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3353
3411
|
}
|
|
3354
3412
|
return Option7.none();
|
|
3355
3413
|
}),
|
|
3356
|
-
deferredDone: (options) =>
|
|
3414
|
+
deferredDone: (options) => Effect17.gen(function* () {
|
|
3357
3415
|
yield* setEncodedExit(store, deferredKey(options.executionId, options.deferredName), options.exit);
|
|
3358
|
-
yield* resumeQueue.offer({ executionId: options.executionId }, { id: options.executionId }).pipe(
|
|
3416
|
+
yield* resumeQueue.offer({ executionId: options.executionId }, { id: options.executionId }).pipe(Effect17.orDie);
|
|
3359
3417
|
yield* resumeExecution(options.executionId);
|
|
3360
3418
|
}),
|
|
3361
3419
|
scheduleClock: (workflow, options) => engine.deferredDone(options.clock.deferred, {
|
|
@@ -3363,17 +3421,17 @@ var layerDurable = Layer6.effect(WorkflowEngine2.WorkflowEngine)(Effect16.gen(fu
|
|
|
3363
3421
|
executionId: options.executionId,
|
|
3364
3422
|
deferredName: options.clock.deferred.name,
|
|
3365
3423
|
exit: Exit4.void
|
|
3366
|
-
}).pipe(
|
|
3424
|
+
}).pipe(Effect17.delay(options.clock.duration), FiberMap.run(clocks, `${options.executionId}/${options.clock.name}`, {
|
|
3367
3425
|
onlyIfMissing: true
|
|
3368
|
-
}),
|
|
3426
|
+
}), Effect17.asVoid)
|
|
3369
3427
|
});
|
|
3370
3428
|
return engine;
|
|
3371
3429
|
}));
|
|
3372
|
-
var executeEncoded = (workflows, executions, store, resume) =>
|
|
3373
|
-
const payload = yield*
|
|
3430
|
+
var executeEncoded = (workflows, executions, store, resume) => Effect17.fnUntraced(function* (workflow, options) {
|
|
3431
|
+
const payload = yield* Effect17.orDie(Schema17.decodeUnknownEffect(PayloadCodec)(options.payload));
|
|
3374
3432
|
const entry = workflows.get(workflow._tag);
|
|
3375
3433
|
if (!entry) {
|
|
3376
|
-
return yield*
|
|
3434
|
+
return yield* Effect17.orDie(Effect17.fail(`Workflow ${workflow._tag} is not registered`));
|
|
3377
3435
|
}
|
|
3378
3436
|
let state = executions.get(options.executionId);
|
|
3379
3437
|
if (!state) {
|
|
@@ -3389,43 +3447,43 @@ var executeEncoded = (workflows, executions, store, resume) => Effect16.fnUntrac
|
|
|
3389
3447
|
yield* resume(state);
|
|
3390
3448
|
}
|
|
3391
3449
|
if (options.discard)
|
|
3392
|
-
return yield*
|
|
3450
|
+
return yield* Effect17.void;
|
|
3393
3451
|
if (!state.fiber) {
|
|
3394
|
-
return yield*
|
|
3452
|
+
return yield* Effect17.orDie(Effect17.fail(`Workflow ${workflow._tag}: execution ${options.executionId} has no fiber`));
|
|
3395
3453
|
}
|
|
3396
3454
|
return yield* Fiber.join(state.fiber);
|
|
3397
3455
|
});
|
|
3398
|
-
var loadRecord = (store, executionId) =>
|
|
3456
|
+
var loadRecord = (store, executionId) => Effect17.gen(function* () {
|
|
3399
3457
|
const persisted = yield* getEncodedExit(store, recordKey(executionId));
|
|
3400
3458
|
if (persisted && Exit4.isSuccess(persisted)) {
|
|
3401
|
-
return yield*
|
|
3459
|
+
return yield* Effect17.orDie(Schema17.decodeUnknownEffect(ExecutionRecord)(persisted.value));
|
|
3402
3460
|
}
|
|
3403
3461
|
return;
|
|
3404
3462
|
});
|
|
3405
|
-
var workflowTagFor = (workflows, store, executionId) =>
|
|
3463
|
+
var workflowTagFor = (workflows, store, executionId) => Effect17.gen(function* () {
|
|
3406
3464
|
const record = yield* loadRecord(store, executionId);
|
|
3407
3465
|
if (!record)
|
|
3408
3466
|
return;
|
|
3409
3467
|
return workflows.get(record.workflowTag)?.workflow;
|
|
3410
3468
|
});
|
|
3411
|
-
var libsqlStack = (options) =>
|
|
3412
|
-
yield*
|
|
3469
|
+
var libsqlStack = (options) => Layer7.unwrap(Effect17.gen(function* () {
|
|
3470
|
+
yield* Effect17.tryPromise({
|
|
3413
3471
|
try: () => prepareFileDatabase(options.url),
|
|
3414
3472
|
catch: (cause) => new Error(String(cause))
|
|
3415
|
-
}).pipe(
|
|
3473
|
+
}).pipe(Effect17.catch(() => Effect17.void));
|
|
3416
3474
|
const storeLayer = PersistedQueue.layerStoreSql();
|
|
3417
|
-
const inner =
|
|
3418
|
-
const ctx = yield*
|
|
3475
|
+
const inner = Layer7.mergeAll(PersistenceNamespace2.layerSql, PersistedQueue.layer.pipe(Layer7.provide(storeLayer)), PersistedQueue.layerCleanup().pipe(Layer7.provide(storeLayer)), layer6).pipe(Layer7.provide(exports_LibsqlClient.layer(options)));
|
|
3476
|
+
const ctx = yield* Effect17.acquireUseRelease(Effect17.tryPromise({
|
|
3419
3477
|
try: () => acquireMigrationLock(options.url),
|
|
3420
|
-
catch: (cause) => new
|
|
3421
|
-
reason: new
|
|
3478
|
+
catch: (cause) => new SqlErrorNamespace2.SqlError({
|
|
3479
|
+
reason: new SqlErrorNamespace2.LockTimeoutError({
|
|
3422
3480
|
cause,
|
|
3423
3481
|
message: "timed out acquiring the jobs-database migration lock",
|
|
3424
3482
|
operation: "libsqlStack"
|
|
3425
3483
|
})
|
|
3426
3484
|
})
|
|
3427
|
-
}), () =>
|
|
3428
|
-
return
|
|
3485
|
+
}), () => Layer7.build(inner), (release) => Effect17.sync(() => release?.()));
|
|
3486
|
+
return Layer7.succeedContext(ctx);
|
|
3429
3487
|
}));
|
|
3430
3488
|
|
|
3431
3489
|
// ../workflow/src/runtime.ts
|
|
@@ -3437,14 +3495,14 @@ var DEFAULT_TIMEOUT = Duration.minutes(5);
|
|
|
3437
3495
|
var DEFAULT_POLL_INTERVAL = Duration.millis(250);
|
|
3438
3496
|
var PROCESS_WINDOW = Duration.seconds(30);
|
|
3439
3497
|
|
|
3440
|
-
class
|
|
3498
|
+
class Service8 extends Context9.Service()("@dimple/workflow/JobRuntime") {
|
|
3441
3499
|
}
|
|
3442
3500
|
var jobsError = (jobId, cause) => {
|
|
3443
3501
|
if (cause instanceof JobFailed || cause instanceof JobTimeout || cause instanceof JobCancelled || cause instanceof QueueError)
|
|
3444
3502
|
return cause;
|
|
3445
3503
|
return new QueueError({ message: cause instanceof Error ? cause.message : String(cause) });
|
|
3446
3504
|
};
|
|
3447
|
-
var
|
|
3505
|
+
var layer7 = (defs) => Layer8.effect(Service8, Effect18.gen(function* () {
|
|
3448
3506
|
const engine = yield* WorkflowEngine3.WorkflowEngine;
|
|
3449
3507
|
const persistence = yield* Persistence2;
|
|
3450
3508
|
const queueFactory = yield* PersistedQueueFactory2;
|
|
@@ -3452,24 +3510,24 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3452
3510
|
for (const def of defs) {
|
|
3453
3511
|
yield* def.register(engine);
|
|
3454
3512
|
}
|
|
3455
|
-
const enqueue =
|
|
3456
|
-
const decoded = yield* Schema18.decodeUnknownEffect(def.payloadSchema)(payload).pipe(
|
|
3513
|
+
const enqueue = Effect18.fn("JobRuntime.enqueue")(function* (def, payload) {
|
|
3514
|
+
const decoded = yield* Schema18.decodeUnknownEffect(def.payloadSchema)(payload).pipe(Effect18.mapError((e) => new QueueError({ message: `invalid payload: ${e.message}` })));
|
|
3457
3515
|
const token = def.idempotencyKey(decoded);
|
|
3458
3516
|
const envelope = { token, payload: decoded, traceId: "" };
|
|
3459
|
-
const jobId = yield* def.workflow.executionId(envelope).pipe(
|
|
3460
|
-
yield* def.workflow.execute(envelope, { discard: true }).pipe(
|
|
3517
|
+
const jobId = yield* def.workflow.executionId(envelope).pipe(Effect18.provideService(WorkflowEngine3.WorkflowEngine, engine), Effect18.mapError((cause) => new QueueError({ message: cause instanceof Error ? cause.message : String(cause) })));
|
|
3518
|
+
yield* def.workflow.execute(envelope, { discard: true }).pipe(Effect18.provideService(WorkflowEngine3.WorkflowEngine, engine), Effect18.mapError((cause) => jobsError(jobId, cause)));
|
|
3461
3519
|
return jobId;
|
|
3462
3520
|
});
|
|
3463
|
-
const wait =
|
|
3521
|
+
const wait = Effect18.fn("JobRuntime.wait")(function* (jobId, options) {
|
|
3464
3522
|
const timeout = Duration.fromInput(options?.timeout ?? DEFAULT_TIMEOUT).pipe(Option8.getOrElse(() => Duration.minutes(5)));
|
|
3465
3523
|
const pollInterval = Duration.fromInput(options?.pollInterval ?? DEFAULT_POLL_INTERVAL).pipe(Option8.getOrElse(() => Duration.millis(250)));
|
|
3466
|
-
const deadline = yield* Clock2.currentTimeMillis.pipe(
|
|
3524
|
+
const deadline = yield* Clock2.currentTimeMillis.pipe(Effect18.map((now) => now + Duration.toMillis(timeout)));
|
|
3467
3525
|
const completionSignal = yield* Deferred2.make();
|
|
3468
3526
|
onExecutionDone(jobId, completionSignal);
|
|
3469
3527
|
try {
|
|
3470
3528
|
while (true) {
|
|
3471
3529
|
if (yield* readMarker(store, jobId, CANCELLED)) {
|
|
3472
|
-
return yield*
|
|
3530
|
+
return yield* Effect18.fail(new JobCancelled({ jobId }));
|
|
3473
3531
|
}
|
|
3474
3532
|
const def = yield* findDef(defs, jobId, store);
|
|
3475
3533
|
if (def) {
|
|
@@ -3480,7 +3538,7 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3480
3538
|
if (Exit5.isSuccess(r.exit))
|
|
3481
3539
|
return r.exit.value;
|
|
3482
3540
|
const failure = Exit5.getCause(r.exit).pipe(Option8.flatMap(Cause.findErrorOption), Option8.map((error) => Option8.getOrElse(Schema18.decodeUnknownOption(Schema18.String)(error), () => String(error))));
|
|
3483
|
-
return yield*
|
|
3541
|
+
return yield* Effect18.fail(new JobFailed({
|
|
3484
3542
|
jobId,
|
|
3485
3543
|
message: Option8.getOrElse(failure, () => "workflow failed")
|
|
3486
3544
|
}));
|
|
@@ -3490,19 +3548,19 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3490
3548
|
const now = yield* Clock2.currentTimeMillis;
|
|
3491
3549
|
if (now >= deadline) {
|
|
3492
3550
|
yield* writeMarker(store, jobId, TIMED_OUT);
|
|
3493
|
-
return yield*
|
|
3551
|
+
return yield* Effect18.fail(new JobTimeout({ jobId, timeout }));
|
|
3494
3552
|
}
|
|
3495
|
-
yield*
|
|
3553
|
+
yield* Effect18.race(Deferred2.await(completionSignal).pipe(Effect18.asVoid), Effect18.sleep(pollInterval));
|
|
3496
3554
|
}
|
|
3497
3555
|
} finally {
|
|
3498
3556
|
offExecutionDone(jobId, completionSignal);
|
|
3499
3557
|
}
|
|
3500
3558
|
});
|
|
3501
|
-
const submit =
|
|
3559
|
+
const submit = Effect18.fn("JobRuntime.submit")(function* (def, payload) {
|
|
3502
3560
|
const jobId = yield* enqueue(def, payload);
|
|
3503
3561
|
return yield* wait(jobId);
|
|
3504
3562
|
});
|
|
3505
|
-
const status =
|
|
3563
|
+
const status = Effect18.fn("JobRuntime.status")(function* (jobId) {
|
|
3506
3564
|
if (yield* readMarker(store, jobId, CANCELLED))
|
|
3507
3565
|
return "cancelled";
|
|
3508
3566
|
if (yield* readMarker(store, jobId, TIMED_OUT))
|
|
@@ -3519,66 +3577,66 @@ var layer6 = (defs) => Layer7.effect(Service7, Effect17.gen(function* () {
|
|
|
3519
3577
|
}
|
|
3520
3578
|
return "pending";
|
|
3521
3579
|
});
|
|
3522
|
-
const cancel =
|
|
3580
|
+
const cancel = Effect18.fn("JobRuntime.cancel")(function* (jobId) {
|
|
3523
3581
|
yield* writeMarker(store, jobId, CANCELLED);
|
|
3524
3582
|
return true;
|
|
3525
3583
|
});
|
|
3526
|
-
const processPending =
|
|
3584
|
+
const processPending = Effect18.fn("JobRuntime.processPending")(function* (options) {
|
|
3527
3585
|
const resumeQueue = yield* queueFactory.make({
|
|
3528
3586
|
name: RESUME_QUEUE,
|
|
3529
3587
|
schema: ResumeSignal,
|
|
3530
3588
|
maxAttempts: 1
|
|
3531
3589
|
});
|
|
3532
3590
|
const window = Duration.toMillis(Duration.fromInput(options?.window ?? PROCESS_WINDOW).pipe(Option8.getOrElse(() => Duration.seconds(30))));
|
|
3533
|
-
const deadline = yield* Clock2.currentTimeMillis.pipe(
|
|
3591
|
+
const deadline = yield* Clock2.currentTimeMillis.pipe(Effect18.map((now) => now + window));
|
|
3534
3592
|
let processed = 0;
|
|
3535
3593
|
while (true) {
|
|
3536
3594
|
const now = yield* Clock2.currentTimeMillis;
|
|
3537
3595
|
if (now >= deadline)
|
|
3538
3596
|
break;
|
|
3539
|
-
const signal = yield* resumeQueue.take((value2) =>
|
|
3597
|
+
const signal = yield* resumeQueue.take((value2) => Effect18.succeed(Option8.some(value2))).pipe(Effect18.mapError((e) => new QueueError({ message: e instanceof Error ? e.message : String(e) })), Effect18.timeoutOrElse({
|
|
3540
3598
|
duration: Duration.millis(100),
|
|
3541
|
-
orElse: () =>
|
|
3599
|
+
orElse: () => Effect18.succeed(Option8.none())
|
|
3542
3600
|
}));
|
|
3543
3601
|
if (Option8.isNone(signal))
|
|
3544
3602
|
continue;
|
|
3545
3603
|
const executionId = signal.value.executionId;
|
|
3546
3604
|
const def = yield* findDef(defs, executionId, store);
|
|
3547
3605
|
if (def) {
|
|
3548
|
-
yield* engine.resume(def.workflow, executionId).pipe(
|
|
3606
|
+
yield* engine.resume(def.workflow, executionId).pipe(Effect18.orDie);
|
|
3549
3607
|
processed++;
|
|
3550
3608
|
}
|
|
3551
3609
|
}
|
|
3552
3610
|
return processed;
|
|
3553
3611
|
});
|
|
3554
|
-
return
|
|
3612
|
+
return Service8.of({ enqueue, wait, submit, status, cancel, processPending });
|
|
3555
3613
|
}));
|
|
3556
|
-
var findDef = (defs, executionId, store) =>
|
|
3614
|
+
var findDef = (defs, executionId, store) => Effect18.gen(function* () {
|
|
3557
3615
|
const persisted = yield* getEncodedExit(store, `${executionId}/record`);
|
|
3558
3616
|
if (!persisted || !Exit5.isSuccess(persisted))
|
|
3559
3617
|
return;
|
|
3560
|
-
const record = yield*
|
|
3618
|
+
const record = yield* Effect18.orDie(Schema18.decodeUnknownEffect(ExecutionRecord)(persisted.value));
|
|
3561
3619
|
return defs.find((d) => d.workflow._tag === record.workflowTag);
|
|
3562
3620
|
});
|
|
3563
3621
|
var layerLibsql = (defs, options) => {
|
|
3564
3622
|
const stack = libsqlStack(options);
|
|
3565
|
-
return
|
|
3623
|
+
return layer7(defs).pipe(Layer8.provideMerge(Layer8.mergeAll(layerDurable.pipe(Layer8.provide(stack)), stack)));
|
|
3566
3624
|
};
|
|
3567
|
-
var configured3 = (defs, options) => make({ name: "JobStack", layer:
|
|
3625
|
+
var configured3 = (defs, options) => make({ name: "JobStack", layer: Layer8.suspend(() => layerLibsql(defs, options)), deps: [] });
|
|
3568
3626
|
var node3 = configured3(queues, { url: JobsConfigDefaults.dbUrl });
|
|
3569
3627
|
// ../workflow/src/registry.ts
|
|
3570
|
-
import { Context as
|
|
3628
|
+
import { Context as Context10, Effect as Effect19, Layer as Layer9, Option as Option9, Ref, Schema as Schema19 } from "effect";
|
|
3571
3629
|
class JobFailure extends Schema19.TaggedError()("JobFailure", {
|
|
3572
3630
|
jobId: Schema19.String,
|
|
3573
3631
|
message: Schema19.String
|
|
3574
3632
|
}) {
|
|
3575
3633
|
}
|
|
3576
3634
|
|
|
3577
|
-
class
|
|
3635
|
+
class Service9 extends Context10.Service()("@dimple/workflow/JobRegistry") {
|
|
3578
3636
|
}
|
|
3579
|
-
var
|
|
3637
|
+
var layer8 = Layer9.effect(Service9, Effect19.gen(function* () {
|
|
3580
3638
|
const handlers = yield* Ref.make(new Map);
|
|
3581
|
-
const register =
|
|
3639
|
+
const register = Effect19.fn("JobRegistry.register")(function* (type, handler) {
|
|
3582
3640
|
const existed = yield* Ref.modify(handlers, (map2) => {
|
|
3583
3641
|
if (map2.has(type)) {
|
|
3584
3642
|
return [true, map2];
|
|
@@ -3586,48 +3644,48 @@ var layer7 = Layer8.effect(Service8, Effect18.gen(function* () {
|
|
|
3586
3644
|
return [false, new Map(map2).set(type, handler)];
|
|
3587
3645
|
});
|
|
3588
3646
|
if (existed) {
|
|
3589
|
-
return yield*
|
|
3647
|
+
return yield* Effect19.fail(new RegistryError({ type, message: `Job type already registered: ${type}` }));
|
|
3590
3648
|
}
|
|
3591
3649
|
});
|
|
3592
|
-
const resolve =
|
|
3593
|
-
const handler = yield* Ref.get(handlers).pipe(
|
|
3650
|
+
const resolve = Effect19.fn("JobRegistry.resolve")(function* (type) {
|
|
3651
|
+
const handler = yield* Ref.get(handlers).pipe(Effect19.map((map2) => Option9.fromNullishOr(map2.get(type))));
|
|
3594
3652
|
if (Option9.isNone(handler)) {
|
|
3595
|
-
return yield*
|
|
3653
|
+
return yield* Effect19.fail(new UnknownJobType({ type }));
|
|
3596
3654
|
}
|
|
3597
3655
|
return handler.value;
|
|
3598
3656
|
});
|
|
3599
|
-
return
|
|
3657
|
+
return Service9.of({ register, resolve });
|
|
3600
3658
|
}));
|
|
3601
|
-
var configured4 = () => make({ name: "JobRegistry", layer:
|
|
3659
|
+
var configured4 = () => make({ name: "JobRegistry", layer: layer8, deps: [] });
|
|
3602
3660
|
var node4 = configured4();
|
|
3603
3661
|
// ../workflow/src/executor.ts
|
|
3604
|
-
import { Cause as Cause2, Duration as Duration2, Effect as
|
|
3662
|
+
import { Cause as Cause2, Duration as Duration2, Effect as Effect20, Layer as Layer10, Schedule } from "effect";
|
|
3605
3663
|
import { DurableQueue as DurableQueue2, WorkflowEngine as WorkflowEngine4 } from "effect/workflow";
|
|
3606
|
-
var runHandler = (def, payload, jobId, maxAttempts) =>
|
|
3607
|
-
const registry = yield*
|
|
3664
|
+
var runHandler = (def, payload, jobId, maxAttempts) => Effect20.gen(function* () {
|
|
3665
|
+
const registry = yield* Service9;
|
|
3608
3666
|
const handler = yield* registry.resolve(def.name);
|
|
3609
|
-
yield*
|
|
3667
|
+
yield* Effect20.logInfo("job.started").pipe(Effect20.annotateLogs({
|
|
3610
3668
|
scope: "@dimple/workflow",
|
|
3611
3669
|
event: "job.started",
|
|
3612
3670
|
job: def.name,
|
|
3613
3671
|
idempotencyKey: jobId
|
|
3614
3672
|
}));
|
|
3615
3673
|
const run = () => handler(payload, { jobId });
|
|
3616
|
-
const outcome = yield* (maxAttempts > 1 ? run().pipe(
|
|
3674
|
+
const outcome = yield* (maxAttempts > 1 ? run().pipe(Effect20.retry({
|
|
3617
3675
|
times: maxAttempts - 1,
|
|
3618
3676
|
schedule: Schedule.spaced(Duration2.millis(250)).pipe(Schedule.jittered)
|
|
3619
|
-
})) : run()).pipe(
|
|
3677
|
+
})) : run()).pipe(Effect20.exit);
|
|
3620
3678
|
if (outcome._tag === "Failure") {
|
|
3621
|
-
yield*
|
|
3679
|
+
yield* Effect20.logError("job.failed").pipe(Effect20.annotateLogs({
|
|
3622
3680
|
scope: "@dimple/workflow",
|
|
3623
3681
|
event: "job.failed",
|
|
3624
3682
|
job: def.name,
|
|
3625
3683
|
idempotencyKey: jobId,
|
|
3626
3684
|
error: Cause2.pretty(outcome.cause)
|
|
3627
3685
|
}));
|
|
3628
|
-
return yield*
|
|
3686
|
+
return yield* Effect20.failCause(outcome.cause).pipe(Effect20.mapError((e) => e.message));
|
|
3629
3687
|
}
|
|
3630
|
-
yield*
|
|
3688
|
+
yield* Effect20.logInfo("job.completed").pipe(Effect20.annotateLogs({
|
|
3631
3689
|
scope: "@dimple/workflow",
|
|
3632
3690
|
event: "job.completed",
|
|
3633
3691
|
job: def.name,
|
|
@@ -3642,23 +3700,23 @@ var workerLayer = (defs, options = {}) => {
|
|
|
3642
3700
|
const layers = defs.map((def) => workerFor(def, options.concurrency ?? 1, maxAttempts));
|
|
3643
3701
|
const [first, ...rest] = layers;
|
|
3644
3702
|
if (!first)
|
|
3645
|
-
return
|
|
3646
|
-
const workers = rest.reduce((acc,
|
|
3647
|
-
return options.logging === undefined ? workers :
|
|
3703
|
+
return Layer10.empty;
|
|
3704
|
+
const workers = rest.reduce((acc, layer9) => Layer10.mergeAll(acc, layer9), first);
|
|
3705
|
+
return options.logging === undefined ? workers : Layer10.provideMerge(layer(options.logging))(workers);
|
|
3648
3706
|
};
|
|
3649
|
-
var configured5 = (defs, options, deps) => make({ name: "Workers", layer:
|
|
3707
|
+
var configured5 = (defs, options, deps) => make({ name: "Workers", layer: Layer10.suspend(() => workerLayer(defs, options)), deps });
|
|
3650
3708
|
// ../store/src/topics/lifecycle.ts
|
|
3651
|
-
var routingFailure3 = (tag) => (effect) => effect.pipe(
|
|
3652
|
-
var isActive =
|
|
3709
|
+
var routingFailure3 = (tag) => (effect) => effect.pipe(Effect21.mapError((cause) => new RoutingFailure({ cause: { operation: tag, cause } })));
|
|
3710
|
+
var isActive = Effect21.fn("topics.isActive")(function* (modelKey) {
|
|
3653
3711
|
const backend = yield* Service2;
|
|
3654
3712
|
const db = backend.db;
|
|
3655
|
-
const rows = yield*
|
|
3713
|
+
const rows = yield* Effect21.tryPromise({
|
|
3656
3714
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and3(eq3(topics.modelKey, modelKey), eq3(topics.status, "active"))).limit(1),
|
|
3657
3715
|
catch: (cause) => cause
|
|
3658
3716
|
}).pipe(routingFailure3("topics.isActive"));
|
|
3659
3717
|
return rows.length > 0;
|
|
3660
3718
|
});
|
|
3661
|
-
var setActiveModel =
|
|
3719
|
+
var setActiveModel = Effect21.fn("topics.setActiveModel")(function* (model, config) {
|
|
3662
3720
|
const backend = yield* Service2;
|
|
3663
3721
|
const db = backend.db;
|
|
3664
3722
|
yield* ensureTopicTree(model, config).pipe(routingFailure3("topics.setActiveModel.seed"));
|
|
@@ -3666,14 +3724,14 @@ var setActiveModel = Effect20.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3666
3724
|
let routed = 0;
|
|
3667
3725
|
let alreadyInTree = 0;
|
|
3668
3726
|
if (memoryIds.length > 0) {
|
|
3669
|
-
const topicRows = yield*
|
|
3727
|
+
const topicRows = yield* Effect21.tryPromise({
|
|
3670
3728
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(eq3(topics.modelKey, model.id)),
|
|
3671
3729
|
catch: (cause) => cause
|
|
3672
3730
|
}).pipe(routingFailure3("topics.setActiveModel.topics"));
|
|
3673
3731
|
const modelTopicKeys = topicRows.map((r) => r.topicKey);
|
|
3674
3732
|
if (modelTopicKeys.length > 0) {
|
|
3675
3733
|
for (const memoryId of memoryIds) {
|
|
3676
|
-
const existing = yield*
|
|
3734
|
+
const existing = yield* Effect21.tryPromise({
|
|
3677
3735
|
try: () => db.select({ topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and3(eq3(topicMemberships.memoryId, memoryId), eq3(topicMemberships.modelKey, model.id), inArray(topicMemberships.topicKey, modelTopicKeys))).limit(1),
|
|
3678
3736
|
catch: (cause) => cause
|
|
3679
3737
|
}).pipe(routingFailure3("topics.setActiveModel.membership"));
|
|
@@ -3691,7 +3749,7 @@ var setActiveModel = Effect20.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3691
3749
|
}
|
|
3692
3750
|
}
|
|
3693
3751
|
const auditTimestamp = yield* Clock3.currentTimeMillis;
|
|
3694
|
-
const deactivated = yield*
|
|
3752
|
+
const deactivated = yield* Effect21.tryPromise({
|
|
3695
3753
|
try: () => db.transaction(async (tx) => {
|
|
3696
3754
|
const others = await tx.selectDistinct({ modelKey: topics.modelKey }).from(topics).where(and3(ne(topics.modelKey, model.id), ne(topics.status, "inactive")));
|
|
3697
3755
|
const affected = others.map((r) => r.modelKey).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
@@ -3714,18 +3772,18 @@ var setActiveModel = Effect20.fn("topics.setActiveModel")(function* (model, conf
|
|
|
3714
3772
|
}).pipe(routingFailure3("topics.setActiveModel.deactivate"));
|
|
3715
3773
|
return { routed, alreadyInTree, deactivated };
|
|
3716
3774
|
});
|
|
3717
|
-
var embedOneMemory = (deps, payload, ctx) =>
|
|
3775
|
+
var embedOneMemory = (deps, payload, ctx) => Effect21.gen(function* () {
|
|
3718
3776
|
const envelope = payload;
|
|
3719
3777
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
3720
3778
|
if (!model) {
|
|
3721
|
-
return yield*
|
|
3779
|
+
return yield* Effect21.fail(new JobFailure({
|
|
3722
3780
|
jobId: ctx.jobId,
|
|
3723
3781
|
message: `memory.embed: unknown model "${envelope.modelKey}"`
|
|
3724
3782
|
}));
|
|
3725
3783
|
}
|
|
3726
|
-
const content = yield* deps.getMemoryContent(envelope.memoryId).pipe(
|
|
3784
|
+
const content = yield* deps.getMemoryContent(envelope.memoryId).pipe(Effect21.catchTag("Store.NotFound", () => Effect21.succeed(undefined)), Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: load content: ${e}` })));
|
|
3727
3785
|
if (content === undefined) {
|
|
3728
|
-
yield*
|
|
3786
|
+
yield* Effect21.logDebug("embed.stale-reoffer").pipe(Effect21.annotateLogs({
|
|
3729
3787
|
scope: "@dimple/store",
|
|
3730
3788
|
event: "embed.stale-reoffer",
|
|
3731
3789
|
memoryId: envelope.memoryId,
|
|
@@ -3733,24 +3791,24 @@ var embedOneMemory = (deps, payload, ctx) => Effect20.gen(function* () {
|
|
|
3733
3791
|
}));
|
|
3734
3792
|
return;
|
|
3735
3793
|
}
|
|
3736
|
-
const vector = yield* deps.embedOne(content, model).pipe(
|
|
3794
|
+
const vector = yield* deps.embedOne(content, model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: embed: ${e}` })));
|
|
3737
3795
|
const backend = yield* Service2;
|
|
3738
3796
|
const db = backend.db;
|
|
3739
|
-
const existing = yield*
|
|
3797
|
+
const existing = yield* Effect21.tryPromise({
|
|
3740
3798
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and3(eq3(topicMemberships.memoryId, envelope.memoryId), eq3(topicMemberships.modelKey, model.id))),
|
|
3741
3799
|
catch: (cause) => new StoreError({ message: `memory.embed: membership check: ${cause}` })
|
|
3742
|
-
}).pipe(
|
|
3743
|
-
yield* ensureTopicTree(model, deps.config).pipe(
|
|
3744
|
-
yield* backend.ensureVectorTable(model).pipe(
|
|
3745
|
-
yield* backend.insertVector(envelope.memoryId, [...vector], model).pipe(
|
|
3746
|
-
yield*
|
|
3800
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: membership check: ${e}` })));
|
|
3801
|
+
yield* ensureTopicTree(model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: seed tree: ${e}` })));
|
|
3802
|
+
yield* backend.ensureVectorTable(model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: vector table: ${e}` })));
|
|
3803
|
+
yield* backend.insertVector(envelope.memoryId, [...vector], model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: insert vector: ${e}` })));
|
|
3804
|
+
yield* Effect21.tryPromise({
|
|
3747
3805
|
try: () => db.update(memories).set({ embedding: [...vector], embeddingModel: model.id }).where(eq3(memories.id, envelope.memoryId)),
|
|
3748
3806
|
catch: (cause) => new StoreError({ message: `memory.embed: stamp embedding: ${cause}` })
|
|
3749
|
-
}).pipe(
|
|
3807
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: stamp embedding: ${e}` })));
|
|
3750
3808
|
if (existing.length === 0) {
|
|
3751
|
-
yield* routeInsert(envelope.memoryId, [...vector], model, deps.config).pipe(
|
|
3809
|
+
yield* routeInsert(envelope.memoryId, [...vector], model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: route: ${e}` })));
|
|
3752
3810
|
}
|
|
3753
|
-
yield*
|
|
3811
|
+
yield* Effect21.logInfo("embed.completed").pipe(Effect21.annotateLogs({
|
|
3754
3812
|
scope: "@dimple/store",
|
|
3755
3813
|
event: "embed.completed",
|
|
3756
3814
|
memoryId: envelope.memoryId,
|
|
@@ -3761,9 +3819,9 @@ var embedOneMemory = (deps, payload, ctx) => Effect20.gen(function* () {
|
|
|
3761
3819
|
});
|
|
3762
3820
|
function memoryEmbedHandler(deps) {
|
|
3763
3821
|
const run = (payload, ctx) => embedOneMemory(deps, payload, ctx);
|
|
3764
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
3822
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect21.provideService(Service2, deps.backend)) : run;
|
|
3765
3823
|
}
|
|
3766
|
-
var logBatchCompleted = (modelKey, embedded, failed, ctx) =>
|
|
3824
|
+
var logBatchCompleted = (modelKey, embedded, failed, ctx) => Effect21.logInfo("embed.batch.completed").pipe(Effect21.annotateLogs({
|
|
3767
3825
|
scope: "@dimple/store",
|
|
3768
3826
|
event: "embed.batch.completed",
|
|
3769
3827
|
jobId: ctx.jobId,
|
|
@@ -3771,15 +3829,15 @@ var logBatchCompleted = (modelKey, embedded, failed, ctx) => Effect20.logInfo("e
|
|
|
3771
3829
|
embedded,
|
|
3772
3830
|
failed
|
|
3773
3831
|
}));
|
|
3774
|
-
var batchFailure = (ctx, failed, total, first) =>
|
|
3832
|
+
var batchFailure = (ctx, failed, total, first) => Effect21.fail(new JobFailure({
|
|
3775
3833
|
jobId: ctx.jobId,
|
|
3776
3834
|
message: `memory.embed.batch: ${failed}/${total} failed — ${first}`
|
|
3777
3835
|
}));
|
|
3778
3836
|
function memoryEmbedBatchHandler(deps) {
|
|
3779
|
-
const run = (payload, ctx) =>
|
|
3837
|
+
const run = (payload, ctx) => Effect21.gen(function* () {
|
|
3780
3838
|
const model = deps.getModelConfig(payload.modelKey);
|
|
3781
3839
|
if (!model) {
|
|
3782
|
-
return yield*
|
|
3840
|
+
return yield* Effect21.fail(new JobFailure({
|
|
3783
3841
|
jobId: ctx.jobId,
|
|
3784
3842
|
message: `memory.embed: unknown model "${payload.modelKey}"`
|
|
3785
3843
|
}));
|
|
@@ -3790,23 +3848,23 @@ function memoryEmbedBatchHandler(deps) {
|
|
|
3790
3848
|
let first;
|
|
3791
3849
|
const embedded = [];
|
|
3792
3850
|
const stale = [];
|
|
3793
|
-
yield*
|
|
3794
|
-
const content = yield* deps.getMemoryContent(memoryId).pipe(
|
|
3851
|
+
yield* Effect21.forEach(payload.memoryIds, (memoryId) => Effect21.gen(function* () {
|
|
3852
|
+
const content = yield* deps.getMemoryContent(memoryId).pipe(Effect21.catchTag("Store.NotFound", () => Effect21.succeed(undefined)), Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: load content: ${e}` })));
|
|
3795
3853
|
if (content === undefined) {
|
|
3796
3854
|
stale.push(memoryId);
|
|
3797
3855
|
return;
|
|
3798
3856
|
}
|
|
3799
|
-
const vector = yield* deps.embedOne(content, model).pipe(
|
|
3857
|
+
const vector = yield* deps.embedOne(content, model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: embed: ${e}` })));
|
|
3800
3858
|
embedded.push({ memoryId, vector });
|
|
3801
|
-
}).pipe(
|
|
3859
|
+
}).pipe(Effect21.exit, Effect21.tap((exit) => {
|
|
3802
3860
|
if (Exit6.isSuccess(exit))
|
|
3803
|
-
return
|
|
3861
|
+
return Effect21.void;
|
|
3804
3862
|
failed += 1;
|
|
3805
3863
|
first ??= Cause3.pretty(exit.cause);
|
|
3806
|
-
return
|
|
3864
|
+
return Effect21.void;
|
|
3807
3865
|
})), { concurrency: 1, discard: true });
|
|
3808
3866
|
for (const memoryId of stale) {
|
|
3809
|
-
yield*
|
|
3867
|
+
yield* Effect21.logDebug("embed.stale-reoffer").pipe(Effect21.annotateLogs({
|
|
3810
3868
|
scope: "@dimple/store",
|
|
3811
3869
|
event: "embed.stale-reoffer",
|
|
3812
3870
|
memoryId,
|
|
@@ -3820,25 +3878,25 @@ function memoryEmbedBatchHandler(deps) {
|
|
|
3820
3878
|
}
|
|
3821
3879
|
return;
|
|
3822
3880
|
}
|
|
3823
|
-
yield* ensureTopicTree(model, deps.config).pipe(
|
|
3824
|
-
yield* backend.ensureVectorTable(model).pipe(
|
|
3881
|
+
yield* ensureTopicTree(model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: seed tree: ${e}` })));
|
|
3882
|
+
yield* backend.ensureVectorTable(model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: vector table: ${e}` })));
|
|
3825
3883
|
const ids = embedded.map((e) => e.memoryId);
|
|
3826
|
-
const members = yield*
|
|
3884
|
+
const members = yield* Effect21.tryPromise({
|
|
3827
3885
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and3(eq3(topicMemberships.modelKey, model.id), inArray(topicMemberships.memoryId, ids))),
|
|
3828
3886
|
catch: (cause) => new StoreError({ message: `memory.embed: membership check: ${cause}` })
|
|
3829
|
-
}).pipe(
|
|
3887
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: membership check: ${e}` })));
|
|
3830
3888
|
const memberSet = new Set(members.map((r) => r.memoryId));
|
|
3831
3889
|
const entries = embedded.map((e) => ({ memoryId: e.memoryId, vector: [...e.vector] }));
|
|
3832
|
-
yield* backend.insertVectors(entries, model).pipe(
|
|
3833
|
-
yield*
|
|
3890
|
+
yield* backend.insertVectors(entries, model).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: insert vectors: ${e}` })));
|
|
3891
|
+
yield* Effect21.tryPromise({
|
|
3834
3892
|
try: () => db.transaction(async (tx) => {
|
|
3835
3893
|
await tx.run(sql3`UPDATE ${memories} SET embedding = CASE id ${sql3.join(entries.map((entry) => sql3`WHEN ${entry.memoryId} THEN ${new Float32Array(entry.vector).buffer}`), sql3` `)} END, embedding_model = ${model.id} WHERE id IN (${sql3.join(entries.map((entry) => sql3`${entry.memoryId}`), sql3`, `)})`);
|
|
3836
3894
|
}),
|
|
3837
3895
|
catch: (cause) => new StoreError({ message: `memory.embed: stamp embeddings: ${cause}` })
|
|
3838
|
-
}).pipe(
|
|
3896
|
+
}).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: stamp embeddings: ${e}` })));
|
|
3839
3897
|
const fresh = entries.filter((entry) => !memberSet.has(entry.memoryId));
|
|
3840
3898
|
if (fresh.length > 0) {
|
|
3841
|
-
yield* routeInsertMany(fresh, model, deps.config).pipe(
|
|
3899
|
+
yield* routeInsertMany(fresh, model, deps.config).pipe(Effect21.mapError((e) => new JobFailure({ jobId: ctx.jobId, message: `memory.embed: route: ${e}` })));
|
|
3842
3900
|
}
|
|
3843
3901
|
yield* logBatchCompleted(payload.modelKey, embedded.length, failed, ctx);
|
|
3844
3902
|
if (failed > 0) {
|
|
@@ -3846,11 +3904,11 @@ function memoryEmbedBatchHandler(deps) {
|
|
|
3846
3904
|
}
|
|
3847
3905
|
return;
|
|
3848
3906
|
});
|
|
3849
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
3907
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect21.provideService(Service2, deps.backend)) : run;
|
|
3850
3908
|
}
|
|
3851
3909
|
|
|
3852
3910
|
// ../store/src/topics/jobs.ts
|
|
3853
|
-
import { Clock as Clock4, Effect as
|
|
3911
|
+
import { Clock as Clock4, Effect as Effect22 } from "effect";
|
|
3854
3912
|
import { and as and4, asc as asc3, eq as eq4, inArray as inArray2, sql as sql5 } from "drizzle-orm";
|
|
3855
3913
|
|
|
3856
3914
|
// ../store/src/topics/topic-vectors.ts
|
|
@@ -3889,13 +3947,13 @@ var childBaseKey = (parent, vectors) => {
|
|
|
3889
3947
|
}
|
|
3890
3948
|
return `t_${(hash >>> 0).toString(16).padStart(8, "0")}`;
|
|
3891
3949
|
};
|
|
3892
|
-
var splitTopic =
|
|
3950
|
+
var splitTopic = Effect22.fn("TopicSplit.splitTopic")(function* (topicKey, model, config) {
|
|
3893
3951
|
const backend = yield* Service2;
|
|
3894
3952
|
const db = backend.db;
|
|
3895
3953
|
yield* backend.ensureTopicVectorTable(model);
|
|
3896
3954
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS;
|
|
3897
3955
|
const hardLimit = config.splitDeferredHardLimit ?? DEFAULT_SPLIT_DEFERRED_HARD_LIMIT;
|
|
3898
|
-
const leafRows = yield*
|
|
3956
|
+
const leafRows = yield* Effect22.tryPromise({
|
|
3899
3957
|
try: () => db.select().from(topics).where(and4(eq4(topics.topicKey, topicKey), eq4(topics.modelKey, model.id))),
|
|
3900
3958
|
catch: (cause) => new StoreError({ message: `splitTopic: load topic: ${cause}` })
|
|
3901
3959
|
});
|
|
@@ -3906,13 +3964,13 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3906
3964
|
if (leaf.modelKey !== model.id) {
|
|
3907
3965
|
return yield* new ModelMismatch({ expected: model.id, actual: leaf.modelKey });
|
|
3908
3966
|
}
|
|
3909
|
-
const memberRows = yield*
|
|
3967
|
+
const memberRows = yield* Effect22.tryPromise({
|
|
3910
3968
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and4(eq4(topicMemberships.topicKey, topicKey), eq4(topicMemberships.modelKey, model.id))).orderBy(asc3(topicMemberships.memoryId)),
|
|
3911
3969
|
catch: (cause) => new StoreError({ message: `splitTopic: load members: ${cause}` })
|
|
3912
3970
|
});
|
|
3913
3971
|
const memberIds = memberRows.map((r) => r.memoryId);
|
|
3914
3972
|
const memberCount = memberIds.length;
|
|
3915
|
-
const growthRows = yield*
|
|
3973
|
+
const growthRows = yield* Effect22.tryPromise({
|
|
3916
3974
|
try: () => db.select().from(topicGrowthState).where(and4(eq4(topicGrowthState.topicKey, topicKey), eq4(topicGrowthState.modelKey, model.id))),
|
|
3917
3975
|
catch: (cause) => new StoreError({ message: `splitTopic: load growth state: ${cause}` })
|
|
3918
3976
|
});
|
|
@@ -3944,7 +4002,7 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3944
4002
|
const forced = memberCount > hardLimit;
|
|
3945
4003
|
if (!plan.accepted && !forced) {
|
|
3946
4004
|
const now2 = yield* Clock4.currentTimeMillis;
|
|
3947
|
-
yield*
|
|
4005
|
+
yield* Effect22.tryPromise({
|
|
3948
4006
|
try: () => db.transaction(async (tx) => {
|
|
3949
4007
|
await tx.insert(topicGrowthState).values({
|
|
3950
4008
|
topicKey,
|
|
@@ -3977,7 +4035,7 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
3977
4035
|
};
|
|
3978
4036
|
}
|
|
3979
4037
|
const now = yield* Clock4.currentTimeMillis;
|
|
3980
|
-
const children = yield*
|
|
4038
|
+
const children = yield* Effect22.tryPromise({
|
|
3981
4039
|
try: () => db.transaction(async (tx) => {
|
|
3982
4040
|
const child0Cf = cfFromVectors(groupVectors(0));
|
|
3983
4041
|
const child1Cf = cfFromVectors(groupVectors(1));
|
|
@@ -4098,34 +4156,34 @@ var splitTopic = Effect21.fn("TopicSplit.splitTopic")(function* (topicKey, model
|
|
|
4098
4156
|
});
|
|
4099
4157
|
return { accepted: true, children: children.keys, childCounts: children.counts };
|
|
4100
4158
|
});
|
|
4101
|
-
var staleSplitNoOp = (notFound, model) =>
|
|
4159
|
+
var staleSplitNoOp = (notFound, model) => Effect22.gen(function* () {
|
|
4102
4160
|
const backend = yield* Service2;
|
|
4103
4161
|
const db = backend.db;
|
|
4104
|
-
const rows = yield*
|
|
4162
|
+
const rows = yield* Effect22.tryPromise({
|
|
4105
4163
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and4(eq4(topics.topicKey, notFound.topicKey), eq4(topics.modelKey, model.id))),
|
|
4106
4164
|
catch: (cause) => new StoreError({ message: `topic.split: stale check: ${cause}` })
|
|
4107
4165
|
});
|
|
4108
4166
|
if (rows.length > 0) {
|
|
4109
4167
|
return { accepted: false, reason: "no-improvement" };
|
|
4110
4168
|
}
|
|
4111
|
-
return yield*
|
|
4169
|
+
return yield* Effect22.fail(notFound);
|
|
4112
4170
|
});
|
|
4113
4171
|
function topicSplitHandler(deps) {
|
|
4114
|
-
const run = (payload, ctx) =>
|
|
4172
|
+
const run = (payload, ctx) => Effect22.gen(function* () {
|
|
4115
4173
|
const envelope = payload;
|
|
4116
4174
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4117
4175
|
if (!model) {
|
|
4118
|
-
return yield*
|
|
4176
|
+
return yield* Effect22.fail(new JobFailure({
|
|
4119
4177
|
jobId: ctx.jobId,
|
|
4120
4178
|
message: `topic.split: unknown model "${envelope.modelKey}"`
|
|
4121
4179
|
}));
|
|
4122
4180
|
}
|
|
4123
|
-
const outcome = yield* splitTopic(envelope.topicId, model, deps.config).pipe(
|
|
4181
|
+
const outcome = yield* splitTopic(envelope.topicId, model, deps.config).pipe(Effect22.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleSplitNoOp(notFound, model)), Effect22.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4124
4182
|
if (outcome.accepted && deps.onUnevenSplit) {
|
|
4125
4183
|
const [n0, n1] = outcome.childCounts;
|
|
4126
4184
|
const [k0] = outcome.children;
|
|
4127
4185
|
if (Math.max(n0, n1) > 2 * Math.min(n0, n1)) {
|
|
4128
|
-
yield* deps.onUnevenSplit(k0, envelope.modelKey).pipe(
|
|
4186
|
+
yield* deps.onUnevenSplit(k0, envelope.modelKey).pipe(Effect22.mapError((error) => new JobFailure({
|
|
4129
4187
|
jobId: ctx.jobId,
|
|
4130
4188
|
message: `topic.split: enqueue rebalance: ${error}`
|
|
4131
4189
|
})));
|
|
@@ -4135,14 +4193,14 @@ function topicSplitHandler(deps) {
|
|
|
4135
4193
|
const maxLeafItems = deps.config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS;
|
|
4136
4194
|
for (let i = 0;i < outcome.children.length; i++) {
|
|
4137
4195
|
if (outcome.childCounts[i] > maxLeafItems) {
|
|
4138
|
-
yield* deps.onChildOverflow(outcome.children[i], envelope.modelKey).pipe(
|
|
4196
|
+
yield* deps.onChildOverflow(outcome.children[i], envelope.modelKey).pipe(Effect22.mapError((error) => new JobFailure({
|
|
4139
4197
|
jobId: ctx.jobId,
|
|
4140
4198
|
message: `topic.split: enqueue child split: ${error}`
|
|
4141
4199
|
})));
|
|
4142
4200
|
}
|
|
4143
4201
|
}
|
|
4144
4202
|
}
|
|
4145
|
-
yield*
|
|
4203
|
+
yield* Effect22.logInfo("split.completed").pipe(Effect22.annotateLogs({
|
|
4146
4204
|
scope: "@dimple/store",
|
|
4147
4205
|
event: "split.completed",
|
|
4148
4206
|
topicId: envelope.topicId,
|
|
@@ -4152,16 +4210,16 @@ function topicSplitHandler(deps) {
|
|
|
4152
4210
|
}));
|
|
4153
4211
|
return;
|
|
4154
4212
|
});
|
|
4155
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4213
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect22.provideService(Service2, deps.backend)) : run;
|
|
4156
4214
|
}
|
|
4157
4215
|
|
|
4158
4216
|
// ../store/src/topics/maintenance.ts
|
|
4159
|
-
import { Clock as Clock5, Effect as
|
|
4217
|
+
import { Clock as Clock5, Effect as Effect23 } from "effect";
|
|
4160
4218
|
import { and as and5, asc as asc4, eq as eq5, ne as ne2 } from "drizzle-orm";
|
|
4161
4219
|
var DEFAULT_MIN_LEAF_ITEMS = 32;
|
|
4162
4220
|
var DEFAULT_MAX_LEAF_ITEMS2 = 128;
|
|
4163
4221
|
var DEFAULT_REBALANCE_COOLDOWN_MS = 60000;
|
|
4164
|
-
var maintenanceError = (tag) => (effect) => effect.pipe(
|
|
4222
|
+
var maintenanceError = (tag) => (effect) => effect.pipe(Effect23.mapError((cause) => new StoreError({ message: `${tag}: ${cause}` })));
|
|
4165
4223
|
var dot2 = (a, b) => {
|
|
4166
4224
|
let sum = 0;
|
|
4167
4225
|
for (let i = 0;i < a.length; i++)
|
|
@@ -4179,12 +4237,12 @@ var resolvedConfig2 = (config) => ({
|
|
|
4179
4237
|
maxLeafItems: config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS2,
|
|
4180
4238
|
rebalanceCooldownMs: config.rebalanceCooldownMs ?? DEFAULT_REBALANCE_COOLDOWN_MS
|
|
4181
4239
|
});
|
|
4182
|
-
var mergeOrBorrow =
|
|
4240
|
+
var mergeOrBorrow = Effect23.fn("topics.mergeOrBorrow")(function* (topicKey, model, config) {
|
|
4183
4241
|
const backend = yield* Service2;
|
|
4184
4242
|
const db = backend.db;
|
|
4185
4243
|
const { minLeafItems, maxLeafItems } = resolvedConfig2(config);
|
|
4186
4244
|
const now = yield* Clock5.currentTimeMillis;
|
|
4187
|
-
const leafRows = yield*
|
|
4245
|
+
const leafRows = yield* Effect23.tryPromise({
|
|
4188
4246
|
try: () => db.select().from(topics).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id))),
|
|
4189
4247
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load topic: ${cause}` })
|
|
4190
4248
|
});
|
|
@@ -4197,7 +4255,7 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4197
4255
|
}
|
|
4198
4256
|
if (leaf.memberCount >= minLeafItems)
|
|
4199
4257
|
return { action: "none" };
|
|
4200
|
-
const parentRows = yield*
|
|
4258
|
+
const parentRows = yield* Effect23.tryPromise({
|
|
4201
4259
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.parentKey)),
|
|
4202
4260
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load parent: ${cause}` })
|
|
4203
4261
|
});
|
|
@@ -4205,16 +4263,16 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4205
4263
|
return yield* logUnderfilled(db, topicKey, model, now, "no-borrowable");
|
|
4206
4264
|
}
|
|
4207
4265
|
const parentKey = parentRows[0].parentKey;
|
|
4208
|
-
const parentRowRows = yield*
|
|
4266
|
+
const parentRowRows = yield* Effect23.tryPromise({
|
|
4209
4267
|
try: () => db.select({ childCount: topics.childCount }).from(topics).where(and5(eq5(topics.topicKey, parentKey), eq5(topics.modelKey, model.id))),
|
|
4210
4268
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load parent row: ${cause}` })
|
|
4211
4269
|
});
|
|
4212
4270
|
const parentRow = parentRowRows[0];
|
|
4213
|
-
const siblingRows = yield*
|
|
4271
|
+
const siblingRows = yield* Effect23.tryPromise({
|
|
4214
4272
|
try: () => db.select().from(topics).where(and5(eq5(topics.modelKey, model.id), eq5(topics.kind, "leaf"), ne2(topics.status, "inactive"))).orderBy(asc4(topics.topicKey)),
|
|
4215
4273
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load siblings: ${cause}` })
|
|
4216
4274
|
});
|
|
4217
|
-
const relationRows = yield*
|
|
4275
|
+
const relationRows = yield* Effect23.tryPromise({
|
|
4218
4276
|
try: () => db.select({ childKey: topicRelations.childKey }).from(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.childKey)),
|
|
4219
4277
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: load relations: ${cause}` })
|
|
4220
4278
|
});
|
|
@@ -4236,7 +4294,7 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4236
4294
|
const membersByTopic = new Map;
|
|
4237
4295
|
const vecById = new Map;
|
|
4238
4296
|
for (const sibling of siblingsOf) {
|
|
4239
|
-
const members = yield*
|
|
4297
|
+
const members = yield* Effect23.tryPromise({
|
|
4240
4298
|
try: () => db.select({
|
|
4241
4299
|
memoryId: topicMemberships.memoryId,
|
|
4242
4300
|
manualLock: topicMemberships.manualLock
|
|
@@ -4249,7 +4307,7 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4249
4307
|
vecById.set(row.memoryId, new Float32Array(row.vector));
|
|
4250
4308
|
}
|
|
4251
4309
|
const movedIds = new Set;
|
|
4252
|
-
const outcome = yield*
|
|
4310
|
+
const outcome = yield* Effect23.tryPromise({
|
|
4253
4311
|
try: () => db.transaction(async (tx) => {
|
|
4254
4312
|
const leafCf0 = cfOf(leaf);
|
|
4255
4313
|
let leafCf = leafCf0;
|
|
@@ -4418,12 +4476,12 @@ var mergeOrBorrow = Effect22.fn("topics.mergeOrBorrow")(function* (topicKey, mod
|
|
|
4418
4476
|
});
|
|
4419
4477
|
return outcome;
|
|
4420
4478
|
});
|
|
4421
|
-
var rebalanceSiblings =
|
|
4479
|
+
var rebalanceSiblings = Effect23.fn("topics.rebalanceSiblings")(function* (topicKey, model, config) {
|
|
4422
4480
|
const backend = yield* Service2;
|
|
4423
4481
|
const db = backend.db;
|
|
4424
4482
|
const { rebalanceCooldownMs } = resolvedConfig2(config);
|
|
4425
4483
|
const now = yield* Clock5.currentTimeMillis;
|
|
4426
|
-
const rows = yield*
|
|
4484
|
+
const rows = yield* Effect23.tryPromise({
|
|
4427
4485
|
try: () => db.select().from(topics).where(and5(eq5(topics.topicKey, topicKey), eq5(topics.modelKey, model.id))),
|
|
4428
4486
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load topic: ${cause}` })
|
|
4429
4487
|
});
|
|
@@ -4432,19 +4490,19 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4432
4490
|
return yield* new TopicNotFound({ topicKey });
|
|
4433
4491
|
if (row.modelKey !== model.id)
|
|
4434
4492
|
return yield* new ModelMismatch({ expected: model.id, actual: row.modelKey });
|
|
4435
|
-
const parentRows = yield*
|
|
4493
|
+
const parentRows = yield* Effect23.tryPromise({
|
|
4436
4494
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and5(eq5(topicRelations.childKey, topicKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.parentKey)),
|
|
4437
4495
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load parent: ${cause}` })
|
|
4438
4496
|
});
|
|
4439
4497
|
if (parentRows.length === 0)
|
|
4440
4498
|
return { action: "none", reason: "balanced" };
|
|
4441
4499
|
const parentKey = parentRows[0].parentKey;
|
|
4442
|
-
const relationRows = yield*
|
|
4500
|
+
const relationRows = yield* Effect23.tryPromise({
|
|
4443
4501
|
try: () => db.select({ childKey: topicRelations.childKey }).from(topicRelations).where(and5(eq5(topicRelations.parentKey, parentKey), eq5(topicRelations.modelKey, model.id), eq5(topicRelations.relationKind, "parent_child"))).orderBy(asc4(topicRelations.childKey)),
|
|
4444
4502
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load relations: ${cause}` })
|
|
4445
4503
|
});
|
|
4446
4504
|
const siblingKeys = relationRows.map((r) => r.childKey);
|
|
4447
|
-
const topicRows = yield*
|
|
4505
|
+
const topicRows = yield* Effect23.tryPromise({
|
|
4448
4506
|
try: () => db.select().from(topics).where(and5(eq5(topics.modelKey, model.id), eq5(topics.kind, "leaf"), ne2(topics.status, "inactive"))),
|
|
4449
4507
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load siblings: ${cause}` })
|
|
4450
4508
|
});
|
|
@@ -4452,7 +4510,7 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4452
4510
|
const siblings = siblingKeys.map((k) => map2.get(k)).filter(Boolean);
|
|
4453
4511
|
if (siblings.length < 2)
|
|
4454
4512
|
return { action: "none", reason: "balanced" };
|
|
4455
|
-
const growthRows = yield*
|
|
4513
|
+
const growthRows = yield* Effect23.tryPromise({
|
|
4456
4514
|
try: () => db.select().from(topicGrowthState).where(and5(eq5(topicGrowthState.topicKey, topicKey), eq5(topicGrowthState.modelKey, model.id))),
|
|
4457
4515
|
catch: (cause) => new StoreError({ message: `rebalanceSiblings: load growth: ${cause}` })
|
|
4458
4516
|
});
|
|
@@ -4477,7 +4535,7 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4477
4535
|
const membersByTopic = new Map;
|
|
4478
4536
|
const vecById = new Map;
|
|
4479
4537
|
for (const sibling of siblings) {
|
|
4480
|
-
const members = yield*
|
|
4538
|
+
const members = yield* Effect23.tryPromise({
|
|
4481
4539
|
try: () => db.select({
|
|
4482
4540
|
memoryId: topicMemberships.memoryId,
|
|
4483
4541
|
manualLock: topicMemberships.manualLock
|
|
@@ -4490,7 +4548,7 @@ var rebalanceSiblings = Effect22.fn("topics.rebalanceSiblings")(function* (topic
|
|
|
4490
4548
|
vecById.set(row2.memoryId, new Float32Array(row2.vector));
|
|
4491
4549
|
}
|
|
4492
4550
|
const movedIds = new Set;
|
|
4493
|
-
const outcome = yield*
|
|
4551
|
+
const outcome = yield* Effect23.tryPromise({
|
|
4494
4552
|
try: () => db.transaction(async (tx) => {
|
|
4495
4553
|
const liveCfs = new Map(siblings.map((s) => [s.topicKey, cfOf(s)]));
|
|
4496
4554
|
const liveList = () => [...liveCfs.entries()].map(([topicKey2, cf]) => ({ topicKey: topicKey2, cf })).sort((a, b) => b.cf.count - a.cf.count || (a.topicKey < b.topicKey ? -1 : 1));
|
|
@@ -4577,7 +4635,7 @@ var loadMembers = async (tx, topicKey, modelKey) => {
|
|
|
4577
4635
|
const rows = await tx.select({ memoryId: topicMemberships.memoryId, manualLock: topicMemberships.manualLock }).from(topicMemberships).where(and5(eq5(topicMemberships.topicKey, topicKey), eq5(topicMemberships.modelKey, modelKey))).orderBy(asc4(topicMemberships.memoryId));
|
|
4578
4636
|
return rows;
|
|
4579
4637
|
};
|
|
4580
|
-
var logUnderfilled = (db, topicKey, model, now, reason) =>
|
|
4638
|
+
var logUnderfilled = (db, topicKey, model, now, reason) => Effect23.tryPromise({
|
|
4581
4639
|
try: () => db.insert(topicChangeLog).values({
|
|
4582
4640
|
topicKey,
|
|
4583
4641
|
operation: "merge",
|
|
@@ -4587,7 +4645,7 @@ var logUnderfilled = (db, topicKey, model, now, reason) => Effect22.tryPromise({
|
|
|
4587
4645
|
createdAt: now
|
|
4588
4646
|
}),
|
|
4589
4647
|
catch: (cause) => new StoreError({ message: `mergeOrBorrow: log underfilled: ${cause}` })
|
|
4590
|
-
}).pipe(
|
|
4648
|
+
}).pipe(Effect23.map(() => ({ action: "underfilled", reason })));
|
|
4591
4649
|
var logUnderfilledTx = async (tx, topicKey, model, now, reason) => {
|
|
4592
4650
|
await tx.insert(topicChangeLog).values({
|
|
4593
4651
|
topicKey,
|
|
@@ -4600,61 +4658,61 @@ var logUnderfilledTx = async (tx, topicKey, model, now, reason) => {
|
|
|
4600
4658
|
return { action: "underfilled", reason };
|
|
4601
4659
|
};
|
|
4602
4660
|
function topicMergeHandler(deps) {
|
|
4603
|
-
const run = (payload, ctx) =>
|
|
4661
|
+
const run = (payload, ctx) => Effect23.gen(function* () {
|
|
4604
4662
|
const envelope = payload;
|
|
4605
4663
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4606
4664
|
if (!model) {
|
|
4607
|
-
return yield*
|
|
4665
|
+
return yield* Effect23.fail(new JobFailure({
|
|
4608
4666
|
jobId: ctx.jobId,
|
|
4609
4667
|
message: `topic.merge: unknown model "${envelope.modelKey}"`
|
|
4610
4668
|
}));
|
|
4611
4669
|
}
|
|
4612
|
-
yield* mergeOrBorrow(envelope.topicId, model, deps.config).pipe(
|
|
4670
|
+
yield* mergeOrBorrow(envelope.topicId, model, deps.config).pipe(Effect23.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleNoOp(notFound, model)), Effect23.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4613
4671
|
return;
|
|
4614
4672
|
});
|
|
4615
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4673
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect23.provideService(Service2, deps.backend)) : run;
|
|
4616
4674
|
}
|
|
4617
4675
|
function topicRebalanceHandler(deps) {
|
|
4618
|
-
const run = (payload, ctx) =>
|
|
4676
|
+
const run = (payload, ctx) => Effect23.gen(function* () {
|
|
4619
4677
|
const envelope = payload;
|
|
4620
4678
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
4621
4679
|
if (!model) {
|
|
4622
|
-
return yield*
|
|
4680
|
+
return yield* Effect23.fail(new JobFailure({
|
|
4623
4681
|
jobId: ctx.jobId,
|
|
4624
4682
|
message: `topic.rebalance: unknown model "${envelope.modelKey}"`
|
|
4625
4683
|
}));
|
|
4626
4684
|
}
|
|
4627
|
-
yield* rebalanceSiblings(envelope.topicId, model, deps.config).pipe(
|
|
4685
|
+
yield* rebalanceSiblings(envelope.topicId, model, deps.config).pipe(Effect23.catchIf((error) => error._tag === "topics.TopicNotFound", (notFound) => staleNoOp(notFound, model)), Effect23.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `${error}` })));
|
|
4628
4686
|
return;
|
|
4629
4687
|
});
|
|
4630
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
4688
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect23.provideService(Service2, deps.backend)) : run;
|
|
4631
4689
|
}
|
|
4632
|
-
var staleNoOp = (notFound, model) =>
|
|
4690
|
+
var staleNoOp = (notFound, model) => Effect23.gen(function* () {
|
|
4633
4691
|
const backend = yield* Service2;
|
|
4634
4692
|
const db = backend.db;
|
|
4635
|
-
const rows = yield*
|
|
4693
|
+
const rows = yield* Effect23.tryPromise({
|
|
4636
4694
|
try: () => db.select({ topicKey: topics.topicKey }).from(topics).where(and5(eq5(topics.topicKey, notFound.topicKey), eq5(topics.modelKey, model.id))),
|
|
4637
4695
|
catch: (cause) => new StoreError({ message: `maintenance: stale check: ${cause}` })
|
|
4638
4696
|
});
|
|
4639
4697
|
if (rows.length > 0) {
|
|
4640
4698
|
return;
|
|
4641
4699
|
}
|
|
4642
|
-
return yield*
|
|
4643
|
-
}).pipe(
|
|
4700
|
+
return yield* Effect23.fail(new JobFailure({ jobId: notFound.topicKey, message: `topic missing: ${notFound.topicKey}` }));
|
|
4701
|
+
}).pipe(Effect23.mapError((error) => error instanceof JobFailure ? error : new JobFailure({ jobId: notFound.topicKey, message: `${error}` })));
|
|
4644
4702
|
|
|
4645
4703
|
// ../store/src/topics/observability.ts
|
|
4646
|
-
import { Effect as
|
|
4704
|
+
import { Effect as Effect24 } from "effect";
|
|
4647
4705
|
import { and as and6, desc, eq as eq6, inArray as inArray3, ne as ne3, sql as sql6 } from "drizzle-orm";
|
|
4648
4706
|
var DEFAULT_MAX_LEAF_ITEMS3 = 128;
|
|
4649
4707
|
var DEFAULT_SPLIT_DEFERRED_HARD_LIMIT2 = 512;
|
|
4650
4708
|
var DEFAULT_CHANGE_LOG_LIMIT = 50;
|
|
4651
|
-
var routingFailure4 = (tag) => (effect) => effect.pipe(
|
|
4709
|
+
var routingFailure4 = (tag) => (effect) => effect.pipe(Effect24.mapError((cause) => new RoutingFailure({ cause: { operation: tag, cause } })));
|
|
4652
4710
|
var modelTopicKeys = (db, modelKey) => db.select({ topicKey: topics.topicKey }).from(topics).where(eq6(topics.modelKey, modelKey));
|
|
4653
|
-
var topicMetrics =
|
|
4711
|
+
var topicMetrics = Effect24.fn("topics.topicMetrics")(function* (modelKey, config) {
|
|
4654
4712
|
const backend = yield* Service2;
|
|
4655
4713
|
const db = backend.db;
|
|
4656
4714
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS3;
|
|
4657
|
-
const rows = yield*
|
|
4715
|
+
const rows = yield* Effect24.tryPromise({
|
|
4658
4716
|
try: () => db.select({
|
|
4659
4717
|
topicKey: topics.topicKey,
|
|
4660
4718
|
kind: topics.kind,
|
|
@@ -4687,12 +4745,12 @@ var topicMetrics = Effect23.fn("topics.topicMetrics")(function* (modelKey, confi
|
|
|
4687
4745
|
deferredLeaves++;
|
|
4688
4746
|
}
|
|
4689
4747
|
leafMemberCounts.sort((a, b) => a - b);
|
|
4690
|
-
const growth = yield*
|
|
4748
|
+
const growth = yield* Effect24.tryPromise({
|
|
4691
4749
|
try: () => db.select({ total: sql6`coalesce(sum(${topicGrowthState.splitRejectedCount}), 0)` }).from(topicGrowthState).where(eq6(topicGrowthState.modelKey, modelKey)),
|
|
4692
4750
|
catch: (cause) => cause
|
|
4693
4751
|
}).pipe(routingFailure4("topics.topicMetrics.growth"));
|
|
4694
4752
|
const splitRejectedTotal = Number(growth[0]?.total ?? 0);
|
|
4695
|
-
const lastLog = yield*
|
|
4753
|
+
const lastLog = yield* Effect24.tryPromise({
|
|
4696
4754
|
try: () => db.select({
|
|
4697
4755
|
operation: topicChangeLog.operation,
|
|
4698
4756
|
createdAt: topicChangeLog.createdAt
|
|
@@ -4712,11 +4770,11 @@ var topicMetrics = Effect23.fn("topics.topicMetrics")(function* (modelKey, confi
|
|
|
4712
4770
|
lastOperation
|
|
4713
4771
|
};
|
|
4714
4772
|
});
|
|
4715
|
-
var topicChangeLog2 =
|
|
4773
|
+
var topicChangeLog2 = Effect24.fn("topics.topicChangeLog")(function* (modelKey, opts) {
|
|
4716
4774
|
const backend = yield* Service2;
|
|
4717
4775
|
const db = backend.db;
|
|
4718
4776
|
const limit = opts?.limit ?? DEFAULT_CHANGE_LOG_LIMIT;
|
|
4719
|
-
const rows = yield*
|
|
4777
|
+
const rows = yield* Effect24.tryPromise({
|
|
4720
4778
|
try: () => {
|
|
4721
4779
|
const scoped = inArray3(topicChangeLog.topicKey, modelTopicKeys(db, modelKey));
|
|
4722
4780
|
const filtered = opts?.operation ? and6(scoped, eq6(topicChangeLog.operation, opts.operation)) : scoped;
|
|
@@ -4726,14 +4784,14 @@ var topicChangeLog2 = Effect23.fn("topics.topicChangeLog")(function* (modelKey,
|
|
|
4726
4784
|
}).pipe(routingFailure4("topics.topicChangeLog"));
|
|
4727
4785
|
return rows;
|
|
4728
4786
|
});
|
|
4729
|
-
var topicHealth =
|
|
4787
|
+
var topicHealth = Effect24.fn("topics.topicHealth")(function* (modelKey, config) {
|
|
4730
4788
|
const backend = yield* Service2;
|
|
4731
4789
|
const db = backend.db;
|
|
4732
4790
|
const maxLeafItems = config.maxLeafItems ?? DEFAULT_MAX_LEAF_ITEMS3;
|
|
4733
4791
|
const hardLimit = config.splitDeferredHardLimit ?? DEFAULT_SPLIT_DEFERRED_HARD_LIMIT2;
|
|
4734
4792
|
const slack = Math.max(2, Math.ceil(hardLimit / 2) - maxLeafItems);
|
|
4735
4793
|
const leafLimit = maxLeafItems + slack;
|
|
4736
|
-
const rows = yield*
|
|
4794
|
+
const rows = yield* Effect24.tryPromise({
|
|
4737
4795
|
try: () => db.select({
|
|
4738
4796
|
topicKey: topics.topicKey,
|
|
4739
4797
|
kind: topics.kind,
|
|
@@ -4743,7 +4801,7 @@ var topicHealth = Effect23.fn("topics.topicHealth")(function* (modelKey, config)
|
|
|
4743
4801
|
}).from(topics).where(and6(eq6(topics.modelKey, modelKey), ne3(topics.status, "inactive"))),
|
|
4744
4802
|
catch: (cause) => cause
|
|
4745
4803
|
}).pipe(routingFailure4("topics.topicHealth.topics"));
|
|
4746
|
-
const countRows = yield*
|
|
4804
|
+
const countRows = yield* Effect24.tryPromise({
|
|
4747
4805
|
try: () => db.select({ topicKey: topicMemberships.topicKey, count: sql6`count(*)` }).from(topicMemberships).where(eq6(topicMemberships.modelKey, modelKey)).groupBy(topicMemberships.topicKey),
|
|
4748
4806
|
catch: (cause) => cause
|
|
4749
4807
|
}).pipe(routingFailure4("topics.topicHealth.memberships"));
|
|
@@ -4768,30 +4826,30 @@ var topicHealth = Effect23.fn("topics.topicHealth")(function* (modelKey, config)
|
|
|
4768
4826
|
});
|
|
4769
4827
|
|
|
4770
4828
|
// ../store/src/graph/observability.ts
|
|
4771
|
-
import { Effect as
|
|
4829
|
+
import { Effect as Effect25 } from "effect";
|
|
4772
4830
|
import { and as and7, eq as eq7, isNull, sql as sql7 } from "drizzle-orm";
|
|
4773
|
-
var graphCoverage =
|
|
4831
|
+
var graphCoverage = Effect25.fn("GraphCoverage.run")(function* (modelKey) {
|
|
4774
4832
|
const backend = yield* Service2;
|
|
4775
4833
|
const db = backend.db;
|
|
4776
|
-
const totalRows = yield*
|
|
4834
|
+
const totalRows = yield* Effect25.tryPromise({
|
|
4777
4835
|
try: () => db.select({ id: memories.id }).from(memories).where(and7(eq7(memories.state, "active"), eq7(memories.embeddingModel, modelKey))),
|
|
4778
4836
|
catch: (cause) => new StoreError({ message: `graphCoverage: total: ${cause}` })
|
|
4779
4837
|
});
|
|
4780
4838
|
const totalMemories = totalRows.length;
|
|
4781
|
-
const isolatedRows = yield* backend.listIsolatedMemories(0, modelKey).pipe(
|
|
4839
|
+
const isolatedRows = yield* backend.listIsolatedMemories(0, modelKey).pipe(Effect25.mapError((cause) => new StoreError({ message: `graphCoverage: isolated: ${cause}` })));
|
|
4782
4840
|
const isolatedMemories = isolatedRows.length;
|
|
4783
4841
|
const coveredMemories = totalMemories - isolatedMemories;
|
|
4784
4842
|
const coveragePercent = totalMemories === 0 ? 0 : Math.round(coveredMemories / totalMemories * 1000) / 10;
|
|
4785
|
-
const pendingRows = yield*
|
|
4843
|
+
const pendingRows = yield* Effect25.tryPromise({
|
|
4786
4844
|
try: () => db.select({ count: sql7`count(*)` }).from(artifactEdges).where(and7(eq7(artifactEdges.status, "pending"), isNull(artifactEdges.expiredAt))),
|
|
4787
4845
|
catch: (cause) => new StoreError({ message: `graphCoverage: pending: ${cause}` })
|
|
4788
4846
|
});
|
|
4789
4847
|
const pendingCandidates = totalMemories === 0 ? 0 : Number(pendingRows[0]?.count ?? 0);
|
|
4790
|
-
const conceptCount = totalMemories === 0 ? 0 : (yield*
|
|
4848
|
+
const conceptCount = totalMemories === 0 ? 0 : (yield* Effect25.tryPromise({
|
|
4791
4849
|
try: () => db.select({ id: concepts.id }).from(concepts),
|
|
4792
4850
|
catch: (cause) => new StoreError({ message: `graphCoverage: concepts: ${cause}` })
|
|
4793
4851
|
})).length;
|
|
4794
|
-
const annotatedMemories = totalMemories === 0 ? 0 : new Set((yield*
|
|
4852
|
+
const annotatedMemories = totalMemories === 0 ? 0 : new Set((yield* Effect25.tryPromise({
|
|
4795
4853
|
try: () => db.select({ memoryId: memoryConcepts.memoryId }).from(memoryConcepts),
|
|
4796
4854
|
catch: (cause) => new StoreError({ message: `graphCoverage: annotations: ${cause}` })
|
|
4797
4855
|
})).map((row) => row.memoryId)).size;
|
|
@@ -5026,25 +5084,25 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5026
5084
|
const db = backend.db;
|
|
5027
5085
|
const topicsConfig = topicsOptions?.config ?? Schema23.decodeUnknownSync(TopicConfig)({});
|
|
5028
5086
|
const topicsEnabled = topicsOptions?.enabled ?? false;
|
|
5029
|
-
const updateTopicVectorForget = (backendImpl, model, topicKey, x) =>
|
|
5030
|
-
const current = yield* backendImpl.fetchTopicVector(topicKey, model).pipe(
|
|
5087
|
+
const updateTopicVectorForget = (backendImpl, model, topicKey, x) => Effect26.gen(function* () {
|
|
5088
|
+
const current = yield* backendImpl.fetchTopicVector(topicKey, model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: load vector: ${cause}` })));
|
|
5031
5089
|
const ls = current ? new Float32Array(current.sumVector) : new Float32Array(model.dimensions);
|
|
5032
5090
|
for (let i = 0;i < ls.length; i++)
|
|
5033
5091
|
ls[i] = ls[i] - x[i];
|
|
5034
|
-
yield* backendImpl.upsertTopicVector(topicKey, Array.from(ls), model).pipe(
|
|
5035
|
-
const relRows = yield*
|
|
5092
|
+
yield* backendImpl.upsertTopicVector(topicKey, Array.from(ls), model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: write vector: ${cause}` })));
|
|
5093
|
+
const relRows = yield* Effect26.tryPromise({
|
|
5036
5094
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and8(eq8(topicRelations.childKey, topicKey), eq8(topicRelations.modelKey, model.id), eq8(topicRelations.relationKind, "parent_child"))),
|
|
5037
5095
|
catch: storeError("Store.forget: load ancestor")
|
|
5038
5096
|
});
|
|
5039
5097
|
let ancestorKey = relRows[0]?.parentKey;
|
|
5040
5098
|
while (ancestorKey !== undefined) {
|
|
5041
5099
|
const key = ancestorKey;
|
|
5042
|
-
const a = yield* backendImpl.fetchTopicVector(key, model).pipe(
|
|
5100
|
+
const a = yield* backendImpl.fetchTopicVector(key, model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: load vector: ${cause}` })));
|
|
5043
5101
|
const als = a ? new Float32Array(a.sumVector) : new Float32Array(model.dimensions);
|
|
5044
5102
|
for (let i = 0;i < als.length; i++)
|
|
5045
5103
|
als[i] = als[i] - x[i];
|
|
5046
|
-
yield* backendImpl.upsertTopicVector(key, Array.from(als), model).pipe(
|
|
5047
|
-
const up = yield*
|
|
5104
|
+
yield* backendImpl.upsertTopicVector(key, Array.from(als), model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget: write vector: ${cause}` })));
|
|
5105
|
+
const up = yield* Effect26.tryPromise({
|
|
5048
5106
|
try: () => db.select({ parentKey: topicRelations.parentKey }).from(topicRelations).where(and8(eq8(topicRelations.childKey, key), eq8(topicRelations.modelKey, model.id), eq8(topicRelations.relationKind, "parent_child"))),
|
|
5049
5107
|
catch: storeError("Store.forget: load ancestor")
|
|
5050
5108
|
});
|
|
@@ -5094,14 +5152,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5094
5152
|
};
|
|
5095
5153
|
const seam = auditOptions?.enqueueAudit;
|
|
5096
5154
|
if (seam !== undefined) {
|
|
5097
|
-
return propagateSeam ? seam(row).pipe(
|
|
5155
|
+
return propagateSeam ? seam(row).pipe(Effect26.mapError((cause) => new StoreError({ message: `audit.enqueue-failed: ${cause}` }))) : seam(row).pipe(Effect26.catch((cause) => Effect26.logError("audit.enqueue-failed").pipe(Effect26.annotateLogs({
|
|
5098
5156
|
scope: "@dimple/store",
|
|
5099
5157
|
event: "audit.enqueue-failed",
|
|
5100
5158
|
mode,
|
|
5101
5159
|
cause: String(cause)
|
|
5102
5160
|
}))));
|
|
5103
5161
|
}
|
|
5104
|
-
return
|
|
5162
|
+
return Effect26.sync(() => {
|
|
5105
5163
|
db.insert(retrievalLog2).values({
|
|
5106
5164
|
id: row.id,
|
|
5107
5165
|
queryHash: row.queryHash,
|
|
@@ -5116,7 +5174,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5116
5174
|
}).catch(() => {});
|
|
5117
5175
|
});
|
|
5118
5176
|
};
|
|
5119
|
-
const logAudit =
|
|
5177
|
+
const logAudit = Effect26.fn("Store.logAudit")(function* (input) {
|
|
5120
5178
|
return yield* logRetrieval(input.query, input.namespace, input.mode, input.resultCount, input.startTime, input.queryProfile, input.sources, input.hashSeed, true);
|
|
5121
5179
|
});
|
|
5122
5180
|
const GRAPH_SUPERSEDE_DEMOTION = graphConfig?.supersedeDemotion ?? 0.5;
|
|
@@ -5127,7 +5185,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5127
5185
|
const graphEnabled = graphConfig?.enabled !== false;
|
|
5128
5186
|
const GRAPH_CANDIDATE_LIMIT_MAX = 200;
|
|
5129
5187
|
const GRAPH_ISOLATED_EXAMPLES = 10;
|
|
5130
|
-
const fetchResultEdges = (ids) =>
|
|
5188
|
+
const fetchResultEdges = (ids) => Effect26.tryPromise({
|
|
5131
5189
|
try: () => db.select({
|
|
5132
5190
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
5133
5191
|
toMemoryId: artifactEdges.toMemoryId,
|
|
@@ -5136,11 +5194,11 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5136
5194
|
}).from(artifactEdges).where(and8(or(inArray4(artifactEdges.fromMemoryId, ids), inArray4(artifactEdges.toMemoryId, ids)), ne4(artifactEdges.status, "rejected"), isNull2(artifactEdges.expiredAt))),
|
|
5137
5195
|
catch: storeError("Store.retrieval post-pass: edges")
|
|
5138
5196
|
});
|
|
5139
|
-
const fetchResultConcepts = (ids) =>
|
|
5197
|
+
const fetchResultConcepts = (ids) => Effect26.tryPromise({
|
|
5140
5198
|
try: () => db.select({ memoryId: memoryConcepts2.memoryId, conceptId: memoryConcepts2.conceptId }).from(memoryConcepts2).where(inArray4(memoryConcepts2.memoryId, ids)),
|
|
5141
5199
|
catch: storeError("Store.retrieval post-pass: concepts")
|
|
5142
5200
|
});
|
|
5143
|
-
const augmentRetrieval = (results) =>
|
|
5201
|
+
const augmentRetrieval = (results) => Effect26.gen(function* () {
|
|
5144
5202
|
if (results.length === 0)
|
|
5145
5203
|
return results;
|
|
5146
5204
|
const ids = results.map((r) => r.memory.id);
|
|
@@ -5150,7 +5208,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5150
5208
|
supersedeDemotion: GRAPH_SUPERSEDE_DEMOTION
|
|
5151
5209
|
});
|
|
5152
5210
|
});
|
|
5153
|
-
const augmentSearchResults =
|
|
5211
|
+
const augmentSearchResults = Effect26.fn("Store.augmentSearchResults")(function* (results) {
|
|
5154
5212
|
return yield* augmentRetrieval(results);
|
|
5155
5213
|
});
|
|
5156
5214
|
const EDGE_KINDS = [
|
|
@@ -5160,7 +5218,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5160
5218
|
"code_calls"
|
|
5161
5219
|
];
|
|
5162
5220
|
const EDGE_SOURCES = ["rule", "llm", "agent"];
|
|
5163
|
-
const write =
|
|
5221
|
+
const write = Effect26.fn("Store.write")(function* (input) {
|
|
5164
5222
|
if (input.content.includes("\x00")) {
|
|
5165
5223
|
return yield* new StoreError({
|
|
5166
5224
|
message: "Store.write: content contains a NUL byte (libSQL would silently truncate it)"
|
|
@@ -5169,7 +5227,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5169
5227
|
const id = crypto.randomUUID();
|
|
5170
5228
|
const now = Date.now();
|
|
5171
5229
|
const metadataBlob = input.typedMetadata !== undefined ? mergeMetadataBlob(input.metadata, input.typedMetadata) : input.metadata ?? null;
|
|
5172
|
-
yield*
|
|
5230
|
+
yield* Effect26.tryPromise({
|
|
5173
5231
|
try: () => db.transaction(async (tx) => {
|
|
5174
5232
|
await tx.insert(memories2).values({
|
|
5175
5233
|
id,
|
|
@@ -5202,7 +5260,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5202
5260
|
embeddingModel: undefined
|
|
5203
5261
|
};
|
|
5204
5262
|
});
|
|
5205
|
-
const writeMany =
|
|
5263
|
+
const writeMany = Effect26.fn("Store.writeMany")(function* (inputs) {
|
|
5206
5264
|
if (inputs.length === 0)
|
|
5207
5265
|
return [];
|
|
5208
5266
|
for (const input of inputs) {
|
|
@@ -5226,7 +5284,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5226
5284
|
metadata: metadataBlob
|
|
5227
5285
|
};
|
|
5228
5286
|
});
|
|
5229
|
-
yield*
|
|
5287
|
+
yield* Effect26.tryPromise({
|
|
5230
5288
|
try: () => db.transaction(async (tx) => {
|
|
5231
5289
|
await tx.insert(memories2).values(rows);
|
|
5232
5290
|
await tx.insert(memoryEvents2).values(rows.map((row) => ({
|
|
@@ -5254,8 +5312,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5254
5312
|
embeddingModel: undefined
|
|
5255
5313
|
}));
|
|
5256
5314
|
});
|
|
5257
|
-
const get =
|
|
5258
|
-
const rows = yield*
|
|
5315
|
+
const get = Effect26.fn("Store.get")(function* (id) {
|
|
5316
|
+
const rows = yield* Effect26.tryPromise({
|
|
5259
5317
|
try: () => db.select(memoryFields).from(memories2).where(eq8(memories2.id, id)).limit(1),
|
|
5260
5318
|
catch: storeError("Store.get")
|
|
5261
5319
|
});
|
|
@@ -5264,8 +5322,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5264
5322
|
return yield* new NotFound({ id });
|
|
5265
5323
|
return fromMemoryRow(row);
|
|
5266
5324
|
});
|
|
5267
|
-
const getByKey =
|
|
5268
|
-
const rows = yield*
|
|
5325
|
+
const getByKey = Effect26.fn("Store.getByKey")(function* (key, namespace) {
|
|
5326
|
+
const rows = yield* Effect26.tryPromise({
|
|
5269
5327
|
try: () => db.select(memoryFields).from(memories2).where(and8(eq8(memories2.logicalKey, key), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).orderBy(desc2(memories2.createdAt)).limit(1),
|
|
5270
5328
|
catch: storeError("Store.getByKey")
|
|
5271
5329
|
});
|
|
@@ -5274,9 +5332,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5274
5332
|
return yield* new NotFound({ id: key });
|
|
5275
5333
|
return fromMemoryRow(row);
|
|
5276
5334
|
});
|
|
5277
|
-
const recency =
|
|
5335
|
+
const recency = Effect26.fn("Store.recency")(function* (namespace, limit) {
|
|
5278
5336
|
const start = Date.now();
|
|
5279
|
-
const rows = yield*
|
|
5337
|
+
const rows = yield* Effect26.tryPromise({
|
|
5280
5338
|
try: () => db.select(memoryFields).from(memories2).where(and8(eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).orderBy(desc2(memories2.createdAt)).limit(limit),
|
|
5281
5339
|
catch: storeError("Store.recency")
|
|
5282
5340
|
});
|
|
@@ -5289,7 +5347,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5289
5347
|
yield* logRetrieval(namespace, namespace, "recency", results.length, start);
|
|
5290
5348
|
return yield* augmentRetrieval(results);
|
|
5291
5349
|
});
|
|
5292
|
-
const searchFts5 =
|
|
5350
|
+
const searchFts5 = Effect26.fn("Store.searchFts5")(function* (query, namespace, limit, opts) {
|
|
5293
5351
|
const start = Date.now();
|
|
5294
5352
|
const rows = yield* backend.ftsSearch(query, namespace, limit, opts?.topicKeys, modelConfig?.id, opts?.contentType, opts?.includeEmbedding === true);
|
|
5295
5353
|
if (opts?.audit !== false) {
|
|
@@ -5320,7 +5378,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5320
5378
|
return results;
|
|
5321
5379
|
return yield* augmentRetrieval(results);
|
|
5322
5380
|
});
|
|
5323
|
-
const searchVector =
|
|
5381
|
+
const searchVector = Effect26.fn("Store.searchVector")(function* (query, namespace, limit, opts) {
|
|
5324
5382
|
if (!modelConfig) {
|
|
5325
5383
|
return yield* new StoreError({
|
|
5326
5384
|
message: "Store.searchVector: no embedding model configured"
|
|
@@ -5330,7 +5388,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5330
5388
|
let allowedMemoryIds;
|
|
5331
5389
|
const conceptIds = opts?.conceptIds;
|
|
5332
5390
|
if (conceptIds !== undefined && conceptIds.length > 0) {
|
|
5333
|
-
const conceptRows = yield*
|
|
5391
|
+
const conceptRows = yield* Effect26.tryPromise({
|
|
5334
5392
|
try: () => db.select({ memoryId: memoryConcepts2.memoryId }).from(memoryConcepts2).where(inArray4(memoryConcepts2.conceptId, conceptIds)),
|
|
5335
5393
|
catch: storeError("Store.searchVector: concepts")
|
|
5336
5394
|
});
|
|
@@ -5367,9 +5425,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5367
5425
|
return results;
|
|
5368
5426
|
return yield* augmentRetrieval(results);
|
|
5369
5427
|
});
|
|
5370
|
-
const supersede =
|
|
5428
|
+
const supersede = Effect26.fn("Store.supersede")(function* (key, namespace) {
|
|
5371
5429
|
const now = Date.now();
|
|
5372
|
-
const result = yield*
|
|
5430
|
+
const result = yield* Effect26.tryPromise({
|
|
5373
5431
|
try: () => db.transaction(async (tx) => {
|
|
5374
5432
|
const active = await tx.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.logicalKey, key), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).limit(1);
|
|
5375
5433
|
if (active.length === 0)
|
|
@@ -5381,7 +5439,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5381
5439
|
if (!result)
|
|
5382
5440
|
return yield* new NotFound({ id: key });
|
|
5383
5441
|
});
|
|
5384
|
-
const decrementTreeCf = (model, membership, x) =>
|
|
5442
|
+
const decrementTreeCf = (model, membership, x) => Effect26.tryPromise({
|
|
5385
5443
|
try: () => db.transaction(async (tx) => {
|
|
5386
5444
|
const leafRows = await tx.select({
|
|
5387
5445
|
topicKey: topics.topicKey,
|
|
@@ -5411,10 +5469,10 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5411
5469
|
}),
|
|
5412
5470
|
catch: storeError("Store: tree CF decrement")
|
|
5413
5471
|
});
|
|
5414
|
-
const forget =
|
|
5472
|
+
const forget = Effect26.fn("Store.forget")(function* (id) {
|
|
5415
5473
|
let membership;
|
|
5416
5474
|
if (topicsEnabled) {
|
|
5417
|
-
const memberRows = yield*
|
|
5475
|
+
const memberRows = yield* Effect26.tryPromise({
|
|
5418
5476
|
try: () => db.select({ topicKey: topicMemberships.topicKey, modelKey: topicMemberships.modelKey }).from(topicMemberships).where(eq8(topicMemberships.memoryId, id)),
|
|
5419
5477
|
catch: storeError("Store.forget: load membership")
|
|
5420
5478
|
});
|
|
@@ -5429,14 +5487,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5429
5487
|
}
|
|
5430
5488
|
}
|
|
5431
5489
|
}
|
|
5432
|
-
const rs = yield*
|
|
5490
|
+
const rs = yield* Effect26.tryPromise({
|
|
5433
5491
|
try: () => db.delete(memories2).where(eq8(memories2.id, id)),
|
|
5434
5492
|
catch: storeError("Store.forget")
|
|
5435
5493
|
});
|
|
5436
5494
|
if (rs.rowsAffected === 0)
|
|
5437
5495
|
return yield* new NotFound({ id });
|
|
5438
5496
|
if (topicsEnabled && membership && modelConfig && modelConfig.id === membership.modelKey) {
|
|
5439
|
-
const leafRows = yield*
|
|
5497
|
+
const leafRows = yield* Effect26.tryPromise({
|
|
5440
5498
|
try: () => db.select({ memberCount: topics.memberCount }).from(topics).where(and8(eq8(topics.topicKey, membership.topicKey), eq8(topics.modelKey, membership.modelKey))),
|
|
5441
5499
|
catch: storeError("Store.forget: load leaf")
|
|
5442
5500
|
});
|
|
@@ -5444,18 +5502,18 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5444
5502
|
if (liveCount < (topicsConfig.minLeafItems ?? 32)) {
|
|
5445
5503
|
const enqueueMerge = topicsOptions?.enqueueMerge;
|
|
5446
5504
|
if (enqueueMerge) {
|
|
5447
|
-
yield* enqueueMerge(membership.topicKey, membership.modelKey).pipe(
|
|
5505
|
+
yield* enqueueMerge(membership.topicKey, membership.modelKey).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.forget enqueueMerge: ${cause}` })));
|
|
5448
5506
|
} else {
|
|
5449
|
-
yield* mergeOrBorrow(membership.topicKey, modelConfig, topicsConfig).pipe(
|
|
5507
|
+
yield* mergeOrBorrow(membership.topicKey, modelConfig, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.forget mergeOrBorrow: ${cause}` })));
|
|
5450
5508
|
}
|
|
5451
5509
|
}
|
|
5452
5510
|
}
|
|
5453
5511
|
});
|
|
5454
|
-
const sweep =
|
|
5512
|
+
const sweep = Effect26.fn("Store.sweep")(function* (namespace) {
|
|
5455
5513
|
return yield* backend.sweep(namespace);
|
|
5456
5514
|
});
|
|
5457
|
-
const events =
|
|
5458
|
-
const rows = yield*
|
|
5515
|
+
const events = Effect26.fn("Store.events")(function* (memoryId) {
|
|
5516
|
+
const rows = yield* Effect26.tryPromise({
|
|
5459
5517
|
try: () => db.select().from(memoryEvents2).where(eq8(memoryEvents2.memoryId, memoryId)).orderBy(asc5(memoryEvents2.createdAt)),
|
|
5460
5518
|
catch: storeError("Store.events")
|
|
5461
5519
|
});
|
|
@@ -5472,7 +5530,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5472
5530
|
});
|
|
5473
5531
|
});
|
|
5474
5532
|
const resolveModel = (modelKey) => modelConfig?.id === modelKey ? modelConfig : undefined;
|
|
5475
|
-
const ensureTopicTree2 =
|
|
5533
|
+
const ensureTopicTree2 = Effect26.fn("Store.ensureTopicTree")(function* (modelKey) {
|
|
5476
5534
|
const model = resolveModel(modelKey);
|
|
5477
5535
|
if (!model) {
|
|
5478
5536
|
return yield* new ModelMismatch({
|
|
@@ -5480,33 +5538,33 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5480
5538
|
actual: modelKey
|
|
5481
5539
|
});
|
|
5482
5540
|
}
|
|
5483
|
-
return yield* ensureTopicTree(model, topicsConfig).pipe(
|
|
5541
|
+
return yield* ensureTopicTree(model, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5484
5542
|
});
|
|
5485
|
-
const resolveTopics2 =
|
|
5543
|
+
const resolveTopics2 = Effect26.fn("Store.resolveTopics")(function* (queryVector, opts) {
|
|
5486
5544
|
if (!modelConfig) {
|
|
5487
5545
|
return yield* new ModelMismatch({
|
|
5488
5546
|
expected: "a configured embedding model",
|
|
5489
5547
|
actual: "(none)"
|
|
5490
5548
|
});
|
|
5491
5549
|
}
|
|
5492
|
-
return yield* resolveTopics(queryVector, modelConfig, topicsConfig, opts).pipe(
|
|
5550
|
+
return yield* resolveTopics(queryVector, modelConfig, topicsConfig, opts).pipe(Effect26.provideService(Service2, backend));
|
|
5493
5551
|
});
|
|
5494
|
-
const topicMembers =
|
|
5495
|
-
const rows = yield*
|
|
5552
|
+
const topicMembers = Effect26.fn("Store.topicMembers")(function* (topicKey) {
|
|
5553
|
+
const rows = yield* Effect26.tryPromise({
|
|
5496
5554
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(eq8(topicMemberships.topicKey, topicKey)).orderBy(asc5(topicMemberships.memoryId)),
|
|
5497
5555
|
catch: storeError("Store.topicMembers")
|
|
5498
5556
|
});
|
|
5499
5557
|
return rows;
|
|
5500
5558
|
});
|
|
5501
|
-
const listTopics =
|
|
5502
|
-
const rows = yield*
|
|
5559
|
+
const listTopics = Effect26.fn("Store.listTopics")(function* (modelKey) {
|
|
5560
|
+
const rows = yield* Effect26.tryPromise({
|
|
5503
5561
|
try: () => db.select().from(topics).where(eq8(topics.modelKey, modelKey)).orderBy(asc5(topics.topicKey)),
|
|
5504
5562
|
catch: storeError("Store.listTopics")
|
|
5505
5563
|
});
|
|
5506
5564
|
return rows;
|
|
5507
5565
|
});
|
|
5508
|
-
const topicStats =
|
|
5509
|
-
const rows = yield*
|
|
5566
|
+
const topicStats = Effect26.fn("Store.topicStats")(function* (modelKey) {
|
|
5567
|
+
const rows = yield* Effect26.tryPromise({
|
|
5510
5568
|
try: () => db.select({ kind: topics.kind, depth: topics.depth }).from(topics).where(eq8(topics.modelKey, modelKey)),
|
|
5511
5569
|
catch: storeError("Store.topicStats")
|
|
5512
5570
|
});
|
|
@@ -5523,7 +5581,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5523
5581
|
}
|
|
5524
5582
|
return { depth, leafCount, internalCount, totalTopics: rows.length };
|
|
5525
5583
|
});
|
|
5526
|
-
const deleteTree =
|
|
5584
|
+
const deleteTree = Effect26.fn("Store.deleteTree")(function* (modelKey) {
|
|
5527
5585
|
const model = resolveModel(modelKey);
|
|
5528
5586
|
if (!model) {
|
|
5529
5587
|
return yield* new ModelMismatch({
|
|
@@ -5532,7 +5590,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5532
5590
|
});
|
|
5533
5591
|
}
|
|
5534
5592
|
yield* backend.deleteTopicVectors(model);
|
|
5535
|
-
yield*
|
|
5593
|
+
yield* Effect26.tryPromise({
|
|
5536
5594
|
try: () => db.transaction(async (tx) => {
|
|
5537
5595
|
const modelTopicKeys2 = tx.select({ topicKey: topics.topicKey }).from(topics).where(eq8(topics.modelKey, modelKey));
|
|
5538
5596
|
await tx.delete(topicChangeLog).where(inArray4(topicChangeLog.topicKey, modelTopicKeys2));
|
|
@@ -5544,7 +5602,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5544
5602
|
catch: storeError("Store.deleteTree")
|
|
5545
5603
|
});
|
|
5546
5604
|
});
|
|
5547
|
-
const processSplits =
|
|
5605
|
+
const processSplits = Effect26.fn("Store.processSplits")(function* (modelKey) {
|
|
5548
5606
|
const model = resolveModel(modelKey);
|
|
5549
5607
|
if (!model) {
|
|
5550
5608
|
return yield* new ModelMismatch({
|
|
@@ -5554,19 +5612,19 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5554
5612
|
}
|
|
5555
5613
|
let processed = 0;
|
|
5556
5614
|
for (let iteration = 0;iteration < 100; iteration++) {
|
|
5557
|
-
const leaves = yield*
|
|
5615
|
+
const leaves = yield* Effect26.tryPromise({
|
|
5558
5616
|
try: () => db.select({ topicKey: topics.topicKey, memberCount: topics.memberCount }).from(topics).where(and8(eq8(topics.modelKey, modelKey), eq8(topics.kind, "leaf"), ne4(topics.status, "inactive"))).orderBy(asc5(topics.topicKey)),
|
|
5559
5617
|
catch: storeError("Store.processSplits")
|
|
5560
5618
|
});
|
|
5561
5619
|
const candidate = leaves.find((leaf) => shouldSplit(leaf.memberCount, topicsConfig));
|
|
5562
5620
|
if (!candidate)
|
|
5563
5621
|
break;
|
|
5564
|
-
yield* splitTopic(candidate.topicKey, model, topicsConfig).pipe(
|
|
5622
|
+
yield* splitTopic(candidate.topicKey, model, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.processSplits: ${cause}` })));
|
|
5565
5623
|
processed++;
|
|
5566
5624
|
}
|
|
5567
5625
|
return processed;
|
|
5568
5626
|
});
|
|
5569
|
-
const mergeOrBorrow2 =
|
|
5627
|
+
const mergeOrBorrow2 = Effect26.fn("Store.mergeOrBorrow")(function* (modelKey, topicKey) {
|
|
5570
5628
|
const model = resolveModel(modelKey);
|
|
5571
5629
|
if (!model) {
|
|
5572
5630
|
return yield* new ModelMismatch({
|
|
@@ -5574,9 +5632,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5574
5632
|
actual: modelKey
|
|
5575
5633
|
});
|
|
5576
5634
|
}
|
|
5577
|
-
return yield* mergeOrBorrow(topicKey, model, topicsConfig).pipe(
|
|
5635
|
+
return yield* mergeOrBorrow(topicKey, model, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.mergeOrBorrow: ${cause}` })));
|
|
5578
5636
|
});
|
|
5579
|
-
const rebalanceSiblings2 =
|
|
5637
|
+
const rebalanceSiblings2 = Effect26.fn("Store.rebalanceSiblings")(function* (modelKey, topicKey) {
|
|
5580
5638
|
const model = resolveModel(modelKey);
|
|
5581
5639
|
if (!model) {
|
|
5582
5640
|
return yield* new ModelMismatch({
|
|
@@ -5584,15 +5642,15 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5584
5642
|
actual: modelKey
|
|
5585
5643
|
});
|
|
5586
5644
|
}
|
|
5587
|
-
return yield* rebalanceSiblings(topicKey, model, topicsConfig).pipe(
|
|
5645
|
+
return yield* rebalanceSiblings(topicKey, model, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.rebalanceSiblings: ${cause}` })));
|
|
5588
5646
|
});
|
|
5589
|
-
const setActiveModel2 =
|
|
5590
|
-
return yield* setActiveModel(model, topicsConfig).pipe(
|
|
5647
|
+
const setActiveModel2 = Effect26.fn("Store.setActiveModel")(function* (model) {
|
|
5648
|
+
return yield* setActiveModel(model, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5591
5649
|
});
|
|
5592
|
-
const isActive2 =
|
|
5593
|
-
return yield* isActive(modelKey).pipe(
|
|
5650
|
+
const isActive2 = Effect26.fn("Store.isActive")(function* (modelKey) {
|
|
5651
|
+
return yield* isActive(modelKey).pipe(Effect26.provideService(Service2, backend));
|
|
5594
5652
|
});
|
|
5595
|
-
const topicMetrics2 =
|
|
5653
|
+
const topicMetrics2 = Effect26.fn("Store.topicMetrics")(function* (modelKey) {
|
|
5596
5654
|
const model = resolveModel(modelKey);
|
|
5597
5655
|
if (!model) {
|
|
5598
5656
|
return yield* new ModelMismatch({
|
|
@@ -5600,9 +5658,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5600
5658
|
actual: modelKey
|
|
5601
5659
|
});
|
|
5602
5660
|
}
|
|
5603
|
-
return yield* topicMetrics(modelKey, topicsConfig).pipe(
|
|
5661
|
+
return yield* topicMetrics(modelKey, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5604
5662
|
});
|
|
5605
|
-
const topicChangeLog3 =
|
|
5663
|
+
const topicChangeLog3 = Effect26.fn("Store.topicChangeLog")(function* (modelKey, opts) {
|
|
5606
5664
|
const model = resolveModel(modelKey);
|
|
5607
5665
|
if (!model) {
|
|
5608
5666
|
return yield* new ModelMismatch({
|
|
@@ -5610,9 +5668,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5610
5668
|
actual: modelKey
|
|
5611
5669
|
});
|
|
5612
5670
|
}
|
|
5613
|
-
return yield* topicChangeLog2(modelKey, opts).pipe(
|
|
5671
|
+
return yield* topicChangeLog2(modelKey, opts).pipe(Effect26.provideService(Service2, backend));
|
|
5614
5672
|
});
|
|
5615
|
-
const topicHealth2 =
|
|
5673
|
+
const topicHealth2 = Effect26.fn("Store.topicHealth")(function* (modelKey) {
|
|
5616
5674
|
const model = resolveModel(modelKey);
|
|
5617
5675
|
if (!model) {
|
|
5618
5676
|
return yield* new ModelMismatch({
|
|
@@ -5620,11 +5678,11 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5620
5678
|
actual: modelKey
|
|
5621
5679
|
});
|
|
5622
5680
|
}
|
|
5623
|
-
return yield* topicHealth(modelKey, topicsConfig).pipe(
|
|
5681
|
+
return yield* topicHealth(modelKey, topicsConfig).pipe(Effect26.provideService(Service2, backend));
|
|
5624
5682
|
});
|
|
5625
|
-
const updateEmbedding =
|
|
5683
|
+
const updateEmbedding = Effect26.fn("Store.updateEmbedding")(function* (id, embedding, modelName, options) {
|
|
5626
5684
|
const now = Date.now();
|
|
5627
|
-
const affected = yield*
|
|
5685
|
+
const affected = yield* Effect26.tryPromise({
|
|
5628
5686
|
try: () => db.transaction(async (tx) => {
|
|
5629
5687
|
const rs = await tx.update(memories2).set({ embedding: [...embedding], embeddingModel: modelName }).where(eq8(memories2.id, id));
|
|
5630
5688
|
if (rs.rowsAffected === 0)
|
|
@@ -5654,7 +5712,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5654
5712
|
if (modelConfig && modelConfig.id === modelName) {
|
|
5655
5713
|
yield* backend.ensureVectorTable(modelConfig);
|
|
5656
5714
|
if (topicsEnabled) {
|
|
5657
|
-
const memberRows = yield*
|
|
5715
|
+
const memberRows = yield* Effect26.tryPromise({
|
|
5658
5716
|
try: () => db.select({
|
|
5659
5717
|
topicKey: topicMemberships.topicKey,
|
|
5660
5718
|
modelKey: topicMemberships.modelKey
|
|
@@ -5669,7 +5727,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5669
5727
|
yield* decrementTreeCf(modelConfig, mem, x);
|
|
5670
5728
|
yield* updateTopicVectorForget(backend, modelConfig, mem.topicKey, x);
|
|
5671
5729
|
}
|
|
5672
|
-
yield*
|
|
5730
|
+
yield* Effect26.tryPromise({
|
|
5673
5731
|
try: () => db.delete(topicMemberships).where(and8(eq8(topicMemberships.memoryId, id), eq8(topicMemberships.modelKey, modelName))),
|
|
5674
5732
|
catch: storeError("Store.updateEmbedding: delete membership")
|
|
5675
5733
|
});
|
|
@@ -5677,26 +5735,26 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5677
5735
|
}
|
|
5678
5736
|
yield* backend.insertVector(id, embedding, modelConfig);
|
|
5679
5737
|
if (topicsEnabled) {
|
|
5680
|
-
const routed = yield* routeInsert(id, embedding, modelConfig, topicsConfig).pipe(
|
|
5738
|
+
const routed = yield* routeInsert(id, embedding, modelConfig, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding topic route: ${cause}` })));
|
|
5681
5739
|
if (options?.scanCandidates !== false && graphEnabled) {
|
|
5682
5740
|
yield* scanLeafCandidates(id, embedding, routed.topicKey, modelConfig);
|
|
5683
5741
|
}
|
|
5684
5742
|
if (routed.overflow) {
|
|
5685
5743
|
const enqueueSplit = topicsOptions?.enqueueSplit;
|
|
5686
5744
|
if (enqueueSplit) {
|
|
5687
|
-
yield* enqueueSplit(routed.topicKey, modelConfig.id).pipe(
|
|
5745
|
+
yield* enqueueSplit(routed.topicKey, modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding enqueueSplit: ${cause}` })));
|
|
5688
5746
|
} else {
|
|
5689
|
-
yield* processSplits(modelConfig.id).pipe(
|
|
5747
|
+
yield* processSplits(modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding processSplits: ${cause}` })));
|
|
5690
5748
|
}
|
|
5691
5749
|
}
|
|
5692
5750
|
}
|
|
5693
5751
|
}
|
|
5694
5752
|
});
|
|
5695
|
-
const updateEmbeddingMany =
|
|
5753
|
+
const updateEmbeddingMany = Effect26.fn("Store.updateEmbeddingMany")(function* (entries, modelName, options) {
|
|
5696
5754
|
if (entries.length === 0)
|
|
5697
5755
|
return;
|
|
5698
5756
|
const now = Date.now();
|
|
5699
|
-
yield*
|
|
5757
|
+
yield* Effect26.tryPromise({
|
|
5700
5758
|
try: () => db.transaction(async (tx) => {
|
|
5701
5759
|
await tx.run(sql8`UPDATE ${memories2} SET embedding = CASE id ${sql8.join(entries.map((entry) => sql8`WHEN ${entry.id} THEN ${new Float32Array(entry.embedding).buffer}`), sql8` `)} END, embedding_model = ${modelName} WHERE id IN (${sql8.join(entries.map((entry) => entry.id), sql8`, `)})`);
|
|
5702
5760
|
await tx.insert(memoryEvents2).values(entries.map((entry) => ({
|
|
@@ -5722,7 +5780,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5722
5780
|
yield* backend.ensureVectorTable(modelConfig);
|
|
5723
5781
|
yield* backend.insertVectors(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig);
|
|
5724
5782
|
if (topicsEnabled) {
|
|
5725
|
-
const routed = yield* routeInsertMany(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig, topicsConfig).pipe(
|
|
5783
|
+
const routed = yield* routeInsertMany(entries.map((entry) => ({ memoryId: entry.id, vector: entry.embedding })), modelConfig, topicsConfig).pipe(Effect26.provideService(Service2, backend), Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany route: ${cause}` })));
|
|
5726
5784
|
if (options?.scanCandidates === true && graphEnabled) {
|
|
5727
5785
|
for (const routedEntry of routed) {
|
|
5728
5786
|
const entry = entries.find((e) => e.id === routedEntry.memoryId);
|
|
@@ -5736,17 +5794,17 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5736
5794
|
const enqueueSplit = topicsOptions?.enqueueSplit;
|
|
5737
5795
|
if (enqueueSplit) {
|
|
5738
5796
|
for (const topicKey of overflowed) {
|
|
5739
|
-
yield* enqueueSplit(topicKey, modelConfig.id).pipe(
|
|
5797
|
+
yield* enqueueSplit(topicKey, modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany enqueueSplit: ${cause}` })));
|
|
5740
5798
|
}
|
|
5741
5799
|
} else {
|
|
5742
|
-
yield* processSplits(modelConfig.id).pipe(
|
|
5800
|
+
yield* processSplits(modelConfig.id).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbeddingMany processSplits: ${cause}` })));
|
|
5743
5801
|
}
|
|
5744
5802
|
}
|
|
5745
5803
|
}
|
|
5746
5804
|
}
|
|
5747
5805
|
});
|
|
5748
|
-
const findStale =
|
|
5749
|
-
const rows = yield*
|
|
5806
|
+
const findStale = Effect26.fn("Store.findStale")(function* (model) {
|
|
5807
|
+
const rows = yield* Effect26.tryPromise({
|
|
5750
5808
|
try: () => db.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.state, "active"), sql8`(embedding IS NULL OR embedding_model IS NULL OR embedding_model != ${model})`)),
|
|
5751
5809
|
catch: storeError("Store.findStale")
|
|
5752
5810
|
});
|
|
@@ -5766,14 +5824,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5766
5824
|
});
|
|
5767
5825
|
return memoryId;
|
|
5768
5826
|
};
|
|
5769
|
-
const reconcile =
|
|
5827
|
+
const reconcile = Effect26.fn("Store.reconcile")(function* (namespace, keyPrefix, keyRoot, walkedRoots, keptKeys) {
|
|
5770
5828
|
const inDomain = (key) => {
|
|
5771
5829
|
const rel = key.slice(keyPrefix.length).split("#")[0];
|
|
5772
5830
|
const filePath = keyRoot === "/" ? `/${rel}` : `${keyRoot}/${rel}`;
|
|
5773
5831
|
return walkedRoots.some((root) => root === "/" ? filePath.startsWith("/") : filePath === root || filePath.startsWith(`${root}/`));
|
|
5774
5832
|
};
|
|
5775
5833
|
const owned = (key) => keptKeys.includes(key);
|
|
5776
|
-
const rows = yield*
|
|
5834
|
+
const rows = yield* Effect26.tryPromise({
|
|
5777
5835
|
try: () => db.select({
|
|
5778
5836
|
id: memories2.id,
|
|
5779
5837
|
logicalKey: memories2.logicalKey,
|
|
@@ -5784,7 +5842,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5784
5842
|
const vanished = rows.filter((r) => inDomain(r.logicalKey) && !owned(r.logicalKey));
|
|
5785
5843
|
if (vanished.length > 0) {
|
|
5786
5844
|
const now = Date.now();
|
|
5787
|
-
yield*
|
|
5845
|
+
yield* Effect26.tryPromise({
|
|
5788
5846
|
try: () => db.transaction(async (tx) => {
|
|
5789
5847
|
for (const row of vanished) {
|
|
5790
5848
|
await flipToSupersededById(tx, row.id, now, "source_removed");
|
|
@@ -5795,7 +5853,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5795
5853
|
}
|
|
5796
5854
|
return vanished;
|
|
5797
5855
|
});
|
|
5798
|
-
const writeSuperseding =
|
|
5856
|
+
const writeSuperseding = Effect26.fn("Store.writeSuperseding")(function* (input) {
|
|
5799
5857
|
if (input.content.includes("\x00")) {
|
|
5800
5858
|
return yield* new StoreError({
|
|
5801
5859
|
message: "Store.writeSuperseding: content contains a NUL byte (libSQL would silently truncate it)"
|
|
@@ -5806,7 +5864,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5806
5864
|
const metadataBlob = input.typedMetadata !== undefined ? mergeMetadataBlob(input.metadata, input.typedMetadata) : input.metadata ?? null;
|
|
5807
5865
|
const namespace = input.namespace ?? "default";
|
|
5808
5866
|
let supersededId;
|
|
5809
|
-
yield*
|
|
5867
|
+
yield* Effect26.tryPromise({
|
|
5810
5868
|
try: () => db.transaction(async (tx) => {
|
|
5811
5869
|
const existing = await tx.select({ id: memories2.id }).from(memories2).where(and8(eq8(memories2.logicalKey, input.logicalKey), eq8(memories2.namespace, namespace), eq8(memories2.state, "active"))).limit(1);
|
|
5812
5870
|
const old = existing[0];
|
|
@@ -5861,7 +5919,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5861
5919
|
...supersededId !== undefined ? { supersededId } : undefined
|
|
5862
5920
|
};
|
|
5863
5921
|
});
|
|
5864
|
-
const addEdge =
|
|
5922
|
+
const addEdge = Effect26.fn("Store.addEdge")(function* (input) {
|
|
5865
5923
|
if (input.fromMemoryId === input.toMemoryId) {
|
|
5866
5924
|
return yield* new StoreError({ message: "Store.addEdge: self-loops are not allowed" });
|
|
5867
5925
|
}
|
|
@@ -5887,7 +5945,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5887
5945
|
message: `Store.addEdge: confidence must be a finite number in [0,1] (got ${String(confidence)})`
|
|
5888
5946
|
});
|
|
5889
5947
|
}
|
|
5890
|
-
const endpoints = yield*
|
|
5948
|
+
const endpoints = yield* Effect26.tryPromise({
|
|
5891
5949
|
try: () => db.select({ id: memories2.id }).from(memories2).where(inArray4(memories2.id, [input.fromMemoryId, input.toMemoryId])),
|
|
5892
5950
|
catch: storeError("Store.addEdge: endpoints")
|
|
5893
5951
|
});
|
|
@@ -5912,7 +5970,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5912
5970
|
createdAt: now,
|
|
5913
5971
|
updatedAt: now
|
|
5914
5972
|
};
|
|
5915
|
-
yield*
|
|
5973
|
+
yield* Effect26.tryPromise({
|
|
5916
5974
|
try: () => db.transaction(async (tx) => {
|
|
5917
5975
|
await tx.insert(artifactEdges).values(row);
|
|
5918
5976
|
if (input.confirmed && input.edgeKind === "memory_supersedes") {
|
|
@@ -5932,17 +5990,17 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5932
5990
|
});
|
|
5933
5991
|
return row;
|
|
5934
5992
|
});
|
|
5935
|
-
const expireEdge =
|
|
5936
|
-
const rs = yield*
|
|
5993
|
+
const expireEdge = Effect26.fn("Store.expireEdge")(function* (id) {
|
|
5994
|
+
const rs = yield* Effect26.tryPromise({
|
|
5937
5995
|
try: () => db.update(artifactEdges).set({ expiredAt: Date.now(), updatedAt: Date.now() }).where(eq8(artifactEdges.id, id)),
|
|
5938
5996
|
catch: storeError("Store.expireEdge")
|
|
5939
5997
|
});
|
|
5940
5998
|
if (rs.rowsAffected === 0)
|
|
5941
5999
|
return yield* new EdgeNotFound({ id });
|
|
5942
6000
|
});
|
|
5943
|
-
const confirmEdge =
|
|
6001
|
+
const confirmEdge = Effect26.fn("Store.confirmEdge")(function* (id) {
|
|
5944
6002
|
const now = Date.now();
|
|
5945
|
-
const found = yield*
|
|
6003
|
+
const found = yield* Effect26.tryPromise({
|
|
5946
6004
|
try: () => db.transaction(async (tx) => {
|
|
5947
6005
|
const rows = await tx.select().from(artifactEdges).where(eq8(artifactEdges.id, id)).limit(1);
|
|
5948
6006
|
const edge = rows[0];
|
|
@@ -5959,15 +6017,15 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5959
6017
|
if (!found)
|
|
5960
6018
|
return yield* new EdgeNotFound({ id });
|
|
5961
6019
|
});
|
|
5962
|
-
const rejectEdge =
|
|
5963
|
-
const rs = yield*
|
|
6020
|
+
const rejectEdge = Effect26.fn("Store.rejectEdge")(function* (id) {
|
|
6021
|
+
const rs = yield* Effect26.tryPromise({
|
|
5964
6022
|
try: () => db.update(artifactEdges).set({ status: "rejected", updatedAt: Date.now() }).where(eq8(artifactEdges.id, id)),
|
|
5965
6023
|
catch: storeError("Store.rejectEdge")
|
|
5966
6024
|
});
|
|
5967
6025
|
if (rs.rowsAffected === 0)
|
|
5968
6026
|
return yield* new EdgeNotFound({ id });
|
|
5969
6027
|
});
|
|
5970
|
-
const listEdges =
|
|
6028
|
+
const listEdges = Effect26.fn("Store.listEdges")(function* (opts) {
|
|
5971
6029
|
const conditions = [];
|
|
5972
6030
|
if (!opts?.includeExpired)
|
|
5973
6031
|
conditions.push(isNull2(artifactEdges.expiredAt));
|
|
@@ -5978,7 +6036,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5978
6036
|
conditions.push(eq8(artifactEdges.edgeKind, opts.edgeKind));
|
|
5979
6037
|
if (opts?.status)
|
|
5980
6038
|
conditions.push(eq8(artifactEdges.status, opts.status));
|
|
5981
|
-
const rows = yield*
|
|
6039
|
+
const rows = yield* Effect26.tryPromise({
|
|
5982
6040
|
try: () => {
|
|
5983
6041
|
const base = db.select().from(artifactEdges);
|
|
5984
6042
|
return (conditions.length === 0 ? base : base.where(and8(...conditions))).orderBy(desc2(artifactEdges.createdAt));
|
|
@@ -5987,7 +6045,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
5987
6045
|
});
|
|
5988
6046
|
return rows;
|
|
5989
6047
|
});
|
|
5990
|
-
const explain =
|
|
6048
|
+
const explain = Effect26.fn("Store.explain")(function* (node5, opts) {
|
|
5991
6049
|
if (node5.kind !== "memory" && node5.kind !== "topic" && node5.kind !== "concept") {
|
|
5992
6050
|
return yield* new StoreError({
|
|
5993
6051
|
message: `Store.explain: unknown kind "${String(node5.kind)}"`
|
|
@@ -6031,7 +6089,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6031
6089
|
pendingConflicts: pending
|
|
6032
6090
|
};
|
|
6033
6091
|
});
|
|
6034
|
-
const writeCandidateEdges = (candidates, now) =>
|
|
6092
|
+
const writeCandidateEdges = (candidates, now) => Effect26.tryPromise({
|
|
6035
6093
|
try: () => db.transaction(async (tx) => {
|
|
6036
6094
|
let inserted = 0;
|
|
6037
6095
|
for (const candidate of candidates) {
|
|
@@ -6061,11 +6119,11 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6061
6119
|
}),
|
|
6062
6120
|
catch: storeError("Store.writeCandidateEdges")
|
|
6063
6121
|
});
|
|
6064
|
-
const reconcileRenameCandidates =
|
|
6122
|
+
const reconcileRenameCandidates = Effect26.fn("Store.reconcileRenameCandidates")(function* (vanished, newUnitIds) {
|
|
6065
6123
|
const withVectors = vanished.filter((v) => v.embedding !== null);
|
|
6066
6124
|
if (withVectors.length === 0 || newUnitIds.length === 0)
|
|
6067
6125
|
return 0;
|
|
6068
|
-
const newRows = yield*
|
|
6126
|
+
const newRows = yield* Effect26.tryPromise({
|
|
6069
6127
|
try: () => db.select({ id: memories2.id, embedding: memories2.embedding }).from(memories2).where(and8(inArray4(memories2.id, newUnitIds))),
|
|
6070
6128
|
catch: storeError("Store.reconcileRenameCandidates")
|
|
6071
6129
|
});
|
|
@@ -6097,16 +6155,16 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6097
6155
|
inserted = yield* writeCandidateEdges(batch, now);
|
|
6098
6156
|
return inserted;
|
|
6099
6157
|
});
|
|
6100
|
-
const scanLeafCandidates = (memoryId, vector, topicKey, model) =>
|
|
6101
|
-
const memberRows = yield*
|
|
6158
|
+
const scanLeafCandidates = (memoryId, vector, topicKey, model) => Effect26.gen(function* () {
|
|
6159
|
+
const memberRows = yield* Effect26.tryPromise({
|
|
6102
6160
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and8(eq8(topicMemberships.topicKey, topicKey), eq8(topicMemberships.modelKey, model.id))).orderBy(asc5(topicMemberships.memoryId)),
|
|
6103
6161
|
catch: storeError("Store.updateEmbedding candidates: members")
|
|
6104
6162
|
});
|
|
6105
6163
|
const memberIds = memberRows.map((r) => r.memoryId);
|
|
6106
6164
|
if (memberIds.length === 0)
|
|
6107
6165
|
return;
|
|
6108
|
-
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(
|
|
6109
|
-
const memoryRows = yield*
|
|
6166
|
+
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.updateEmbedding candidates: vectors: ${cause}` })));
|
|
6167
|
+
const memoryRows = yield* Effect26.tryPromise({
|
|
6110
6168
|
try: () => db.select({
|
|
6111
6169
|
id: memories2.id,
|
|
6112
6170
|
logicalKey: memories2.logicalKey,
|
|
@@ -6115,7 +6173,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6115
6173
|
}).from(memories2).where(and8(inArray4(memories2.id, memberIds), eq8(memories2.state, "active"))),
|
|
6116
6174
|
catch: storeError("Store.updateEmbedding candidates: memories")
|
|
6117
6175
|
});
|
|
6118
|
-
const selfRows = yield*
|
|
6176
|
+
const selfRows = yield* Effect26.tryPromise({
|
|
6119
6177
|
try: () => db.select({
|
|
6120
6178
|
logicalKey: memories2.logicalKey,
|
|
6121
6179
|
createdAt: memories2.createdAt,
|
|
@@ -6143,9 +6201,9 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6143
6201
|
const now = Date.now();
|
|
6144
6202
|
yield* writeCandidateEdges(candidates, now);
|
|
6145
6203
|
});
|
|
6146
|
-
const graphCandidates =
|
|
6204
|
+
const graphCandidates = Effect26.fn("Store.graphCandidates")(function* (opts) {
|
|
6147
6205
|
const limit = Math.max(1, Math.min(opts?.limit ?? 50, GRAPH_CANDIDATE_LIMIT_MAX));
|
|
6148
|
-
const rows = yield*
|
|
6206
|
+
const rows = yield* Effect26.tryPromise({
|
|
6149
6207
|
try: () => db.select({
|
|
6150
6208
|
id: artifactEdges.id,
|
|
6151
6209
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
@@ -6157,7 +6215,7 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6157
6215
|
}).from(artifactEdges).where(and8(eq8(artifactEdges.status, "pending"), isNull2(artifactEdges.expiredAt))).orderBy(desc2(artifactEdges.createdAt), asc5(artifactEdges.id)).limit(limit),
|
|
6158
6216
|
catch: storeError("Store.graphCandidates: pending")
|
|
6159
6217
|
});
|
|
6160
|
-
const isolatedRows = yield* backend.listIsolatedMemories(0).pipe(
|
|
6218
|
+
const isolatedRows = yield* backend.listIsolatedMemories(0).pipe(Effect26.mapError((cause) => new StoreError({ message: `Store.graphCandidates: isolated: ${cause}` })));
|
|
6161
6219
|
const isolatedRowsDecoded = Schema23.decodeUnknownSync(Schema23.Array(Schema23.Struct({ id: Schema23.String, createdAt: Schema23.Number })))(isolatedRows);
|
|
6162
6220
|
const isolated = isolatedRowsDecoded.map((row) => ({
|
|
6163
6221
|
id: row.id,
|
|
@@ -6177,13 +6235,13 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6177
6235
|
oldestIsolated: isolated.slice(0, GRAPH_ISOLATED_EXAMPLES)
|
|
6178
6236
|
};
|
|
6179
6237
|
});
|
|
6180
|
-
const graphCoverage2 =
|
|
6181
|
-
return yield* graphCoverage(modelKey).pipe(
|
|
6238
|
+
const graphCoverage2 = Effect26.fn("Store.graphCoverage")(function* (modelKey) {
|
|
6239
|
+
return yield* graphCoverage(modelKey).pipe(Effect26.provideService(Service2, backend));
|
|
6182
6240
|
});
|
|
6183
|
-
const createConcept =
|
|
6241
|
+
const createConcept = Effect26.fn("Store.createConcept")(function* (name, embedding) {
|
|
6184
6242
|
const id = crypto.randomUUID();
|
|
6185
6243
|
const now = Date.now();
|
|
6186
|
-
yield*
|
|
6244
|
+
yield* Effect26.tryPromise({
|
|
6187
6245
|
try: () => db.insert(concepts2).values({
|
|
6188
6246
|
id,
|
|
6189
6247
|
name,
|
|
@@ -6203,8 +6261,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6203
6261
|
lastSeen: now
|
|
6204
6262
|
};
|
|
6205
6263
|
});
|
|
6206
|
-
const getConcept =
|
|
6207
|
-
const rows = yield*
|
|
6264
|
+
const getConcept = Effect26.fn("Store.getConcept")(function* (id) {
|
|
6265
|
+
const rows = yield* Effect26.tryPromise({
|
|
6208
6266
|
try: () => db.select().from(concepts2).where(eq8(concepts2.id, id)).limit(1),
|
|
6209
6267
|
catch: storeError("Store.getConcept")
|
|
6210
6268
|
});
|
|
@@ -6220,8 +6278,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6220
6278
|
lastSeen: row.lastSeen
|
|
6221
6279
|
};
|
|
6222
6280
|
});
|
|
6223
|
-
const listConcepts =
|
|
6224
|
-
const rows = yield*
|
|
6281
|
+
const listConcepts = Effect26.fn("Store.listConcepts")(function* () {
|
|
6282
|
+
const rows = yield* Effect26.tryPromise({
|
|
6225
6283
|
try: () => db.select().from(concepts2),
|
|
6226
6284
|
catch: storeError("Store.listConcepts")
|
|
6227
6285
|
});
|
|
@@ -6234,8 +6292,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6234
6292
|
lastSeen: row.lastSeen
|
|
6235
6293
|
}));
|
|
6236
6294
|
});
|
|
6237
|
-
const annotateMemory =
|
|
6238
|
-
yield*
|
|
6295
|
+
const annotateMemory = Effect26.fn("Store.annotateMemory")(function* (memoryId, conceptId, confidence) {
|
|
6296
|
+
yield* Effect26.tryPromise({
|
|
6239
6297
|
try: () => db.insert(memoryConcepts2).values({
|
|
6240
6298
|
conceptId,
|
|
6241
6299
|
memoryId,
|
|
@@ -6245,14 +6303,14 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6245
6303
|
catch: storeError("Store.annotateMemory")
|
|
6246
6304
|
});
|
|
6247
6305
|
});
|
|
6248
|
-
const getMemoryAnnotations =
|
|
6249
|
-
const rows = yield*
|
|
6306
|
+
const getMemoryAnnotations = Effect26.fn("Store.getMemoryAnnotations")(function* (memoryId) {
|
|
6307
|
+
const rows = yield* Effect26.tryPromise({
|
|
6250
6308
|
try: () => db.select({ conceptId: memoryConcepts2.conceptId }).from(memoryConcepts2).where(eq8(memoryConcepts2.memoryId, memoryId)),
|
|
6251
6309
|
catch: storeError("Store.getMemoryAnnotations")
|
|
6252
6310
|
});
|
|
6253
6311
|
return rows.map((r) => r.conceptId);
|
|
6254
6312
|
});
|
|
6255
|
-
const queryByMetadata =
|
|
6313
|
+
const queryByMetadata = Effect26.fn("Store.queryByMetadata")(function* (filter) {
|
|
6256
6314
|
const rows = yield* backend.queryByMetadata(filter);
|
|
6257
6315
|
const typed = rows;
|
|
6258
6316
|
return typed.map((row) => fromMemoryRow({
|
|
@@ -6269,8 +6327,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6269
6327
|
embeddingModel: row.embedding_model
|
|
6270
6328
|
}));
|
|
6271
6329
|
});
|
|
6272
|
-
const getMemoryMetadata =
|
|
6273
|
-
const rows = yield*
|
|
6330
|
+
const getMemoryMetadata = Effect26.fn("Store.getMemoryMetadata")(function* (id) {
|
|
6331
|
+
const rows = yield* Effect26.tryPromise({
|
|
6274
6332
|
try: () => db.select({ metadata: memories2.metadata }).from(memories2).where(eq8(memories2.id, id)).limit(1),
|
|
6275
6333
|
catch: storeError("Store.getMemoryMetadata")
|
|
6276
6334
|
});
|
|
@@ -6279,8 +6337,8 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6279
6337
|
return yield* new NotFound({ id });
|
|
6280
6338
|
return parseMetadata(row.metadata);
|
|
6281
6339
|
});
|
|
6282
|
-
const setMemoryMetadata =
|
|
6283
|
-
const updated = yield*
|
|
6340
|
+
const setMemoryMetadata = Effect26.fn("Store.setMemoryMetadata")(function* (id, metadata) {
|
|
6341
|
+
const updated = yield* Effect26.tryPromise({
|
|
6284
6342
|
try: () => db.transaction(async (tx) => {
|
|
6285
6343
|
const rows = await tx.select({ metadata: memories2.metadata }).from(memories2).where(eq8(memories2.id, id)).limit(1);
|
|
6286
6344
|
const row = rows[0];
|
|
@@ -6347,17 +6405,17 @@ var makeStore = (backend, modelConfig, topicsOptions, graphConfig, auditOptions,
|
|
|
6347
6405
|
graphCoverage: graphCoverage2
|
|
6348
6406
|
};
|
|
6349
6407
|
};
|
|
6350
|
-
var
|
|
6351
|
-
const base =
|
|
6408
|
+
var layer9 = (url, options) => {
|
|
6409
|
+
const base = Layer11.effect(Service5, Effect26.gen(function* () {
|
|
6352
6410
|
const backend = yield* Service2;
|
|
6353
6411
|
return Service5.of(makeStore(backend, options?.embeddingModelConfig, options?.topics, options?.graphConfig, options?.enqueueAudit !== undefined ? { enqueueAudit: options.enqueueAudit } : undefined, options?.defaultTtlSecs));
|
|
6354
|
-
})).pipe(
|
|
6412
|
+
})).pipe(Layer11.provideMerge(layer3(url, options)));
|
|
6355
6413
|
const config = options?.logging ?? silentConfig;
|
|
6356
|
-
return
|
|
6414
|
+
return Layer11.provideMerge(layer(config))(base);
|
|
6357
6415
|
};
|
|
6358
|
-
var configured6 = (url, options) => make({ name: "Store", layer:
|
|
6416
|
+
var configured6 = (url, options) => make({ name: "Store", layer: Layer11.suspend(() => layer9(url, options)), deps: [] });
|
|
6359
6417
|
var node5 = configured6("file:./.dimple/dimple.db");
|
|
6360
|
-
var loggingFromConfig =
|
|
6418
|
+
var loggingFromConfig = Layer11.unwrap(Effect26.map(Effect26.service(Service4), (cfg) => layer(cfg.config.logging ?? silentConfig)));
|
|
6361
6419
|
// ../store/src/topics/index.ts
|
|
6362
6420
|
var exports_topics = {};
|
|
6363
6421
|
__export(exports_topics, {
|
|
@@ -6416,15 +6474,15 @@ __export(exports_topics, {
|
|
|
6416
6474
|
});
|
|
6417
6475
|
|
|
6418
6476
|
// ../store/src/audit.ts
|
|
6419
|
-
import { Effect as
|
|
6477
|
+
import { Effect as Effect27 } from "effect";
|
|
6420
6478
|
function auditWriteHandler(deps = {}) {
|
|
6421
|
-
const run = (payload, ctx) =>
|
|
6479
|
+
const run = (payload, ctx) => Effect27.gen(function* () {
|
|
6422
6480
|
const envelope = payload;
|
|
6423
6481
|
const backend = yield* Service2;
|
|
6424
|
-
yield* backend.writeAuditRows(envelope.rows).pipe(
|
|
6482
|
+
yield* backend.writeAuditRows(envelope.rows).pipe(Effect27.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `audit.write: ${error}` })));
|
|
6425
6483
|
return;
|
|
6426
6484
|
});
|
|
6427
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6485
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect27.provideService(Service2, deps.backend)) : run;
|
|
6428
6486
|
}
|
|
6429
6487
|
// ../store/src/graph/index.ts
|
|
6430
6488
|
var exports_graph = {};
|
|
@@ -6459,7 +6517,7 @@ __export(exports_graph, {
|
|
|
6459
6517
|
});
|
|
6460
6518
|
|
|
6461
6519
|
// ../store/src/graph/concepts.ts
|
|
6462
|
-
import { Clock as Clock6, Effect as
|
|
6520
|
+
import { Clock as Clock6, Effect as Effect28 } from "effect";
|
|
6463
6521
|
import { and as and9, asc as asc6, eq as eq9, isNotNull, sql as sql9 } from "drizzle-orm";
|
|
6464
6522
|
var DEFAULT_CLUSTER_K = 8;
|
|
6465
6523
|
var DEFAULT_CLUSTER_MIN_SIZE = 3;
|
|
@@ -6582,11 +6640,11 @@ var conceptConfidence = (vector, centroid) => {
|
|
|
6582
6640
|
return 0;
|
|
6583
6641
|
return Math.min(1, Math.max(0, dot3(vector, centroid) / Math.sqrt(norm)));
|
|
6584
6642
|
};
|
|
6585
|
-
var clusterLeaf =
|
|
6643
|
+
var clusterLeaf = Effect28.fn("ConceptCluster.clusterLeaf")(function* (topicKey, model, config) {
|
|
6586
6644
|
const backend = yield* Service2;
|
|
6587
6645
|
const db = backend.db;
|
|
6588
6646
|
const now = yield* Clock6.currentTimeMillis;
|
|
6589
|
-
const memberRows = yield*
|
|
6647
|
+
const memberRows = yield* Effect28.tryPromise({
|
|
6590
6648
|
try: () => db.select({ memoryId: topicMemberships.memoryId }).from(topicMemberships).where(and9(eq9(topicMemberships.topicKey, topicKey), eq9(topicMemberships.modelKey, model.id))).orderBy(asc6(topicMemberships.memoryId)),
|
|
6591
6649
|
catch: (cause) => new StoreError({ message: `clusterLeaf: load members: ${cause}` })
|
|
6592
6650
|
});
|
|
@@ -6605,11 +6663,11 @@ var clusterLeaf = Effect27.fn("ConceptCluster.clusterLeaf")(function* (topicKey,
|
|
|
6605
6663
|
return { concepts: 0, annotated: 0 };
|
|
6606
6664
|
return yield* persistClusters(db, members, config, now);
|
|
6607
6665
|
});
|
|
6608
|
-
var clusterCorpus =
|
|
6666
|
+
var clusterCorpus = Effect28.fn("ConceptCluster.clusterCorpus")(function* (model, config) {
|
|
6609
6667
|
const backend = yield* Service2;
|
|
6610
6668
|
const db = backend.db;
|
|
6611
6669
|
const now = yield* Clock6.currentTimeMillis;
|
|
6612
|
-
const rows = yield*
|
|
6670
|
+
const rows = yield* Effect28.tryPromise({
|
|
6613
6671
|
try: () => db.select({ id: memories.id, embedding: memories.embedding }).from(memories).where(and9(eq9(memories.state, "active"), isNotNull(memories.embedding))).orderBy(asc6(memories.id)),
|
|
6614
6672
|
catch: (cause) => new StoreError({ message: `clusterCorpus: load memories: ${cause}` })
|
|
6615
6673
|
});
|
|
@@ -6626,7 +6684,7 @@ var clusterCorpus = Effect27.fn("ConceptCluster.clusterCorpus")(function* (model
|
|
|
6626
6684
|
return { concepts: 0, annotated: 0 };
|
|
6627
6685
|
return yield* persistClusters(db, members, config, now);
|
|
6628
6686
|
});
|
|
6629
|
-
var persistClusters = (db, members, config, now) =>
|
|
6687
|
+
var persistClusters = (db, members, config, now) => Effect28.tryPromise({
|
|
6630
6688
|
try: () => db.transaction(async (tx) => {
|
|
6631
6689
|
const { k, minClusterSize } = resolvedClusterConfig(config);
|
|
6632
6690
|
const vectors = members.map((m) => m.vector);
|
|
@@ -6677,21 +6735,21 @@ var persistClusters = (db, members, config, now) => Effect27.tryPromise({
|
|
|
6677
6735
|
catch: (cause) => new StoreError({ message: `concept cluster: persist: ${cause}` })
|
|
6678
6736
|
});
|
|
6679
6737
|
function conceptClusterHandler(deps) {
|
|
6680
|
-
const run = (payload, ctx) =>
|
|
6738
|
+
const run = (payload, ctx) => Effect28.gen(function* () {
|
|
6681
6739
|
const envelope = payload;
|
|
6682
6740
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
6683
6741
|
if (!model) {
|
|
6684
|
-
return yield*
|
|
6742
|
+
return yield* Effect28.fail(new JobFailure({
|
|
6685
6743
|
jobId: ctx.jobId,
|
|
6686
6744
|
message: `concept.cluster: unknown model "${envelope.modelKey}"`
|
|
6687
6745
|
}));
|
|
6688
6746
|
}
|
|
6689
6747
|
if (envelope.topicKey !== undefined) {
|
|
6690
|
-
yield* clusterLeaf(envelope.topicKey, model, deps.config).pipe(
|
|
6748
|
+
yield* clusterLeaf(envelope.topicKey, model, deps.config).pipe(Effect28.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `concept.cluster: ${error}` })));
|
|
6691
6749
|
} else {
|
|
6692
|
-
yield* clusterCorpus(model, deps.config).pipe(
|
|
6750
|
+
yield* clusterCorpus(model, deps.config).pipe(Effect28.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `concept.cluster: ${error}` })));
|
|
6693
6751
|
}
|
|
6694
|
-
yield*
|
|
6752
|
+
yield* Effect28.logInfo("concepts.clustered").pipe(Effect28.annotateLogs({
|
|
6695
6753
|
scope: "@dimple/store",
|
|
6696
6754
|
event: "concepts.clustered",
|
|
6697
6755
|
modelKey: envelope.modelKey,
|
|
@@ -6699,10 +6757,10 @@ function conceptClusterHandler(deps) {
|
|
|
6699
6757
|
}));
|
|
6700
6758
|
return;
|
|
6701
6759
|
});
|
|
6702
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
6760
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect28.provideService(Service2, deps.backend)) : run;
|
|
6703
6761
|
}
|
|
6704
6762
|
// ../store/src/graph/dream.ts
|
|
6705
|
-
import { Clock as Clock7, Effect as
|
|
6763
|
+
import { Clock as Clock7, Effect as Effect29 } from "effect";
|
|
6706
6764
|
import { and as and10, asc as asc7, eq as eq10, gt, inArray as inArray5, isNull as isNull3, sql as sql10 } from "drizzle-orm";
|
|
6707
6765
|
var dreamStateDefaults = {
|
|
6708
6766
|
minHours: 24,
|
|
@@ -6813,18 +6871,18 @@ var writeDerivedMemoryTx = async (tx, content, sources, now) => {
|
|
|
6813
6871
|
}
|
|
6814
6872
|
return { inserted };
|
|
6815
6873
|
};
|
|
6816
|
-
var runDream =
|
|
6874
|
+
var runDream = Effect29.fn("MemoryDream.run")(function* (model, classify) {
|
|
6817
6875
|
const backend = yield* Service2;
|
|
6818
6876
|
const db = backend.db;
|
|
6819
6877
|
const now = yield* Clock7.currentTimeMillis;
|
|
6820
6878
|
const { minHours, minNewMemories, maxPairs, maxDerived } = dreamStateDefaults;
|
|
6821
|
-
const stateRows = yield*
|
|
6879
|
+
const stateRows = yield* Effect29.tryPromise({
|
|
6822
6880
|
try: () => db.select().from(dreamState).where(eq10(dreamState.modelKey, model.id)),
|
|
6823
6881
|
catch: (cause) => new StoreError({ message: `memory.dream: load state: ${cause}` })
|
|
6824
6882
|
});
|
|
6825
6883
|
const lastDreamAt = stateRows[0]?.lastDreamAt ?? 0;
|
|
6826
6884
|
if (now - lastDreamAt < minHours * 3600000) {
|
|
6827
|
-
yield*
|
|
6885
|
+
yield* Effect29.logDebug("dream.skipped").pipe(Effect29.annotateLogs({ scope: "@dimple/store", event: "dream.skipped", reason: "cooldown" }));
|
|
6828
6886
|
return {
|
|
6829
6887
|
skipped: true,
|
|
6830
6888
|
reason: "cooldown",
|
|
@@ -6833,13 +6891,13 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6833
6891
|
superseded: 0
|
|
6834
6892
|
};
|
|
6835
6893
|
}
|
|
6836
|
-
const sinceCountRows = yield*
|
|
6894
|
+
const sinceCountRows = yield* Effect29.tryPromise({
|
|
6837
6895
|
try: () => db.select({ count: sql10`count(*)` }).from(memories).where(gt(memories.createdAt, lastDreamAt)),
|
|
6838
6896
|
catch: (cause) => new StoreError({ message: `memory.dream: count new: ${cause}` })
|
|
6839
6897
|
});
|
|
6840
6898
|
const memoriesSinceWatermark = Number(sinceCountRows[0]?.count ?? 0);
|
|
6841
6899
|
if (memoriesSinceWatermark < minNewMemories) {
|
|
6842
|
-
yield*
|
|
6900
|
+
yield* Effect29.logDebug("dream.skipped").pipe(Effect29.annotateLogs({
|
|
6843
6901
|
scope: "@dimple/store",
|
|
6844
6902
|
event: "dream.skipped",
|
|
6845
6903
|
reason: "insufficient-new-memories",
|
|
@@ -6853,7 +6911,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6853
6911
|
superseded: 0
|
|
6854
6912
|
};
|
|
6855
6913
|
}
|
|
6856
|
-
const pendingRows = yield*
|
|
6914
|
+
const pendingRows = yield* Effect29.tryPromise({
|
|
6857
6915
|
try: () => db.select({
|
|
6858
6916
|
fromMemoryId: artifactEdges.fromMemoryId,
|
|
6859
6917
|
toMemoryId: artifactEdges.toMemoryId
|
|
@@ -6866,7 +6924,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6866
6924
|
};
|
|
6867
6925
|
for (const row of pendingRows)
|
|
6868
6926
|
addPair(row.fromMemoryId, row.toMemoryId);
|
|
6869
|
-
const sinceRows = yield*
|
|
6927
|
+
const sinceRows = yield* Effect29.tryPromise({
|
|
6870
6928
|
try: () => db.select({
|
|
6871
6929
|
id: memories.id,
|
|
6872
6930
|
logicalKey: memories.logicalKey,
|
|
@@ -6877,19 +6935,19 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6877
6935
|
});
|
|
6878
6936
|
if (sinceRows.length > 0) {
|
|
6879
6937
|
const sinceIds = sinceRows.map((r) => r.id);
|
|
6880
|
-
const membershipRows = yield*
|
|
6938
|
+
const membershipRows = yield* Effect29.tryPromise({
|
|
6881
6939
|
try: () => db.select({ memoryId: topicMemberships.memoryId, topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and10(eq10(topicMemberships.modelKey, model.id), inArray5(topicMemberships.memoryId, sinceIds))),
|
|
6882
6940
|
catch: (cause) => new StoreError({ message: `memory.dream: scan memberships: ${cause}` })
|
|
6883
6941
|
});
|
|
6884
6942
|
const leafKeys = [...new Set(membershipRows.map((m) => m.topicKey))];
|
|
6885
6943
|
if (leafKeys.length > 0) {
|
|
6886
|
-
const mateRows = yield*
|
|
6944
|
+
const mateRows = yield* Effect29.tryPromise({
|
|
6887
6945
|
try: () => db.select({ memoryId: topicMemberships.memoryId, topicKey: topicMemberships.topicKey }).from(topicMemberships).where(and10(eq10(topicMemberships.modelKey, model.id), inArray5(topicMemberships.topicKey, leafKeys))),
|
|
6888
6946
|
catch: (cause) => new StoreError({ message: `memory.dream: scan mates: ${cause}` })
|
|
6889
6947
|
});
|
|
6890
6948
|
const memberIds = [...new Set([...sinceIds, ...mateRows.map((m) => m.memoryId)])];
|
|
6891
|
-
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(
|
|
6892
|
-
const memoryRows = yield*
|
|
6949
|
+
const vectorRows = yield* backend.fetchMemoryVectors(memberIds, model).pipe(Effect29.mapError((cause) => new StoreError({ message: `memory.dream: scan vectors: ${cause}` })));
|
|
6950
|
+
const memoryRows = yield* Effect29.tryPromise({
|
|
6893
6951
|
try: () => db.select({
|
|
6894
6952
|
id: memories.id,
|
|
6895
6953
|
logicalKey: memories.logicalKey,
|
|
@@ -6946,7 +7004,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6946
7004
|
}
|
|
6947
7005
|
const orderedPairs = [...pairSet.values()].sort((a, b) => cmp3(a.toMemoryId, b.toMemoryId) || cmp3(a.fromMemoryId, b.fromMemoryId)).slice(0, maxPairs);
|
|
6948
7006
|
const endpointIds = [...new Set(orderedPairs.flatMap((p) => [p.fromMemoryId, p.toMemoryId]))];
|
|
6949
|
-
const contentRows = endpointIds.length > 0 ? yield*
|
|
7007
|
+
const contentRows = endpointIds.length > 0 ? yield* Effect29.tryPromise({
|
|
6950
7008
|
try: () => db.select({ id: memories.id, content: memories.content }).from(memories).where(inArray5(memories.id, endpointIds)),
|
|
6951
7009
|
catch: (cause) => new StoreError({ message: `memory.dream: pair contents: ${cause}` })
|
|
6952
7010
|
}) : [];
|
|
@@ -6966,12 +7024,12 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6966
7024
|
}
|
|
6967
7025
|
let classification = { relations: [], derived: [] };
|
|
6968
7026
|
if (dreamPairs.length > 0) {
|
|
6969
|
-
classification = yield* classify.classify(dreamPairs, { maxDerived }).pipe(
|
|
7027
|
+
classification = yield* classify.classify(dreamPairs, { maxDerived }).pipe(Effect29.mapError((error) => new StoreError({ message: `memory.dream: classify: ${error}` })));
|
|
6970
7028
|
}
|
|
6971
7029
|
const endpointIdsOf = [
|
|
6972
7030
|
...new Set(classification.relations.flatMap((r) => [r.fromMemoryId, r.toMemoryId]))
|
|
6973
7031
|
];
|
|
6974
|
-
const endpointRows = endpointIdsOf.length > 0 ? yield*
|
|
7032
|
+
const endpointRows = endpointIdsOf.length > 0 ? yield* Effect29.tryPromise({
|
|
6975
7033
|
try: () => db.select({ id: memories.id, metadata: memories.metadata }).from(memories).where(inArray5(memories.id, endpointIdsOf)),
|
|
6976
7034
|
catch: (cause) => new StoreError({ message: `memory.dream: endpoints: ${cause}` })
|
|
6977
7035
|
}) : [];
|
|
@@ -6989,7 +7047,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
6989
7047
|
}
|
|
6990
7048
|
const acceptedDerived = [];
|
|
6991
7049
|
for (const derived of classification.derived) {
|
|
6992
|
-
const sourceRows = derived.sources.length > 0 ? yield*
|
|
7050
|
+
const sourceRows = derived.sources.length > 0 ? yield* Effect29.tryPromise({
|
|
6993
7051
|
try: () => db.select({ id: memories.id }).from(memories).where(inArray5(memories.id, derived.sources)),
|
|
6994
7052
|
catch: (cause) => new StoreError({ message: `memory.dream: derived sources: ${cause}` })
|
|
6995
7053
|
}) : [];
|
|
@@ -7001,7 +7059,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7001
7059
|
}
|
|
7002
7060
|
let superseded = 0;
|
|
7003
7061
|
let created = 0;
|
|
7004
|
-
yield*
|
|
7062
|
+
yield* Effect29.tryPromise({
|
|
7005
7063
|
try: () => db.transaction(async (tx) => {
|
|
7006
7064
|
for (const relation of acceptedRelations) {
|
|
7007
7065
|
const result = await writeRelationEdgeTx(tx, relation, now);
|
|
@@ -7026,7 +7084,7 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7026
7084
|
}),
|
|
7027
7085
|
catch: (cause) => new StoreError({ message: `memory.dream: persist: ${cause}` })
|
|
7028
7086
|
});
|
|
7029
|
-
yield*
|
|
7087
|
+
yield* Effect29.logInfo("dream.completed").pipe(Effect29.annotateLogs({
|
|
7030
7088
|
scope: "@dimple/store",
|
|
7031
7089
|
event: "dream.completed",
|
|
7032
7090
|
modelKey: model.id,
|
|
@@ -7037,24 +7095,24 @@ var runDream = Effect28.fn("MemoryDream.run")(function* (model, classify) {
|
|
|
7037
7095
|
return { skipped: false, classified: dreamPairs.length, created, superseded };
|
|
7038
7096
|
});
|
|
7039
7097
|
function graphDreamHandler(deps) {
|
|
7040
|
-
const run = (payload, ctx) =>
|
|
7098
|
+
const run = (payload, ctx) => Effect29.gen(function* () {
|
|
7041
7099
|
const envelope = payload;
|
|
7042
7100
|
const model = deps.getModelConfig(envelope.modelKey);
|
|
7043
7101
|
if (!model) {
|
|
7044
|
-
return yield*
|
|
7102
|
+
return yield* Effect29.fail(new JobFailure({
|
|
7045
7103
|
jobId: ctx.jobId,
|
|
7046
7104
|
message: `memory.dream: unknown model "${envelope.modelKey}"`
|
|
7047
7105
|
}));
|
|
7048
7106
|
}
|
|
7049
|
-
return yield* runDream(model, deps.classify).pipe(
|
|
7107
|
+
return yield* runDream(model, deps.classify).pipe(Effect29.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `memory.dream: ${error}` })));
|
|
7050
7108
|
});
|
|
7051
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
7109
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect29.provideService(Service2, deps.backend)) : run;
|
|
7052
7110
|
}
|
|
7053
7111
|
// ../store/src/graph/repair.ts
|
|
7054
|
-
import { Clock as Clock8, Effect as
|
|
7112
|
+
import { Clock as Clock8, Effect as Effect30 } from "effect";
|
|
7055
7113
|
import { and as and11, eq as eq11, isNotNull as isNotNull2, lt, or as or2, sql as sql11 } from "drizzle-orm";
|
|
7056
7114
|
var DEFAULT_EXPIRED_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
7057
|
-
var runRepair = (db, includePurgeExpired, expiredGraceMs, now) =>
|
|
7115
|
+
var runRepair = (db, includePurgeExpired, expiredGraceMs, now) => Effect30.tryPromise({
|
|
7058
7116
|
try: () => db.transaction(async (tx) => {
|
|
7059
7117
|
let orphanedEdges = 0;
|
|
7060
7118
|
let staleAnnotations = 0;
|
|
@@ -7148,10 +7206,10 @@ var runRepair = (db, includePurgeExpired, expiredGraceMs, now) => Effect29.tryPr
|
|
|
7148
7206
|
catch: (cause) => new StoreError({ message: `graph.repair: ${cause}` })
|
|
7149
7207
|
});
|
|
7150
7208
|
function graphRepairHandler(deps) {
|
|
7151
|
-
const run = (payload, ctx) =>
|
|
7209
|
+
const run = (payload, ctx) => Effect30.gen(function* () {
|
|
7152
7210
|
const envelope = payload;
|
|
7153
7211
|
if (!deps.getModelConfig(envelope.modelKey)) {
|
|
7154
|
-
return yield*
|
|
7212
|
+
return yield* Effect30.fail(new JobFailure({
|
|
7155
7213
|
jobId: ctx.jobId,
|
|
7156
7214
|
message: `graph.repair: unknown model "${envelope.modelKey}"`
|
|
7157
7215
|
}));
|
|
@@ -7159,8 +7217,8 @@ function graphRepairHandler(deps) {
|
|
|
7159
7217
|
const backend = yield* Service2;
|
|
7160
7218
|
const db = backend.db;
|
|
7161
7219
|
const now = yield* Clock8.currentTimeMillis;
|
|
7162
|
-
const result = yield* runRepair(db, deps.includePurgeExpired ?? false, deps.expiredGraceMs ?? DEFAULT_EXPIRED_GRACE_MS, now).pipe(
|
|
7163
|
-
yield*
|
|
7220
|
+
const result = yield* runRepair(db, deps.includePurgeExpired ?? false, deps.expiredGraceMs ?? DEFAULT_EXPIRED_GRACE_MS, now).pipe(Effect30.mapError((error) => new JobFailure({ jobId: ctx.jobId, message: `graph.repair: ${error}` })));
|
|
7221
|
+
yield* Effect30.logInfo("repair.completed").pipe(Effect30.annotateLogs({
|
|
7164
7222
|
scope: "@dimple/store",
|
|
7165
7223
|
event: "repair.completed",
|
|
7166
7224
|
modelKey: envelope.modelKey,
|
|
@@ -7170,7 +7228,7 @@ function graphRepairHandler(deps) {
|
|
|
7170
7228
|
}));
|
|
7171
7229
|
return result;
|
|
7172
7230
|
});
|
|
7173
|
-
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(
|
|
7231
|
+
return deps.backend ? (payload, ctx) => run(payload, ctx).pipe(Effect30.provideService(Service2, deps.backend)) : run;
|
|
7174
7232
|
}
|
|
7175
7233
|
// ../store/src/models.ts
|
|
7176
7234
|
import { Schema as Schema24 } from "effect";
|
|
@@ -7249,21 +7307,21 @@ var RetrievalLogEntry = Schema24.Struct({
|
|
|
7249
7307
|
createdAt: Schema24.Number
|
|
7250
7308
|
});
|
|
7251
7309
|
// ../embedder/src/provider.ts
|
|
7252
|
-
import { Context as
|
|
7310
|
+
import { Context as Context11, Schema as Schema25 } from "effect";
|
|
7253
7311
|
|
|
7254
7312
|
class EmbeddingError extends Schema25.TaggedError()("EmbeddingProvider.EmbeddingError", { message: Schema25.String, cause: Schema25.optional(Schema25.Defect()) }) {
|
|
7255
7313
|
}
|
|
7256
7314
|
|
|
7257
|
-
class
|
|
7315
|
+
class Service10 extends Context11.Service()("@dimple/embedder/EmbeddingModelRegistry") {
|
|
7258
7316
|
}
|
|
7259
7317
|
// ../embedder/src/transformers.ts
|
|
7260
|
-
import { Config as Config5, Effect as
|
|
7318
|
+
import { Config as Config5, Effect as Effect33, Layer as Layer12, Option as Option11, Schema as Schema26 } from "effect";
|
|
7261
7319
|
import { existsSync as existsSync2, readFileSync } from "node:fs";
|
|
7262
7320
|
import { join as join2 } from "node:path";
|
|
7263
7321
|
import { pathToFileURL } from "node:url";
|
|
7264
7322
|
|
|
7265
7323
|
// ../embedder/src/auto-install.ts
|
|
7266
|
-
import { Effect as
|
|
7324
|
+
import { Effect as Effect32 } from "effect";
|
|
7267
7325
|
|
|
7268
7326
|
// ../installer/src/index.ts
|
|
7269
7327
|
import { existsSync } from "node:fs";
|
|
@@ -7354,7 +7412,7 @@ var installOptionalPeers = (packages, opts) => {
|
|
|
7354
7412
|
return run;
|
|
7355
7413
|
};
|
|
7356
7414
|
// ../embedder/src/auto-install.ts
|
|
7357
|
-
var installTransformers = (cwd) =>
|
|
7415
|
+
var installTransformers = (cwd) => Effect32.tryPromise({
|
|
7358
7416
|
try: () => installOptionalPeers(["@huggingface/transformers"], { cwd, timeoutMs: 300000 }),
|
|
7359
7417
|
catch: (cause) => new EmbeddingError({
|
|
7360
7418
|
message: cause instanceof InstallerError ? `${cause.message} — or set \`autoInstall: false\` and configure a hosted embedding provider` : `auto-install of @huggingface/transformers failed: ${String(cause)} — run it manually ` + "or set `autoInstall: false` and configure a hosted embedding provider"
|
|
@@ -7423,16 +7481,32 @@ var importFromDisk = () => new Promise((resolve, reject) => {
|
|
|
7423
7481
|
reject(cause);
|
|
7424
7482
|
}
|
|
7425
7483
|
});
|
|
7426
|
-
var buildSessionOptions = (opts) =>
|
|
7427
|
-
|
|
7428
|
-
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
}
|
|
7434
|
-
|
|
7435
|
-
|
|
7484
|
+
var buildSessionOptions = (opts) => {
|
|
7485
|
+
const cuda = opts.gpuMemLimit !== undefined || opts.arenaExtendStrategy !== undefined;
|
|
7486
|
+
return {
|
|
7487
|
+
enableCpuMemArena: opts.enableCpuMemArena ?? false,
|
|
7488
|
+
...opts.intraOpNumThreads !== undefined ? { intraOpNumThreads: opts.intraOpNumThreads } : undefined,
|
|
7489
|
+
...opts.interOpNumThreads !== undefined ? { interOpNumThreads: opts.interOpNumThreads } : undefined,
|
|
7490
|
+
...opts.executionMode !== undefined ? { executionMode: opts.executionMode } : undefined,
|
|
7491
|
+
...opts.graphOptimizationLevel !== undefined ? { graphOptimizationLevel: opts.graphOptimizationLevel } : undefined,
|
|
7492
|
+
...opts.enableMemPattern !== undefined ? { enableMemPattern: opts.enableMemPattern } : undefined,
|
|
7493
|
+
...cuda ? {
|
|
7494
|
+
executionProviders: [
|
|
7495
|
+
{
|
|
7496
|
+
name: "cuda",
|
|
7497
|
+
device_id: 0,
|
|
7498
|
+
...opts.gpuMemLimit !== undefined ? { gpu_mem_limit: opts.gpuMemLimit } : undefined,
|
|
7499
|
+
...opts.arenaExtendStrategy !== undefined ? { arena_extend_strategy: opts.arenaExtendStrategy } : undefined
|
|
7500
|
+
}
|
|
7501
|
+
]
|
|
7502
|
+
} : undefined
|
|
7503
|
+
};
|
|
7504
|
+
};
|
|
7505
|
+
var applyWasmOptions = (env, wasm) => {
|
|
7506
|
+
if (wasm?.numThreads === undefined)
|
|
7507
|
+
return;
|
|
7508
|
+
env.backends.onnx.wasm.numThreads = wasm.numThreads;
|
|
7509
|
+
};
|
|
7436
7510
|
var defaults2 = {
|
|
7437
7511
|
modelId: "Xenova/all-MiniLM-L6-v2",
|
|
7438
7512
|
device: "cpu",
|
|
@@ -7454,40 +7528,38 @@ var outputVectors = (result) => {
|
|
|
7454
7528
|
};
|
|
7455
7529
|
var make6 = (options) => {
|
|
7456
7530
|
let pipe = null;
|
|
7457
|
-
const getPipeline =
|
|
7531
|
+
const getPipeline = Effect33.fnUntraced(function* () {
|
|
7458
7532
|
if (pipe)
|
|
7459
7533
|
return pipe;
|
|
7460
|
-
const loadModule = () =>
|
|
7461
|
-
let transformers = yield* loadModule().pipe(
|
|
7534
|
+
const loadModule = () => Effect33.tryPromise(() => import("@huggingface/transformers"));
|
|
7535
|
+
let transformers = yield* loadModule().pipe(Effect33.catch(() => {
|
|
7462
7536
|
if (options.autoInstall === false) {
|
|
7463
|
-
return
|
|
7537
|
+
return Effect33.fail(new EmbeddingError({
|
|
7464
7538
|
message: "@huggingface/transformers is not installed — run `bun add @huggingface/transformers` " + "to use local embeddings (or configure a hosted embedding provider)"
|
|
7465
7539
|
}));
|
|
7466
7540
|
}
|
|
7467
|
-
return
|
|
7468
|
-
yield*
|
|
7541
|
+
return Effect33.gen(function* () {
|
|
7542
|
+
yield* Effect33.logInfo("auto-installing @huggingface/transformers (first local-embed use — scripts skipped)").pipe(Effect33.annotateLogs({ scope: "@dimple/embedder", event: "embedder.auto-install" }));
|
|
7469
7543
|
yield* installTransformers(process.cwd());
|
|
7470
|
-
return yield*
|
|
7544
|
+
return yield* Effect33.tryPromise(importFromDisk).pipe(Effect33.mapError(() => new EmbeddingError({
|
|
7471
7545
|
message: "@huggingface/transformers still unavailable after auto-install — run `bun add " + "@huggingface/transformers` manually (or set `autoInstall: false` and configure a hosted embedding provider)"
|
|
7472
7546
|
})));
|
|
7473
7547
|
});
|
|
7474
7548
|
}));
|
|
7475
7549
|
if (options.cacheDir)
|
|
7476
7550
|
transformers.env.cacheDir = options.cacheDir;
|
|
7477
|
-
|
|
7551
|
+
applyWasmOptions(transformers.env, options.wasm);
|
|
7552
|
+
pipe = yield* Effect33.tryPromise(() => transformers.pipeline("feature-extraction", options.modelId ?? defaults2.modelId, {
|
|
7478
7553
|
device: options.device ?? defaults2.device,
|
|
7479
7554
|
dtype: options.dtype ?? defaults2.dtype,
|
|
7480
|
-
session_options: {
|
|
7481
|
-
|
|
7482
|
-
...options.sessionOptions !== undefined ? buildSessionOptions(options.sessionOptions) : undefined
|
|
7483
|
-
}
|
|
7484
|
-
})).pipe(Effect32.mapError((cause) => new EmbeddingError({ message: `load model: ${cause}` })));
|
|
7555
|
+
session_options: buildSessionOptions(options.sessionOptions ?? {})
|
|
7556
|
+
})).pipe(Effect33.mapError((cause) => new EmbeddingError({ message: `load model: ${cause}` })));
|
|
7485
7557
|
return pipe;
|
|
7486
7558
|
});
|
|
7487
|
-
const embed =
|
|
7559
|
+
const embed = Effect33.fn("TransformersEmbedding.embed")(function* (texts) {
|
|
7488
7560
|
const started = Date.now();
|
|
7489
7561
|
const p = yield* getPipeline();
|
|
7490
|
-
const result = yield*
|
|
7562
|
+
const result = yield* Effect33.tryPromise({
|
|
7491
7563
|
try: () => p(texts, {
|
|
7492
7564
|
pooling: options.pooling ?? defaults2.pooling,
|
|
7493
7565
|
normalize: options.normalize ?? defaults2.normalize
|
|
@@ -7502,7 +7574,7 @@ var make6 = (options) => {
|
|
|
7502
7574
|
});
|
|
7503
7575
|
}
|
|
7504
7576
|
}
|
|
7505
|
-
yield*
|
|
7577
|
+
yield* Effect33.logDebug("embedder.embed").pipe(Effect33.annotateLogs({
|
|
7506
7578
|
scope: "@dimple/embedder",
|
|
7507
7579
|
event: "embedder.embed",
|
|
7508
7580
|
count: texts.length,
|
|
@@ -7510,9 +7582,9 @@ var make6 = (options) => {
|
|
|
7510
7582
|
}));
|
|
7511
7583
|
return vectors;
|
|
7512
7584
|
});
|
|
7513
|
-
const embedSingle =
|
|
7585
|
+
const embedSingle = Effect33.fn("TransformersEmbedding.embedSingle")(function* (text4) {
|
|
7514
7586
|
const p = yield* getPipeline();
|
|
7515
|
-
const result = yield*
|
|
7587
|
+
const result = yield* Effect33.tryPromise({
|
|
7516
7588
|
try: () => p(text4, {
|
|
7517
7589
|
pooling: options.pooling ?? defaults2.pooling,
|
|
7518
7590
|
normalize: options.normalize ?? defaults2.normalize
|
|
@@ -7540,14 +7612,14 @@ var makeTransformersModel = (options = {}) => {
|
|
|
7540
7612
|
};
|
|
7541
7613
|
};
|
|
7542
7614
|
// ../embedder/src/registry.ts
|
|
7543
|
-
import { Effect as
|
|
7615
|
+
import { Effect as Effect34, Layer as Layer13 } from "effect";
|
|
7544
7616
|
// ../core/src/hosted-embedding.ts
|
|
7545
7617
|
import { embedMany } from "ai";
|
|
7546
7618
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
7547
7619
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
7548
7620
|
import { createCohere } from "@ai-sdk/cohere";
|
|
7549
7621
|
import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
|
|
7550
|
-
import { Effect as
|
|
7622
|
+
import { Effect as Effect35 } from "effect";
|
|
7551
7623
|
var makeHostedEmbeddingModel = (options) => {
|
|
7552
7624
|
const provider = options.provider ?? "openai";
|
|
7553
7625
|
const model = provider === "google" ? createGoogleGenerativeAI({
|
|
@@ -7578,8 +7650,8 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
7578
7650
|
return { openaiCompatible: { dimensions: options.dimensions } };
|
|
7579
7651
|
}
|
|
7580
7652
|
})();
|
|
7581
|
-
const embed =
|
|
7582
|
-
const result = yield*
|
|
7653
|
+
const embed = Effect35.fn("HostedEmbedding.embed")(function* (texts) {
|
|
7654
|
+
const result = yield* Effect35.tryPromise({
|
|
7583
7655
|
try: () => embedMany({ model, values: [...texts], providerOptions }),
|
|
7584
7656
|
catch: (cause) => new EmbeddingError({ message: `hosted embedMany: ${cause}` })
|
|
7585
7657
|
});
|
|
@@ -7592,7 +7664,7 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
7592
7664
|
}
|
|
7593
7665
|
return result.embeddings.map((e) => new Float32Array(e));
|
|
7594
7666
|
});
|
|
7595
|
-
const embedSingle =
|
|
7667
|
+
const embedSingle = Effect35.fn("HostedEmbedding.embedSingle")(function* (text4) {
|
|
7596
7668
|
const results = yield* embed([text4]);
|
|
7597
7669
|
return results[0];
|
|
7598
7670
|
});
|
|
@@ -7606,7 +7678,7 @@ var makeHostedEmbeddingModel = (options) => {
|
|
|
7606
7678
|
};
|
|
7607
7679
|
|
|
7608
7680
|
// ../retriever/src/hybrid.ts
|
|
7609
|
-
import { Clock as Clock9, Context as
|
|
7681
|
+
import { Clock as Clock9, Context as Context12, Effect as Effect36, Layer as Layer14, Schema as Schema27 } from "effect";
|
|
7610
7682
|
|
|
7611
7683
|
// ../retriever/src/rrf.ts
|
|
7612
7684
|
function fuse(denseRanks, fts5Ranks, k, limit) {
|
|
@@ -7649,17 +7721,17 @@ class HybridError extends Schema27.TaggedError()("HybridRetrieval.HybridError",
|
|
|
7649
7721
|
}) {
|
|
7650
7722
|
}
|
|
7651
7723
|
|
|
7652
|
-
class
|
|
7724
|
+
class Service11 extends Context12.Service()("@dimple/retriever/HybridRetrieval") {
|
|
7653
7725
|
}
|
|
7654
|
-
var wrapError = (tag) => (effect) => effect.pipe(
|
|
7655
|
-
var
|
|
7726
|
+
var wrapError = (tag) => (effect) => effect.pipe(Effect36.mapError((cause) => new HybridError({ message: `${tag}: ${cause}` })));
|
|
7727
|
+
var layer12 = (options) => Layer14.effect(Service11, Effect36.gen(function* () {
|
|
7656
7728
|
const store = yield* Service5;
|
|
7657
|
-
const registry = yield*
|
|
7729
|
+
const registry = yield* Service10;
|
|
7658
7730
|
const defaultId = yield* registry.defaultModelId.pipe(wrapError("defaultModelId"));
|
|
7659
7731
|
const defaultModel = yield* registry.getModel(defaultId).pipe(wrapError("getModel"));
|
|
7660
7732
|
const defaultK = options?.defaultK ?? 60;
|
|
7661
7733
|
const defaultLimit = options?.defaultLimit ?? 10;
|
|
7662
|
-
const hybrid =
|
|
7734
|
+
const hybrid = Effect36.fn("HybridRetrieval.hybrid")(function* (query2, options2) {
|
|
7663
7735
|
const namespace = options2?.namespace ?? "default";
|
|
7664
7736
|
const limit = options2?.limit ?? defaultLimit;
|
|
7665
7737
|
const k = options2?.k ?? defaultK;
|
|
@@ -7762,7 +7834,7 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7762
7834
|
fts: fts5Results.map((s) => s.memory.id)
|
|
7763
7835
|
})
|
|
7764
7836
|
}).pipe(wrapError("logAudit"));
|
|
7765
|
-
yield*
|
|
7837
|
+
yield* Effect36.logInfo("hybrid.search").pipe(Effect36.annotateLogs({
|
|
7766
7838
|
scope: "@dimple/retriever",
|
|
7767
7839
|
event: "hybrid.search",
|
|
7768
7840
|
mode,
|
|
@@ -7772,16 +7844,16 @@ var layer11 = (options) => Layer13.effect(Service10, Effect35.gen(function* () {
|
|
|
7772
7844
|
}));
|
|
7773
7845
|
return results;
|
|
7774
7846
|
});
|
|
7775
|
-
return
|
|
7847
|
+
return Service11.of({ hybrid });
|
|
7776
7848
|
}));
|
|
7777
|
-
var hybridRetrievalLayer =
|
|
7849
|
+
var hybridRetrievalLayer = layer12();
|
|
7778
7850
|
var configured8 = (options, deps) => make({
|
|
7779
7851
|
name: "HybridRetrieval",
|
|
7780
|
-
layer: options === undefined ? hybridRetrievalLayer :
|
|
7852
|
+
layer: options === undefined ? hybridRetrievalLayer : Layer14.suspend(() => layer12(options)),
|
|
7781
7853
|
deps
|
|
7782
7854
|
});
|
|
7783
7855
|
// ../llm/src/llm.ts
|
|
7784
|
-
import { Cause as Cause4, Effect as
|
|
7856
|
+
import { Cause as Cause4, Effect as Effect37, Option as Option12, Schema as Schema28 } from "effect";
|
|
7785
7857
|
import {
|
|
7786
7858
|
generateText as generateTextCall,
|
|
7787
7859
|
NoSuchToolError,
|
|
@@ -7829,9 +7901,9 @@ var makeLLM = (model, defaults3 = {}) => {
|
|
|
7829
7901
|
...reasoningEffort !== undefined ? { providerOptions: { openaiCompatible: { reasoningEffort } } } : undefined
|
|
7830
7902
|
};
|
|
7831
7903
|
};
|
|
7832
|
-
const generateText =
|
|
7904
|
+
const generateText = Effect37.fn("LLM.generateText")(function* (input) {
|
|
7833
7905
|
const started = Date.now();
|
|
7834
|
-
const text4 = yield*
|
|
7906
|
+
const text4 = yield* Effect37.tryPromise({
|
|
7835
7907
|
try: () => generateTextCall({
|
|
7836
7908
|
model,
|
|
7837
7909
|
prompt: input.prompt,
|
|
@@ -7840,16 +7912,16 @@ var makeLLM = (model, defaults3 = {}) => {
|
|
|
7840
7912
|
}).then((result) => result.text),
|
|
7841
7913
|
catch: (cause) => new LLMError({ message: "LLM text generation failed", cause: Cause4.die(cause) })
|
|
7842
7914
|
});
|
|
7843
|
-
yield*
|
|
7915
|
+
yield* Effect37.logDebug("llm.generate-text").pipe(Effect37.annotateLogs({
|
|
7844
7916
|
scope: "@dimple/llm",
|
|
7845
7917
|
event: "llm.generate-text",
|
|
7846
7918
|
latencyMs: Date.now() - started
|
|
7847
7919
|
}));
|
|
7848
7920
|
return text4;
|
|
7849
7921
|
});
|
|
7850
|
-
const generateObject =
|
|
7922
|
+
const generateObject = Effect37.fn("LLM.generateObject")(function* (input) {
|
|
7851
7923
|
const started = Date.now();
|
|
7852
|
-
const result = yield*
|
|
7924
|
+
const result = yield* Effect37.tryPromise({
|
|
7853
7925
|
try: async () => {
|
|
7854
7926
|
try {
|
|
7855
7927
|
const result3 = await generateTextCall({
|
|
@@ -7892,7 +7964,7 @@ ${schemaJson}`,
|
|
|
7892
7964
|
},
|
|
7893
7965
|
catch: (cause) => new LLMError({ message: "LLM structured generation failed", cause: Cause4.die(cause) })
|
|
7894
7966
|
});
|
|
7895
|
-
yield*
|
|
7967
|
+
yield* Effect37.logDebug("llm.generate-object").pipe(Effect37.annotateLogs({
|
|
7896
7968
|
scope: "@dimple/llm",
|
|
7897
7969
|
event: "llm.generate-object",
|
|
7898
7970
|
latencyMs: Date.now() - started
|
|
@@ -7912,7 +7984,7 @@ var openAICompatible = (options) => {
|
|
|
7912
7984
|
return provider.languageModel(options.modelId ?? "default");
|
|
7913
7985
|
};
|
|
7914
7986
|
// ../llm/src/dream-classifier.ts
|
|
7915
|
-
import { Effect as
|
|
7987
|
+
import { Effect as Effect38 } from "effect";
|
|
7916
7988
|
import { z as z2 } from "zod";
|
|
7917
7989
|
var EDGE_KINDS = ["memory_supersedes", "memory_contradicts", "memory_supports"];
|
|
7918
7990
|
var EDGE_KIND_SET = new Set(EDGE_KINDS);
|
|
@@ -7959,7 +8031,7 @@ var buildPrompt = (pairs, maxDerived) => [
|
|
|
7959
8031
|
].join(`
|
|
7960
8032
|
`);
|
|
7961
8033
|
var makeDreamClassifier = (deps) => {
|
|
7962
|
-
const classify =
|
|
8034
|
+
const classify = Effect38.fn("DreamClassifier.classify")(function* (pairs, opts) {
|
|
7963
8035
|
const system = deps.modelId === undefined ? SYSTEM_PROMPT : `${SYSTEM_PROMPT}
|
|
7964
8036
|
Judging for model: ${deps.modelId}`;
|
|
7965
8037
|
const judged = yield* deps.llm.generateObject({
|
|
@@ -7989,11 +8061,11 @@ Judging for model: ${deps.modelId}`;
|
|
|
7989
8061
|
return { classify };
|
|
7990
8062
|
};
|
|
7991
8063
|
// ../llm/src/service.ts
|
|
7992
|
-
import { Config as Config6, Context as
|
|
7993
|
-
class
|
|
8064
|
+
import { Config as Config6, Context as Context13, Effect as Effect39, Layer as Layer15 } from "effect";
|
|
8065
|
+
class Service12 extends Context13.Service()("@dimple/LLM") {
|
|
7994
8066
|
}
|
|
7995
8067
|
// ../core/src/layer.ts
|
|
7996
|
-
var embedderFromConfig = (model) => model.provider === "transformers" ?
|
|
8068
|
+
var embedderFromConfig = (model) => model.provider === "transformers" ? Effect40.succeed(makeTransformersModel({
|
|
7997
8069
|
modelId: model.model,
|
|
7998
8070
|
dimensions: model.dimensions,
|
|
7999
8071
|
...model.device !== undefined ? { device: model.device } : undefined,
|
|
@@ -8002,41 +8074,42 @@ var embedderFromConfig = (model) => model.provider === "transformers" ? Effect39
|
|
|
8002
8074
|
...model.normalize !== undefined ? { normalize: model.normalize } : undefined,
|
|
8003
8075
|
...model.cacheDir !== undefined ? { cacheDir: model.cacheDir } : undefined,
|
|
8004
8076
|
...model.autoInstall !== undefined ? { autoInstall: model.autoInstall } : undefined,
|
|
8005
|
-
...model.sessionOptions !== undefined ? { sessionOptions: model.sessionOptions } : undefined
|
|
8006
|
-
|
|
8077
|
+
...model.sessionOptions !== undefined ? { sessionOptions: model.sessionOptions } : undefined,
|
|
8078
|
+
...model.wasm !== undefined ? { wasm: model.wasm } : undefined
|
|
8079
|
+
})) : model.provider === "openai" ? model.apiKey === undefined ? Effect40.fail(new EmbeddingError({ message: `OpenAI model ${model.id} requires an apiKey` })) : Effect40.succeed(makeHostedEmbeddingModel({
|
|
8007
8080
|
modelId: model.id,
|
|
8008
8081
|
model: model.model,
|
|
8009
8082
|
dimensions: model.dimensions,
|
|
8010
8083
|
apiKey: model.apiKey,
|
|
8011
8084
|
...model.provider !== "openai" ? { provider: model.provider } : undefined,
|
|
8012
8085
|
...model.baseURL !== undefined ? { baseURL: model.baseURL } : undefined
|
|
8013
|
-
})) :
|
|
8014
|
-
var
|
|
8086
|
+
})) : Effect40.fail(new EmbeddingError({ message: `Unknown provider: ${model.provider}` }));
|
|
8087
|
+
var layer14 = (options) => {
|
|
8015
8088
|
const topicConfig = Schema29.decodeUnknownSync(TopicConfig)(options.topics?.config ?? {});
|
|
8016
8089
|
const graphConfig = Schema29.decodeUnknownSync(exports_graph.GraphConfig)(options.graph ?? {});
|
|
8017
8090
|
const models2 = new Map(options.models.map((m) => [m.id, m]));
|
|
8018
8091
|
const defaultModel = models2.get(options.defaultModelId);
|
|
8019
8092
|
const loggingConfig = options.logging ?? silentConfig;
|
|
8020
8093
|
const getModelConfig = (key) => models2.get(key);
|
|
8021
|
-
const storeWithSeams =
|
|
8022
|
-
const rt = yield*
|
|
8023
|
-
const enqueueSplit = (topicKey, modelKey) => rt.enqueue(TopicSplit, { topicId: topicKey, modelKey }).pipe(
|
|
8024
|
-
const enqueueMerge = (topicKey, modelKey) => rt.enqueue(TopicMerge, { topicId: topicKey, modelKey }).pipe(
|
|
8094
|
+
const storeWithSeams = Layer16.unwrap(Effect40.gen(function* () {
|
|
8095
|
+
const rt = yield* Service8;
|
|
8096
|
+
const enqueueSplit = (topicKey, modelKey) => rt.enqueue(TopicSplit, { topicId: topicKey, modelKey }).pipe(Effect40.asVoid);
|
|
8097
|
+
const enqueueMerge = (topicKey, modelKey) => rt.enqueue(TopicMerge, { topicId: topicKey, modelKey }).pipe(Effect40.asVoid);
|
|
8025
8098
|
const auditBuffer = [];
|
|
8026
|
-
const flushAudit =
|
|
8099
|
+
const flushAudit = Effect40.gen(function* () {
|
|
8027
8100
|
if (auditBuffer.length === 0)
|
|
8028
8101
|
return;
|
|
8029
8102
|
const rows = auditBuffer.splice(0, auditBuffer.length);
|
|
8030
|
-
yield* rt.enqueue(AuditWrite, { rows }).pipe(
|
|
8103
|
+
yield* rt.enqueue(AuditWrite, { rows }).pipe(Effect40.asVoid);
|
|
8031
8104
|
});
|
|
8032
|
-
const enqueueAudit = (row) =>
|
|
8105
|
+
const enqueueAudit = (row) => Effect40.gen(function* () {
|
|
8033
8106
|
auditBuffer.push(row);
|
|
8034
8107
|
if (auditBuffer.length >= 50) {
|
|
8035
8108
|
yield* flushAudit;
|
|
8036
8109
|
}
|
|
8037
8110
|
});
|
|
8038
|
-
yield* flushAudit.pipe(
|
|
8039
|
-
return
|
|
8111
|
+
yield* flushAudit.pipe(Effect40.catchCause((cause) => Effect40.logWarning("audit flush failed", cause)), Effect40.repeat(Schedule2.spaced(1000)), Effect40.forkScoped);
|
|
8112
|
+
return layer9(options.storeUrl, {
|
|
8040
8113
|
...options.defaultTtlSecs !== undefined ? { defaultTtlSecs: options.defaultTtlSecs } : undefined,
|
|
8041
8114
|
...options.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : undefined,
|
|
8042
8115
|
releaseClient: true,
|
|
@@ -8054,48 +8127,49 @@ var layer13 = (options) => {
|
|
|
8054
8127
|
...options.audit !== false ? { enqueueAudit } : undefined
|
|
8055
8128
|
});
|
|
8056
8129
|
}));
|
|
8057
|
-
const embedderRegistry =
|
|
8130
|
+
const embedderRegistry = Layer16.effect(Service10, Effect40.sync(() => Service10.of({
|
|
8058
8131
|
getModel: (id) => {
|
|
8059
8132
|
const model = models2.get(id);
|
|
8060
|
-
return model !== undefined ? options.embedderFor(model) :
|
|
8133
|
+
return model !== undefined ? options.embedderFor(model) : Effect40.fail(new EmbeddingError({ message: `Unknown model: ${id}` }));
|
|
8061
8134
|
},
|
|
8062
|
-
defaultModelId:
|
|
8063
|
-
registerModel: () =>
|
|
8135
|
+
defaultModelId: Effect40.succeed(options.defaultModelId),
|
|
8136
|
+
registerModel: () => Effect40.void
|
|
8064
8137
|
})));
|
|
8065
|
-
const coreService =
|
|
8138
|
+
const coreService = Layer16.effect(Service, Effect40.gen(function* () {
|
|
8066
8139
|
if (defaultModel === undefined) {
|
|
8067
8140
|
return yield* new CoreConfigError({
|
|
8068
8141
|
message: `@dimple/core: unknown defaultModelId "${options.defaultModelId}"`
|
|
8069
8142
|
});
|
|
8070
8143
|
}
|
|
8071
8144
|
const store2 = yield* Service5;
|
|
8072
|
-
const rt = yield*
|
|
8073
|
-
const
|
|
8145
|
+
const rt = yield* Service8;
|
|
8146
|
+
const queueStats = yield* Service7;
|
|
8147
|
+
const registry = yield* Service9;
|
|
8074
8148
|
const backend = yield* Service2;
|
|
8075
|
-
const hybrid = yield*
|
|
8149
|
+
const hybrid = yield* Service11;
|
|
8076
8150
|
const logging = yield* Service3;
|
|
8077
8151
|
const extraByName = new Map((options.extraHandlers ?? []).map((h) => [h.name, h.handler]));
|
|
8078
|
-
const register = (name, handler) => extraByName.has(name) ?
|
|
8152
|
+
const register = (name, handler) => extraByName.has(name) ? Effect40.void : registry.register(name, handler);
|
|
8079
8153
|
yield* register("topic.split", exports_topics.topicSplitHandler({
|
|
8080
8154
|
getModelConfig,
|
|
8081
8155
|
config: topicConfig,
|
|
8082
8156
|
backend,
|
|
8083
|
-
onUnevenSplit: (childKey, modelKey) => rt.enqueue(TopicRebalance, { topicId: childKey, modelKey }).pipe(
|
|
8084
|
-
onChildOverflow: (childKey, modelKey) => rt.enqueue(TopicSplit, { topicId: childKey, modelKey }).pipe(
|
|
8157
|
+
onUnevenSplit: (childKey, modelKey) => rt.enqueue(TopicRebalance, { topicId: childKey, modelKey }).pipe(Effect40.asVoid),
|
|
8158
|
+
onChildOverflow: (childKey, modelKey) => rt.enqueue(TopicSplit, { topicId: childKey, modelKey }).pipe(Effect40.asVoid)
|
|
8085
8159
|
}));
|
|
8086
8160
|
yield* register("topic.merge", exports_topics.topicMergeHandler({ getModelConfig, config: topicConfig, backend }));
|
|
8087
8161
|
yield* register("topic.rebalance", exports_topics.topicRebalanceHandler({ getModelConfig, config: topicConfig, backend }));
|
|
8088
8162
|
yield* register("memory.embed", exports_topics.memoryEmbedHandler({
|
|
8089
8163
|
getModelConfig,
|
|
8090
|
-
embedOne: (text4, model) => options.embedderFor(model).pipe(
|
|
8091
|
-
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(
|
|
8164
|
+
embedOne: (text4, model) => options.embedderFor(model).pipe(Effect40.flatMap((m) => m.embedSingle(text4))),
|
|
8165
|
+
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(Effect40.map((m) => m.content)),
|
|
8092
8166
|
config: topicConfig,
|
|
8093
8167
|
backend
|
|
8094
8168
|
}));
|
|
8095
8169
|
yield* register("memory.embed.batch", exports_topics.memoryEmbedBatchHandler({
|
|
8096
8170
|
getModelConfig,
|
|
8097
|
-
embedOne: (text4, model) => options.embedderFor(model).pipe(
|
|
8098
|
-
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(
|
|
8171
|
+
embedOne: (text4, model) => options.embedderFor(model).pipe(Effect40.flatMap((m) => m.embedSingle(text4))),
|
|
8172
|
+
getMemoryContent: (memoryId) => store2.get(memoryId).pipe(Effect40.map((m) => m.content)),
|
|
8099
8173
|
config: topicConfig,
|
|
8100
8174
|
backend
|
|
8101
8175
|
}));
|
|
@@ -8110,12 +8184,12 @@ var layer13 = (options) => {
|
|
|
8110
8184
|
yield* register("memory.dream", (payload, ctx) => {
|
|
8111
8185
|
const envelope = payload;
|
|
8112
8186
|
if (getModelConfig(envelope.modelKey) === undefined) {
|
|
8113
|
-
return
|
|
8187
|
+
return Effect40.fail(new JobFailure({
|
|
8114
8188
|
jobId: ctx.jobId,
|
|
8115
8189
|
message: `memory.dream: unknown model "${envelope.modelKey}"`
|
|
8116
8190
|
}));
|
|
8117
8191
|
}
|
|
8118
|
-
return
|
|
8192
|
+
return Effect40.fail(new JobFailure({
|
|
8119
8193
|
jobId: ctx.jobId,
|
|
8120
8194
|
message: "memory.dream: LLM not configured — provide `llm` to @dimple/core"
|
|
8121
8195
|
}));
|
|
@@ -8130,6 +8204,7 @@ var layer13 = (options) => {
|
|
|
8130
8204
|
store: store2,
|
|
8131
8205
|
hybrid,
|
|
8132
8206
|
runtime: rt,
|
|
8207
|
+
queueStats,
|
|
8133
8208
|
registry,
|
|
8134
8209
|
models: models2,
|
|
8135
8210
|
defaultModel,
|
|
@@ -8159,7 +8234,7 @@ var layer13 = (options) => {
|
|
|
8159
8234
|
});
|
|
8160
8235
|
return compile(group([jobStack, jobRegistry, store, embedder, retrieval, workers, core]));
|
|
8161
8236
|
};
|
|
8162
|
-
var layerConfig7 = (options) =>
|
|
8237
|
+
var layerConfig7 = (options) => Layer16.unwrap(Effect40.gen(function* () {
|
|
8163
8238
|
const cfg = yield* Service4;
|
|
8164
8239
|
const embedding = cfg.config.embedding;
|
|
8165
8240
|
if (embedding === undefined || embedding.models.length === 0) {
|
|
@@ -8174,14 +8249,21 @@ var layerConfig7 = (options) => Layer15.unwrap(Effect39.gen(function* () {
|
|
|
8174
8249
|
});
|
|
8175
8250
|
}
|
|
8176
8251
|
const llm = cfg.config.llm;
|
|
8177
|
-
|
|
8252
|
+
const models2 = embedding.models.map((model) => {
|
|
8253
|
+
const sessionOptions = mergeSessionOptions(embedding.sessionOptions, model.sessionOptions);
|
|
8254
|
+
return {
|
|
8255
|
+
...model,
|
|
8256
|
+
...sessionOptions !== undefined ? { sessionOptions } : undefined
|
|
8257
|
+
};
|
|
8258
|
+
});
|
|
8259
|
+
return layer14({
|
|
8178
8260
|
storeUrl: cfg.config.storeUrl,
|
|
8179
8261
|
...cfg.config.defaultK !== undefined ? { defaultK: cfg.config.defaultK } : undefined,
|
|
8180
8262
|
...cfg.config.defaultLimit !== undefined ? { defaultLimit: cfg.config.defaultLimit } : undefined,
|
|
8181
8263
|
...cfg.config.defaultTtlSecs !== undefined ? { defaultTtlSecs: cfg.config.defaultTtlSecs } : undefined,
|
|
8182
8264
|
jobsUrl,
|
|
8183
8265
|
...options?.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : undefined,
|
|
8184
|
-
models:
|
|
8266
|
+
models: models2,
|
|
8185
8267
|
defaultModelId: embedding.defaultModel,
|
|
8186
8268
|
embedderFor: embedderFromConfig,
|
|
8187
8269
|
...cfg.config.topics !== undefined ? { topics: { config: cfg.config.topics } } : undefined,
|
|
@@ -8316,10 +8398,10 @@ var remapError = (cause) => {
|
|
|
8316
8398
|
};
|
|
8317
8399
|
|
|
8318
8400
|
// src/make.ts
|
|
8319
|
-
import { Effect as
|
|
8401
|
+
import { Effect as Effect51, Exit as Exit8, Layer as Layer27, ManagedRuntime, Scope as Scope5 } from "effect";
|
|
8320
8402
|
|
|
8321
8403
|
// ../extractor/src/code-extractor.ts
|
|
8322
|
-
import { Context as
|
|
8404
|
+
import { Context as Context14, Effect as Effect41, Layer as Layer17 } from "effect";
|
|
8323
8405
|
|
|
8324
8406
|
// ../extractor/src/models/extraction.ts
|
|
8325
8407
|
import { Schema as Schema31 } from "effect";
|
|
@@ -8763,11 +8845,11 @@ function extract3(filePath, tree2) {
|
|
|
8763
8845
|
}
|
|
8764
8846
|
|
|
8765
8847
|
// ../extractor/src/code-extractor.ts
|
|
8766
|
-
class
|
|
8848
|
+
class Service13 extends Context14.Service()("@dimple/extractor/CodeExtractor") {
|
|
8767
8849
|
}
|
|
8768
|
-
var
|
|
8769
|
-
extract:
|
|
8770
|
-
const result = yield*
|
|
8850
|
+
var layer15 = Layer17.effect(Service13, Effect41.succeed(Service13.of({
|
|
8851
|
+
extract: Effect41.fn("CodeExtractor.extract")(function* (request) {
|
|
8852
|
+
const result = yield* Effect41.try({
|
|
8771
8853
|
try: () => extract3(request.filePath, request.tree),
|
|
8772
8854
|
catch: (cause) => new ExtractionError({ message: `Extraction failed for ${request.filePath}`, cause })
|
|
8773
8855
|
});
|
|
@@ -8783,10 +8865,10 @@ var layer14 = Layer16.effect(Service12, Effect40.succeed(Service12.of({
|
|
|
8783
8865
|
});
|
|
8784
8866
|
})
|
|
8785
8867
|
})));
|
|
8786
|
-
var configured10 = () => make({ name: "CodeExtractor", layer:
|
|
8868
|
+
var configured10 = () => make({ name: "CodeExtractor", layer: layer15, deps: [] });
|
|
8787
8869
|
var node6 = configured10();
|
|
8788
8870
|
// ../extractor/src/content-handler.ts
|
|
8789
|
-
import { Context as
|
|
8871
|
+
import { Context as Context15, Layer as Layer18, Schema as Schema32 } from "effect";
|
|
8790
8872
|
|
|
8791
8873
|
class ContentError extends Schema32.TaggedError()("Content.ContentError", {
|
|
8792
8874
|
contentType: Schema32.String,
|
|
@@ -8794,22 +8876,22 @@ class ContentError extends Schema32.TaggedError()("Content.ContentError", {
|
|
|
8794
8876
|
}) {
|
|
8795
8877
|
}
|
|
8796
8878
|
|
|
8797
|
-
class
|
|
8879
|
+
class Service14 extends Context15.Service()("@dimple/extractor/ContentHandler") {
|
|
8798
8880
|
}
|
|
8799
|
-
var
|
|
8881
|
+
var layer16 = Layer18.succeed(Service14, Service14.of({
|
|
8800
8882
|
contentType: "text/plain",
|
|
8801
8883
|
renderText: (content) => content,
|
|
8802
8884
|
extractText: (content) => content
|
|
8803
8885
|
}));
|
|
8804
8886
|
// ../parser/src/code-parser.ts
|
|
8805
|
-
import { Context as
|
|
8887
|
+
import { Context as Context21, Effect as Effect48, Layer as Layer25 } from "effect";
|
|
8806
8888
|
import { Parser as Parser2 } from "web-tree-sitter";
|
|
8807
8889
|
|
|
8808
8890
|
// ../parser/src/language-store.ts
|
|
8809
|
-
import { Cache, Context as
|
|
8891
|
+
import { Cache, Context as Context20, Effect as Effect47, Layer as Layer24 } from "effect";
|
|
8810
8892
|
|
|
8811
8893
|
// ../wasm/src/wasm-registry.ts
|
|
8812
|
-
import { Context as
|
|
8894
|
+
import { Context as Context16, Effect as Effect42, Layer as Layer19, Schema as Schema36 } from "effect";
|
|
8813
8895
|
|
|
8814
8896
|
// ../shared/src/errors/wasm.ts
|
|
8815
8897
|
import { Schema as Schema33 } from "effect";
|
|
@@ -8909,9 +8991,9 @@ for (const grammar of grammars.grammars) {
|
|
|
8909
8991
|
registryMap.set(grammar.lang, grammar);
|
|
8910
8992
|
}
|
|
8911
8993
|
|
|
8912
|
-
class
|
|
8913
|
-
static layer =
|
|
8914
|
-
resolve:
|
|
8994
|
+
class Service15 extends Context16.Service()("@dimple/wasm/TreeSitterRegistry") {
|
|
8995
|
+
static layer = Layer19.succeed(Service15, Service15.of({
|
|
8996
|
+
resolve: Effect42.fn("TreeSitterRegistry.resolve")(function* (lang) {
|
|
8915
8997
|
const found = registryMap.get(lang);
|
|
8916
8998
|
if (found === undefined) {
|
|
8917
8999
|
return yield* new RegistryError2({ message: `Language not found in registry: ${lang}` });
|
|
@@ -8921,11 +9003,11 @@ class Service14 extends Context15.Service()("@dimple/wasm/TreeSitterRegistry") {
|
|
|
8921
9003
|
}));
|
|
8922
9004
|
}
|
|
8923
9005
|
// ../wasm/src/wasm-downloader.ts
|
|
8924
|
-
import { Context as
|
|
8925
|
-
class
|
|
8926
|
-
static layer =
|
|
8927
|
-
download:
|
|
8928
|
-
const response = yield*
|
|
9006
|
+
import { Context as Context17, Effect as Effect43, Layer as Layer20 } from "effect";
|
|
9007
|
+
class Service16 extends Context17.Service()("@dimple/wasm/WasmDownloader") {
|
|
9008
|
+
static layer = Layer20.effect(Service16, Effect43.succeed(Service16.of({
|
|
9009
|
+
download: Effect43.fn("WasmDownloader.download")(function* (url) {
|
|
9010
|
+
const response = yield* Effect43.tryPromise({
|
|
8929
9011
|
try: () => fetch(url),
|
|
8930
9012
|
catch: (cause) => new WasmDownloadError({
|
|
8931
9013
|
message: `Failed to fetch from ${url}`,
|
|
@@ -8939,7 +9021,7 @@ class Service15 extends Context16.Service()("@dimple/wasm/WasmDownloader") {
|
|
|
8939
9021
|
url
|
|
8940
9022
|
});
|
|
8941
9023
|
}
|
|
8942
|
-
const arrayBuffer = yield*
|
|
9024
|
+
const arrayBuffer = yield* Effect43.tryPromise({
|
|
8943
9025
|
try: () => response.arrayBuffer(),
|
|
8944
9026
|
catch: (cause) => new WasmDownloadError({
|
|
8945
9027
|
message: `Failed to read response from ${url}`,
|
|
@@ -8952,7 +9034,7 @@ class Service15 extends Context16.Service()("@dimple/wasm/WasmDownloader") {
|
|
|
8952
9034
|
})));
|
|
8953
9035
|
}
|
|
8954
9036
|
// ../wasm/src/wasm-cache.ts
|
|
8955
|
-
import { Context as
|
|
9037
|
+
import { Context as Context18, Effect as Effect44, Layer as Layer21 } from "effect";
|
|
8956
9038
|
import { existsSync as existsSync3, mkdirSync, renameSync, writeFileSync } from "node:fs";
|
|
8957
9039
|
import { join as join3 } from "node:path";
|
|
8958
9040
|
|
|
@@ -9047,37 +9129,37 @@ function envPaths(name, { suffix = "nodejs" } = {}) {
|
|
|
9047
9129
|
// ../wasm/src/wasm-cache.ts
|
|
9048
9130
|
var paths = envPaths("dimple");
|
|
9049
9131
|
|
|
9050
|
-
class
|
|
9132
|
+
class Service17 extends Context18.Service()("@dimple/wasm/WasmCache") {
|
|
9051
9133
|
}
|
|
9052
|
-
var
|
|
9053
|
-
ensure:
|
|
9134
|
+
var layer17 = Layer21.effect(Service17, Effect44.succeed(Service17.of({
|
|
9135
|
+
ensure: Effect44.fn("WasmCache.ensure")(function* (lang, version, download) {
|
|
9054
9136
|
const versionDir = join3(paths.cache, "tree-sitter", lang, version);
|
|
9055
9137
|
const finalPath = join3(versionDir, "parser.wasm");
|
|
9056
|
-
const exists = yield*
|
|
9138
|
+
const exists = yield* Effect44.try({
|
|
9057
9139
|
try: () => existsSync3(finalPath),
|
|
9058
9140
|
catch: (cause) => new WasmCacheError({ message: `Failed to stat ${finalPath}`, cause })
|
|
9059
9141
|
});
|
|
9060
9142
|
if (exists)
|
|
9061
9143
|
return finalPath;
|
|
9062
|
-
yield*
|
|
9144
|
+
yield* Effect44.try({
|
|
9063
9145
|
try: () => mkdirSync(versionDir, { recursive: true }),
|
|
9064
9146
|
catch: (cause) => new WasmCacheError({ message: `Failed to create ${versionDir}`, cause })
|
|
9065
9147
|
});
|
|
9066
9148
|
const tempPath = join3(versionDir, `.tmp-${process.pid}`);
|
|
9067
9149
|
const wasm = yield* download;
|
|
9068
|
-
yield*
|
|
9150
|
+
yield* Effect44.try({
|
|
9069
9151
|
try: () => writeFileSync(tempPath, wasm),
|
|
9070
9152
|
catch: (cause) => new WasmCacheError({ message: `Failed to write ${tempPath}`, cause })
|
|
9071
9153
|
});
|
|
9072
|
-
const renamed = yield*
|
|
9154
|
+
const renamed = yield* Effect44.try({
|
|
9073
9155
|
try: () => {
|
|
9074
9156
|
renameSync(tempPath, finalPath);
|
|
9075
9157
|
return true;
|
|
9076
9158
|
},
|
|
9077
9159
|
catch: (cause) => new WasmCacheError({ message: `Failed to rename ${tempPath}`, cause })
|
|
9078
|
-
}).pipe(
|
|
9160
|
+
}).pipe(Effect44.catchCause(() => Effect44.succeed(false)));
|
|
9079
9161
|
if (!renamed) {
|
|
9080
|
-
const existsAfter = yield*
|
|
9162
|
+
const existsAfter = yield* Effect44.try({
|
|
9081
9163
|
try: () => existsSync3(finalPath),
|
|
9082
9164
|
catch: (cause) => new WasmCacheError({ message: `Failed to stat ${finalPath}`, cause })
|
|
9083
9165
|
});
|
|
@@ -9091,72 +9173,72 @@ var layer16 = Layer20.effect(Service16, Effect43.succeed(Service16.of({
|
|
|
9091
9173
|
})
|
|
9092
9174
|
})));
|
|
9093
9175
|
// ../wasm/src/grammar-store.ts
|
|
9094
|
-
import { Context as
|
|
9095
|
-
class
|
|
9176
|
+
import { Context as Context19, Effect as Effect45, Layer as Layer22 } from "effect";
|
|
9177
|
+
class Service18 extends Context19.Service()("@dimple/wasm/GrammarStore") {
|
|
9096
9178
|
}
|
|
9097
|
-
var
|
|
9098
|
-
const registry = yield*
|
|
9099
|
-
const downloader = yield*
|
|
9100
|
-
const cache = yield*
|
|
9101
|
-
const ensure =
|
|
9102
|
-
const metadata = yield* registry.resolve(lang).pipe(
|
|
9179
|
+
var layer18 = Layer22.effect(Service18, Effect45.gen(function* () {
|
|
9180
|
+
const registry = yield* Service15;
|
|
9181
|
+
const downloader = yield* Service16;
|
|
9182
|
+
const cache = yield* Service17;
|
|
9183
|
+
const ensure = Effect45.fn("GrammarStore.ensure")(function* (lang) {
|
|
9184
|
+
const metadata = yield* registry.resolve(lang).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
9103
9185
|
language: lang,
|
|
9104
9186
|
message: `Failed to resolve grammar metadata for: ${lang}`,
|
|
9105
9187
|
cause
|
|
9106
9188
|
})));
|
|
9107
|
-
const wasmBytes = yield* downloader.download(metadata.url).pipe(
|
|
9189
|
+
const wasmBytes = yield* downloader.download(metadata.url).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
9108
9190
|
language: lang,
|
|
9109
9191
|
message: `Failed to download grammar for: ${lang}`,
|
|
9110
9192
|
cause
|
|
9111
9193
|
})));
|
|
9112
|
-
const path2 = yield* cache.ensure(lang, metadata.version,
|
|
9194
|
+
const path2 = yield* cache.ensure(lang, metadata.version, Effect45.succeed(wasmBytes)).pipe(Effect45.mapError((cause) => new GrammarProvisionError({
|
|
9113
9195
|
language: lang,
|
|
9114
9196
|
message: `Failed to provision grammar for: ${lang}`,
|
|
9115
9197
|
cause
|
|
9116
9198
|
})));
|
|
9117
9199
|
return path2;
|
|
9118
9200
|
});
|
|
9119
|
-
return
|
|
9201
|
+
return Service18.of({ ensure });
|
|
9120
9202
|
}));
|
|
9121
|
-
var allLive =
|
|
9203
|
+
var allLive = layer18.pipe(Layer22.provideMerge(Layer22.mergeAll(Service15.layer, Service16.layer, layer17)));
|
|
9122
9204
|
var configured11 = () => make({ name: "GrammarStore", layer: allLive, deps: [] });
|
|
9123
9205
|
var node7 = configured11();
|
|
9124
9206
|
// ../parser/src/language-store.ts
|
|
9125
9207
|
import { Language } from "web-tree-sitter";
|
|
9126
9208
|
|
|
9127
9209
|
// ../parser/src/tree-sitter-engine.ts
|
|
9128
|
-
import { Effect as
|
|
9210
|
+
import { Effect as Effect46, Layer as Layer23 } from "effect";
|
|
9129
9211
|
import { Parser } from "web-tree-sitter";
|
|
9130
|
-
var
|
|
9212
|
+
var layer19 = Layer23.effectDiscard(Effect46.tryPromise({
|
|
9131
9213
|
try: () => Parser.init(),
|
|
9132
9214
|
catch: (cause) => new ParserInitError({ message: "Failed to initialize tree sitter", cause })
|
|
9133
9215
|
}));
|
|
9134
9216
|
|
|
9135
9217
|
// ../parser/src/language-store.ts
|
|
9136
|
-
class
|
|
9218
|
+
class Service19 extends Context20.Service()("@dimple/parser/LanguageStore") {
|
|
9137
9219
|
}
|
|
9138
|
-
var
|
|
9139
|
-
const grammarStore = yield*
|
|
9220
|
+
var layer20 = Layer24.effect(Service19, Effect47.gen(function* () {
|
|
9221
|
+
const grammarStore = yield* Service18;
|
|
9140
9222
|
const resolveLang = (lang) => lang === "jsx" ? "javascript" : lang;
|
|
9141
9223
|
const cache = yield* Cache.make({
|
|
9142
9224
|
capacity: 100,
|
|
9143
9225
|
timeToLive: Infinity,
|
|
9144
|
-
lookup: (lang) =>
|
|
9226
|
+
lookup: (lang) => Effect47.gen(function* () {
|
|
9145
9227
|
const grammarLang = resolveLang(lang);
|
|
9146
|
-
const wasmPath = yield* grammarStore.ensure(grammarLang).pipe(
|
|
9228
|
+
const wasmPath = yield* grammarStore.ensure(grammarLang).pipe(Effect47.mapError((e) => new LanguageLoadError({
|
|
9147
9229
|
lang,
|
|
9148
9230
|
message: `Failed to provision grammar for ${lang}`,
|
|
9149
9231
|
cause: e
|
|
9150
9232
|
})));
|
|
9151
|
-
return yield*
|
|
9233
|
+
return yield* Effect47.tryPromise({
|
|
9152
9234
|
try: () => Language.load(wasmPath),
|
|
9153
9235
|
catch: (cause) => new LanguageLoadError({ lang, message: `Failed to load language: ${lang}`, cause })
|
|
9154
9236
|
});
|
|
9155
9237
|
})
|
|
9156
9238
|
});
|
|
9157
9239
|
const get = (lang) => Cache.get(cache, resolveLang(lang));
|
|
9158
|
-
return
|
|
9159
|
-
})).pipe(
|
|
9240
|
+
return Service19.of({ get });
|
|
9241
|
+
})).pipe(Layer24.provide(Layer24.mergeAll(allLive, layer19)));
|
|
9160
9242
|
|
|
9161
9243
|
// ../parser/src/models/treeHandle.ts
|
|
9162
9244
|
import { Schema as Schema37 } from "effect";
|
|
@@ -9169,13 +9251,13 @@ var TreeHandle = Schema37.Struct({
|
|
|
9169
9251
|
});
|
|
9170
9252
|
|
|
9171
9253
|
// ../parser/src/code-parser.ts
|
|
9172
|
-
class
|
|
9254
|
+
class Service20 extends Context21.Service()("@dimple/parser/CodeParser") {
|
|
9173
9255
|
}
|
|
9174
|
-
var
|
|
9175
|
-
const languageStore = yield*
|
|
9256
|
+
var layer21 = Layer25.effect(Service20, Effect48.gen(function* () {
|
|
9257
|
+
const languageStore = yield* Service19;
|
|
9176
9258
|
const parserByLanguage = new Map;
|
|
9177
|
-
const parse2 =
|
|
9178
|
-
const language = yield* languageStore.get(request.lang).pipe(
|
|
9259
|
+
const parse2 = Effect48.fn("CodeParser.parse")(function* (request) {
|
|
9260
|
+
const language = yield* languageStore.get(request.lang).pipe(Effect48.mapError((cause) => new ParseError({
|
|
9179
9261
|
message: `Failed to load language '${request.lang}' for parsing`,
|
|
9180
9262
|
cause
|
|
9181
9263
|
})));
|
|
@@ -9191,9 +9273,9 @@ var layer20 = Layer24.effect(Service19, Effect47.gen(function* () {
|
|
|
9191
9273
|
}
|
|
9192
9274
|
return TreeHandle.make({ tree: tree2, lang: request.lang, source: request.source });
|
|
9193
9275
|
});
|
|
9194
|
-
return
|
|
9195
|
-
})).pipe(
|
|
9196
|
-
var configured12 = () => make({ name: "CodeParser", layer:
|
|
9276
|
+
return Service20.of({ parse: parse2 });
|
|
9277
|
+
})).pipe(Layer25.provide(layer20));
|
|
9278
|
+
var configured12 = () => make({ name: "CodeParser", layer: layer21, deps: [] });
|
|
9197
9279
|
var node8 = configured12();
|
|
9198
9280
|
// ../parser/src/models/parseRequest.ts
|
|
9199
9281
|
import { Schema as Schema38 } from "effect";
|
|
@@ -9202,15 +9284,15 @@ var ParseRequest = Schema38.Struct({
|
|
|
9202
9284
|
source: Schema38.String
|
|
9203
9285
|
});
|
|
9204
9286
|
// src/config.ts
|
|
9205
|
-
import { Effect as
|
|
9287
|
+
import { Effect as Effect49, Layer as Layer26, Option as Option14, Schema as Schema39 } from "effect";
|
|
9206
9288
|
var isString = (value2) => Option14.isSome(Schema39.decodeUnknownOption(Schema39.String)(value2));
|
|
9207
|
-
var resolveConfigInput =
|
|
9208
|
-
return yield* (isString(input) ? loadConfigFromFile(input) : loadConfig(input)).pipe(
|
|
9289
|
+
var resolveConfigInput = Effect49.fn("Sdk.resolveConfigInput")(function* (input) {
|
|
9290
|
+
return yield* (isString(input) ? loadConfigFromFile(input) : loadConfig(input)).pipe(Effect49.mapError((error) => new ConfigError({ message: error.message })));
|
|
9209
9291
|
});
|
|
9210
|
-
var configLayerFromInput = (input) =>
|
|
9292
|
+
var configLayerFromInput = (input) => Layer26.effect(Service4, resolveConfigInput(input).pipe(Effect49.map((config) => Service4.of({ config }))));
|
|
9211
9293
|
|
|
9212
9294
|
// src/dimple.ts
|
|
9213
|
-
import { Cause as Cause5, Effect as
|
|
9295
|
+
import { Cause as Cause5, Effect as Effect50, Exit as Exit7, Option as Option15, Schema as Schema40 } from "effect";
|
|
9214
9296
|
|
|
9215
9297
|
// src/indexer.ts
|
|
9216
9298
|
import { execFileSync } from "node:child_process";
|
|
@@ -9459,7 +9541,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9459
9541
|
}
|
|
9460
9542
|
return;
|
|
9461
9543
|
};
|
|
9462
|
-
const runQuery = (text4, options) => run(
|
|
9544
|
+
const runQuery = (text4, options) => run(Effect50.gen(function* () {
|
|
9463
9545
|
if (!asString(text4)) {
|
|
9464
9546
|
return yield* new InputError({ message: "query: text must be a string" });
|
|
9465
9547
|
}
|
|
@@ -9472,7 +9554,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9472
9554
|
let rows = yield* core().hybrid.hybrid(text4, {
|
|
9473
9555
|
...rest,
|
|
9474
9556
|
...capLimit !== undefined ? { limit: capLimit } : undefined
|
|
9475
|
-
}).pipe(
|
|
9557
|
+
}).pipe(Effect50.map((r) => [...r]));
|
|
9476
9558
|
let truncated = false;
|
|
9477
9559
|
if (budget?.maxChars !== undefined && budget.maxChars > 0) {
|
|
9478
9560
|
let used = 0;
|
|
@@ -9559,7 +9641,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9559
9641
|
}
|
|
9560
9642
|
return;
|
|
9561
9643
|
};
|
|
9562
|
-
const writeUnit = (input, embed) =>
|
|
9644
|
+
const writeUnit = (input, embed) => Effect50.gen(function* () {
|
|
9563
9645
|
const mem = yield* core().store.write(input);
|
|
9564
9646
|
if (embed) {
|
|
9565
9647
|
yield* core().store.ensureTopicTree(core().defaultModel.id);
|
|
@@ -9569,8 +9651,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9569
9651
|
}
|
|
9570
9652
|
return mem;
|
|
9571
9653
|
});
|
|
9572
|
-
const upsertUnit = (input, embed) =>
|
|
9573
|
-
const existing = yield* core().store.getByKey(input.logicalKey, input.namespace ?? "default").pipe(
|
|
9654
|
+
const upsertUnit = (input, embed) => Effect50.gen(function* () {
|
|
9655
|
+
const existing = yield* core().store.getByKey(input.logicalKey, input.namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9574
9656
|
if (existing !== undefined && existing.content === input.content) {
|
|
9575
9657
|
const stamp = codeVersionOf(input.metadata);
|
|
9576
9658
|
if (stamp !== undefined && codeVersionOf(existing.metadata) === undefined) {
|
|
@@ -9635,7 +9717,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9635
9717
|
return;
|
|
9636
9718
|
};
|
|
9637
9719
|
const messageOf3 = (cause) => cause instanceof Error ? cause.message : String(cause);
|
|
9638
|
-
const resolveMemory = (idOrKey) => core().store.get(idOrKey).pipe(
|
|
9720
|
+
const resolveMemory = (idOrKey) => core().store.get(idOrKey).pipe(Effect50.catchTag("Store.NotFound", () => core().store.getByKey(idOrKey, "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.fail(new NotFoundError({ id: idOrKey, message: `Memory not found: ${idOrKey}` }))))));
|
|
9639
9721
|
const codeVersionOf = (blob) => {
|
|
9640
9722
|
if (blob === undefined || blob === null)
|
|
9641
9723
|
return;
|
|
@@ -9655,13 +9737,13 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9655
9737
|
get defaultModelId() {
|
|
9656
9738
|
return core().defaultModel.id;
|
|
9657
9739
|
},
|
|
9658
|
-
write: (input, options) => run(
|
|
9740
|
+
write: (input, options) => run(Effect50.gen(function* () {
|
|
9659
9741
|
const invalid = validateWriteInput(input);
|
|
9660
9742
|
if (invalid !== undefined)
|
|
9661
9743
|
return yield* invalid;
|
|
9662
9744
|
return yield* writeUnit(input, options?.embed !== false);
|
|
9663
9745
|
})),
|
|
9664
|
-
writeMany: (units, options) => run(
|
|
9746
|
+
writeMany: (units, options) => run(Effect50.gen(function* () {
|
|
9665
9747
|
if (!Array.isArray(units)) {
|
|
9666
9748
|
return yield* new InputError({ message: "writeMany: units must be an array" });
|
|
9667
9749
|
}
|
|
@@ -9681,7 +9763,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9681
9763
|
vectors.push(...vs);
|
|
9682
9764
|
}
|
|
9683
9765
|
yield* core().store.updateEmbeddingMany(mems.map((mem, i) => ({ id: mem.id, embedding: [...vectors[i]] })), core().defaultModel.id);
|
|
9684
|
-
return yield*
|
|
9766
|
+
return yield* Effect50.forEach(mems, (mem) => core().store.get(mem.id), {
|
|
9685
9767
|
discard: false
|
|
9686
9768
|
});
|
|
9687
9769
|
}
|
|
@@ -9689,8 +9771,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9689
9771
|
})),
|
|
9690
9772
|
get: (id) => run(resolveMemory(id)),
|
|
9691
9773
|
query: runQuery,
|
|
9692
|
-
forget: (id) => run(resolveMemory(id).pipe(
|
|
9693
|
-
index: (options) => run(
|
|
9774
|
+
forget: (id) => run(resolveMemory(id).pipe(Effect50.flatMap((memory) => core().store.forget(memory.id)))),
|
|
9775
|
+
index: (options) => run(Effect50.gen(function* () {
|
|
9694
9776
|
const invalid = validateIndexOptions(options);
|
|
9695
9777
|
if (invalid !== undefined)
|
|
9696
9778
|
return yield* invalid;
|
|
@@ -9702,7 +9784,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9702
9784
|
const maxFileBytes = options.maxFileBytes ?? 64 * 1024;
|
|
9703
9785
|
const maxChunkBytes = options.maxChunkBytes ?? 8 * 1024;
|
|
9704
9786
|
const extensions = [...options.extensions ?? INDEXABLE_EXTENSIONS];
|
|
9705
|
-
const walk = yield*
|
|
9787
|
+
const walk = yield* Effect50.try({
|
|
9706
9788
|
try: () => walkFiles([...options.paths], extensions),
|
|
9707
9789
|
catch: (cause) => new InputError({
|
|
9708
9790
|
message: `index: failed to walk paths: ${messageOf3(cause)}`
|
|
@@ -9735,8 +9817,8 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9735
9817
|
if (language === undefined)
|
|
9736
9818
|
continue;
|
|
9737
9819
|
const chunkKeyOf = (chunkId) => `${keyPrefix}${relative}#${stableChunkId(chunkId)}`;
|
|
9738
|
-
const exit = yield*
|
|
9739
|
-
const source = yield*
|
|
9820
|
+
const exit = yield* Effect50.exit(Effect50.gen(function* () {
|
|
9821
|
+
const source = yield* Effect50.try({
|
|
9740
9822
|
try: () => readCapped(filePath, maxFileBytes),
|
|
9741
9823
|
catch: (cause) => new InputError({ message: `read failed: ${messageOf3(cause)}` })
|
|
9742
9824
|
});
|
|
@@ -9815,7 +9897,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9815
9897
|
unitIds.set(key, id);
|
|
9816
9898
|
freshIds.push(id);
|
|
9817
9899
|
};
|
|
9818
|
-
const upsertAll = (candidates2) =>
|
|
9900
|
+
const upsertAll = (candidates2) => Effect50.gen(function* () {
|
|
9819
9901
|
const ids = [];
|
|
9820
9902
|
for (const spec of candidates2) {
|
|
9821
9903
|
const outcome = yield* upsertUnit(spec.input, embedUnits);
|
|
@@ -9827,7 +9909,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9827
9909
|
if (deferEmbed) {
|
|
9828
9910
|
const fresh = [];
|
|
9829
9911
|
for (const spec of specs) {
|
|
9830
|
-
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(
|
|
9912
|
+
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9831
9913
|
if (existing === undefined) {
|
|
9832
9914
|
fresh.push(spec);
|
|
9833
9915
|
continue;
|
|
@@ -9855,7 +9937,7 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9855
9937
|
} else {
|
|
9856
9938
|
const fresh = [];
|
|
9857
9939
|
for (const spec of specs) {
|
|
9858
|
-
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(
|
|
9940
|
+
const existing = yield* core().store.getByKey(spec.key, namespace ?? "default").pipe(Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9859
9941
|
if (existing === undefined) {
|
|
9860
9942
|
fresh.push(spec);
|
|
9861
9943
|
continue;
|
|
@@ -9923,14 +10005,14 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9923
10005
|
const resolveKey = (key) => {
|
|
9924
10006
|
const fromMap = unitIds.get(key);
|
|
9925
10007
|
if (fromMap !== undefined)
|
|
9926
|
-
return
|
|
9927
|
-
return core().store.getByKey(key, namespace ?? "default").pipe(
|
|
10008
|
+
return Effect50.succeed(fromMap);
|
|
10009
|
+
return core().store.getByKey(key, namespace ?? "default").pipe(Effect50.map((m) => m.id), Effect50.catchTag("Store.NotFound", () => Effect50.succeed(undefined)));
|
|
9928
10010
|
};
|
|
9929
|
-
const edgeEffect = (edge) =>
|
|
9930
|
-
const [from, to] = yield*
|
|
10011
|
+
const edgeEffect = (edge) => Effect50.gen(function* () {
|
|
10012
|
+
const [from, to] = yield* Effect50.all([resolveKey(edge.from), resolveKey(edge.to)]);
|
|
9931
10013
|
if (from === undefined || to === undefined || from === to)
|
|
9932
10014
|
return;
|
|
9933
|
-
yield*
|
|
10015
|
+
yield* Effect50.exit(core().store.addEdge({
|
|
9934
10016
|
fromMemoryId: from,
|
|
9935
10017
|
toMemoryId: to,
|
|
9936
10018
|
edgeKind: edge.edgeKind,
|
|
@@ -9940,10 +10022,10 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9940
10022
|
}));
|
|
9941
10023
|
});
|
|
9942
10024
|
for (const edge of pendingEdges) {
|
|
9943
|
-
yield*
|
|
10025
|
+
yield* Effect50.exit(edgeEffect({ ...edge, edgeKind: "memory_supports" }));
|
|
9944
10026
|
}
|
|
9945
10027
|
for (const edge of pendingCallEdges) {
|
|
9946
|
-
yield*
|
|
10028
|
+
yield* Effect50.exit(edgeEffect({ ...edge, edgeKind: "code_calls" }));
|
|
9947
10029
|
}
|
|
9948
10030
|
const EMBED_BATCH = 512;
|
|
9949
10031
|
if (deferEmbed) {
|
|
@@ -9967,21 +10049,27 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9967
10049
|
...deferEmbed ? { pendingEmbeddings: pendingEmbedIds.length } : undefined
|
|
9968
10050
|
};
|
|
9969
10051
|
})),
|
|
9970
|
-
updateEmbedding: (id, embedding, modelKey) => run(
|
|
10052
|
+
updateEmbedding: (id, embedding, modelKey) => run(Effect50.gen(function* () {
|
|
9971
10053
|
if (!Array.isArray(embedding) || embedding.length === 0 || embedding.some((v) => !asNumber(v) || !Number.isFinite(v))) {
|
|
9972
10054
|
return yield* new InputError({
|
|
9973
10055
|
message: "updateEmbedding: embedding must be a non-empty array of finite numbers"
|
|
9974
10056
|
});
|
|
9975
10057
|
}
|
|
9976
10058
|
const key = modelKey ?? core().defaultModel.id;
|
|
10059
|
+
const model = core().models.get(key);
|
|
10060
|
+
if (model !== undefined && embedding.length !== model.dimensions) {
|
|
10061
|
+
return yield* new InputError({
|
|
10062
|
+
message: `updateEmbedding: model "${key}" expects ${model.dimensions}-dimensional vectors, got ${embedding.length}`
|
|
10063
|
+
});
|
|
10064
|
+
}
|
|
9977
10065
|
const memory = yield* resolveMemory(id);
|
|
9978
10066
|
yield* core().store.ensureTopicTree(key);
|
|
9979
10067
|
yield* core().store.updateEmbedding(memory.id, [...embedding], key);
|
|
9980
10068
|
})),
|
|
9981
|
-
embed: (text4) => run(core().defaultEmbedder.embedSingle(text4).pipe(
|
|
9982
|
-
dream: (modelKey) => run(core().runtime.enqueue(MemoryDream, { modelKey, runId: crypto.randomUUID() }).pipe(
|
|
10069
|
+
embed: (text4) => run(core().defaultEmbedder.embedSingle(text4).pipe(Effect50.map((vector) => [...vector]))),
|
|
10070
|
+
dream: (modelKey) => run(core().runtime.enqueue(MemoryDream, { modelKey, runId: crypto.randomUUID() }).pipe(Effect50.flatMap((jobId) => core().runtime.wait(jobId)), Effect50.asVoid)),
|
|
9983
10071
|
dbSchema: () => run(core().store.listSchema()),
|
|
9984
|
-
repair: (modelKey) => run(core().runtime.enqueue(GraphRepair, { modelKey, runId: crypto.randomUUID() }).pipe(
|
|
10072
|
+
repair: (modelKey) => run(core().runtime.enqueue(GraphRepair, { modelKey, runId: crypto.randomUUID() }).pipe(Effect50.flatMap((jobId) => core().runtime.wait(jobId)), Effect50.asVoid)),
|
|
9985
10073
|
explain: (input, options) => run(core().store.explain(input, {
|
|
9986
10074
|
...options?.edgeKinds !== undefined ? { edgeKinds: options.edgeKinds } : undefined,
|
|
9987
10075
|
...options?.maxDepth !== undefined ? { maxDepth: options.maxDepth } : undefined,
|
|
@@ -9989,15 +10077,19 @@ var makeFacade = (state, reload, dispose) => {
|
|
|
9989
10077
|
...options?.maxTotal !== undefined ? { maxTotal: options.maxTotal } : undefined,
|
|
9990
10078
|
...options?.includePending !== undefined ? { includePending: options.includePending } : undefined
|
|
9991
10079
|
})),
|
|
9992
|
-
health: (modelKey) => run(
|
|
10080
|
+
health: (modelKey) => run(Effect50.gen(function* () {
|
|
10081
|
+
const topics3 = yield* core().store.topicHealth(modelKey);
|
|
10082
|
+
const jobs = yield* core().queueStats.pending();
|
|
10083
|
+
return { ...topics3, jobs };
|
|
10084
|
+
})),
|
|
9993
10085
|
graph: {
|
|
9994
|
-
addEdge: (input) => run(
|
|
10086
|
+
addEdge: (input) => run(Effect50.gen(function* () {
|
|
9995
10087
|
const invalid = validateAddEdgeInput(input);
|
|
9996
10088
|
if (invalid !== undefined)
|
|
9997
10089
|
return yield* invalid;
|
|
9998
10090
|
return yield* core().store.addEdge(input);
|
|
9999
10091
|
})),
|
|
10000
|
-
candidates: () => run(core().store.graphCandidates()),
|
|
10092
|
+
candidates: (options) => run(core().store.graphCandidates(options)),
|
|
10001
10093
|
coverage: (modelKey) => run(core().store.graphCoverage(modelKey)),
|
|
10002
10094
|
confirm: (id) => run(core().store.confirmEdge(id)),
|
|
10003
10095
|
reject: (id) => run(core().store.rejectEdge(id)),
|
|
@@ -10033,28 +10125,28 @@ var diffConfig = (oldC, newC) => {
|
|
|
10033
10125
|
};
|
|
10034
10126
|
var RELOAD_DRAIN_MS = 30000;
|
|
10035
10127
|
var bootStack = async (options) => {
|
|
10036
|
-
const scope = await
|
|
10128
|
+
const scope = await Effect51.runPromise(Scope5.make());
|
|
10037
10129
|
let runtime;
|
|
10038
10130
|
try {
|
|
10039
10131
|
const coreNode = make({
|
|
10040
10132
|
name: "SdkCore",
|
|
10041
10133
|
deps: [],
|
|
10042
|
-
layer: "core" in options ?
|
|
10134
|
+
layer: "core" in options ? layer14(options.core) : layerConfig7(options.migrationFolder !== undefined ? { migrationFolder: options.migrationFolder } : {}).pipe(Layer27.provide(configLayerFromInput(options.config)))
|
|
10043
10135
|
});
|
|
10044
|
-
const appLayer = compile(group([coreNode, node8, node6])).pipe(
|
|
10136
|
+
const appLayer = compile(group([coreNode, node8, node6])).pipe(Layer27.provide(Layer27.succeed(Scope5.Scope, scope)));
|
|
10045
10137
|
const rt = ManagedRuntime.make(appLayer);
|
|
10046
10138
|
runtime = rt;
|
|
10047
|
-
const [core, parser, extractor] = await rt.runPromise(
|
|
10139
|
+
const [core, parser, extractor] = await rt.runPromise(Effect51.all([Service, Service20, Service13]));
|
|
10048
10140
|
return {
|
|
10049
10141
|
runPromise: (effect) => rt.runPromise(effect),
|
|
10050
10142
|
disposeEffect: rt.disposeEffect,
|
|
10051
10143
|
core,
|
|
10052
10144
|
parser,
|
|
10053
10145
|
extractor,
|
|
10054
|
-
dispose: () =>
|
|
10146
|
+
dispose: () => Effect51.runPromise(Scope5.close(scope, Exit8.void).pipe(Effect51.andThen(rt.disposeEffect)))
|
|
10055
10147
|
};
|
|
10056
10148
|
} catch (cause) {
|
|
10057
|
-
await
|
|
10149
|
+
await Effect51.runPromise(Scope5.close(scope, Exit8.void).pipe(Effect51.andThen(runtime?.disposeEffect ?? Effect51.void))).catch(() => {
|
|
10058
10150
|
return;
|
|
10059
10151
|
});
|
|
10060
10152
|
return Promise.reject(cause);
|
|
@@ -10065,7 +10157,7 @@ var makeDimple = async (options) => {
|
|
|
10065
10157
|
const migrationFolder = "config" in options ? options.migrationFolder : undefined;
|
|
10066
10158
|
let currentConfig;
|
|
10067
10159
|
if (configSource !== undefined) {
|
|
10068
|
-
currentConfig = await
|
|
10160
|
+
currentConfig = await Effect51.runPromise(resolveConfigInput(configSource)).catch((cause) => Promise.reject(remapError(cause)));
|
|
10069
10161
|
}
|
|
10070
10162
|
let current = await bootStack(configSource !== undefined && currentConfig !== undefined ? {
|
|
10071
10163
|
config: currentConfig,
|
|
@@ -10087,7 +10179,7 @@ var makeDimple = async (options) => {
|
|
|
10087
10179
|
}
|
|
10088
10180
|
const stack = current;
|
|
10089
10181
|
inflight2 += 1;
|
|
10090
|
-
return stack.runPromise(effect.pipe(
|
|
10182
|
+
return stack.runPromise(effect.pipe(Effect51.mapError(remapError))).catch((cause) => {
|
|
10091
10183
|
return Promise.reject(remapError(cause));
|
|
10092
10184
|
}).finally(() => {
|
|
10093
10185
|
inflight2 -= 1;
|
|
@@ -10143,7 +10235,7 @@ var makeDimple = async (options) => {
|
|
|
10143
10235
|
}));
|
|
10144
10236
|
}
|
|
10145
10237
|
const nextInput = input ?? configSource;
|
|
10146
|
-
const resolved = await
|
|
10238
|
+
const resolved = await Effect51.runPromise(resolveConfigInput(nextInput));
|
|
10147
10239
|
if (currentConfig !== undefined && stableKey(resolved) === stableKey(currentConfig)) {
|
|
10148
10240
|
return {
|
|
10149
10241
|
config: resolved,
|
|
@@ -10166,10 +10258,10 @@ var makeDimple = async (options) => {
|
|
|
10166
10258
|
currentConfig = resolved;
|
|
10167
10259
|
await old.dispose();
|
|
10168
10260
|
if (changes.storeUrl) {
|
|
10169
|
-
await
|
|
10261
|
+
await Effect51.runPromise(Effect51.logWarning(`reload: store database switched "${oldStoreUrl}" → "${resolved.storeUrl}" — ` + "the old db is preserved but no longer served (nothing was migrated)").pipe(Effect51.annotateLogs({ scope: "@dimple/sdk", event: "reload.store.switch" })));
|
|
10170
10262
|
}
|
|
10171
10263
|
if (changes.jobsUrl) {
|
|
10172
|
-
await
|
|
10264
|
+
await Effect51.runPromise(Effect51.logWarning(`reload: jobs database switched "${oldJobsUrl}" → "${resolved.jobs?.dbUrl}" — ` + "pending jobs remain durable in the old db").pipe(Effect51.annotateLogs({ scope: "@dimple/sdk", event: "reload.jobs.switch" })));
|
|
10173
10265
|
}
|
|
10174
10266
|
return { config: resolved, changed: changes, reloaded: true };
|
|
10175
10267
|
} finally {
|