@ponraaj/dimple-sdk 0.0.0-alpha.1 → 0.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -0
- package/dist/dimple.d.ts +113 -5
- package/dist/errors.d.ts +24 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3055 -1524
- package/dist/indexer.d.ts +9 -0
- package/dist/internal/config/schema.d.ts +87 -0
- package/dist/internal/core/hosted-embedding.d.ts +33 -0
- package/dist/internal/embedder/auto-install.d.ts +10 -0
- package/dist/internal/embedder/index.d.ts +1 -2
- package/dist/internal/embedder/transformers.d.ts +40 -2
- package/dist/internal/extractor/models/extraction.d.ts +14 -0
- package/dist/internal/installer/index.d.ts +26 -0
- package/dist/internal/logger/index.d.ts +1 -1
- package/dist/internal/logger/service.d.ts +31 -12
- package/dist/internal/logger/types.d.ts +15 -3
- package/dist/internal/store/audit.d.ts +7 -1
- package/dist/internal/store/db-backend.d.ts +46 -8
- package/dist/internal/store/graph/candidates.d.ts +3 -1
- package/dist/internal/store/graph/concepts.d.ts +14 -5
- package/dist/internal/store/graph/dream.d.ts +7 -1
- package/dist/internal/store/graph/repair.d.ts +7 -1
- package/dist/internal/store/graph/traversal.d.ts +1 -1
- package/dist/internal/store/index.d.ts +1 -0
- package/dist/internal/store/models.d.ts +4 -3
- package/dist/internal/store/schema/graph.d.ts +7 -7
- package/dist/internal/store/schema/libsql.d.ts +4 -4
- package/dist/internal/store/schema/metadata.d.ts +4 -1
- package/dist/internal/store/service.d.ts +83 -4
- package/dist/internal/store/topics/jobs.d.ts +7 -1
- package/dist/internal/store/topics/lifecycle.d.ts +7 -1
- package/dist/internal/store/topics/maintenance.d.ts +8 -2
- package/dist/internal/store/topics/routing.d.ts +33 -0
- package/dist/internal/store/topics/topic-vectors.d.ts +10 -3
- package/dist/internal/store/topics/tree.d.ts +15 -9
- package/dist/internal/workflow/executor.d.ts +1 -1
- package/dist/internal/workflow/persistence.d.ts +1 -1
- package/dist/internal/workflow/queue.d.ts +6 -3
- package/dist/internal/workflow/registry.d.ts +1 -1
- package/dist/internal/workflow/signals.d.ts +21 -0
- package/dist/make.d.ts +27 -0
- package/package.json +12 -4
- package/dist/internal/embedder/openai.d.ts +0 -8
|
@@ -5,6 +5,7 @@ export { topics };
|
|
|
5
5
|
export { auditWriteHandler, type AuditWriteHandler } from "./audit.ts";
|
|
6
6
|
export { TopicConfig } from "./schema/topics.ts";
|
|
7
7
|
export type { AddEdgeInput, ExplainResult } from "./service.ts";
|
|
8
|
+
export type { ArtifactEdge, EdgeKind, EdgeSource } from "./schema/graph.ts";
|
|
8
9
|
export type { TopicHealth } from "./topics/observability.ts";
|
|
9
10
|
export type { TopicConfig as TopicConfigType } from "./schema/topics.ts";
|
|
10
11
|
export type { RetrievalAuditRow } from "./models.ts";
|
|
@@ -11,7 +11,7 @@ export declare const MemoryUnit: Schema.Struct<{
|
|
|
11
11
|
readonly contentType: Schema.String;
|
|
12
12
|
readonly state: Schema.Literals<readonly ["active", "superseded", "archived", "forgotten"]>;
|
|
13
13
|
readonly createdAt: Schema.Number;
|
|
14
|
-
readonly ttlSecs: Schema.Number
|
|
14
|
+
readonly ttlSecs: Schema.NullOr<Schema.Number>;
|
|
15
15
|
readonly namespace: Schema.String;
|
|
16
16
|
readonly metadata: Schema.optional<Schema.String>;
|
|
17
17
|
readonly embedding: Schema.optional<Schema.$Array<Schema.Number>>;
|
|
@@ -24,7 +24,7 @@ export declare const WriteMemoryInput: Schema.Struct<{
|
|
|
24
24
|
readonly content: Schema.String;
|
|
25
25
|
readonly contentType: Schema.optional<Schema.String>;
|
|
26
26
|
readonly namespace: Schema.optional<Schema.String>;
|
|
27
|
-
readonly ttlSecs: Schema.optional<Schema.Number
|
|
27
|
+
readonly ttlSecs: Schema.optional<Schema.NullOr<Schema.Number>>;
|
|
28
28
|
readonly metadata: Schema.optional<Schema.String>;
|
|
29
29
|
/** Typed overlay merged INTO the raw metadata blob on write (unknown blob
|
|
30
30
|
* keys preserved. Takes precedence over `metadata`. */
|
|
@@ -38,6 +38,7 @@ export declare const WriteMemoryInput: Schema.Struct<{
|
|
|
38
38
|
readonly confidence: Schema.Number;
|
|
39
39
|
readonly affordances: Schema.$Array<Schema.String>;
|
|
40
40
|
readonly language: Schema.optional<Schema.String>;
|
|
41
|
+
readonly codeVersion: Schema.optional<Schema.String>;
|
|
41
42
|
readonly pinned: Schema.optional<Schema.Boolean>;
|
|
42
43
|
}>>;
|
|
43
44
|
}>;
|
|
@@ -51,7 +52,7 @@ export declare const ScoredMemory: Schema.Struct<{
|
|
|
51
52
|
readonly contentType: Schema.String;
|
|
52
53
|
readonly state: Schema.Literals<readonly ["active", "superseded", "archived", "forgotten"]>;
|
|
53
54
|
readonly createdAt: Schema.Number;
|
|
54
|
-
readonly ttlSecs: Schema.Number
|
|
55
|
+
readonly ttlSecs: Schema.NullOr<Schema.Number>;
|
|
55
56
|
readonly namespace: Schema.String;
|
|
56
57
|
readonly metadata: Schema.optional<Schema.String>;
|
|
57
58
|
readonly embedding: Schema.optional<Schema.$Array<Schema.Number>>;
|
|
@@ -74,14 +74,14 @@ export declare const artifactEdges: import("drizzle-orm/sqlite-core").SQLiteTabl
|
|
|
74
74
|
name: string;
|
|
75
75
|
tableName: "artifact_edges";
|
|
76
76
|
dataType: "string enum";
|
|
77
|
-
data: "memory_contradicts" | "memory_supersedes" | "memory_supports";
|
|
77
|
+
data: "code_calls" | "memory_contradicts" | "memory_supersedes" | "memory_supports";
|
|
78
78
|
driverParam: string;
|
|
79
79
|
notNull: true;
|
|
80
80
|
hasDefault: false;
|
|
81
81
|
isPrimaryKey: false;
|
|
82
82
|
isAutoincrement: false;
|
|
83
83
|
hasRuntimeDefault: false;
|
|
84
|
-
enumValues: ["memory_supports", "memory_supersedes", "memory_contradicts"];
|
|
84
|
+
enumValues: ["memory_supports", "memory_supersedes", "memory_contradicts", "code_calls"];
|
|
85
85
|
baseColumn: never;
|
|
86
86
|
identity: undefined;
|
|
87
87
|
generated: undefined;
|
|
@@ -218,7 +218,7 @@ export declare const artifactEdges: import("drizzle-orm/sqlite-core").SQLiteTabl
|
|
|
218
218
|
dialect: 'sqlite';
|
|
219
219
|
}>;
|
|
220
220
|
/** The memory↔memory edge kinds `artifact_edges` can hold (the only kinds in the lean table). */
|
|
221
|
-
export type EdgeKind = "memory_supports" | "memory_supersedes" | "memory_contradicts";
|
|
221
|
+
export type EdgeKind = "memory_supports" | "memory_supersedes" | "memory_contradicts" | "code_calls";
|
|
222
222
|
/** Writers of edges (provenance on every row). */
|
|
223
223
|
export type EdgeSource = "rule" | "llm" | "agent";
|
|
224
224
|
export declare const ArtifactEdgeInsert: import("drizzle-orm/effect-schema").BuildSchema<"insert", {
|
|
@@ -274,14 +274,14 @@ export declare const ArtifactEdgeInsert: import("drizzle-orm/effect-schema").Bui
|
|
|
274
274
|
name: string;
|
|
275
275
|
tableName: "artifact_edges";
|
|
276
276
|
dataType: "string enum";
|
|
277
|
-
data: "memory_contradicts" | "memory_supersedes" | "memory_supports";
|
|
277
|
+
data: "code_calls" | "memory_contradicts" | "memory_supersedes" | "memory_supports";
|
|
278
278
|
driverParam: string;
|
|
279
279
|
notNull: true;
|
|
280
280
|
hasDefault: false;
|
|
281
281
|
isPrimaryKey: false;
|
|
282
282
|
isAutoincrement: false;
|
|
283
283
|
hasRuntimeDefault: false;
|
|
284
|
-
enumValues: ["memory_supports", "memory_supersedes", "memory_contradicts"];
|
|
284
|
+
enumValues: ["memory_supports", "memory_supersedes", "memory_contradicts", "code_calls"];
|
|
285
285
|
baseColumn: never;
|
|
286
286
|
identity: undefined;
|
|
287
287
|
generated: undefined;
|
|
@@ -468,14 +468,14 @@ export declare const ArtifactEdgeSelect: import("drizzle-orm/effect-schema").Bui
|
|
|
468
468
|
name: string;
|
|
469
469
|
tableName: "artifact_edges";
|
|
470
470
|
dataType: "string enum";
|
|
471
|
-
data: "memory_contradicts" | "memory_supersedes" | "memory_supports";
|
|
471
|
+
data: "code_calls" | "memory_contradicts" | "memory_supersedes" | "memory_supports";
|
|
472
472
|
driverParam: string;
|
|
473
473
|
notNull: true;
|
|
474
474
|
hasDefault: false;
|
|
475
475
|
isPrimaryKey: false;
|
|
476
476
|
isAutoincrement: false;
|
|
477
477
|
hasRuntimeDefault: false;
|
|
478
|
-
enumValues: ["memory_supports", "memory_supersedes", "memory_contradicts"];
|
|
478
|
+
enumValues: ["memory_supports", "memory_supersedes", "memory_contradicts", "code_calls"];
|
|
479
479
|
baseColumn: never;
|
|
480
480
|
identity: undefined;
|
|
481
481
|
generated: undefined;
|
|
@@ -104,7 +104,7 @@ export declare const memories: import("drizzle-orm/sqlite-core").SQLiteTableWith
|
|
|
104
104
|
dataType: "number int53";
|
|
105
105
|
data: number;
|
|
106
106
|
driverParam: number;
|
|
107
|
-
notNull:
|
|
107
|
+
notNull: false;
|
|
108
108
|
hasDefault: true;
|
|
109
109
|
isPrimaryKey: false;
|
|
110
110
|
isAutoincrement: false;
|
|
@@ -847,7 +847,7 @@ export declare const MemoryInsert: import("drizzle-orm/effect-schema").BuildSche
|
|
|
847
847
|
dataType: "number int53";
|
|
848
848
|
data: number;
|
|
849
849
|
driverParam: number;
|
|
850
|
-
notNull:
|
|
850
|
+
notNull: false;
|
|
851
851
|
hasDefault: true;
|
|
852
852
|
isPrimaryKey: false;
|
|
853
853
|
isAutoincrement: false;
|
|
@@ -1025,7 +1025,7 @@ export declare const MemorySelect: import("drizzle-orm/effect-schema").BuildSche
|
|
|
1025
1025
|
dataType: "number int53";
|
|
1026
1026
|
data: number;
|
|
1027
1027
|
driverParam: number;
|
|
1028
|
-
notNull:
|
|
1028
|
+
notNull: false;
|
|
1029
1029
|
hasDefault: true;
|
|
1030
1030
|
isPrimaryKey: false;
|
|
1031
1031
|
isAutoincrement: false;
|
|
@@ -1203,7 +1203,7 @@ export declare const MemoryUpdate: import("drizzle-orm/effect-schema").BuildSche
|
|
|
1203
1203
|
dataType: "number int53";
|
|
1204
1204
|
data: number;
|
|
1205
1205
|
driverParam: number;
|
|
1206
|
-
notNull:
|
|
1206
|
+
notNull: false;
|
|
1207
1207
|
hasDefault: true;
|
|
1208
1208
|
isPrimaryKey: false;
|
|
1209
1209
|
isAutoincrement: false;
|
|
@@ -27,6 +27,9 @@ export declare const MemoryMetadata: Schema.Struct<{
|
|
|
27
27
|
readonly affordances: Schema.$Array<Schema.String>;
|
|
28
28
|
/** Code-specific: "typescript" | "rust" | ... */
|
|
29
29
|
readonly language: Schema.optional<Schema.String>;
|
|
30
|
+
/** Code-specific: the repo HEAD at index time (the indexer's
|
|
31
|
+
* provenance stamp — which commit this unit came from). */
|
|
32
|
+
readonly codeVersion: Schema.optional<Schema.String>;
|
|
30
33
|
/** Writer-set pin: the memory is deliberately stable — automatic graph
|
|
31
34
|
* lifecycle moves should leave it alone. */
|
|
32
35
|
readonly pinned: Schema.optional<Schema.Boolean>;
|
|
@@ -48,4 +51,4 @@ export declare const encodeMetadata: (metadata: MemoryMetadata) => string;
|
|
|
48
51
|
* does not declare (producer-specific extensions) survives. A malformed
|
|
49
52
|
* existing blob is treated as empty — the merge never throws.
|
|
50
53
|
*/
|
|
51
|
-
export declare const mergeMetadataBlob: (existing: string | null | undefined, typed: MemoryMetadata) => string;
|
|
54
|
+
export declare const mergeMetadataBlob: (existing: string | null | undefined, typed: Partial<MemoryMetadata>) => string;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Context, Effect } from "effect";
|
|
2
|
-
import type { MemoryUnit, MemoryEvent, WriteMemoryInput, SweepResult, Concept } from "./models.ts";
|
|
2
|
+
import type { MemoryUnit, MemoryEvent, WriteMemoryInput, SweepResult, Concept, ScoredMemory } from "./models.ts";
|
|
3
3
|
import type { StoreError, NotFound, DuplicateActiveKey } from "./error.ts";
|
|
4
|
+
import type { MetadataFilter } from "./db-backend.ts";
|
|
4
5
|
import type { TopicError } from "./topics/errors.ts";
|
|
5
6
|
import type { MaintenanceOutcome, RebalanceOutcome } from "./topics/maintenance.ts";
|
|
6
7
|
import type { EmbeddingModelConfig } from "./internal/config/index.js";
|
|
@@ -65,21 +66,83 @@ export interface ExplainResult {
|
|
|
65
66
|
}
|
|
66
67
|
export interface Interface {
|
|
67
68
|
readonly write: (input: WriteMemoryInput) => Effect.Effect<MemoryUnit, StoreError | DuplicateActiveKey>;
|
|
69
|
+
/** Bulk write: batch insert in ONE transaction (all-or-nothing; a
|
|
70
|
+
* duplicate logical key fails the whole batch with DuplicateActiveKey).
|
|
71
|
+
* The per-write graph candidate scan is NOT part of this — pair with
|
|
72
|
+
* `updateEmbedding(..., { scanCandidates: false })` for bulk ingest. */
|
|
73
|
+
readonly writeMany: (inputs: readonly WriteMemoryInput[]) => Effect.Effect<readonly MemoryUnit[], StoreError | DuplicateActiveKey>;
|
|
74
|
+
/** Upsert for the index path — the identity rule (same logical key ⇒
|
|
75
|
+
* confirmed `memory_supersedes`) made reachable for ACTIVE keys: demote
|
|
76
|
+
* the active holder of the key (if any), write the new unit, and link
|
|
77
|
+
* old → new with a confirmed rule edge, all in ONE transaction. Returns
|
|
78
|
+
* the new memory plus the superseded id (when a predecessor existed). */
|
|
79
|
+
readonly writeSuperseding: (input: WriteMemoryInput) => Effect.Effect<{
|
|
80
|
+
readonly memory: MemoryUnit;
|
|
81
|
+
readonly supersededId?: string;
|
|
82
|
+
}, StoreError | DuplicateActiveKey>;
|
|
83
|
+
/** Index reconciliation — demote every ACTIVE unit under `keyPrefix`
|
|
84
|
+
* that is (a) INSIDE the walked domain (its file path is under one of
|
|
85
|
+
* `walkedRoots`, joined against `keyRoot`) and (b) not among `keptKeys`
|
|
86
|
+
* (this run's unit keys — files AND chunks: a chunk the extractor no
|
|
87
|
+
* longer emits, e.g. a deleted function inside a surviving file, is
|
|
88
|
+
* vanished). Demotion is `source_removed` (one transaction). Returns the demoted
|
|
89
|
+
* rows for the rename-candidate scan. Units outside the walked domain
|
|
90
|
+
* (subtree runs) and normal (non-index) memories are never touched. */
|
|
91
|
+
readonly reconcile: (namespace: string, keyPrefix: string, keyRoot: string, walkedRoots: readonly string[], keptKeys: readonly string[]) => Effect.Effect<readonly {
|
|
92
|
+
readonly id: string;
|
|
93
|
+
readonly logicalKey: string;
|
|
94
|
+
readonly embedding: number[] | null;
|
|
95
|
+
}[], StoreError>;
|
|
96
|
+
/** Rename/move detection (human-gated): cosine each vanished unit's
|
|
97
|
+
* stored embedding against this run's new units; ≥ the near-duplicate
|
|
98
|
+
* band → a PENDING `memory_supersedes` candidate (source rule, never
|
|
99
|
+
* auto-confirms — a rename is a guess). Returns the new-candidate count. */
|
|
100
|
+
readonly reconcileRenameCandidates: (vanished: readonly {
|
|
101
|
+
readonly id: string;
|
|
102
|
+
readonly embedding: number[] | null;
|
|
103
|
+
}[], newUnitIds: readonly string[]) => Effect.Effect<number, StoreError>;
|
|
68
104
|
readonly get: (id: string) => Effect.Effect<MemoryUnit, NotFound | StoreError>;
|
|
69
105
|
readonly getByKey: (key: string, namespace: string) => Effect.Effect<MemoryUnit, NotFound | StoreError>;
|
|
70
106
|
readonly recency: (namespace: string, limit: number) => Effect.Effect<ReadonlyArray<AugmentedMemory>, StoreError>;
|
|
71
107
|
readonly searchFts5: (query: string, namespace: string, limit: number, opts?: {
|
|
72
108
|
readonly topicKeys?: ReadonlyArray<string>;
|
|
109
|
+
/** Content-type filter: one kind or a list (code-vs-message
|
|
110
|
+
* separation lever at query time). */
|
|
111
|
+
readonly contentType?: string | ReadonlyArray<string>;
|
|
73
112
|
/** `false` suppresses the per-leg audit row (fused-row callers). */
|
|
74
113
|
readonly audit?: boolean;
|
|
114
|
+
/** Include the 384-float embedding on results (default false). */
|
|
115
|
+
readonly includeEmbedding?: boolean;
|
|
116
|
+
/** `false` skips the per-leg graph augmentation (fused-row callers). */
|
|
117
|
+
readonly augment?: boolean;
|
|
75
118
|
}) => Effect.Effect<ReadonlyArray<AugmentedMemory>, StoreError>;
|
|
76
119
|
readonly searchVector: (query: ReadonlyArray<number>, namespace: string, limit: number, opts?: {
|
|
77
120
|
readonly topicKeys?: ReadonlyArray<string>;
|
|
78
121
|
/** Restrict to memories annotated with ANY of these concept ids. */
|
|
79
122
|
readonly conceptIds?: ReadonlyArray<string>;
|
|
123
|
+
/** Content-type filter: one kind or a list (code-vs-message
|
|
124
|
+
* separation lever at query time). */
|
|
125
|
+
readonly contentType?: string | ReadonlyArray<string>;
|
|
80
126
|
/** `false` suppresses the per-leg audit row (fused-row callers). */
|
|
81
127
|
readonly audit?: boolean;
|
|
128
|
+
/** Include the 384-float embedding on results (default false). */
|
|
129
|
+
readonly includeEmbedding?: boolean;
|
|
130
|
+
/** `false` skips the per-leg graph augmentation (fused-row callers). */
|
|
131
|
+
readonly augment?: boolean;
|
|
82
132
|
}) => Effect.Effect<ReadonlyArray<AugmentedMemory>, StoreError>;
|
|
133
|
+
/** One-pass graph augmentation over an already-ranked result set (the
|
|
134
|
+
* hybrid path: the per-leg augment passes are skipped and the FUSED set
|
|
135
|
+
* gets one edge fetch + one concept fetch instead of two of each over
|
|
136
|
+
* 2× the rows). */
|
|
137
|
+
readonly augmentSearchResults: (results: ReadonlyArray<ScoredMemory>) => Effect.Effect<ReadonlyArray<AugmentedMemory>, StoreError>;
|
|
138
|
+
/** The DB's tables + columns (`dimple db-schema` surface). */
|
|
139
|
+
readonly listSchema: () => Effect.Effect<ReadonlyArray<{
|
|
140
|
+
readonly table: string;
|
|
141
|
+
readonly columns: ReadonlyArray<{
|
|
142
|
+
readonly name: string;
|
|
143
|
+
readonly type: string;
|
|
144
|
+
}>;
|
|
145
|
+
}>, StoreError>;
|
|
83
146
|
/** One fused retrieval-audit row (the hybrid path's audit contract). */
|
|
84
147
|
readonly logAudit: (input: {
|
|
85
148
|
readonly query: string;
|
|
@@ -95,16 +158,32 @@ export interface Interface {
|
|
|
95
158
|
readonly forget: (id: string) => Effect.Effect<void, NotFound | StoreError>;
|
|
96
159
|
readonly sweep: (namespace: string) => Effect.Effect<SweepResult, StoreError>;
|
|
97
160
|
readonly events: (memoryId: string) => Effect.Effect<readonly MemoryEvent[], StoreError>;
|
|
98
|
-
readonly updateEmbedding: (id: string, embedding: ReadonlyArray<number>, model: string
|
|
161
|
+
readonly updateEmbedding: (id: string, embedding: ReadonlyArray<number>, model: string, options?: {
|
|
162
|
+
readonly scanCandidates?: boolean;
|
|
163
|
+
}) => Effect.Effect<void, NotFound | StoreError>;
|
|
164
|
+
/** Bulk variant of `updateEmbedding` — the writeMany path. One
|
|
165
|
+
* transaction for all memory updates + events, batch vector insert,
|
|
166
|
+
* and `routeInsertMany` (tree read once, all insertions computed in
|
|
167
|
+
* memory, one transaction). For NEW memories: the idempotent re-embed
|
|
168
|
+
* decrement is deliberately NOT part of the batch (use
|
|
169
|
+
* `updateEmbedding` for re-embedding existing routed memories).
|
|
170
|
+
* `scanCandidates` defaults to false-ish via the config: it runs only
|
|
171
|
+
* when explicitly requested AND `graph.enabled` is not false. */
|
|
172
|
+
readonly updateEmbeddingMany: (entries: ReadonlyArray<{
|
|
173
|
+
readonly id: string;
|
|
174
|
+
readonly embedding: ReadonlyArray<number>;
|
|
175
|
+
}>, model: string, options?: {
|
|
176
|
+
readonly scanCandidates?: boolean;
|
|
177
|
+
}) => Effect.Effect<void, StoreError>;
|
|
99
178
|
readonly findStale: (model: string) => Effect.Effect<readonly string[], StoreError>;
|
|
100
179
|
readonly createConcept: (name: string, embedding: ReadonlyArray<number>) => Effect.Effect<Concept, StoreError>;
|
|
101
180
|
readonly getConcept: (id: string) => Effect.Effect<Concept, NotFound | StoreError>;
|
|
102
181
|
readonly listConcepts: () => Effect.Effect<readonly Concept[], StoreError>;
|
|
103
182
|
readonly annotateMemory: (memoryId: string, conceptId: string, confidence?: number) => Effect.Effect<void, NotFound | StoreError>;
|
|
104
183
|
readonly getMemoryAnnotations: (memoryId: string) => Effect.Effect<readonly string[], StoreError>;
|
|
105
|
-
readonly queryByMetadata: (filter:
|
|
184
|
+
readonly queryByMetadata: (filter: MetadataFilter) => Effect.Effect<readonly MemoryUnit[], StoreError>;
|
|
106
185
|
readonly getMemoryMetadata: (id: string) => Effect.Effect<MemoryMetadata | undefined, NotFound | StoreError>;
|
|
107
|
-
readonly setMemoryMetadata: (id: string, metadata: MemoryMetadata) => Effect.Effect<void, NotFound | StoreError>;
|
|
186
|
+
readonly setMemoryMetadata: (id: string, metadata: Partial<MemoryMetadata>) => Effect.Effect<void, NotFound | StoreError>;
|
|
108
187
|
readonly ensureTopicTree: (modelKey: string) => Effect.Effect<{
|
|
109
188
|
rootKey: string;
|
|
110
189
|
unassignedKey: string;
|
|
@@ -127,8 +127,14 @@ export interface TopicSplitHandlerDeps {
|
|
|
127
127
|
*/
|
|
128
128
|
readonly backend?: DbBackendInterface;
|
|
129
129
|
}
|
|
130
|
+
/** The decoded `topic.split` payload — mirrors the queue definition's
|
|
131
|
+
* schema (`{ topicId, modelKey }`); the executor decodes before calling. */
|
|
132
|
+
export interface TopicSplitPayload {
|
|
133
|
+
readonly topicId: string;
|
|
134
|
+
readonly modelKey: string;
|
|
135
|
+
}
|
|
130
136
|
/** The `topic.split` handler shape — `unknown` success like `JobRegistry.Handler`. */
|
|
131
|
-
export type TopicSplitHandler = (payload:
|
|
137
|
+
export type TopicSplitHandler = (payload: TopicSplitPayload, ctx: JobContext) => Effect.Effect<unknown, JobFailure, DbBackend>;
|
|
132
138
|
/**
|
|
133
139
|
* Factory for the `topic.split` job handler.
|
|
134
140
|
*
|
|
@@ -112,8 +112,14 @@ export interface MemoryEmbedHandlerDeps {
|
|
|
112
112
|
/** Backend captured at registration (see {@link topicSplitHandler}). */
|
|
113
113
|
readonly backend?: DbBackendInterface;
|
|
114
114
|
}
|
|
115
|
+
/** The decoded `memory.embed` payload — mirrors the queue definition's
|
|
116
|
+
* schema (`{ memoryId, modelKey }`); the executor decodes before calling. */
|
|
117
|
+
export interface MemoryEmbedPayload {
|
|
118
|
+
readonly memoryId: string;
|
|
119
|
+
readonly modelKey: string;
|
|
120
|
+
}
|
|
115
121
|
/** The `memory.embed` handler shape (see jobs.ts). */
|
|
116
|
-
export type MemoryEmbedHandler = (payload:
|
|
122
|
+
export type MemoryEmbedHandler = (payload: MemoryEmbedPayload, ctx: JobContext) => Effect.Effect<unknown, JobFailure, DbBackend>;
|
|
117
123
|
/**
|
|
118
124
|
* Factory for the `memory.embed` job handler: re-embeds one memory with the
|
|
119
125
|
* target model and routes it into that model's tree. The success schema is
|
|
@@ -89,8 +89,14 @@ export interface TopicMaintenanceHandlerDeps {
|
|
|
89
89
|
/** Backend captured at registration (see {@link topicSplitHandler}). */
|
|
90
90
|
readonly backend?: DbBackendInterface;
|
|
91
91
|
}
|
|
92
|
-
/** The `topic.merge` / `topic.rebalance`
|
|
93
|
-
|
|
92
|
+
/** The decoded `topic.merge` / `topic.rebalance` payload — mirrors the
|
|
93
|
+
* queue definitions' schema (`{ topicId, modelKey }`); the executor
|
|
94
|
+
* decodes before calling. */
|
|
95
|
+
export interface TopicMaintenancePayload {
|
|
96
|
+
readonly topicId: string;
|
|
97
|
+
readonly modelKey: string;
|
|
98
|
+
}
|
|
99
|
+
export type TopicMaintenanceHandler = (payload: TopicMaintenancePayload, ctx: JobContext) => Effect.Effect<unknown, JobFailure, DbBackend>;
|
|
94
100
|
/**
|
|
95
101
|
* Factory for the `topic.merge` job handler: resolves the model, runs
|
|
96
102
|
* {@link mergeOrBorrow}, returns `undefined` (the definition's success
|
|
@@ -88,3 +88,36 @@ export declare const routeInsert: (memoryId: string, vector: ReadonlyArray<numbe
|
|
|
88
88
|
topicKey: string;
|
|
89
89
|
overflow: boolean;
|
|
90
90
|
}, TopicError | StoreError, DbBackend>;
|
|
91
|
+
/**
|
|
92
|
+
* Batch routing — the bulk-ingest path (`Store.updateEmbeddingMany`).
|
|
93
|
+
*
|
|
94
|
+
* Sequential `routeInsert` re-reads the WHOLE tree (all topics, all
|
|
95
|
+
* relations, all centroid sum-vectors) and commits its own transaction
|
|
96
|
+
* PER UNIT — measured ~42 ms/unit under bulk ingest, dominating the
|
|
97
|
+
* write path. This variant:
|
|
98
|
+
*
|
|
99
|
+
* 1. Reads the tree structure + sum-vectors ONCE.
|
|
100
|
+
* 2. Routes every entry against an IN-MEMORY mirror of the tree state,
|
|
101
|
+
* applying each unit's CF updates locally before the next descent —
|
|
102
|
+
* so the descents are bit-identical to sequential routeInsert calls
|
|
103
|
+
* against the DB (same reads, same writes, same order).
|
|
104
|
+
* 3. Applies ALL membership inserts + topic CF updates in ONE
|
|
105
|
+
* transaction (per-topic deltas accumulated in memory).
|
|
106
|
+
* 4. Writes the derived sum vectors in ONE batched upsert (backend
|
|
107
|
+
* `upsertTopicVectors`), still after commit (backend not
|
|
108
|
+
* transaction-aware — see module docs).
|
|
109
|
+
*
|
|
110
|
+
* Contract differences vs sequential: all-or-nothing (a validation
|
|
111
|
+
* failure or structural corruption rejects the whole batch), and the
|
|
112
|
+
* batch does NOT handle re-embedding (no membership decrement path) —
|
|
113
|
+
* it is for NEW memories; re-embedding existing ones goes through
|
|
114
|
+
* `routeInsert`/`Store.updateEmbedding`.
|
|
115
|
+
*/
|
|
116
|
+
export declare const routeInsertMany: (entries: readonly {
|
|
117
|
+
readonly memoryId: string;
|
|
118
|
+
readonly vector: ReadonlyArray<number>;
|
|
119
|
+
}[], model: EmbeddingModelConfig, config: TopicConfig) => Effect.Effect<readonly {
|
|
120
|
+
readonly memoryId: string;
|
|
121
|
+
readonly topicKey: string;
|
|
122
|
+
readonly overflow: boolean;
|
|
123
|
+
}[], TopicError | StoreError, DbBackend>;
|
|
@@ -9,10 +9,17 @@ import { type SQLWrapper } from "drizzle-orm";
|
|
|
9
9
|
/** Per-model topic vector table name (mirrors libsql-backend.ts — canonical). */
|
|
10
10
|
export declare const topicVectorTableName: (model: EmbeddingModelConfig) => string;
|
|
11
11
|
/** vectorType → libSQL conversion function for the sum-vector column. */
|
|
12
|
-
export declare const VECTOR_CONVERT_FNS:
|
|
13
|
-
|
|
12
|
+
export declare const VECTOR_CONVERT_FNS: {
|
|
13
|
+
readonly float32: "vector32";
|
|
14
|
+
readonly float16: "vector16";
|
|
15
|
+
readonly float8: "vector8";
|
|
16
|
+
readonly float1bit: "vector1bit";
|
|
17
|
+
};
|
|
18
|
+
/** Anything with a `.run(sql)` — a Drizzle tx or the db handle. The raw
|
|
19
|
+
* result is discarded by every caller (DELETE statements only), so the
|
|
20
|
+
* contract is fire-and-forget. */
|
|
14
21
|
export interface SqlRunner {
|
|
15
|
-
readonly run: (query: string | SQLWrapper<unknown>) =>
|
|
22
|
+
readonly run: (query: string | SQLWrapper<unknown>) => void;
|
|
16
23
|
}
|
|
17
24
|
/** Delete one topic's sum-vector row (merge absorbs the sibling's row). */
|
|
18
25
|
export declare const deleteTopicVectorRow: (runner: SqlRunner, model: EmbeddingModelConfig, topicKey: string) => Promise<void>;
|
|
@@ -91,10 +91,13 @@ export declare const canonicalOrder: (vectors: readonly Float32Array[]) => reado
|
|
|
91
91
|
* For fewer than 2 vectors returns `{ a: 0, b: 0 }` (degenerate — callers
|
|
92
92
|
* guard on counts before splitting). Never mutates input.
|
|
93
93
|
*/
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
94
|
+
/** The farthest-seed pair — `a` is fixed by the canonical order; `b` is
|
|
95
|
+
* the vector with the lowest dot product against `a`. */
|
|
96
|
+
export interface FarthestPair {
|
|
97
|
+
readonly a: number;
|
|
98
|
+
readonly b: number;
|
|
99
|
+
}
|
|
100
|
+
export declare const farthestPair: (vectors: readonly Float32Array[]) => FarthestPair;
|
|
98
101
|
/**
|
|
99
102
|
* Assign every vector to group 0 or 1 by higher dot product against the
|
|
100
103
|
* seed vectors at `seedA` / `seedB`; ties go to group 0 (smaller index).
|
|
@@ -119,11 +122,14 @@ export declare const assignNearest: (vectors: readonly Float32Array[], seedA: nu
|
|
|
119
122
|
*
|
|
120
123
|
* Never mutates inputs.
|
|
121
124
|
*/
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
/** Outcome of the Lloyd refinement — the assignment plus each group's
|
|
126
|
+
* CF triplet (the split-plan acceptance gates read these). */
|
|
127
|
+
export interface LloydRefineResult {
|
|
128
|
+
readonly groups: Uint8Array;
|
|
129
|
+
readonly leftCf: CfTriplet;
|
|
130
|
+
readonly rightCf: CfTriplet;
|
|
131
|
+
}
|
|
132
|
+
export declare const lloydRefine: (vectors: readonly Float32Array[], groups: Uint8Array, passes: number) => LloydRefineResult;
|
|
127
133
|
/** Outcome of {@link splitPlan} — accepted splits plus the acceptance gate diagnostics. */
|
|
128
134
|
export interface SplitPlanResult {
|
|
129
135
|
readonly accepted: boolean;
|
|
@@ -38,7 +38,7 @@ export declare const workerLayer: (defs: readonly JobDefinition.Any[], options?:
|
|
|
38
38
|
* Requires the full stack (JobRuntime + engine + queue + registry); use
|
|
39
39
|
* `workerLayer` for persistent processes instead.
|
|
40
40
|
*/
|
|
41
|
-
export declare const inline: (def: JobDefinition.Any, payload:
|
|
41
|
+
export declare const inline: <P>(def: JobDefinition.Any, payload: P, options?: {
|
|
42
42
|
readonly concurrency?: number;
|
|
43
43
|
readonly maxAttempts?: number;
|
|
44
44
|
}) => Effect.Effect<unknown, unknown, WorkflowEngine.WorkflowEngine | PersistedQueue.PersistedQueueFactory | JobRegistry | JobRuntime>;
|
|
@@ -56,7 +56,7 @@ export interface ResumeSignal extends Schema.Schema.Type<typeof ResumeSignal> {
|
|
|
56
56
|
*/
|
|
57
57
|
export declare const ExecutionRecord: Schema.Struct<{
|
|
58
58
|
readonly workflowTag: Schema.String;
|
|
59
|
-
readonly payload: Schema.$Record<Schema.String, Schema.
|
|
59
|
+
readonly payload: Schema.$Record<Schema.String, Schema.Codec<Schema.Json, Schema.Json, never, never>>;
|
|
60
60
|
}>;
|
|
61
61
|
export interface ExecutionRecord extends Schema.Schema.Type<typeof ExecutionRecord> {
|
|
62
62
|
}
|
|
@@ -75,8 +75,11 @@ export declare class JobDefinition<Payload extends Workflow.AnyStructSchema = Wo
|
|
|
75
75
|
/**
|
|
76
76
|
* Decodes an envelope back to `{ payload }`. Built in `make` where the
|
|
77
77
|
* schema types are concrete; the erased `Any` form widens the result.
|
|
78
|
+
* The engine delivers the workflow payload per its `register` contract
|
|
79
|
+
* (the decoded envelope type); the runtime decode below re-validates it
|
|
80
|
+
* against the envelope schema before the payload reaches the queue.
|
|
78
81
|
*/
|
|
79
|
-
readonly decodeEnvelope: (envelope:
|
|
82
|
+
readonly decodeEnvelope: (envelope: EnvelopeSchema<Payload>["Type"]) => Effect.Effect<{
|
|
80
83
|
readonly payload: Payload["Type"];
|
|
81
84
|
}, never, Payload["DecodingServices"] | Payload["EncodingServices"]>;
|
|
82
85
|
constructor(args: {
|
|
@@ -87,7 +90,7 @@ export declare class JobDefinition<Payload extends Workflow.AnyStructSchema = Wo
|
|
|
87
90
|
readonly successSchema: Success;
|
|
88
91
|
readonly errorSchema: Error;
|
|
89
92
|
readonly idempotencyKey: (payload: Payload["Type"]) => string;
|
|
90
|
-
readonly decodeEnvelope: (envelope:
|
|
93
|
+
readonly decodeEnvelope: (envelope: EnvelopeSchema<Payload>["Type"]) => Effect.Effect<{
|
|
91
94
|
readonly payload: Payload["Type"];
|
|
92
95
|
}, never, Payload["DecodingServices"] | Payload["EncodingServices"]>;
|
|
93
96
|
});
|
|
@@ -145,7 +148,7 @@ export declare namespace JobDefinition {
|
|
|
145
148
|
readonly successSchema: Schema.Top;
|
|
146
149
|
readonly errorSchema: Schema.Top;
|
|
147
150
|
readonly idempotencyKey: (payload: any) => string;
|
|
148
|
-
readonly decodeEnvelope: (envelope:
|
|
151
|
+
readonly decodeEnvelope: (envelope: any) => Effect.Effect<{
|
|
149
152
|
readonly payload: unknown;
|
|
150
153
|
}, never, unknown>;
|
|
151
154
|
readonly register: (engine: WorkflowEngine.WorkflowEngine["Service"]) => Effect.Effect<void, never, Scope.Scope | PersistedQueue.PersistedQueueFactory>;
|
|
@@ -29,7 +29,7 @@ export interface JobContext {
|
|
|
29
29
|
* decoded payload and returns the job's success value (or fails with
|
|
30
30
|
* `JobFailure`); the runtime owns retries/leases.
|
|
31
31
|
*/
|
|
32
|
-
export type Handler = (payload:
|
|
32
|
+
export type Handler = (payload: any, ctx: JobContext) => Effect.Effect<unknown, JobFailure>;
|
|
33
33
|
/**
|
|
34
34
|
* JobRegistry service contract.
|
|
35
35
|
*
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process execution-completion signals — the event side of `Runtime.wait`.
|
|
3
|
+
*
|
|
4
|
+
* `wait` used to poll the persisted `${jobId}/result` key every 250ms. The
|
|
5
|
+
* engine knows the exact completion moment, so it now notifies a per-
|
|
6
|
+
* executionId set of Deferreds; `wait` races the signal against its poll
|
|
7
|
+
* interval (the poll stays as the fallback for cross-process waits, where
|
|
8
|
+
* the completing engine lives in another process).
|
|
9
|
+
*
|
|
10
|
+
* Plain module-level Map — no Effect service: the signal is a fire-and-
|
|
11
|
+
* forget wakeup, and the subscribers are runtime-layer Deferreds. Cleanup
|
|
12
|
+
* on unsubscribe keeps the map from leaking job ids.
|
|
13
|
+
*/
|
|
14
|
+
import { Deferred, Effect } from "effect";
|
|
15
|
+
/** Subscribe to one execution's completion. */
|
|
16
|
+
export declare const onExecutionDone: (executionId: string, deferred: Deferred.Deferred<void>) => void;
|
|
17
|
+
/** Unsubscribe (wait's finally — the poll fallback may still be running). */
|
|
18
|
+
export declare const offExecutionDone: (executionId: string, deferred: Deferred.Deferred<void>) => void;
|
|
19
|
+
/** Fire every subscriber's deferred (the engine's completion tap) — an
|
|
20
|
+
* Effect the caller forks, so no Effect.run* leaks into package code. */
|
|
21
|
+
export declare const notifyExecutionDone: (executionId: string) => Effect.Effect<void>;
|
package/dist/make.d.ts
CHANGED
|
@@ -16,6 +16,12 @@ export type MakeDimpleOptions = {
|
|
|
16
16
|
} | {
|
|
17
17
|
readonly core: CoreOptions;
|
|
18
18
|
};
|
|
19
|
+
/** What changed between the live config and the resolved one. */
|
|
20
|
+
export interface ConfigDiff {
|
|
21
|
+
readonly storeUrl: boolean;
|
|
22
|
+
readonly jobsUrl: boolean;
|
|
23
|
+
readonly rest: boolean;
|
|
24
|
+
}
|
|
19
25
|
/**
|
|
20
26
|
* Create the Dimple facade — resolves once the composed system is fully
|
|
21
27
|
* booted. One runtime is built from the application layer; `runPromise`
|
|
@@ -23,5 +29,26 @@ export type MakeDimpleOptions = {
|
|
|
23
29
|
* Boot failures reject with the same closed `DimpleError` surface as the
|
|
24
30
|
* facade methods (never raw Effect internals) and close the owned
|
|
25
31
|
* resources (scope + runtime) before rethrowing.
|
|
32
|
+
*
|
|
33
|
+
* ### Reload (see `Dimple.reload`)
|
|
34
|
+
*
|
|
35
|
+
* The facade owns a mutable `LiveStack` cell. A reload:
|
|
36
|
+
*
|
|
37
|
+
* 1. re-resolves the config source TYPED (schema errors reject before
|
|
38
|
+
* anything pauses — a bad config has zero impact);
|
|
39
|
+
* 2. pauses: new calls wait on the gate, calls already in flight settle on
|
|
40
|
+
* the OLD stack;
|
|
41
|
+
* 3. boots a fresh stack from the re-resolved config;
|
|
42
|
+
* 4. drains (waits the in-flight count to zero, bounded by
|
|
43
|
+
* `RELOAD_DRAIN_MS`), then swaps the cell and disposes the old stack
|
|
44
|
+
* (worker fleet stops, clients close);
|
|
45
|
+
* 5. releases the gate. Boot or drain failure leaves the old stack
|
|
46
|
+
* untouched — the reload rejects and the system keeps serving the
|
|
47
|
+
* previous config (blue-green).
|
|
48
|
+
*
|
|
49
|
+
* Durable jobs make the pause safe: a handler interrupted by the old
|
|
50
|
+
* stack's disposal has its lease expire and is re-claimed after boot;
|
|
51
|
+
* handlers are idempotent by design (transactions roll back, watermarks
|
|
52
|
+
* advance only on success).
|
|
26
53
|
*/
|
|
27
54
|
export declare const makeDimple: (options: MakeDimpleOptions) => Promise<Dimple>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ponraaj/dimple-sdk",
|
|
3
|
-
"version": "0.0.0-alpha.
|
|
3
|
+
"version": "0.0.0-alpha.10",
|
|
4
4
|
"private": false,
|
|
5
5
|
"files": [
|
|
6
6
|
"dist"
|
|
@@ -29,10 +29,12 @@
|
|
|
29
29
|
"typecheck": "tsgo --noEmit"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@ai-sdk/
|
|
33
|
-
"@
|
|
32
|
+
"@ai-sdk/amazon-bedrock": "^5.0.46",
|
|
33
|
+
"@ai-sdk/cohere": "^3",
|
|
34
|
+
"@ai-sdk/google": "^2",
|
|
35
|
+
"@ai-sdk/openai-compatible": "^3.0.21",
|
|
34
36
|
"@libsql/client": "^0.17.4",
|
|
35
|
-
"ai": "^
|
|
37
|
+
"ai": "^7.0.50",
|
|
36
38
|
"drizzle-orm": "^1.0.0-rc.4",
|
|
37
39
|
"effect": "4.0.0-beta.103",
|
|
38
40
|
"jsonc-parser": "^3.3.1",
|
|
@@ -40,6 +42,12 @@
|
|
|
40
42
|
"zod": "^4.4.3"
|
|
41
43
|
},
|
|
42
44
|
"peerDependencies": {
|
|
45
|
+
"@huggingface/transformers": "^4.0.0",
|
|
43
46
|
"typescript": "^5"
|
|
47
|
+
},
|
|
48
|
+
"peerDependenciesMeta": {
|
|
49
|
+
"@huggingface/transformers": {
|
|
50
|
+
"optional": true
|
|
51
|
+
}
|
|
44
52
|
}
|
|
45
53
|
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { type EmbeddingModel } from "./provider.ts";
|
|
2
|
-
export interface OpenAIOptions {
|
|
3
|
-
readonly modelId: string;
|
|
4
|
-
readonly model: string;
|
|
5
|
-
readonly dimensions: number;
|
|
6
|
-
readonly apiKey: string;
|
|
7
|
-
}
|
|
8
|
-
export declare const makeOpenAIModel: (options: OpenAIOptions) => EmbeddingModel;
|