@ponraaj/dimple-sdk 0.0.0-alpha.10 → 0.0.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/dimple.d.ts +13 -0
- package/dist/errors.js +8330 -0
- package/dist/index.js +1936 -1688
- package/dist/internal/config/index.d.ts +1 -1
- package/dist/internal/config/service.d.ts +4 -0
- package/dist/internal/core/core.d.ts +8 -8
- package/dist/internal/core/hosted-embedding.d.ts +1 -1
- package/dist/internal/core/index.d.ts +6 -0
- package/dist/internal/core/layer.d.ts +17 -8
- package/dist/internal/embedder/auto-install.d.ts +23 -5
- package/dist/internal/embedder/index.d.ts +1 -1
- package/dist/internal/embedder/registry.d.ts +7 -0
- package/dist/internal/embedder/transformers.d.ts +6 -2
- package/dist/internal/extractor/code-extractor.d.ts +11 -1
- package/dist/internal/extractor/index.d.ts +1 -1
- package/dist/internal/extractor/languages/index.d.ts +1 -1
- package/dist/internal/extractor/languages/javascript.d.ts +1 -1
- package/dist/internal/extractor/languages/typescript.d.ts +1 -1
- package/dist/internal/installer/index.d.ts +11 -0
- package/dist/internal/llm/dream-classifier.d.ts +1 -1
- package/dist/internal/llm/index.d.ts +1 -1
- package/dist/internal/llm/service.d.ts +7 -1
- package/dist/internal/logger/index.d.ts +1 -1
- package/dist/internal/logger/service.d.ts +14 -3
- package/dist/internal/parser/code-parser.d.ts +12 -2
- package/dist/internal/parser/index.d.ts +3 -3
- package/dist/internal/parser/language-store.d.ts +3 -3
- package/dist/internal/parser/tree-sitter-engine.d.ts +1 -1
- package/dist/internal/retriever/hybrid.d.ts +21 -7
- package/dist/internal/retriever/index.d.ts +1 -1
- package/dist/internal/shared/effect/node.d.ts +155 -0
- package/dist/internal/shared/file-database.d.ts +17 -0
- package/dist/internal/shared/file-lock.d.ts +20 -0
- package/dist/internal/shared/schema/defaults.d.ts +13 -0
- package/dist/internal/store/audit.d.ts +1 -1
- package/dist/internal/store/db-backend.d.ts +2 -2
- package/dist/internal/store/graph/concepts.d.ts +2 -2
- package/dist/internal/store/graph/dream.d.ts +2 -2
- package/dist/internal/store/graph/errors.d.ts +1 -1
- package/dist/internal/store/graph/repair.d.ts +2 -2
- package/dist/internal/store/index.d.ts +3 -3
- package/dist/internal/store/libsql-backend.d.ts +1 -1
- package/dist/internal/store/schema/topics.d.ts +0 -12
- package/dist/internal/store/service.d.ts +6 -5
- package/dist/internal/store/store.d.ts +24 -5
- package/dist/internal/store/topics/errors.d.ts +1 -1
- package/dist/internal/store/topics/index.d.ts +2 -2
- package/dist/internal/store/topics/jobs.d.ts +2 -2
- package/dist/internal/store/topics/lifecycle.d.ts +27 -2
- package/dist/internal/store/topics/maintenance.d.ts +2 -2
- package/dist/internal/store/topics/query.d.ts +1 -1
- package/dist/internal/store/topics/routing.d.ts +1 -1
- package/dist/internal/store/topics/topic-vectors.d.ts +1 -1
- package/dist/internal/workflow/engine.d.ts +27 -3
- package/dist/internal/workflow/executor.d.ts +19 -5
- package/dist/internal/workflow/index.d.ts +5 -5
- package/dist/internal/workflow/persistence.d.ts +2 -2
- package/dist/internal/workflow/queue.d.ts +23 -2
- package/dist/internal/workflow/registry.d.ts +10 -0
- package/dist/internal/workflow/runtime.d.ts +22 -4
- package/package.json +23 -7
- package/dist/internal/wasm/grammar-store.d.ts +0 -14
- package/dist/internal/wasm/index.d.ts +0 -5
- package/dist/internal/wasm/wasm-cache.d.ts +0 -17
- package/dist/internal/wasm/wasm-downloader.d.ts +0 -10
- package/dist/internal/wasm/wasm-registry.d.ts +0 -16
- package/dist/internal/workflow/queue-store.d.ts +0 -5
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-process migration lock shared by the store backend and the workflow
|
|
3
|
+
* queue stack. Both run `CREATE TABLE`-style migrations on a `file:` database
|
|
4
|
+
* at boot; two runtimes starting together race them and the loser fails with
|
|
5
|
+
* a raw "Failed to execute statement". An exclusive lock file next to the
|
|
6
|
+
* database serializes them. Remote URLs skip locking — the server owns its
|
|
7
|
+
* migrations.
|
|
8
|
+
*
|
|
9
|
+
* `node:fs` is imported lazily: callers bundle for workerd, and the lock is
|
|
10
|
+
* only ever reached for `file:` URLs.
|
|
11
|
+
*/
|
|
12
|
+
export interface FileLockOptions {
|
|
13
|
+
/** Steal a lock whose mtime is older than this (a crashed owner). */
|
|
14
|
+
readonly staleMs?: number;
|
|
15
|
+
/** Give up after this long and throw (default 30s). */
|
|
16
|
+
readonly timeoutMs?: number;
|
|
17
|
+
}
|
|
18
|
+
/** Acquire `<db>.migrate-lock` for a `file:` URL. Remote URLs resolve to
|
|
19
|
+
* `undefined` (nothing to release). */
|
|
20
|
+
export declare const acquireMigrationLock: (url: string) => Promise<(() => void) | undefined>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Defaulted optional struct fields for Effect 4.0.0.
|
|
4
|
+
*
|
|
5
|
+
* `Schema.optionalWith` does not exist in this version; `optional` plus
|
|
6
|
+
* `withDecodingDefaultType` is the equivalent: the key may be absent or
|
|
7
|
+
* `undefined` on input, the default fills it during decoding, and the
|
|
8
|
+
* decoded type keeps the field required. Curried so the default thunk is
|
|
9
|
+
* contextually typed against the already-fixed schema type.
|
|
10
|
+
*/
|
|
11
|
+
export declare const optionalWithDefault: <S extends Schema.Constraint>(schema: S) => (options: {
|
|
12
|
+
default: () => Schema.Schema.Type<S>;
|
|
13
|
+
}) => Schema.withDecodingDefaultType<Schema.optional<S>>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Effect } from "effect";
|
|
2
2
|
import type { RetrievalAuditRow } from "./models.ts";
|
|
3
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
3
|
+
import { JobFailure, type Handler, type JobContext } from "../workflow/index.js";
|
|
4
4
|
import { Service as DbBackend } from "./db-backend.ts";
|
|
5
5
|
import type { Interface as DbBackendInterface } from "./db-backend.ts";
|
|
6
6
|
/** The decoded `audit.write` payload — mirrors the queue definition's
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Context, Effect } from "effect";
|
|
2
2
|
import type { StoreError } from "./error.ts";
|
|
3
3
|
import type { RetrievalAuditRow, SweepResult } from "./models.ts";
|
|
4
|
-
import type { EmbeddingModelConfig } from "
|
|
4
|
+
import type { EmbeddingModelConfig } from "../config/index.js";
|
|
5
5
|
/** Opaque handle to a Drizzle database instance.
|
|
6
6
|
* Concrete type varies by backend dialect (LibSQLDatabase, PgDatabase, etc.).
|
|
7
7
|
* Standard CRUD uses Drizzle's polymorphic query builder — no dialect-specific cast needed
|
|
@@ -29,7 +29,7 @@ export interface TopicVectorRow {
|
|
|
29
29
|
/** A memory embedding row from a per-model memory vector table (raw SQL boundary shape). */
|
|
30
30
|
export interface MemoryVectorRow {
|
|
31
31
|
readonly memoryId: string;
|
|
32
|
-
readonly vector: number[];
|
|
32
|
+
readonly vector: readonly number[];
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
35
|
* Canonical row of a `graphNeighbors` traversal.
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
* the active model stays 384-dim.
|
|
40
40
|
*/
|
|
41
41
|
import { Effect } from "effect";
|
|
42
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
43
|
-
import type { EmbeddingModelConfig } from "
|
|
42
|
+
import { JobFailure, type Handler, type JobContext } from "../../workflow/index.js";
|
|
43
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
44
44
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
45
45
|
import type { Interface as DbBackendInterface } from "../db-backend.ts";
|
|
46
46
|
import { StoreError } from "../error.ts";
|
|
@@ -58,8 +58,8 @@
|
|
|
58
58
|
* Effect Clock (observability only).
|
|
59
59
|
*/
|
|
60
60
|
import { Effect } from "effect";
|
|
61
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
62
|
-
import type { EmbeddingModelConfig } from "
|
|
61
|
+
import { JobFailure, type Handler, type JobContext } from "../../workflow/index.js";
|
|
62
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
63
63
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
64
64
|
import type { Interface as DbBackendInterface } from "../db-backend.ts";
|
|
65
65
|
/** One memory pair judged by the classifier (older first — the direction
|
|
@@ -4,7 +4,7 @@ declare const EdgeNotFound_base: Schema.Class<EdgeNotFound, Schema.TaggedStruct<
|
|
|
4
4
|
}>, import("effect/Cause").YieldableError>;
|
|
5
5
|
/**
|
|
6
6
|
* Graph error types (mirror of `topics/errors.ts`). Each distinct failure
|
|
7
|
-
* mode gets its own `Schema.
|
|
7
|
+
* mode gets its own `Schema.TaggedError` so callers can `catchTag`
|
|
8
8
|
* precisely; they ride the public `Store` error channel alongside
|
|
9
9
|
* `StoreError`.
|
|
10
10
|
*/
|
|
@@ -38,8 +38,8 @@
|
|
|
38
38
|
* simply finds nothing to fix).
|
|
39
39
|
*/
|
|
40
40
|
import { Effect } from "effect";
|
|
41
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
42
|
-
import type { EmbeddingModelConfig } from "
|
|
41
|
+
import { JobFailure, type Handler, type JobContext } from "../../workflow/index.js";
|
|
42
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
43
43
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
44
44
|
import type { Interface as DbBackendInterface } from "../db-backend.ts";
|
|
45
45
|
/** How many rows of each class one repair pass fixed. */
|
|
@@ -28,12 +28,12 @@ export { Service as DbBackend, type DrizzleDB } from "./db-backend.ts";
|
|
|
28
28
|
export { Service as Store } from "./service.ts";
|
|
29
29
|
export * from "./models.ts";
|
|
30
30
|
export { layer as dbBackendLayer, layerConfig as dbBackendLayerConfig, layerFromConfig as dbBackendLayerFromConfig, } from "./libsql-backend.ts";
|
|
31
|
-
export { layer as storeLayer, layerConfig as storeLayerConfig, layerFromConfig as storeLayerFromConfig, } from "./store.ts";
|
|
31
|
+
export { layer as storeLayer, layerConfig as storeLayerConfig, layerFromConfig as storeLayerFromConfig, configured as storeConfigured, node as storeNode, } from "./store.ts";
|
|
32
32
|
export declare const layer: (url: string, options?: {
|
|
33
33
|
migrationFolder?: string;
|
|
34
|
-
}) => import("effect/Layer").Layer<import("
|
|
34
|
+
}) => import("effect/Layer").Layer<import("../logger/index.js").Logging | import("./db-backend.ts").Service | import("./service.ts").Service, import("./error.ts").StoreError, never>;
|
|
35
35
|
export declare const layerConfig: (config: Config.Wrap<{
|
|
36
36
|
url: string;
|
|
37
37
|
}>, options?: {
|
|
38
38
|
migrationFolder?: string;
|
|
39
|
-
}) => import("effect/Layer").Layer<import("
|
|
39
|
+
}) => import("effect/Layer").Layer<import("../logger/index.js").Logging | import("./db-backend.ts").Service | import("./service.ts").Service, Config.ConfigError | import("./error.ts").StoreError, never>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Layer, Config, Scope } from "effect";
|
|
2
2
|
import { StoreError } from "./error.ts";
|
|
3
3
|
import { Service } from "./db-backend.ts";
|
|
4
|
-
import { Service as ConfigService } from "
|
|
4
|
+
import { Service as ConfigService } from "../config/index.js";
|
|
5
5
|
export declare const layer: (url: string, options?: {
|
|
6
6
|
migrationFolder?: string;
|
|
7
7
|
releaseClient?: boolean;
|
|
@@ -1,16 +1,4 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
|
-
/**
|
|
3
|
-
* Mirrors `Schema.optionalWith(schema, { default })` from newer Effect versions.
|
|
4
|
-
*
|
|
5
|
-
* Effect 4.0.0-beta.99 (the pinned version) does not ship `optionalWith`;
|
|
6
|
-
* `optional` + `withDecodingDefaultType` is semantically identical: the key
|
|
7
|
-
* may be absent or `undefined` on input, the default fills it during
|
|
8
|
-
* decoding, and the decoded type keeps the field required. Curried so the
|
|
9
|
-
* default thunk is contextually typed against the already-fixed schema type.
|
|
10
|
-
*/
|
|
11
|
-
export declare const optionalWithDefault: <S extends Schema.Constraint>(schema: S) => (options: {
|
|
12
|
-
default: () => Schema.Schema.Type<S>;
|
|
13
|
-
}) => Schema.withDecodingDefaultType<Schema.optional<S>>;
|
|
14
2
|
/**
|
|
15
3
|
* The deterministic topic hierarchy. One ACTIVE tree per model;
|
|
16
4
|
* retired trees keep their rows (status `inactive`) so the switch is
|
|
@@ -4,7 +4,7 @@ import type { StoreError, NotFound, DuplicateActiveKey } from "./error.ts";
|
|
|
4
4
|
import type { MetadataFilter } from "./db-backend.ts";
|
|
5
5
|
import type { TopicError } from "./topics/errors.ts";
|
|
6
6
|
import type { MaintenanceOutcome, RebalanceOutcome } from "./topics/maintenance.ts";
|
|
7
|
-
import type { EmbeddingModelConfig } from "
|
|
7
|
+
import type { EmbeddingModelConfig } from "../config/index.js";
|
|
8
8
|
import type { Topic, TopicChangeLog } from "./schema/topics.ts";
|
|
9
9
|
import type { ArtifactEdge, EdgeKind, EdgeSource } from "./schema/graph.ts";
|
|
10
10
|
import type { AugmentedMemory } from "./graph/retrieval-post-pass.ts";
|
|
@@ -130,10 +130,11 @@ export interface Interface {
|
|
|
130
130
|
/** `false` skips the per-leg graph augmentation (fused-row callers). */
|
|
131
131
|
readonly augment?: boolean;
|
|
132
132
|
}) => Effect.Effect<ReadonlyArray<AugmentedMemory>, StoreError>;
|
|
133
|
-
/** One-pass graph augmentation over an already-ranked result set
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
133
|
+
/** One-pass graph augmentation over an already-ranked result set. The
|
|
134
|
+
* store exposes it for fused-row callers; the hybrid retriever does not
|
|
135
|
+
* call it yet, so fused results carry no graph fields. A wired caller
|
|
136
|
+
* gets one edge fetch + one concept fetch over the fused rows instead of
|
|
137
|
+
* two of each per leg. */
|
|
137
138
|
readonly augmentSearchResults: (results: ReadonlyArray<ScoredMemory>) => Effect.Effect<ReadonlyArray<AugmentedMemory>, StoreError>;
|
|
138
139
|
/** The DB's tables + columns (`dimple db-schema` surface). */
|
|
139
140
|
readonly listSchema: () => Effect.Effect<ReadonlyArray<{
|
|
@@ -4,9 +4,9 @@ import type { RetrievalAuditRow } from "./models.ts";
|
|
|
4
4
|
import { Service as DbBackend } from "./db-backend.ts";
|
|
5
5
|
import { Service } from "./service.ts";
|
|
6
6
|
import { StoreError } from "./error.ts";
|
|
7
|
-
import { Service as ConfigService } from "
|
|
8
|
-
import { type LogConfig, type Logging } from "
|
|
9
|
-
import type { EmbeddingModelConfig } from "
|
|
7
|
+
import { Service as ConfigService } from "../config/index.js";
|
|
8
|
+
import { type LogConfig, type Logging } from "../logger/index.js";
|
|
9
|
+
import type { EmbeddingModelConfig } from "../config/index.js";
|
|
10
10
|
import type { TopicConfig as TopicConfigType } from "./schema/topics.ts";
|
|
11
11
|
import { GraphConfig } from "./schema/graph.ts";
|
|
12
12
|
/**
|
|
@@ -57,14 +57,33 @@ export declare const layer: (url: string, options?: {
|
|
|
57
57
|
/** Default TTL for writes that omit `ttlSecs` (the config's
|
|
58
58
|
* `defaultTtlSecs`; default 604800). */
|
|
59
59
|
defaultTtlSecs?: number;
|
|
60
|
-
}) => Layer.Layer<Logging | DbBackend | Service, StoreError,
|
|
60
|
+
}) => Layer.Layer<Logging | DbBackend | Service, StoreError, never>;
|
|
61
|
+
/**
|
|
62
|
+
* OpenCode-style node declaration for {@link layer} (the layer graph module
|
|
63
|
+
* at `@dimple/shared/effect`).
|
|
64
|
+
*
|
|
65
|
+
* `deps` stays empty on purpose: the store's production seams
|
|
66
|
+
* (`topics.enqueueSplit` / `enqueueMerge` / `enqueueAudit`) are effects
|
|
67
|
+
* built over `JobRuntime`, so the composition root owns the wiring — a node
|
|
68
|
+
* cannot own a dependency it is itself configured with. The options type is
|
|
69
|
+
* read off `layer` (`Parameters<typeof layer>[1]`) so the declaration can
|
|
70
|
+
* never drift from the layer it wraps.
|
|
71
|
+
*/
|
|
72
|
+
export declare const configured: (url: string, options?: Parameters<typeof layer>[1]) => import("../core/index.js").LayerProvider<Logging | DbBackend | Service, StoreError, undefined>;
|
|
73
|
+
/** The default store node — the local libSQL database under `.dimple/`. */
|
|
74
|
+
export declare const node: import("../core/index.js").LayerProvider<Logging | DbBackend | Service, StoreError, undefined>;
|
|
61
75
|
export declare const layerConfig: (config: Config.Wrap<{
|
|
62
76
|
url: string;
|
|
63
77
|
}>, options?: {
|
|
64
78
|
migrationFolder?: string;
|
|
65
79
|
embeddingModelConfig?: EmbeddingModelConfig;
|
|
66
|
-
}) => Layer.Layer<Logging | DbBackend | Service, Config.ConfigError | StoreError,
|
|
80
|
+
}) => Layer.Layer<Logging | DbBackend | Service, Config.ConfigError | StoreError, never>;
|
|
67
81
|
export declare const layerFromConfig: (options?: {
|
|
68
82
|
migrationFolder?: string;
|
|
69
83
|
}) => Layer.Layer<Service | DbBackend | Logging, StoreError, ConfigService | Scope>;
|
|
84
|
+
/** The dimple.jsonc `logging` section, installed unconditionally — a
|
|
85
|
+
* DISABLED config when the section is absent (silent-by-default: library
|
|
86
|
+
* logging is opt-in, and every `Effect.log*` in the system routes through
|
|
87
|
+
* `@dimple/logger` either way). The silent config itself comes from the
|
|
88
|
+
* logger package, the one source of truth. */
|
|
70
89
|
export declare const loggingFromConfig: Layer.Layer<Logging, never, ConfigService | Scope>;
|
|
@@ -4,7 +4,7 @@ declare const TopicNotFound_base: Schema.Class<TopicNotFound, Schema.TaggedStruc
|
|
|
4
4
|
}>, import("effect/Cause").YieldableError>;
|
|
5
5
|
/**
|
|
6
6
|
* Topic-tree error types. Each distinct failure mode gets its
|
|
7
|
-
* own `Schema.
|
|
7
|
+
* own `Schema.TaggedError` so callers can `catchTag` precisely.
|
|
8
8
|
* `TopicError` is the tagged union; individual classes are exported for
|
|
9
9
|
* pattern matching.
|
|
10
10
|
*/
|
|
@@ -12,8 +12,8 @@ export * from "./tree.ts";
|
|
|
12
12
|
export * from "./routing.ts";
|
|
13
13
|
export * from "./query.ts";
|
|
14
14
|
export * from "./lifecycle.ts";
|
|
15
|
-
export { memoryEmbedHandler } from "./lifecycle.ts";
|
|
16
|
-
export type { MemoryEmbedHandler, MemoryEmbedHandlerDeps } from "./lifecycle.ts";
|
|
15
|
+
export { memoryEmbedHandler, memoryEmbedBatchHandler } from "./lifecycle.ts";
|
|
16
|
+
export type { MemoryEmbedHandler, MemoryEmbedHandlerDeps, MemoryEmbedBatchHandler, MemoryEmbedBatchPayload, } from "./lifecycle.ts";
|
|
17
17
|
export * from "./errors.ts";
|
|
18
18
|
export { splitTopic, topicSplitHandler, shouldSplit } from "./jobs.ts";
|
|
19
19
|
export type { SplitOutcome, TopicSplitHandler, TopicSplitHandlerDeps } from "./jobs.ts";
|
|
@@ -46,8 +46,8 @@
|
|
|
46
46
|
* (the canonical source — keep both in sync).
|
|
47
47
|
*/
|
|
48
48
|
import { Effect } from "effect";
|
|
49
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
50
|
-
import type { EmbeddingModelConfig } from "
|
|
49
|
+
import { JobFailure, type Handler, type JobContext } from "../../workflow/index.js";
|
|
50
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
51
51
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
52
52
|
import type { Interface as DbBackendInterface } from "../db-backend.ts";
|
|
53
53
|
import { StoreError } from "../error.ts";
|
|
@@ -56,12 +56,12 @@
|
|
|
56
56
|
* (observability only — never read by routing).
|
|
57
57
|
*/
|
|
58
58
|
import { Effect } from "effect";
|
|
59
|
-
import type { EmbeddingModelConfig } from "
|
|
59
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
60
60
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
61
61
|
import type { TopicConfig } from "../schema/topics.ts";
|
|
62
62
|
import type { TopicError } from "./errors.ts";
|
|
63
63
|
import { NotFound, StoreError } from "../error.ts";
|
|
64
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
64
|
+
import { JobFailure, type Handler, type JobContext } from "../../workflow/index.js";
|
|
65
65
|
import type { Interface as DbBackendInterface } from "../db-backend.ts";
|
|
66
66
|
/** Result of {@link setActiveModel}. */
|
|
67
67
|
export interface SetActiveModelResult {
|
|
@@ -120,6 +120,16 @@ export interface MemoryEmbedPayload {
|
|
|
120
120
|
}
|
|
121
121
|
/** The `memory.embed` handler shape (see jobs.ts). */
|
|
122
122
|
export type MemoryEmbedHandler = (payload: MemoryEmbedPayload, ctx: JobContext) => Effect.Effect<unknown, JobFailure, DbBackend>;
|
|
123
|
+
/** The decoded `memory.embed.batch` payload — mirrors the queue
|
|
124
|
+
* definition's schema; `nonce` only keeps distinct enqueues distinct. */
|
|
125
|
+
export interface MemoryEmbedBatchPayload {
|
|
126
|
+
readonly memoryIds: readonly string[];
|
|
127
|
+
readonly modelKey: string;
|
|
128
|
+
readonly nonce: string;
|
|
129
|
+
}
|
|
130
|
+
/** The `memory.embed.batch` handler shape — the same deps as the per-memory
|
|
131
|
+
* handler; ids are embedded one at a time, in payload order. */
|
|
132
|
+
export type MemoryEmbedBatchHandler = (payload: MemoryEmbedBatchPayload, ctx: JobContext) => Effect.Effect<unknown, JobFailure, DbBackend>;
|
|
123
133
|
/**
|
|
124
134
|
* Factory for the `memory.embed` job handler: re-embeds one memory with the
|
|
125
135
|
* target model and routes it into that model's tree. The success schema is
|
|
@@ -130,3 +140,18 @@ export declare function memoryEmbedHandler(deps: MemoryEmbedHandlerDeps & {
|
|
|
130
140
|
readonly backend: DbBackendInterface;
|
|
131
141
|
}): Handler;
|
|
132
142
|
export declare function memoryEmbedHandler(deps: MemoryEmbedHandlerDeps): MemoryEmbedHandler;
|
|
143
|
+
/**
|
|
144
|
+
* Factory for the `memory.embed.batch` job handler: the same outcome as the
|
|
145
|
+
* per-memory job for a whole list of ids, with the STORE phase batched —
|
|
146
|
+
* one tree seed, one vector-table seed, one membership SELECT, one vector
|
|
147
|
+
* insert, one routing pass, one embedding-column UPDATE for the whole batch
|
|
148
|
+
* (per-id repeats of those dominated a 235-unit drain). The EMBED step stays
|
|
149
|
+
* per id (one model forward pass per text). A stale id (memory gone) is a
|
|
150
|
+
* success no-op, exactly like the per-memory job's; a real failure fails the
|
|
151
|
+
* ONE job, naming how many ids failed and the first failure — ids that did
|
|
152
|
+
* succeed stay committed, and a redelivery re-runs them idempotently.
|
|
153
|
+
*/
|
|
154
|
+
export declare function memoryEmbedBatchHandler(deps: MemoryEmbedHandlerDeps & {
|
|
155
|
+
readonly backend: DbBackendInterface;
|
|
156
|
+
}): Handler;
|
|
157
|
+
export declare function memoryEmbedBatchHandler(deps: MemoryEmbedHandlerDeps): MemoryEmbedBatchHandler;
|
|
@@ -31,8 +31,8 @@
|
|
|
31
31
|
* Effect Clock (captured outside the tx closure) — observability only.
|
|
32
32
|
*/
|
|
33
33
|
import { Effect } from "effect";
|
|
34
|
-
import { JobFailure, type Handler, type JobContext } from "
|
|
35
|
-
import type { EmbeddingModelConfig } from "
|
|
34
|
+
import { JobFailure, type Handler, type JobContext } from "../../workflow/index.js";
|
|
35
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
36
36
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
37
37
|
import type { Interface as DbBackendInterface } from "../db-backend.ts";
|
|
38
38
|
import { StoreError } from "../error.ts";
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* ranking still runs in libSQL (`vectorSearch` with `topic_key IN (...)`).
|
|
25
25
|
*/
|
|
26
26
|
import { Effect } from "effect";
|
|
27
|
-
import type { EmbeddingModelConfig } from "
|
|
27
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
28
28
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
29
29
|
import type { TopicConfig } from "../schema/topics.ts";
|
|
30
30
|
import type { TopicError } from "./errors.ts";
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
* transaction-aware vector methods, move the upserts onto the tx.
|
|
39
39
|
*/
|
|
40
40
|
import { Effect } from "effect";
|
|
41
|
-
import type { EmbeddingModelConfig } from "
|
|
41
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
42
42
|
import { Service as DbBackend } from "../db-backend.ts";
|
|
43
43
|
import type { TopicConfig } from "../schema/topics.ts";
|
|
44
44
|
import type { TopicError } from "./errors.ts";
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* dynamic (not Drizzle tables), so every writer goes through this raw-SQL
|
|
5
5
|
* helper — one canonical place for the table name + conversion function.
|
|
6
6
|
*/
|
|
7
|
-
import type { EmbeddingModelConfig } from "
|
|
7
|
+
import type { EmbeddingModelConfig } from "../../config/index.js";
|
|
8
8
|
import { type SQLWrapper } from "drizzle-orm";
|
|
9
9
|
/** Per-model topic vector table name (mirrors libsql-backend.ts — canonical). */
|
|
10
10
|
export declare const topicVectorTableName: (model: EmbeddingModelConfig) => string;
|
|
@@ -28,9 +28,9 @@
|
|
|
28
28
|
* `Schema.Unknown` Persistable keys — see persistence.ts for why.
|
|
29
29
|
*/
|
|
30
30
|
import { Layer } from "effect";
|
|
31
|
-
import { WorkflowEngine } from "effect/
|
|
32
|
-
import { Persistence as PersistenceNamespace, PersistedQueue } from "effect/
|
|
33
|
-
import { SqlError as SqlErrorNamespace } from "effect/
|
|
31
|
+
import { WorkflowEngine } from "effect/workflow";
|
|
32
|
+
import { Persistence as PersistenceNamespace, PersistedQueue } from "effect/persistence";
|
|
33
|
+
import { SqlError as SqlErrorNamespace } from "effect/sql";
|
|
34
34
|
type SqlError = SqlErrorNamespace.SqlError;
|
|
35
35
|
/**
|
|
36
36
|
* Durable WorkflowEngine layer.
|
|
@@ -40,6 +40,14 @@ type SqlError = SqlErrorNamespace.SqlError;
|
|
|
40
40
|
* (same `Encoded` contract into `makeUnsafe`).
|
|
41
41
|
*/
|
|
42
42
|
export declare const layerDurable: Layer.Layer<WorkflowEngine.WorkflowEngine, never, PersistedQueue.PersistedQueueFactory | PersistenceNamespace.Persistence>;
|
|
43
|
+
/**
|
|
44
|
+
* Shared libSQL stack: SQL persistence + resume-queue store over one
|
|
45
|
+
* `LibsqlClient`. Provides `Persistence | PersistedQueueFactory`, dep-free.
|
|
46
|
+
*
|
|
47
|
+
* The single libSQL connection is shared by the engine's store and the
|
|
48
|
+
* resume queue — both `Persistence.layerSql` and `PersistedQueue.layerStoreSql`
|
|
49
|
+
* create their own tables in the same file.
|
|
50
|
+
*/
|
|
43
51
|
export declare const libsqlStack: (options: {
|
|
44
52
|
readonly url: string;
|
|
45
53
|
}) => Layer.Layer<PersistedQueue.PersistedQueueFactory | PersistenceNamespace.Persistence, SqlErrorNamespace.SqlError, never>;
|
|
@@ -51,4 +59,20 @@ export declare const libsqlStack: (options: {
|
|
|
51
59
|
export declare const layerDurableLibsql: (options: {
|
|
52
60
|
readonly url: string;
|
|
53
61
|
}) => Layer.Layer<WorkflowEngine.WorkflowEngine, SqlError>;
|
|
62
|
+
/**
|
|
63
|
+
* OpenCode-style node declarations for the two libSQL-backed layers above.
|
|
64
|
+
*
|
|
65
|
+
* The convention's single `configured` name is per-layer, and this module
|
|
66
|
+
* exposes two, so each is declared under its layer's name; the shape
|
|
67
|
+
* (`make({ name, layer })`, empty `deps` — the root wires the runtime, the
|
|
68
|
+
* registry and the store) is unchanged. Neither gets a default `node`: both
|
|
69
|
+
* require a `url`, and a node that guessed one would silently point a
|
|
70
|
+
* durable engine at the wrong database.
|
|
71
|
+
*/
|
|
72
|
+
export declare const libsqlStackConfigured: (options: {
|
|
73
|
+
readonly url: string;
|
|
74
|
+
}) => import("../core/index.js").LayerProvider<PersistedQueue.PersistedQueueFactory | PersistenceNamespace.Persistence, SqlErrorNamespace.SqlError, undefined>;
|
|
75
|
+
export declare const layerDurableLibsqlConfigured: (options: {
|
|
76
|
+
readonly url: string;
|
|
77
|
+
}) => import("../core/index.js").LayerProvider<WorkflowEngine.WorkflowEngine, SqlErrorNamespace.SqlError, undefined>;
|
|
54
78
|
export {};
|
|
@@ -14,10 +14,11 @@
|
|
|
14
14
|
* resumes the awaiting workflow (facade `submit`) and persists the result.
|
|
15
15
|
*/
|
|
16
16
|
import { Duration, Effect, Layer } from "effect";
|
|
17
|
-
import { WorkflowEngine } from "effect/
|
|
18
|
-
import { PersistedQueue } from "effect/
|
|
19
|
-
import { type LogConfig } from "
|
|
20
|
-
import type
|
|
17
|
+
import { WorkflowEngine } from "effect/workflow";
|
|
18
|
+
import { PersistedQueue } from "effect/persistence";
|
|
19
|
+
import { type LogConfig } from "../logger/index.js";
|
|
20
|
+
import { type Graph } from "../shared/effect/node.js";
|
|
21
|
+
import { type JobDefinition } from "./queue.ts";
|
|
21
22
|
import { Service as JobRegistry } from "./registry.ts";
|
|
22
23
|
import { Service as JobRuntime } from "./runtime.ts";
|
|
23
24
|
export declare const workerLayer: (defs: readonly JobDefinition.Any[], options?: {
|
|
@@ -27,7 +28,20 @@ export declare const workerLayer: (defs: readonly JobDefinition.Any[], options?:
|
|
|
27
28
|
* [1, 100]. */
|
|
28
29
|
readonly maxAttempts?: number;
|
|
29
30
|
readonly logging?: LogConfig;
|
|
30
|
-
}) => Layer.Layer<never, never, PersistedQueue.PersistedQueueFactory | JobRegistry | WorkflowEngine.WorkflowEngine
|
|
31
|
+
}) => Layer.Layer<never, never, PersistedQueue.PersistedQueueFactory | JobRegistry | WorkflowEngine.WorkflowEngine>;
|
|
32
|
+
/**
|
|
33
|
+
* OpenCode-style node declaration for {@link workerLayer}.
|
|
34
|
+
*
|
|
35
|
+
* Workers need `WorkflowEngine` + `PersistedQueueFactory` (the job stack) and
|
|
36
|
+
* `JobRegistry`, and those providers are root-owned state: a package default
|
|
37
|
+
* would build a SECOND job stack and bind the worker to it, so there is no
|
|
38
|
+
* `node` export here — the caller passes them as the `deps` tuple and `make`
|
|
39
|
+
* checks that they cover the layer's requirements. The tuple elements may be
|
|
40
|
+
* any graph that provides at least the named services, and their build errors
|
|
41
|
+
* are preserved on the node. The options type is read off `workerLayer` so the
|
|
42
|
+
* declaration cannot drift from the layer it wraps.
|
|
43
|
+
*/
|
|
44
|
+
export declare const configured: <const RuntimeE, const RegistryE>(defs: readonly JobDefinition.Any[], options: Parameters<typeof workerLayer>[1], deps: readonly [Graph<PersistedQueue.PersistedQueueFactory | WorkflowEngine.WorkflowEngine, RuntimeE>, Graph<JobRegistry, RegistryE>]) => import("../core/index.js").LayerProvider<never, RegistryE | RuntimeE, undefined>;
|
|
31
45
|
/**
|
|
32
46
|
* Inline mode — run the worker in the same fiber scope as the submit.
|
|
33
47
|
*
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Barrel: named exports with aliases (project convention).
|
|
5
5
|
*/
|
|
6
|
-
export { JobDefinition, Envelope, TopicSplit, MemoryEmbed, TopicMerge, TopicRebalance, ConceptCluster, MemoryDream, GraphRepair, AuditWrite, type AuditRow, queues, type EnvelopeSchema, } from "./queue.ts";
|
|
7
|
-
export { Service as JobRuntime, layer as jobRuntimeLayer, type Interface as JobRuntimeInterface, type JobStatus, } from "./runtime.ts";
|
|
8
|
-
export { layerDurable, layerDurableLibsql } from "./engine.ts";
|
|
9
|
-
export { Service as JobRegistry, layer as jobRegistryLayer, JobFailure, type Handler, type JobContext, } from "./registry.ts";
|
|
10
|
-
export { workerLayer, inline, processQueues } from "./executor.ts";
|
|
6
|
+
export { JobDefinition, Envelope, TopicSplit, MemoryEmbed, MemoryEmbedBatch, TopicMerge, TopicRebalance, ConceptCluster, MemoryDream, GraphRepair, AuditWrite, type AuditRow, queues, type EnvelopeSchema, } from "./queue.ts";
|
|
7
|
+
export { Service as JobRuntime, layer as jobRuntimeLayer, configured as jobRuntimeConfigured, node as jobRuntimeNode, type Interface as JobRuntimeInterface, type JobStatus, } from "./runtime.ts";
|
|
8
|
+
export { layerDurable, layerDurableLibsql, libsqlStackConfigured, layerDurableLibsqlConfigured, } from "./engine.ts";
|
|
9
|
+
export { Service as JobRegistry, layer as jobRegistryLayer, configured as jobRegistryConfigured, node as jobRegistryNode, JobFailure, type Handler, type JobContext, } from "./registry.ts";
|
|
10
|
+
export { workerLayer, configured as workersConfigured, inline, processQueues } from "./executor.ts";
|
|
11
11
|
export { JobsConfig, JobsConfigDefaults, resolveJobsConfig } from "./config.ts";
|
|
12
12
|
export { QueueError, UnknownJobType, RegistryError, ShutdownError, JobFailed, JobTimeout, JobCancelled, type JobRuntimeError, type JobsError, } from "./errors.ts";
|
|
13
13
|
export { STORE_ID, RESUME_QUEUE, ResumeSignal, ExecutionRecord, encodedKey, resultCodecFor, payloadCodecFor, } from "./persistence.ts";
|
|
@@ -32,9 +32,9 @@
|
|
|
32
32
|
*
|
|
33
33
|
*/
|
|
34
34
|
import { Effect, Exit, Option, Schema } from "effect";
|
|
35
|
-
import { Persistable, Persistence as PersistenceNamespace } from "effect/
|
|
35
|
+
import { Persistable, Persistence as PersistenceNamespace } from "effect/persistence";
|
|
36
36
|
type PersistenceStore = PersistenceNamespace.PersistenceStore;
|
|
37
|
-
import { Workflow } from "effect/
|
|
37
|
+
import { Workflow } from "effect/workflow";
|
|
38
38
|
/** Single store id shared by engine + facade (one table partition in jobs.db). */
|
|
39
39
|
export declare const STORE_ID = "dimple/jobs";
|
|
40
40
|
/**
|
|
@@ -30,8 +30,8 @@
|
|
|
30
30
|
*
|
|
31
31
|
*/
|
|
32
32
|
import { Effect, Schema, Scope } from "effect";
|
|
33
|
-
import { DurableQueue, Workflow, WorkflowEngine } from "effect/
|
|
34
|
-
import { PersistedQueue } from "effect/
|
|
33
|
+
import { DurableQueue, Workflow, WorkflowEngine } from "effect/workflow";
|
|
34
|
+
import { PersistedQueue } from "effect/persistence";
|
|
35
35
|
/**
|
|
36
36
|
* Envelope schema factory — wraps a job's payload schema with the runtime
|
|
37
37
|
* fields the workflow machinery needs.
|
|
@@ -173,6 +173,23 @@ export declare const MemoryEmbed: JobDefinition<Schema.Struct<{
|
|
|
173
173
|
memoryId: Schema.String;
|
|
174
174
|
modelKey: Schema.String;
|
|
175
175
|
}>, Schema.Void, Schema.String>;
|
|
176
|
+
/**
|
|
177
|
+
* Batch embedding backfill job — `memory.embed.batch`. Same work as
|
|
178
|
+
* `memory.embed`, one job for many memories: the bulk ingest path
|
|
179
|
+
* (`Dimple.index` with `deferEmbed`) would otherwise enqueue and drain one
|
|
180
|
+
* job per unit while the walk is still running.
|
|
181
|
+
*
|
|
182
|
+
* `nonce` keeps distinct enqueues distinct (the key function stays
|
|
183
|
+
* deterministic over the payload, so it can never collapse two different
|
|
184
|
+
* batches into one execution id); the handler is idempotent PER MEMORY, so a
|
|
185
|
+
* redelivered or overlapping batch re-embeds an already-embedded unit for
|
|
186
|
+
* nothing rather than corrupting it.
|
|
187
|
+
*/
|
|
188
|
+
export declare const MemoryEmbedBatch: JobDefinition<Schema.Struct<{
|
|
189
|
+
memoryIds: Schema.$Array<Schema.String>;
|
|
190
|
+
modelKey: Schema.String;
|
|
191
|
+
nonce: Schema.String;
|
|
192
|
+
}>, Schema.Void, Schema.String>;
|
|
176
193
|
/** Underflow maintenance job — `topic.merge` (borrow-then-merge). */
|
|
177
194
|
export declare const TopicMerge: JobDefinition<Schema.Struct<{
|
|
178
195
|
topicId: Schema.String;
|
|
@@ -270,6 +287,10 @@ export declare const queues: readonly [JobDefinition<Schema.Struct<{
|
|
|
270
287
|
}>, Schema.Void, Schema.String>, JobDefinition<Schema.Struct<{
|
|
271
288
|
memoryId: Schema.String;
|
|
272
289
|
modelKey: Schema.String;
|
|
290
|
+
}>, Schema.Void, Schema.String>, JobDefinition<Schema.Struct<{
|
|
291
|
+
memoryIds: Schema.$Array<Schema.String>;
|
|
292
|
+
modelKey: Schema.String;
|
|
293
|
+
nonce: Schema.String;
|
|
273
294
|
}>, Schema.Void, Schema.String>, JobDefinition<Schema.Struct<{
|
|
274
295
|
topicId: Schema.String;
|
|
275
296
|
modelKey: Schema.String;
|
|
@@ -54,4 +54,14 @@ export declare class Service extends Service_base {
|
|
|
54
54
|
* unknown resolve fails with `UnknownJobType`.
|
|
55
55
|
*/
|
|
56
56
|
export declare const layer: Layer.Layer<Service, never, never>;
|
|
57
|
+
/**
|
|
58
|
+
* OpenCode-style node declaration for the {@link layer} job registry.
|
|
59
|
+
*
|
|
60
|
+
* `deps` is empty: the registry is dependency-free by construction (handlers
|
|
61
|
+
* are registered by the composition root, which owns the runtime and the
|
|
62
|
+
* store they need).
|
|
63
|
+
*/
|
|
64
|
+
export declare const configured: () => import("../core/index.js").LayerProvider<Service, never, undefined>;
|
|
65
|
+
/** The default registry node — the one the app graph always wires. */
|
|
66
|
+
export declare const node: import("../core/index.js").LayerProvider<Service, never, undefined>;
|
|
57
67
|
export {};
|
|
@@ -51,10 +51,10 @@
|
|
|
51
51
|
*
|
|
52
52
|
*/
|
|
53
53
|
import { Context, Duration, Effect, Layer, Schema } from "effect";
|
|
54
|
-
import { Workflow, WorkflowEngine } from "effect/
|
|
55
|
-
import { PersistedQueue, Persistence as PersistenceNamespace } from "effect/
|
|
56
|
-
import { SqlError as SqlErrorNamespace } from "effect/
|
|
57
|
-
import type
|
|
54
|
+
import { Workflow, WorkflowEngine } from "effect/workflow";
|
|
55
|
+
import { PersistedQueue, Persistence as PersistenceNamespace } from "effect/persistence";
|
|
56
|
+
import { SqlError as SqlErrorNamespace } from "effect/sql";
|
|
57
|
+
import { type JobDefinition } from "./queue.ts";
|
|
58
58
|
import { type JobsError } from "./errors.ts";
|
|
59
59
|
type SqlError = SqlErrorNamespace.SqlError;
|
|
60
60
|
/**
|
|
@@ -158,4 +158,22 @@ export declare const layer: (defs: readonly JobDefinition.Any[]) => Layer.Layer<
|
|
|
158
158
|
export declare const layerLibsql: (defs: readonly JobDefinition.Any[], options: {
|
|
159
159
|
readonly url: string;
|
|
160
160
|
}) => Layer.Layer<Service | WorkflowEngine.WorkflowEngine | PersistenceNamespace.Persistence | PersistedQueue.PersistedQueueFactory, SqlError>;
|
|
161
|
+
/**
|
|
162
|
+
* OpenCode-style node declaration for {@link layerLibsql}.
|
|
163
|
+
*
|
|
164
|
+
* `deps` is empty: the libSQL client, the durable engine, persistence and
|
|
165
|
+
* the queue factory all arrive from the node's own layer; the root only adds
|
|
166
|
+
* what the *consumers* (workers, the store's job seams) need.
|
|
167
|
+
*/
|
|
168
|
+
export declare const configured: (defs: readonly JobDefinition.Any[], options: {
|
|
169
|
+
readonly url: string;
|
|
170
|
+
}) => import("../core/index.js").LayerProvider<PersistedQueue.PersistedQueueFactory | PersistenceNamespace.Persistence | Service | WorkflowEngine.WorkflowEngine, SqlErrorNamespace.SqlError, undefined>;
|
|
171
|
+
/**
|
|
172
|
+
* The default job-stack node — every built-in definition over the package's
|
|
173
|
+
* documented jobs database default (`JobsConfigDefaults.dbUrl`,
|
|
174
|
+
* `file:./.dimple/jobs.db`), the same value `resolveJobsConfig` fills when
|
|
175
|
+
* the `jobs` section omits `dbUrl`. Nothing is connected at import: the
|
|
176
|
+
* node only holds a lazily-built layer.
|
|
177
|
+
*/
|
|
178
|
+
export declare const node: import("../core/index.js").LayerProvider<PersistedQueue.PersistedQueueFactory | PersistenceNamespace.Persistence | Service | WorkflowEngine.WorkflowEngine, SqlErrorNamespace.SqlError, undefined>;
|
|
161
179
|
export {};
|