@ponraaj/dimple-sdk 0.0.0-alpha.1 → 0.0.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -0
- package/dist/dimple.d.ts +116 -5
- package/dist/errors.d.ts +24 -2
- package/dist/errors.js +8218 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3584 -1999
- package/dist/indexer.d.ts +9 -0
- package/dist/internal/config/index.d.ts +1 -1
- package/dist/internal/config/schema.d.ts +87 -0
- package/dist/internal/config/service.d.ts +4 -0
- package/dist/internal/core/core.d.ts +8 -8
- package/dist/internal/core/hosted-embedding.d.ts +33 -0
- package/dist/internal/core/index.d.ts +6 -0
- package/dist/internal/core/layer.d.ts +17 -8
- package/dist/internal/embedder/auto-install.d.ts +28 -0
- package/dist/internal/embedder/index.d.ts +2 -3
- package/dist/internal/embedder/registry.d.ts +7 -0
- package/dist/internal/embedder/transformers.d.ts +40 -2
- package/dist/internal/extractor/code-extractor.d.ts +11 -1
- package/dist/internal/extractor/index.d.ts +1 -1
- package/dist/internal/extractor/languages/index.d.ts +1 -1
- package/dist/internal/extractor/languages/javascript.d.ts +1 -1
- package/dist/internal/extractor/languages/typescript.d.ts +1 -1
- package/dist/internal/extractor/models/extraction.d.ts +14 -0
- package/dist/internal/installer/index.d.ts +37 -0
- package/dist/internal/llm/dream-classifier.d.ts +1 -1
- package/dist/internal/llm/index.d.ts +1 -1
- package/dist/internal/llm/service.d.ts +7 -1
- package/dist/internal/logger/index.d.ts +1 -1
- package/dist/internal/logger/service.d.ts +44 -14
- package/dist/internal/logger/types.d.ts +15 -3
- package/dist/internal/parser/code-parser.d.ts +12 -2
- package/dist/internal/parser/index.d.ts +3 -3
- package/dist/internal/parser/language-store.d.ts +3 -3
- package/dist/internal/parser/tree-sitter-engine.d.ts +1 -1
- package/dist/internal/retriever/hybrid.d.ts +21 -7
- package/dist/internal/retriever/index.d.ts +1 -1
- package/dist/internal/shared/effect/node.d.ts +155 -0
- package/dist/internal/shared/file-database.d.ts +17 -0
- package/dist/internal/shared/file-lock.d.ts +20 -0
- package/dist/internal/shared/schema/defaults.d.ts +13 -0
- package/dist/internal/store/audit.d.ts +8 -2
- package/dist/internal/store/db-backend.d.ts +48 -10
- package/dist/internal/store/graph/candidates.d.ts +3 -1
- package/dist/internal/store/graph/concepts.d.ts +16 -7
- package/dist/internal/store/graph/dream.d.ts +9 -3
- package/dist/internal/store/graph/errors.d.ts +1 -1
- package/dist/internal/store/graph/repair.d.ts +9 -3
- package/dist/internal/store/graph/traversal.d.ts +1 -1
- package/dist/internal/store/index.d.ts +4 -3
- package/dist/internal/store/libsql-backend.d.ts +1 -1
- package/dist/internal/store/models.d.ts +4 -3
- package/dist/internal/store/schema/graph.d.ts +7 -7
- package/dist/internal/store/schema/libsql.d.ts +4 -4
- package/dist/internal/store/schema/metadata.d.ts +4 -1
- package/dist/internal/store/schema/topics.d.ts +0 -12
- package/dist/internal/store/service.d.ts +85 -5
- package/dist/internal/store/store.d.ts +24 -5
- package/dist/internal/store/topics/errors.d.ts +1 -1
- package/dist/internal/store/topics/jobs.d.ts +9 -3
- package/dist/internal/store/topics/lifecycle.d.ts +9 -3
- package/dist/internal/store/topics/maintenance.d.ts +10 -4
- package/dist/internal/store/topics/query.d.ts +1 -1
- package/dist/internal/store/topics/routing.d.ts +34 -1
- package/dist/internal/store/topics/topic-vectors.d.ts +11 -4
- package/dist/internal/store/topics/tree.d.ts +15 -9
- package/dist/internal/workflow/engine.d.ts +27 -3
- package/dist/internal/workflow/executor.d.ts +20 -6
- package/dist/internal/workflow/index.d.ts +4 -4
- package/dist/internal/workflow/persistence.d.ts +3 -3
- package/dist/internal/workflow/queue.d.ts +8 -5
- package/dist/internal/workflow/registry.d.ts +11 -1
- package/dist/internal/workflow/runtime.d.ts +22 -4
- package/dist/internal/workflow/signals.d.ts +21 -0
- package/dist/make.d.ts +27 -0
- package/package.json +34 -10
- package/dist/internal/embedder/openai.d.ts +0 -8
- package/dist/internal/wasm/grammar-store.d.ts +0 -14
- package/dist/internal/wasm/index.d.ts +0 -5
- package/dist/internal/wasm/wasm-cache.d.ts +0 -17
- package/dist/internal/wasm/wasm-downloader.d.ts +0 -10
- package/dist/internal/wasm/wasm-registry.d.ts +0 -16
- package/dist/internal/workflow/queue-store.d.ts +0 -5
package/dist/indexer.d.ts
CHANGED
|
@@ -25,11 +25,20 @@ export interface WalkResult {
|
|
|
25
25
|
* SAME file always gets the SAME key regardless of which path list was
|
|
26
26
|
* passed (idempotent re-indexing across invocation shapes). */
|
|
27
27
|
readonly root: string;
|
|
28
|
+
/** The resolved input paths — the reconcile DOMAIN: units whose file
|
|
29
|
+
* path is outside every root are untouched by subtree runs. */
|
|
30
|
+
readonly roots: readonly string[];
|
|
28
31
|
}
|
|
29
32
|
/** Recursively collect indexable files from paths (dirs or files). */
|
|
30
33
|
export declare const walkFiles: (paths: readonly string[], extensions: readonly string[]) => WalkResult;
|
|
31
34
|
/** Read a file's text, capped at `maxBytes` (string slice = char-safe). */
|
|
32
35
|
export declare const readCapped: (path: string, maxBytes: number) => string;
|
|
36
|
+
/** The repository's HEAD commit for provenance stamping — soft-failed
|
|
37
|
+
* (non-git trees, missing git, permission errors → null). Walks UP from
|
|
38
|
+
* the first indexed file (git finds the enclosing repo), so it works for
|
|
39
|
+
* absolute-path trees whose key root is "/". Platform side; the facade
|
|
40
|
+
* never touches node APIs. */
|
|
41
|
+
export declare const currentCommit: (firstFile: string | undefined) => string | null;
|
|
33
42
|
/** Resolve a file's relative import specifiers to indexed-tree files
|
|
34
43
|
* (bare/absolute specifiers are external modules — no node_modules
|
|
35
44
|
* resolution). */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { DimpleConfig, EmbeddingModelConfig, EmbeddingConfig, IndexConfig, JobsConfig, TopicsConfig, } from "./schema.ts";
|
|
2
2
|
export type { DimpleConfig as DimpleConfigType, EmbeddingModelConfig as EmbeddingModelConfigType, EmbeddingConfig as EmbeddingConfigType, IndexConfig as IndexConfigType, JobsConfig as JobsConfigType, TopicsConfig as TopicsConfigType, } from "./schema.ts";
|
|
3
|
-
export { Service, layer } from "./service.ts";
|
|
3
|
+
export { Service, layer, configured as configConfigured, node as configNode } from "./service.ts";
|
|
4
4
|
export { loadConfig, loadConfigFromFile } from "./loader.ts";
|
|
@@ -13,11 +13,40 @@ export declare const EmbeddingModelConfig: Schema.Struct<{
|
|
|
13
13
|
readonly dimensions: Schema.Number;
|
|
14
14
|
readonly vectorType: Schema.optional<Schema.String>;
|
|
15
15
|
readonly apiKey: Schema.optional<Schema.String>;
|
|
16
|
+
/** Hosted providers: OpenAI-compatible endpoint override (default
|
|
17
|
+
* `https://api.openai.com/v1`). */
|
|
18
|
+
readonly baseURL: Schema.optional<Schema.String>;
|
|
16
19
|
readonly indexConfig: Schema.optional<Schema.Struct<{
|
|
17
20
|
readonly compressNeighbors: Schema.optional<Schema.String>;
|
|
18
21
|
readonly maxNeighbors: Schema.optional<Schema.Number>;
|
|
19
22
|
readonly searchL: Schema.optional<Schema.Number>;
|
|
20
23
|
}>>;
|
|
24
|
+
/** Transformers.js runtime options — per-model, all optional (defaults:
|
|
25
|
+
* device "cpu", dtype "q8", pooling "mean", normalize true). `dtype`
|
|
26
|
+
* quantizes the MODEL WEIGHTS (inference memory/speed), never the
|
|
27
|
+
* output vectors; unsupported devices fail typed at first use. */
|
|
28
|
+
readonly device: Schema.optional<Schema.Literals<readonly ["auto", "gpu", "cpu", "wasm", "webgpu", "cuda", "dml", "coreml", "webnn"]>>;
|
|
29
|
+
readonly dtype: Schema.optional<Schema.Literals<readonly ["fp32", "fp16", "q8", "int8", "uint8", "q4", "bnb4", "q4f16", "q2", "q2f16", "q1", "q1f16"]>>;
|
|
30
|
+
readonly pooling: Schema.optional<Schema.Literals<readonly ["mean", "cls", "none"]>>;
|
|
31
|
+
readonly normalize: Schema.optional<Schema.Boolean>;
|
|
32
|
+
readonly cacheDir: Schema.optional<Schema.String>;
|
|
33
|
+
/** ONNX Runtime CUDA session options — the verified fix for GPUs with
|
|
34
|
+
* limited VRAM (≤8GB): ORT's default arena is unbounded and grows by
|
|
35
|
+
* powers of two, so batch inference OOMs mid-run and poisons the
|
|
36
|
+
* process for later queries. `gpuMemLimit` caps the arena (bytes);
|
|
37
|
+
* `arenaExtendStrategy` picks the growth policy (0 = kNextPowerOfTwo,
|
|
38
|
+
* 1 = kSameAsRequested). CPU / ample-VRAM GPUs are unaffected (ORT
|
|
39
|
+
* defaults preserved when absent). */
|
|
40
|
+
readonly sessionOptions: Schema.optional<Schema.Struct<{
|
|
41
|
+
readonly gpuMemLimit: Schema.optional<Schema.Number>;
|
|
42
|
+
readonly arenaExtendStrategy: Schema.optional<Schema.Literals<readonly [0, 1]>>;
|
|
43
|
+
}>>;
|
|
44
|
+
/** First-use auto-install of the optional `@huggingface/transformers`
|
|
45
|
+
* peer (default true) — the package is installed with the project's
|
|
46
|
+
* own package manager on the first local-embed use (scripts skipped,
|
|
47
|
+
* proven unnecessary for embedding). Set false to require a manual
|
|
48
|
+
* install. */
|
|
49
|
+
readonly autoInstall: Schema.optional<Schema.Boolean>;
|
|
21
50
|
}>;
|
|
22
51
|
export interface EmbeddingModelConfig extends Schema.Schema.Type<typeof EmbeddingModelConfig> {
|
|
23
52
|
}
|
|
@@ -30,11 +59,40 @@ export declare const EmbeddingConfig: Schema.Struct<{
|
|
|
30
59
|
readonly dimensions: Schema.Number;
|
|
31
60
|
readonly vectorType: Schema.optional<Schema.String>;
|
|
32
61
|
readonly apiKey: Schema.optional<Schema.String>;
|
|
62
|
+
/** Hosted providers: OpenAI-compatible endpoint override (default
|
|
63
|
+
* `https://api.openai.com/v1`). */
|
|
64
|
+
readonly baseURL: Schema.optional<Schema.String>;
|
|
33
65
|
readonly indexConfig: Schema.optional<Schema.Struct<{
|
|
34
66
|
readonly compressNeighbors: Schema.optional<Schema.String>;
|
|
35
67
|
readonly maxNeighbors: Schema.optional<Schema.Number>;
|
|
36
68
|
readonly searchL: Schema.optional<Schema.Number>;
|
|
37
69
|
}>>;
|
|
70
|
+
/** Transformers.js runtime options — per-model, all optional (defaults:
|
|
71
|
+
* device "cpu", dtype "q8", pooling "mean", normalize true). `dtype`
|
|
72
|
+
* quantizes the MODEL WEIGHTS (inference memory/speed), never the
|
|
73
|
+
* output vectors; unsupported devices fail typed at first use. */
|
|
74
|
+
readonly device: Schema.optional<Schema.Literals<readonly ["auto", "gpu", "cpu", "wasm", "webgpu", "cuda", "dml", "coreml", "webnn"]>>;
|
|
75
|
+
readonly dtype: Schema.optional<Schema.Literals<readonly ["fp32", "fp16", "q8", "int8", "uint8", "q4", "bnb4", "q4f16", "q2", "q2f16", "q1", "q1f16"]>>;
|
|
76
|
+
readonly pooling: Schema.optional<Schema.Literals<readonly ["mean", "cls", "none"]>>;
|
|
77
|
+
readonly normalize: Schema.optional<Schema.Boolean>;
|
|
78
|
+
readonly cacheDir: Schema.optional<Schema.String>;
|
|
79
|
+
/** ONNX Runtime CUDA session options — the verified fix for GPUs with
|
|
80
|
+
* limited VRAM (≤8GB): ORT's default arena is unbounded and grows by
|
|
81
|
+
* powers of two, so batch inference OOMs mid-run and poisons the
|
|
82
|
+
* process for later queries. `gpuMemLimit` caps the arena (bytes);
|
|
83
|
+
* `arenaExtendStrategy` picks the growth policy (0 = kNextPowerOfTwo,
|
|
84
|
+
* 1 = kSameAsRequested). CPU / ample-VRAM GPUs are unaffected (ORT
|
|
85
|
+
* defaults preserved when absent). */
|
|
86
|
+
readonly sessionOptions: Schema.optional<Schema.Struct<{
|
|
87
|
+
readonly gpuMemLimit: Schema.optional<Schema.Number>;
|
|
88
|
+
readonly arenaExtendStrategy: Schema.optional<Schema.Literals<readonly [0, 1]>>;
|
|
89
|
+
}>>;
|
|
90
|
+
/** First-use auto-install of the optional `@huggingface/transformers`
|
|
91
|
+
* peer (default true) — the package is installed with the project's
|
|
92
|
+
* own package manager on the first local-embed use (scripts skipped,
|
|
93
|
+
* proven unnecessary for embedding). Set false to require a manual
|
|
94
|
+
* install. */
|
|
95
|
+
readonly autoInstall: Schema.optional<Schema.Boolean>;
|
|
38
96
|
}>>;
|
|
39
97
|
}>;
|
|
40
98
|
export interface EmbeddingConfig extends Schema.Schema.Type<typeof EmbeddingConfig> {
|
|
@@ -110,11 +168,40 @@ export declare const DimpleConfig: Schema.Struct<{
|
|
|
110
168
|
readonly dimensions: Schema.Number;
|
|
111
169
|
readonly vectorType: Schema.optional<Schema.String>;
|
|
112
170
|
readonly apiKey: Schema.optional<Schema.String>;
|
|
171
|
+
/** Hosted providers: OpenAI-compatible endpoint override (default
|
|
172
|
+
* `https://api.openai.com/v1`). */
|
|
173
|
+
readonly baseURL: Schema.optional<Schema.String>;
|
|
113
174
|
readonly indexConfig: Schema.optional<Schema.Struct<{
|
|
114
175
|
readonly compressNeighbors: Schema.optional<Schema.String>;
|
|
115
176
|
readonly maxNeighbors: Schema.optional<Schema.Number>;
|
|
116
177
|
readonly searchL: Schema.optional<Schema.Number>;
|
|
117
178
|
}>>;
|
|
179
|
+
/** Transformers.js runtime options — per-model, all optional (defaults:
|
|
180
|
+
* device "cpu", dtype "q8", pooling "mean", normalize true). `dtype`
|
|
181
|
+
* quantizes the MODEL WEIGHTS (inference memory/speed), never the
|
|
182
|
+
* output vectors; unsupported devices fail typed at first use. */
|
|
183
|
+
readonly device: Schema.optional<Schema.Literals<readonly ["auto", "gpu", "cpu", "wasm", "webgpu", "cuda", "dml", "coreml", "webnn"]>>;
|
|
184
|
+
readonly dtype: Schema.optional<Schema.Literals<readonly ["fp32", "fp16", "q8", "int8", "uint8", "q4", "bnb4", "q4f16", "q2", "q2f16", "q1", "q1f16"]>>;
|
|
185
|
+
readonly pooling: Schema.optional<Schema.Literals<readonly ["mean", "cls", "none"]>>;
|
|
186
|
+
readonly normalize: Schema.optional<Schema.Boolean>;
|
|
187
|
+
readonly cacheDir: Schema.optional<Schema.String>;
|
|
188
|
+
/** ONNX Runtime CUDA session options — the verified fix for GPUs with
|
|
189
|
+
* limited VRAM (≤8GB): ORT's default arena is unbounded and grows by
|
|
190
|
+
* powers of two, so batch inference OOMs mid-run and poisons the
|
|
191
|
+
* process for later queries. `gpuMemLimit` caps the arena (bytes);
|
|
192
|
+
* `arenaExtendStrategy` picks the growth policy (0 = kNextPowerOfTwo,
|
|
193
|
+
* 1 = kSameAsRequested). CPU / ample-VRAM GPUs are unaffected (ORT
|
|
194
|
+
* defaults preserved when absent). */
|
|
195
|
+
readonly sessionOptions: Schema.optional<Schema.Struct<{
|
|
196
|
+
readonly gpuMemLimit: Schema.optional<Schema.Number>;
|
|
197
|
+
readonly arenaExtendStrategy: Schema.optional<Schema.Literals<readonly [0, 1]>>;
|
|
198
|
+
}>>;
|
|
199
|
+
/** First-use auto-install of the optional `@huggingface/transformers`
|
|
200
|
+
* peer (default true) — the package is installed with the project's
|
|
201
|
+
* own package manager on the first local-embed use (scripts skipped,
|
|
202
|
+
* proven unnecessary for embedding). Set false to require a manual
|
|
203
|
+
* install. */
|
|
204
|
+
readonly autoInstall: Schema.optional<Schema.Boolean>;
|
|
118
205
|
}>>;
|
|
119
206
|
}>>;
|
|
120
207
|
readonly jobs: Schema.optional<Schema.Struct<{
|
|
@@ -7,4 +7,8 @@ declare const Service_base: Context.ServiceClass<Service, "@dimple/Config", Inte
|
|
|
7
7
|
export declare class Service extends Service_base {
|
|
8
8
|
}
|
|
9
9
|
export declare const layer: (overrides?: Partial<DimpleConfigType>) => Layer.Layer<Service, Error>;
|
|
10
|
+
/** The Config layer-graph node. `deps` stays empty — the composition root
|
|
11
|
+
* wires dependencies, per the layer-graph convention. */
|
|
12
|
+
export declare const configured: (overrides?: Partial<DimpleConfigType>) => import("../core/index.js").LayerProvider<Service, Error, undefined>;
|
|
13
|
+
export declare const node: import("../core/index.js").LayerProvider<Service, Error, undefined>;
|
|
10
14
|
export {};
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { Context, Effect, Schema } from "effect";
|
|
2
|
-
import type { EmbeddingModelConfig } from "
|
|
3
|
-
import type { LLM } from "
|
|
4
|
-
import { EmbeddingError, type EmbeddingModel } from "
|
|
5
|
-
import type { LogConfig, Logging } from "
|
|
6
|
-
import { Store, graph, type TopicConfig } from "
|
|
7
|
-
import { HybridRetrieval } from "
|
|
8
|
-
import { JobRuntime, JobRegistry, type Handler, type JobDefinition } from "
|
|
2
|
+
import type { EmbeddingModelConfig } from "../config/index.js";
|
|
3
|
+
import type { LLM } from "../llm/index.js";
|
|
4
|
+
import { EmbeddingError, type EmbeddingModel } from "../embedder/index.js";
|
|
5
|
+
import type { LogConfig, Logging } from "../logger/index.js";
|
|
6
|
+
import { Store, graph, type TopicConfig } from "../store/index.js";
|
|
7
|
+
import { HybridRetrieval } from "../retriever/index.js";
|
|
8
|
+
import { JobRuntime, JobRegistry, type Handler, type JobDefinition } from "../workflow/index.js";
|
|
9
9
|
/** Topic-tree config (the store's schema, defaults decoded by the layer). */
|
|
10
|
-
export type { TopicConfig } from "
|
|
10
|
+
export type { TopicConfig } from "../store/index.js";
|
|
11
11
|
/** Graph config (the store's schema — `graph.GraphConfig`). */
|
|
12
12
|
export type GraphConfig = graph.GraphConfig;
|
|
13
13
|
/**
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type EmbeddingModel } from "../embedder/index.js";
|
|
2
|
+
export interface HostedEmbeddingOptions {
|
|
3
|
+
readonly modelId: string;
|
|
4
|
+
readonly model: string;
|
|
5
|
+
readonly dimensions: number;
|
|
6
|
+
readonly apiKey: string;
|
|
7
|
+
/** The AI SDK provider family — see
|
|
8
|
+
* https://ai-sdk.dev/providers for the full provider list. */
|
|
9
|
+
readonly provider?: "openai" | "google" | "cohere" | "bedrock";
|
|
10
|
+
/** OpenAI-compatible endpoint override — ONLY needed for custom
|
|
11
|
+
* endpoints (vLLM/ollama/…). The preconfigured providers have their
|
|
12
|
+
* base URL baked in. */
|
|
13
|
+
readonly baseURL?: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Hosted embeddings — the `ai` SDK IS the implementation (`embedMany`),
|
|
17
|
+
* this adapter only bridges it to the dimple `EmbeddingModel` interface.
|
|
18
|
+
*
|
|
19
|
+
* Provider mapping (config `provider` field):
|
|
20
|
+
* - `"openai"` — openai-compatible provider; `baseURL` defaults to
|
|
21
|
+
* `https://api.openai.com/v1`, override for any compatible endpoint
|
|
22
|
+
* - `"google"` — native `@ai-sdk/google` (gemini-embedding-*), no baseURL
|
|
23
|
+
* - `"cohere"` — native `@ai-sdk/cohere` (embed-*-v3.0), no baseURL
|
|
24
|
+
* - `"bedrock"` — native `@ai-sdk/amazon-bedrock` (titan/nova/cohere
|
|
25
|
+
* models), no baseURL — credentials come from AWS env (or the provider
|
|
26
|
+
* settings)
|
|
27
|
+
*
|
|
28
|
+
* `dimensions` is sent as a provider option where the API supports it
|
|
29
|
+
* (openaiCompatible.dimensions / google.outputDimensionality /
|
|
30
|
+
* cohere.outputDimension) and ALWAYS enforced on the output (some
|
|
31
|
+
* endpoints ignore the option).
|
|
32
|
+
*/
|
|
33
|
+
export declare const makeHostedEmbeddingModel: (options: HostedEmbeddingOptions) => EmbeddingModel;
|
|
@@ -1,3 +1,9 @@
|
|
|
1
1
|
export { Service as Core, make as makeCore, type Interface as CoreInterface, type CoreOptions, } from "./core.ts";
|
|
2
2
|
export { layer as coreLayer, layerConfig as coreLayerConfig } from "./layer.ts";
|
|
3
|
+
export type { CoreGraphServices } from "./layer.ts";
|
|
4
|
+
/** The layer-graph surface, as the reference exposes it: one value namespace
|
|
5
|
+
* whose members are the `make`/`group`/`compile` constructors and the
|
|
6
|
+
* `Graph`/`Node`/`Provider`/`Replacement` types. */
|
|
7
|
+
export * as LayerNode from "../shared/effect/node.js";
|
|
8
|
+
export { compile as compileLayers, group as groupLayers, make as makeLayer, tags as layerTags, unbound as unboundLayer, type Graph as LayerGraph, type Provider as LayerProvider, type Replacement as LayerReplacement, } from "../shared/effect/node.js";
|
|
3
9
|
export { CoreConfigError } from "./core.ts";
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { Layer } from "effect";
|
|
2
|
-
import { Store, DbBackend } from "
|
|
3
|
-
import { EmbeddingError, EmbeddingModelRegistry } from "
|
|
4
|
-
import { HybridRetrieval } from "
|
|
5
|
-
import { JobRuntime, JobRegistry } from "
|
|
6
|
-
import { Logging } from "
|
|
7
|
-
import { Service as ConfigService } from "
|
|
2
|
+
import { Store, DbBackend } from "../store/index.js";
|
|
3
|
+
import { EmbeddingError, EmbeddingModelRegistry } from "../embedder/index.js";
|
|
4
|
+
import { HybridRetrieval } from "../retriever/index.js";
|
|
5
|
+
import { JobRuntime, JobRegistry } from "../workflow/index.js";
|
|
6
|
+
import { Logging } from "../logger/index.js";
|
|
7
|
+
import { Service as ConfigService } from "../config/index.js";
|
|
8
8
|
import type { CoreOptions } from "./core.ts";
|
|
9
9
|
import { CoreConfigError } from "./core.ts";
|
|
10
10
|
import { Service } from "./core.ts";
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Everything the composed graph exposes. Every node is a root, so the
|
|
13
|
+
* compiled group merges each node's outputs: the Core service plus the
|
|
14
|
+
* store (with `DbBackend` and the single `Logging` install), the embedder
|
|
15
|
+
* registry, hybrid retrieval, and the job stack (runtime, registry, durable
|
|
16
|
+
* engine, persistence, queue factory). The workers node contributes no
|
|
17
|
+
* service (it runs fibers).
|
|
18
|
+
*/
|
|
19
|
+
export type CoreGraphServices = Layer.Success<ReturnType<typeof layer>>;
|
|
20
|
+
export declare const layer: (options: CoreOptions) => Layer.Layer<import("effect/persistence/PersistedQueue").PersistedQueueFactory | import("effect/persistence/Persistence").Persistence | Logging | DbBackend | JobRuntime | JobRegistry | Store | EmbeddingModelRegistry | HybridRetrieval | Service | import("effect/workflow/WorkflowEngine").WorkflowEngine, CoreConfigError | EmbeddingError | import("../retriever/index.js").HybridError | import("../workflow/index.js").RegistryError | import("effect/sql/SqlError").SqlError | import("../store/index.js").StoreError, never>;
|
|
12
21
|
/**
|
|
13
22
|
* Config-driven variant — the `dimple.jsonc` path (the SDK's
|
|
14
23
|
* `makeDimple({ config })`): reads the resolved `DimpleConfig` from
|
|
@@ -18,4 +27,4 @@ export declare const layer: (options: CoreOptions) => Layer.Layer<import("effect
|
|
|
18
27
|
*/
|
|
19
28
|
export declare const layerConfig: (options?: {
|
|
20
29
|
readonly migrationFolder?: string;
|
|
21
|
-
}) => Layer.Layer<import("effect/
|
|
30
|
+
}) => Layer.Layer<import("effect/persistence/PersistedQueue").PersistedQueueFactory | import("effect/persistence/Persistence").Persistence | Logging | DbBackend | JobRuntime | JobRegistry | Store | EmbeddingModelRegistry | HybridRetrieval | Service | import("effect/workflow/WorkflowEngine").WorkflowEngine, CoreConfigError | EmbeddingError | import("../retriever/index.js").HybridError | import("../workflow/index.js").RegistryError | import("effect/sql/SqlError").SqlError | import("../store/index.js").StoreError, ConfigService>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* First-use auto-install of the OPTIONAL `@huggingface/transformers` peer.
|
|
3
|
+
*
|
|
4
|
+
* `@huggingface/transformers` is an optional peer (its tree pulls the
|
|
5
|
+
* `sharp` + `onnxruntime-node` install scripts, which npm 12 blocks by
|
|
6
|
+
* default and npm 11 runs with an advisory warning — npm RFC #0054). When
|
|
7
|
+
* the config declares a `transformers` provider model,
|
|
8
|
+
* the intent is explicit — so the FIRST local-embed use installs the
|
|
9
|
+
* package automatically with the project's own package manager, instead
|
|
10
|
+
* of failing with a manual step.
|
|
11
|
+
*
|
|
12
|
+
* Thin wrapper over the shared `@dimple/installer` machinery (manager
|
|
13
|
+
* detection, script-skipping argv, timeout-kill, in-flight dedup, win32
|
|
14
|
+
* shims, pnpm `-w`); failures map onto the typed `EmbeddingError` with
|
|
15
|
+
* the manual command.
|
|
16
|
+
*/
|
|
17
|
+
import { Effect } from "effect";
|
|
18
|
+
import { EmbeddingError } from "./provider.ts";
|
|
19
|
+
import { type PackageManager } from "../installer/index.js";
|
|
20
|
+
export type { InstallTarget, PackageManager } from "../installer/index.js";
|
|
21
|
+
export { detectPackageManager } from "../installer/index.js";
|
|
22
|
+
/** The install argv for a manager with the transformers package (kept for
|
|
23
|
+
* callers/tests that assert the shape). */
|
|
24
|
+
export declare const installCommand: (pm: PackageManager) => readonly [string, ...string[]];
|
|
25
|
+
/** Install transformers at the project root detected from `cwd`. The
|
|
26
|
+
* shared installer kills the child on timeout and dedups concurrent
|
|
27
|
+
* first-use installs; failures fail typed with the manual command. */
|
|
28
|
+
export declare const installTransformers: (cwd: string) => Effect.Effect<void, EmbeddingError>;
|
|
@@ -1,4 +1,3 @@
|
|
|
1
1
|
export { Service as EmbeddingModelRegistry, type Interface as EmbeddingModelRegistryInterface, type EmbeddingModel, EmbeddingError, } from "./provider.ts";
|
|
2
|
-
export { layer as transformersEmbeddingLayer, layerConfig as transformersEmbeddingLayerConfig, makeTransformersModel, } from "./transformers.ts";
|
|
3
|
-
export {
|
|
4
|
-
export { layer as registryLayer, type ModelSpec } from "./registry.ts";
|
|
2
|
+
export { layer as transformersEmbeddingLayer, layerConfig as transformersEmbeddingLayerConfig, makeTransformersModel, buildSessionOptions, } from "./transformers.ts";
|
|
3
|
+
export { layer as registryLayer, configured as registryConfigured, type ModelSpec, } from "./registry.ts";
|
|
@@ -8,3 +8,10 @@ export interface ModelSpec {
|
|
|
8
8
|
readonly apiKey?: string;
|
|
9
9
|
}
|
|
10
10
|
export declare const layer: (...models: ModelSpec[]) => Layer.Layer<Service, EmbeddingError>;
|
|
11
|
+
/**
|
|
12
|
+
* Node declaration over `layer`, for the composition root: the graph wires
|
|
13
|
+
* the dependencies, so `deps` stays empty here. The variadic model list is
|
|
14
|
+
* the declaration's whole configuration — no `node` default, because an
|
|
15
|
+
* empty registry has no models and `defaultModelId` fails typed.
|
|
16
|
+
*/
|
|
17
|
+
export declare const configured: (...models: ModelSpec[]) => import("../core/index.js").LayerProvider<Service, EmbeddingError, undefined>;
|
|
@@ -1,16 +1,54 @@
|
|
|
1
1
|
import { Config, Layer } from "effect";
|
|
2
|
-
import type { DeviceType } from "@huggingface/transformers";
|
|
3
2
|
import { Service, type EmbeddingModel } from "./provider.ts";
|
|
3
|
+
/** Mirror of transformers.js's `DeviceType` (the config schema carries the
|
|
4
|
+
* same literals) — kept local so the optional peer's types never appear
|
|
5
|
+
* in this module's surface. */
|
|
6
|
+
type DeviceType = "auto" | "gpu" | "cpu" | "wasm" | "webgpu" | "cuda" | "dml" | "coreml" | "webnn";
|
|
7
|
+
/** The ORT session-options shape transformers.js forwards to
|
|
8
|
+
* onnxruntime-node (executionProviders with a bounded CUDA arena). */
|
|
9
|
+
interface OrtSessionOptions {
|
|
10
|
+
readonly executionProviders: readonly [
|
|
11
|
+
{
|
|
12
|
+
readonly name: string;
|
|
13
|
+
readonly device_id: number;
|
|
14
|
+
readonly gpu_mem_limit?: number;
|
|
15
|
+
readonly arena_extend_strategy?: 0 | 1;
|
|
16
|
+
}
|
|
17
|
+
];
|
|
18
|
+
}
|
|
19
|
+
/** Map the focused dimple-level pair to ORT's snake_case session keys.
|
|
20
|
+
* Both fields absent → `undefined`, so the caller can skip the whole
|
|
21
|
+
* sessionOptions (ORT defaults preserved). */
|
|
22
|
+
export declare const buildSessionOptions: (opts: {
|
|
23
|
+
readonly gpuMemLimit?: number | undefined;
|
|
24
|
+
readonly arenaExtendStrategy?: 0 | 1 | undefined;
|
|
25
|
+
}) => OrtSessionOptions;
|
|
4
26
|
interface TransformersOptions {
|
|
5
27
|
readonly modelId?: string;
|
|
6
28
|
readonly device?: DeviceType;
|
|
7
|
-
readonly dtype?: "fp32" | "fp16" | "q8" | "q4";
|
|
29
|
+
readonly dtype?: "fp32" | "fp16" | "q8" | "int8" | "uint8" | "q4" | "bnb4" | "q4f16" | "q2" | "q2f16" | "q1" | "q1f16";
|
|
8
30
|
readonly pooling?: "mean" | "cls" | "none";
|
|
9
31
|
readonly normalize?: boolean;
|
|
10
32
|
readonly cacheDir?: string;
|
|
33
|
+
/** ONNX Runtime CUDA session options — the verified fix for GPUs with
|
|
34
|
+
* limited VRAM (≤8GB): ORT's default arena is unbounded (grows by
|
|
35
|
+
* powers of two), so batch inference OOMs mid-run and leaves the
|
|
36
|
+
* process unable to allocate even a cublas handle afterwards. Forwarded
|
|
37
|
+
* to the pipeline's sessionOptions; absent = ORT defaults (CPU and
|
|
38
|
+
* ample-VRAM GPUs unaffected). */
|
|
39
|
+
readonly sessionOptions?: {
|
|
40
|
+
readonly gpuMemLimit?: number | undefined;
|
|
41
|
+
readonly arenaExtendStrategy?: 0 | 1 | undefined;
|
|
42
|
+
};
|
|
11
43
|
/** Declared output dimension — validated against the model's actual
|
|
12
44
|
* output on first embed (a mismatch fails typed, precisely). */
|
|
13
45
|
readonly dimensions?: number;
|
|
46
|
+
/** First-use auto-install of the optional `@huggingface/transformers`
|
|
47
|
+
* peer (default true): the config declaring a transformers provider is
|
|
48
|
+
* explicit intent — the missing package is installed with the project's
|
|
49
|
+
* own package manager (scripts skipped; proven unnecessary for
|
|
50
|
+
* embedding). Set false to require a manual install. */
|
|
51
|
+
readonly autoInstall?: boolean;
|
|
14
52
|
}
|
|
15
53
|
export declare const makeTransformersModel: (options?: TransformersOptions) => EmbeddingModel;
|
|
16
54
|
export declare const layer: (options?: TransformersOptions) => Layer.Layer<Service>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Context, Effect, Layer } from "effect";
|
|
2
|
-
import type { TreeHandle } from "
|
|
2
|
+
import type { TreeHandle } from "../parser/index.js";
|
|
3
3
|
import { ExtractedFile, ExtractionError } from "./models/extraction";
|
|
4
4
|
export interface ExtractRequest {
|
|
5
5
|
readonly filePath: string;
|
|
@@ -12,4 +12,14 @@ declare const Service_base: Context.ServiceClass<Service, "@dimple/extractor/Cod
|
|
|
12
12
|
export declare class Service extends Service_base {
|
|
13
13
|
}
|
|
14
14
|
export declare const layer: Layer.Layer<Service, never, never>;
|
|
15
|
+
/**
|
|
16
|
+
* Layer-graph declaration for `CodeExtractor`. The layer has no
|
|
17
|
+
* requirements of its own (a `TreeHandle` is passed per call, not
|
|
18
|
+
* injected), so `deps` stays empty and the composition root wires the
|
|
19
|
+
* parser that produces those handles.
|
|
20
|
+
*/
|
|
21
|
+
export declare const configured: () => import("../core/index.js").LayerProvider<Service, never, undefined>;
|
|
22
|
+
/** The default declaration — the live implementation the composition root
|
|
23
|
+
* picks unless it replaces this node. */
|
|
24
|
+
export declare const node: import("../core/index.js").LayerProvider<Service, never, undefined>;
|
|
15
25
|
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { Service as CodeExtractor, layer as codeExtractorLayer } from "./code-extractor";
|
|
1
|
+
export { Service as CodeExtractor, layer as codeExtractorLayer, configured as codeExtractorConfigured, node as codeExtractorNode, } from "./code-extractor";
|
|
2
2
|
export type { ExtractRequest } from "./code-extractor";
|
|
3
3
|
export { ExtractedFile, ExtractedChunk, ImportInfo, ExtractionError } from "./models/extraction";
|
|
4
4
|
export { Service as ContentHandler, layer as contentHandlerLayer, ContentError, } from "./content-handler";
|
|
@@ -15,6 +15,13 @@ export declare const ExtractedChunk: Schema.Struct<{
|
|
|
15
15
|
readonly endLine: Schema.Number;
|
|
16
16
|
readonly language: Schema.String;
|
|
17
17
|
readonly docstring: Schema.optional<Schema.String>;
|
|
18
|
+
/** Call sites inside this definition — the code_calls edge source:
|
|
19
|
+
* `[{ name, line }]`, callee names only (member/property calls unwrap
|
|
20
|
+
* to the property name). */
|
|
21
|
+
readonly calls: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
22
|
+
readonly name: Schema.String;
|
|
23
|
+
readonly line: Schema.Number;
|
|
24
|
+
}>>>;
|
|
18
25
|
}>;
|
|
19
26
|
export interface ExtractedChunk extends Schema.Schema.Type<typeof ExtractedChunk> {
|
|
20
27
|
}
|
|
@@ -40,6 +47,13 @@ export declare const ExtractedFile: Schema.Struct<{
|
|
|
40
47
|
readonly endLine: Schema.Number;
|
|
41
48
|
readonly language: Schema.String;
|
|
42
49
|
readonly docstring: Schema.optional<Schema.String>;
|
|
50
|
+
/** Call sites inside this definition — the code_calls edge source:
|
|
51
|
+
* `[{ name, line }]`, callee names only (member/property calls unwrap
|
|
52
|
+
* to the property name). */
|
|
53
|
+
readonly calls: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
54
|
+
readonly name: Schema.String;
|
|
55
|
+
readonly line: Schema.Number;
|
|
56
|
+
}>>>;
|
|
43
57
|
}>>;
|
|
44
58
|
readonly imports: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
45
59
|
readonly source: Schema.String;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export type PackageManager = "bun" | "npm" | "pnpm" | "yarn";
|
|
2
|
+
export interface InstallTarget {
|
|
3
|
+
readonly pm: PackageManager;
|
|
4
|
+
readonly root: string;
|
|
5
|
+
}
|
|
6
|
+
export declare class InstallerError extends Error {
|
|
7
|
+
constructor(message: string);
|
|
8
|
+
}
|
|
9
|
+
/** Detect the project's package manager by walking UP from `cwd` for a
|
|
10
|
+
* lockfile. NO lockfile → npm (the most universal — a bun fallback broke
|
|
11
|
+
* Node-only machines with ENOENT). */
|
|
12
|
+
export declare const detectPackageManager: (cwd: string) => InstallTarget;
|
|
13
|
+
/** The executable name for a manager. win32 ships `.cmd` shims for
|
|
14
|
+
* npm/pnpm/yarn — the bare names fail with ENOENT. */
|
|
15
|
+
export declare const resolveExecutable: (pm: PackageManager, platform?: NodeJS.Platform) => string;
|
|
16
|
+
/** The base install argv for a manager. Scripts are skipped everywhere;
|
|
17
|
+
* pnpm at a workspace root additionally gets `-w`. */
|
|
18
|
+
export declare const installArgs: (pm: PackageManager, packages: readonly string[], root: string) => string[];
|
|
19
|
+
/** The slice of a spawned child the installer uses; tests inject a fake. */
|
|
20
|
+
export interface InstallChild {
|
|
21
|
+
on(event: "error", listener: (cause: Error) => void): void;
|
|
22
|
+
on(event: "exit", listener: (code: number | null, signal: string | null) => void): void;
|
|
23
|
+
kill(signal?: NodeJS.Signals): boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface InstallOptions {
|
|
26
|
+
readonly cwd?: string;
|
|
27
|
+
/** Hard deadline; the child is SIGKILLed when it trips (default 5 min). */
|
|
28
|
+
readonly timeoutMs?: number;
|
|
29
|
+
/** Injectable for tests (defaults to process.platform). */
|
|
30
|
+
readonly platform?: NodeJS.Platform;
|
|
31
|
+
/** Injectable for tests; defaults to `node:child_process.spawn`. */
|
|
32
|
+
readonly spawnImpl?: (command: string, args: readonly string[], options: {
|
|
33
|
+
readonly cwd: string;
|
|
34
|
+
readonly stdio: "ignore";
|
|
35
|
+
}) => InstallChild;
|
|
36
|
+
}
|
|
37
|
+
export declare const installOptionalPeers: (packages: readonly string[], opts?: InstallOptions) => Promise<void>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LLM } from "./llm.ts";
|
|
2
|
-
import type { DreamClassifier } from "
|
|
2
|
+
import type { DreamClassifier } from "../store/index.js";
|
|
3
3
|
/**
|
|
4
4
|
* Builds a `DreamClassifier` from a generic `LLM`. `modelId` (when given)
|
|
5
5
|
* is mentioned in the system prompt so the model knows which embedding
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { LLMError, makeLLM, type GenerateTextInput, type GenerateObjectInput, type LLM, } from "./llm.ts";
|
|
2
2
|
export { openAICompatible, type OpenAICompatibleOptions } from "./provider.ts";
|
|
3
3
|
export { makeDreamClassifier } from "./dream-classifier.ts";
|
|
4
|
-
export { Service as LLMService, type Interface as LLMServiceInterface, type LLMOptions, layer as llmLayer, layerConfig as llmLayerConfig, layerFromConfig as llmLayerFromConfig, } from "./service.ts";
|
|
4
|
+
export { Service as LLMService, type Interface as LLMServiceInterface, type LLMOptions, layer as llmLayer, layerConfig as llmLayerConfig, layerFromConfig as llmLayerFromConfig, configured as llmConfigured, } from "./service.ts";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Config, Context, Layer } from "effect";
|
|
2
|
-
import { Service as ConfigService } from "
|
|
2
|
+
import { Service as ConfigService } from "../config/index.js";
|
|
3
3
|
import { type LLM, type LLMDefaults } from "./llm.ts";
|
|
4
4
|
/**
|
|
5
5
|
* Connection options for the OpenAI-compatible LLM layer: the chat
|
|
@@ -26,4 +26,10 @@ export declare const layerConfig: (config: Config.Wrap<LLMOptions>) => Layer.Lay
|
|
|
26
26
|
* section). Fails with a clear error when the section is absent.
|
|
27
27
|
*/
|
|
28
28
|
export declare const layerFromConfig: () => Layer.Layer<Service, Error, ConfigService>;
|
|
29
|
+
/**
|
|
30
|
+
* Node declaration over `layer`, for the composition root: the graph wires
|
|
31
|
+
* the dependencies, so `deps` stays empty here. `LLMOptions` is all-required
|
|
32
|
+
* (endpoint, key, model), so there is no `node` default to declare.
|
|
33
|
+
*/
|
|
34
|
+
export declare const configured: (options: LLMOptions) => import("../core/index.js").LayerProvider<Service, never, undefined>;
|
|
29
35
|
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { Service as Logging, layer as loggingLayer, layerConfig as loggingLayerConfig,
|
|
1
|
+
export { Service as Logging, layer as loggingLayer, layerConfig as loggingLayerConfig, configured as loggingConfigured, node as loggingNode, silentConfig as loggingSilentConfig, makeEffect as makeLogging, consoleSink, fileSink, effectLogger, type Sink, type Interface as LoggingInterface, type StructuredOutput, } from "./service.ts";
|
|
2
2
|
export { LogConfig, LogLevel, SEV_NUM, LEVEL_ORDER, type LogEntry } from "./types.ts";
|
|
3
3
|
export { RingBuffer, type RingFrame, type Subscriber } from "./ring.ts";
|
|
4
4
|
export { renderJson, renderPretty } from "./format.ts";
|