@ponraaj/dimple-sdk 0.0.0-alpha.4 → 0.0.0-alpha.5

package/README.md

@@ -12,6 +12,25 @@ thin facade over it.
 
 > **Status**: alpha (`0.0.0-alpha.n`). The API is stabilizing but may change.
 
+### The one-liner
+
+```ts
+import { makeDimple } from "@ponraaj/dimple-sdk"; // ← makeDimple, not dimple
+
+const dimple = await makeDimple({ config: "./dimple.jsonc" });
+const hits = await dimple.query("what did we decide about X", {
+  limit: 5,
+  contentType: "text/plain", // ← code-vs-message separation
+}); // query(text, options) — text is the FIRST positional argument
+await dimple.dispose();
+```
+
+Calling conventions: every method takes **plain arguments** (`query(text,
+opts)`, `write(input, opts)`, `index({ paths })`), every failure is a
+rejected promise with a closed `_tag` (`"Sdk.NotFound"`, `"Sdk.InputError"`,
+…), and `dispose()` is **required** before `process.exit(0)` — the durable
+worker keeps the event loop alive (`dispose()` stops it; it is idempotent).
+
 ## Install
 
 ```bash
@@ -38,7 +57,9 @@ const mem = await dimple.write({
 });
 
 // hybrid semantic + FTS search
-const results = await dimple.query("how does the pipeline deploy", { limit: 3 });
+const results = await dimple.query("how does the pipeline deploy", {
+  limit: 3,
+});
 
 // index real code — files + extracted functions/classes/methods become memories
 const indexed = await dimple.index({ paths: ["./src"] });
@@ -116,6 +137,53 @@ const dimple = await makeDimple({
 | `defaultK` / `defaultLimit` | numbers                                                                                                                         | retrieval defaults                         |
 | `defaultTtlSecs`            | number                                                                                                                          | write TTL default                          |
 
+### Remote databases (Turso)
+
+`storeUrl` and `jobs.dbUrl` accept **remote libSQL URLs** (`libsql://`,
+`https://`, `ws://` — Turso or any libsql-compatible host). The same
+engine runs everything the local file does — FTS5, vector search, the
+topic tree, durable jobs — over the wire. Verified end-to-end against
+the real Turso service (`tests/store/turso-remote.test.ts`).
+
+Turso tokens are **per-database**, so each URL carries its own token.
+`dimple.jsonc` resolves `${ENV_VAR}` in values, so tokens stay out of
+config files:
+
+```jsonc
+{
+  "storeUrl": "libsql://<org>-<memdb>.turso.io?authToken=${TURSO_MEMORIES_TOKEN}",
+  "jobs": {
+    "dbUrl": "libsql://<org>-<jobsdb>.turso.io?authToken=${TURSO_JOBS_TOKEN}",
+  },
+  "embedding": {
+    /* hosted providers for serverless — local models are
+                    local-only (auto-install needs Node) */
+  },
+}
+```
+
+Notes:
+
+- The **jobs DB must also be remote** when the store is remote (the
+  durable queue + workflow state are cross-invocation)
+- `file:`-only behaviors (parent-dir creation, migration lock) skip
+  for remote URLs
+- A single-DB setup can share one URL + one token (`TURSO_TOKEN`)
+- For Cloudflare Workers the client swaps to the pure-JS entry
+  (`@libsql/client/web` — one bundler alias, no source changes)
+
+### Embedding models
+
+Local embeddings (`provider: "transformers"`) auto-install on first
+use — `@huggingface/transformers` is an **optional peer** so hosted-only
+consumers never pull its tree (the `sharp` + `onnxruntime-node`
+postinstall scripts that abort npm 11 installs). The first `embed()`
+installs it with the project's own package manager, scripts skipped
+(proven unnecessary for embedding), and retries in-process. Offline or
+restricted environments get a typed `Sdk.Embedding` error with the manual
+command. Set `autoInstall: false` on the model to disable:
+`embedding.models[].autoInstall: false`
+
 ### Embedding models
 
 Two providers — **local** (Transformers.js ONNX, runs on your CPU, no
@@ -223,6 +291,11 @@ active `logicalKey` → `Sdk.DuplicateKey` (carries `logicalKey`/`namespace`).
 **Memory expiry**: `ttlSecs` (or the config `defaultTtlSecs`) schedules
 expiry; a sweep job retires expired memories in the background.
 
+`query(text, { budget: { maxUnits, maxChars, maxMs } })` — LLM-facing
+context guardrail: caps the row count, truncates result content to the
+total char budget, and marks `results.truncated` when the soft time cap
+is exceeded.
+
 ### `get(id)` / `forget(id)`
 
 Fetch one memory; soft-delete (cascades edges, rebalances the tree).
@@ -260,7 +333,7 @@ await dimple.index({
   maxChunkBytes?: 8 * 1024,
   extensions?: [".ts", ".tsx", ".js", ".jsx", …],
 });
-// → { files, indexed, skipped, unsupported, failed: [{ filePath, message }] }
+// → { files, indexed, updated, reconciled, skipped, commit, unsupported, failed }
 ```
 
 - Tree-sitter parse (TypeScript/JavaScript/TSX/JSX) → the extractor
@@ -271,8 +344,27 @@ await dimple.index({
 - AST-derived graph: relative imports resolve to indexed files and
   class→method containment become **confirmed `memory_supports` edges**
   (`source: "rule"`)
-- **Idempotent**: keys are cwd-anchored + deterministic — re-indexing the
-  same tree skips existing units
+- **Idempotent + upserting**: keys are cwd-anchored + deterministic
+  (chunk keys drop the line number, so edits ABOVE a definition update
+  it instead of duplicating it). Re-indexing skips unchanged units
+  (`skipped`), and changed units are **superseded + rewritten under the
+  same key** (`updated`) with a confirmed identity `memory_supersedes`
+  edge — stale copies never accumulate
+- **Edges re-form after `forget`**: phase-2 resolves endpoints by logical
+  key against the store (not just the current run), so a re-created unit
+  re-connects to its pre-existing dependencies
+- **Reconcile + rename detection**: units whose source vanished (deleted
+  file/definition) are demoted (`reconciled`, `source_removed` — gone
+  from search; domain-scoped to the walked roots, so `index lib` never
+  touches other files). A vanished unit whose embedding is a near-duplicate
+  of a new unit becomes a PENDING `memory_supersedes` (reason rename) —
+  `graph confirm` completes it. Code units are TTL-exempt
+  (`ttlSecs: null` — the index owns their lifecycle); `commit` stamps the
+  repo HEAD into every unit's metadata as `codeVersion`
+- **`code_calls` edges**: every call site inside a definition resolves to
+  the callee's chunk (same file, then imported files) → confirmed edges —
+  `explain(id, { edgeKinds: ["code_calls"] })` gives one-hop impact
+  analysis
 - Per-file failures are collected, never fatal
 
 ### `updateEmbedding(id, embedding, modelKey?)` / `embed(text)`
@@ -288,13 +380,19 @@ requires the `llm` config section) or the graph repair pass. Each call
 gets a fresh run id (always executes). Failures are typed:
 `Sdk.LLMNotConfigured`, `Sdk.JobFailed`.
 
-### `explain({ kind: "memory" | "topic" | "concept", id })`
+### `explain({ kind: "memory" | "topic" | "concept", id }, options?)`
 
 Graph neighborhood traversal — `{ nodes: [{ kind, id, depth, edgeKind }] }`.
+`options.edgeKinds` scopes the answer (memory-only navigation: pass the
+`memory_*` kinds; `code_calls` shows the call graph).
 
 ### `health(modelKey)` / `graph.*`
 
 - `health` — topic-tree invariant sweep (`warnCount` + violation strings)
+- `graph.confirm(id)` / `graph.reject(id)` / `graph.expire(id)` — the
+  pending-edge lifecycle: promote to confirmed (a `memory_supersedes`
+  confirm demotes the old endpoint in the same transaction), reject, or
+  soft-delete. `Sdk.EdgeNotFound` for stale ids
 - `graph.addEdge({ fromMemoryId, toMemoryId, edgeKind: "memory_supports" |
 "memory_supersedes" | "memory_contradicts", source: "rule" | "llm" |
 "agent", weight?, confidence?, confirmed? })` — explicit edges; typed
@@ -302,6 +400,42 @@ Graph neighborhood traversal — `{ nodes: [{ kind, id, depth, edgeKind }] }`.
 - `graph.candidates()` — pending structural candidates + isolated count
 - `graph.coverage(modelKey)` — coverage %, pending, concept count
 
+### `reload(input?)` — apply a config change
+
+Pauses and re-boots with the re-resolved config:
+
+1. the config source is re-resolved **typed** — a `dimple.jsonc` path is
+   re-read from disk, an inline object re-validated (a bad config rejects
+   the reload with **zero impact** — nothing pauses);
+2. an **identical** config resolves as `{ reloaded: false }` — no pause,
+   no restart (safe to call from a file watcher);
+3. new calls wait, in-flight calls settle on the old stack;
+4. a fresh stack boots (new worker fleet, embedders, logger — jobs are
+   durable, so an interrupted handler's lease expires and re-claims after
+   boot);
+5. the swap: the old stack is disposed, the new one serves.
+
+Resolves with `{ config, changed, reloaded }` — `changed` flags which
+sections actually moved (`storeUrl` / `jobsUrl` / `rest`).
+
+Works for `{ config: … }`-built dimples (reload with no argument re-reads
+the original source). `{ core: … }`-built dimples have no config source
+and reject with `Sdk.Reload`. Boot failure leaves the previous system
+running (blue-green); a drain that cannot settle in-flight calls (30s)
+rejects with `Sdk.Reload` and nothing is torn down.
+
+**What happens to data when the database URL changes:** the old database
+file is preserved on disk but no longer served — nothing is migrated or
+merged, the new db is bootstrapped empty (a warning is logged for the
+switch; roll back with `reload(oldConfig)`). Pending jobs in a switched
+`jobs.dbUrl` stay durable in the old db, re-claimable by switching back
+or draining it externally.
+
+```ts
+await dimple.reload();                        // re-read dimple.jsonc
+await dimple.reload({ storeUrl: "file:…", … }); // or a fresh config
+```
+
 ### `dispose()`
 
 Stops the worker, closes sinks + the store. Idempotent; post-dispose calls
@@ -355,6 +489,65 @@ query history, persisted asynchronously (never blocks the query).
 Every release creates a GitHub Release with the per-version changes (the
 tag-based publish workflow): repo → **Releases** page.
 
+## Agent integration patterns
+
+These are the shapes that work well when the consumer is another agent
+(a harness) rather than a human with a REPL.
+
+### Session-start injection
+
+The guaranteed-usage path: a warmup query returns a compact memory block
+the harness injects as context, and the pending-graph review list rides
+along:
+
+```ts
+const [context, pending] = await Promise.all([
+  dimple.query("ongoing work and decisions", { limit: 8, budget: { maxChars: 4000 } }),
+  dimple.graph.candidates(),
+]);
+// context  → inject the top-N content strings into the system prompt
+// pending  → offer each candidate as a confirm/reject decision
+```
+
+`graph.candidates()` pending `memory_supersedes` edges from the rename
+scanner (rule 0.9x near-duplicates) are the highest-value batch to review.
+
+### Supersedes recipe
+
+`addEdge({ … edgeKind: "memory_supersedes", confirmed: true })` performs
+the full state flip in one transaction (the old memory demotes
+immediately) — it is safe to use directly. For human-gated flows the
+pending → confirm pair keeps the old memory ranking until a reviewer
+agrees:
+
+```ts
+const edge = await dimple.graph.addEdge({
+  fromMemoryId: oldId,
+  toMemoryId: newId,
+  edgeKind: "memory_supersedes",
+  source: "agent",
+}); // pending — nothing changed yet
+await dimple.graph.confirm(edge.id); // ← the flip happens here
+```
+
+### Namespaces as the code/message partition
+
+Until a query-time content-type filter exists, namespaces give clean
+isolation: `index({ paths, namespace: "code" })` and
+`write({ …, namespace: "messages" })` partition the store; query with
+`namespace: "code"` for code and `namespace: "messages"` for
+conversations. (`query({ contentType })` now does the same within one
+namespace — the two levers compose.)
+
+### Remember
+
+- `dimple.reload(newConfig)` swaps the whole stack blue-green (validate →
+  pause → boot → drain → swap); a failed boot leaves the old stack
+  serving.
+- JSON output is compact by default: CLI `--json` omits embeddings (pass
+  `--embedding` to include them); the SDK always keeps them on
+  `memory.embedding`.
+
 ## Notes
 
 - Determinism: same data + same config ⇒ same tree, same clusters, same

package/dist/dimple.d.ts

@@ -1,9 +1,11 @@
-import { ManagedRuntime } from "effect";
+import { Effect } from "effect";
 import type { QueryProfile } from "./internal/retriever/index.js";
 import type { CoreInterface } from "./internal/core/index.js";
 import type { CodeExtractor } from "./internal/extractor/index.js";
 import type { CodeParser } from "./internal/parser/index.js";
+import type { DimpleConfig } from "./internal/config/index.js";
 import type { AddEdgeInput, AugmentedMemory, ExplainResult, GraphCandidatesResult, GraphCoverage, MemoryUnit, TopicHealth, WriteMemoryInput } from "./internal/store/index.js";
+import type { ConfigInput } from "./config.ts";
 import { type DimpleError } from "./errors.ts";
 /** Options for a `query` call — mirrors the hybrid retriever's options. */
 export interface QueryOptions {
@@ -16,6 +18,19 @@ export interface QueryOptions {
     readonly topics?: {
         readonly topicKeys?: ReadonlyArray<string>;
     };
+    /** Content-type filter: one kind or a list (e.g. `"text/x-typescript"`
+     * or `["text/plain", "text/markdown"]`) — the code-vs-message
+     * separation lever at query time. */
+    readonly contentType?: string | readonly string[];
+    /** Retrieval context budget (LLM-facing guardrail): `maxUnits` caps the
+     * row count, `maxChars` truncates result content to fit the total char
+     * budget, `maxMs` marks the result truncated when the query exceeds it.
+     * Truncation is surfaced as `results.truncated` on the returned array. */
+    readonly budget?: {
+        readonly maxUnits?: number;
+        readonly maxChars?: number;
+        readonly maxMs?: number;
+    };
 }
 /** Options for a `write` call. */
 export interface WriteOptions {
@@ -49,22 +64,58 @@ export interface IndexResult {
     readonly files: number;
     /** Memory units written (file unit + extracted chunks). */
     readonly indexed: number;
-    /** Units skipped as already-indexed (duplicate logical keys). */
+    /** Units whose active copy changed content — the old unit was
+     * superseded (confirmed `memory_supersedes` edge) and the new unit
+     * written under the same key. */
+    readonly updated: number;
+    /** Units whose source vanished from disk (deleted/moved file or
+     * definition) — demoted to superseded (`source_removed`); they no
+     * longer match queries. */
+    readonly reconciled: number;
+    /** The repo HEAD at index time (null outside a git tree) — stamped into
+     * every file/chunk unit's metadata as `codeVersion`. */
+    /** Units skipped as already-indexed (unchanged content under the same
+     * logical key). */
     readonly skipped: number;
     /** Files skipped because the extension is not indexable. */
     readonly unsupported: number;
+    readonly commit: string | null;
     /** Files that could not be walked/read/parsed/extracted/written. */
     readonly failed: readonly {
         readonly filePath: string;
         readonly message: string;
     }[];
 }
+/** What a `reload` actually changed — so callers can react (alert, log,
+ * refuse) instead of guessing. */
+export interface ReloadChanges {
+    /** The store database URL changed — the new db was bootstrapped (empty
+     * schema); the old db file is preserved on disk but no longer served.
+     * Nothing is migrated or merged. */
+    readonly storeUrl: boolean;
+    /** The jobs database URL changed — pending jobs stay durable in the
+     * old db (re-claimable by switching back or draining it externally). */
+    readonly jobsUrl: boolean;
+    /** Any other section changed (models, topics, graph, logging, …). */
+    readonly rest: boolean;
+}
+/** Result of a `reload`. */
+export interface ReloadResult {
+    readonly config: DimpleConfig;
+    readonly changed: ReloadChanges;
+    /** `false` when the resolved config was identical to the live one —
+     * nothing was paused, booted, or restarted (safe for watchers). */
+    readonly reloaded: boolean;
+}
 /**
  * The promise-based SDK facade — the ONLY surface app code touches.
  * Every method runs the underlying Effect at the boundary, remaps errors
  * onto the closed `DimpleError` hierarchy, and never leaks Effect types.
  */
 export interface Dimple {
+    /** The config-default embedding model's key — what `dream`/`repair`/
+     * `stats` use when no model is passed. */
+    readonly defaultModelId: string;
     readonly write: (input: WriteMemoryInput, options?: WriteOptions) => Promise<MemoryUnit>;
     readonly get: (id: string) => Promise<MemoryUnit>;
     readonly query: (text: string, options?: QueryOptions) => Promise<AugmentedMemory[]>;
@@ -83,19 +134,66 @@ export interface Dimple {
     readonly embed: (text: string) => Promise<number[]>;
     readonly dream: (modelKey: string) => Promise<void>;
     readonly repair: (modelKey: string) => Promise<void>;
+    /** The store's tables + columns (snake_case DB names — the SDK exposes
+     * camelCase domain names; `dimple db-schema` prints this map). */
+    readonly dbSchema: () => Promise<ReadonlyArray<{
+        readonly table: string;
+        readonly columns: ReadonlyArray<{
+            readonly name: string;
+            readonly type: string;
+        }>;
+    }>>;
+    /** Options for `explain` — pass `edgeKinds` for a scoped answer (the
+     * common ask is memory-only navigation: the three `memory_*` kinds keep
+     * the topic/concept legs out). */
     readonly explain: (input: {
         readonly kind: "memory" | "topic" | "concept";
         readonly id: string;
+    }, options?: {
+        readonly edgeKinds?: readonly string[];
+        readonly maxDepth?: number;
+        readonly maxNeighbors?: number;
+        readonly maxTotal?: number;
+        readonly includePending?: boolean;
     }) => Promise<ExplainResult>;
     readonly health: (modelKey: string) => Promise<TopicHealth>;
     readonly graph: {
         readonly addEdge: (input: AddEdgeInput) => Promise<unknown>;
         readonly candidates: () => Promise<GraphCandidatesResult>;
         readonly coverage: (modelKey: string) => Promise<GraphCoverage>;
+        /** Promote a pending edge to confirmed (for `memory_supersedes` the
+         * old endpoint flips to superseded in the same transaction). */
+        readonly confirm: (id: string) => Promise<void>;
+        /** Reject a pending edge (removed from the pending review set). */
+        readonly reject: (id: string) => Promise<void>;
+        /** Soft-delete an edge (`expired_at` set; excluded from traversal). */
+        readonly expire: (id: string) => Promise<void>;
     };
+    /**
+     * Apply a config change: pause (new calls wait, in-flight calls settle),
+     * boot a fresh stack with the re-resolved config, and swap it in.
+     *
+     * With no argument the ORIGINAL config source is re-resolved — a
+     * `dimple.jsonc` FILE PATH is re-read from disk, an inline object is
+     * re-validated. A config that fails validation or boot rejects the
+     * reload and leaves the current system untouched. An identical config
+     * resolves as `{ reloaded: false }` — no pause, no restart.
+     */
+    readonly reload: (input?: ConfigInput) => Promise<ReloadResult>;
     /** Stops the worker, closes the sinks and the store — idempotent. */
     readonly dispose: () => Promise<void>;
 }
+/**
+ * The live handles the facade reads at call time. `makeDimple` swaps the
+ * stack under this state on `reload`, so methods always hit the stack that
+ * is current when they run.
+ */
+export interface FacadeState {
+    readonly run: <A, E>(effect: Effect.Effect<A, E>) => Promise<A>;
+    readonly core: () => CoreInterface;
+    readonly parser: () => CodeParser["Service"];
+    readonly extractor: () => CodeExtractor["Service"];
+}
 /** Build the promise facade over the booted core + runtime. */
-export declare const makeFacade: <ER>(runtime: ManagedRuntime.ManagedRuntime<never, ER>, core: CoreInterface, parser: CodeParser["Service"], extractor: CodeExtractor["Service"], dispose: () => Promise<void>) => Dimple;
+export declare const makeFacade: (state: FacadeState, reload: (input?: ConfigInput) => Promise<ReloadResult>, dispose: () => Promise<void>) => Dimple;
 export type { DimpleError };

package/dist/errors.d.ts

@@ -45,19 +45,41 @@ declare const DuplicateEdgeError_base: Schema.Class<DuplicateEdgeError, Schema.T
 /** A duplicate memory↔memory edge (same from/to/kind already exists). */
 export declare class DuplicateEdgeError extends DuplicateEdgeError_base {
 }
+declare const EdgeNotFoundError_base: Schema.Class<EdgeNotFoundError, Schema.TaggedStruct<"Sdk.EdgeNotFound", {
+    readonly id: Schema.optional<Schema.String>;
+    readonly message: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/** The edge does not exist (confirm/reject/expire on a stale id). */
+export declare class EdgeNotFoundError extends EdgeNotFoundError_base {
+}
 declare const StoreFailure_base: Schema.Class<StoreFailure, Schema.TaggedStruct<"Sdk.StoreFailure", {
     readonly message: Schema.String;
 }>, import("effect/Cause").YieldableError>;
 /** Anything else — the typed fallback that keeps the union closed. */
 export declare class StoreFailure extends StoreFailure_base {
 }
+declare const ReloadError_base: Schema.Class<ReloadError, Schema.TaggedStruct<"Sdk.Reload", {
+    readonly message: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/** `reload` was unavailable (no config source, disposed) or failed to
+ * drain in-flight calls before the swap. */
+export declare class ReloadError extends ReloadError_base {
+}
+declare const EmbeddingFailure_base: Schema.Class<EmbeddingFailure, Schema.TaggedStruct<"Sdk.Embedding", {
+    readonly message: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/** Embedding pipeline failure — the `@huggingface/transformers` package is
+ * missing (local embeddings), the model failed to load, or the output
+ * dimensions mismatch the config. */
+export declare class EmbeddingFailure extends EmbeddingFailure_base {
+}
 declare const InputError_base: Schema.Class<InputError, Schema.TaggedStruct<"Sdk.InputError", {
     readonly message: Schema.String;
 }>, import("effect/Cause").YieldableError>;
 /** Invalid SDK call input (bad `limit`, `k`, profile shape, …). */
 export declare class InputError extends InputError_base {
 }
-export type DimpleError = ConfigError | NotFoundError | DuplicateKeyError | DuplicateEdgeError | LLMNotConfiguredError | JobFailedError | StoreFailure | InputError;
+export type DimpleError = ConfigError | NotFoundError | DuplicateKeyError | DuplicateEdgeError | EdgeNotFoundError | LLMNotConfiguredError | EmbeddingFailure | JobFailedError | StoreFailure | ReloadError | InputError;
 /** Map an underlying failure onto the closed SDK hierarchy. */
 export declare const remapError: (error: unknown) => DimpleError;
 export {};

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export { makeDimple, type MakeDimpleOptions } from "./make.ts";
 export { type ConfigInput } from "./config.ts";
-export { type Dimple, type QueryOptions, type IndexOptions, type IndexResult } from "./dimple.ts";
-export { ConfigError, NotFoundError, DuplicateKeyError, DuplicateEdgeError, LLMNotConfiguredError, JobFailedError, StoreFailure, InputError, remapError, type DimpleError, } from "./errors.ts";
+export type { DimpleConfig } from "./internal/config/index.js";
+export { type Dimple, type QueryOptions, type IndexOptions, type IndexResult, type ReloadChanges, type ReloadResult, } from "./dimple.ts";
+export { ConfigError, NotFoundError, DuplicateKeyError, DuplicateEdgeError, EdgeNotFoundError, LLMNotConfiguredError, JobFailedError, StoreFailure, ReloadError, EmbeddingFailure, InputError, remapError, type DimpleError, } from "./errors.ts";