dsh-library 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.es.md +41 -34
- package/README.hi.md +41 -34
- package/README.md +42 -35
- package/README.pt.md +41 -34
- package/README.zh.md +42 -35
- package/lib/index.js +257 -20
- package/lib/types/config.d.ts +19 -1
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/embedding.d.ts +110 -0
- package/lib/types/embedding.d.ts.map +1 -1
- package/lib/types/events.d.ts +68 -0
- package/lib/types/events.d.ts.map +1 -0
- package/lib/types/index.d.ts +12 -28
- package/lib/types/index.d.ts.map +1 -1
- package/package.json +6 -2
package/lib/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { MessageId } from "@deepseek-ai/dsh-llm";
|
|
|
3
3
|
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
4
4
|
import z from "@deepseek-ai/schemastery";
|
|
5
5
|
import { createHash, randomUUID } from "node:crypto";
|
|
6
|
+
import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
|
|
6
7
|
//#region node_modules/.pnpm/zod@4.4.3/node_modules/zod/v4/core/core.js
|
|
7
8
|
var _a$1;
|
|
8
9
|
function $constructor(name, initializer, params) {
|
|
@@ -4011,6 +4012,10 @@ const LIBRARY_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
|
4011
4012
|
const DEFAULT_MAX_FILE_BYTES = 5242880;
|
|
4012
4013
|
/** Absolute safety bound on one stored chunk (chunkSize must stay below it). */
|
|
4013
4014
|
const MAX_CHUNK_CHARS = 4e3;
|
|
4015
|
+
/** Local Ollama base URL (zero cloud — only this localhost endpoint is ever contacted). */
|
|
4016
|
+
const DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434";
|
|
4017
|
+
/** Default Ollama embedding model. */
|
|
4018
|
+
const DEFAULT_OLLAMA_MODEL = "nomic-embed-text";
|
|
4014
4019
|
/** Embedder subprocess budget. */
|
|
4015
4020
|
const DEFAULT_EMBEDDING_TIMEOUT_MS = 3e4;
|
|
4016
4021
|
const DEFAULT_EMBEDDING_GRACE_MS = 1e3;
|
|
@@ -4029,14 +4034,24 @@ const Config = z.object({
|
|
|
4029
4034
|
maxFileBytes: z.number().default(DEFAULT_MAX_FILE_BYTES),
|
|
4030
4035
|
embedding: z.object({
|
|
4031
4036
|
dims: z.number().default(256),
|
|
4037
|
+
provider: z.union([
|
|
4038
|
+
z.const("hash"),
|
|
4039
|
+
z.const("command"),
|
|
4040
|
+
z.const("ollama")
|
|
4041
|
+
]).default("hash"),
|
|
4032
4042
|
command: z.string(),
|
|
4043
|
+
ollamaUrl: z.string().default(DEFAULT_OLLAMA_URL),
|
|
4044
|
+
ollamaModel: z.string().default(DEFAULT_OLLAMA_MODEL),
|
|
4033
4045
|
timeoutMs: z.number().default(DEFAULT_EMBEDDING_TIMEOUT_MS),
|
|
4034
4046
|
graceMs: z.number().default(DEFAULT_EMBEDDING_GRACE_MS),
|
|
4035
4047
|
maxOutputBytes: z.number().default(DEFAULT_EMBEDDING_MAX_OUTPUT_BYTES),
|
|
4036
4048
|
maxBatchItems: z.number().default(64)
|
|
4037
4049
|
}).default({
|
|
4038
4050
|
dims: 256,
|
|
4051
|
+
provider: "hash",
|
|
4039
4052
|
command: "",
|
|
4053
|
+
ollamaUrl: DEFAULT_OLLAMA_URL,
|
|
4054
|
+
ollamaModel: DEFAULT_OLLAMA_MODEL,
|
|
4040
4055
|
timeoutMs: DEFAULT_EMBEDDING_TIMEOUT_MS,
|
|
4041
4056
|
graceMs: DEFAULT_EMBEDDING_GRACE_MS,
|
|
4042
4057
|
maxOutputBytes: DEFAULT_EMBEDDING_MAX_OUTPUT_BYTES,
|
|
@@ -4127,9 +4142,19 @@ function resolveConfig(config = {}) {
|
|
|
4127
4142
|
["embedding.maxBatchItems", embedding.maxBatchItems ?? 64]
|
|
4128
4143
|
]) assertPositiveInteger(name, value);
|
|
4129
4144
|
const rawCommand = embedding.command;
|
|
4130
|
-
const command = rawCommand === void 0 || rawCommand.trim().length === 0 ? void 0 : rawCommand;
|
|
4131
|
-
|
|
4132
|
-
|
|
4145
|
+
const command = rawCommand === void 0 || rawCommand.trim().length === 0 ? void 0 : rawCommand.trim();
|
|
4146
|
+
const rawProvider = embedding.provider;
|
|
4147
|
+
let provider;
|
|
4148
|
+
if (command !== void 0) provider = "command";
|
|
4149
|
+
else if (rawProvider === void 0 || rawProvider === "hash") provider = "hash";
|
|
4150
|
+
else if (rawProvider === "command") throw new TypeError("dsh-library: embedding.provider=command requires embedding.command to be set");
|
|
4151
|
+
else if (rawProvider === "ollama") provider = "ollama";
|
|
4152
|
+
else throw new TypeError(`dsh-library: embedding.provider must be hash|command|ollama, got ${JSON.stringify(rawProvider)}`);
|
|
4153
|
+
const ollamaUrl = (embedding.ollamaUrl ?? "http://127.0.0.1:11434").trim();
|
|
4154
|
+
const ollamaModel = (embedding.ollamaModel ?? "nomic-embed-text").trim();
|
|
4155
|
+
if (provider === "ollama") {
|
|
4156
|
+
if (ollamaUrl.length === 0) throw new TypeError("dsh-library: embedding.ollamaUrl must be a non-empty URL when provider=ollama");
|
|
4157
|
+
if (ollamaModel.length === 0) throw new TypeError("dsh-library: embedding.ollamaModel must be a non-empty model name when provider=ollama");
|
|
4133
4158
|
}
|
|
4134
4159
|
const search = config.search ?? {};
|
|
4135
4160
|
const topK = search.topK ?? 8;
|
|
@@ -4171,7 +4196,10 @@ function resolveConfig(config = {}) {
|
|
|
4171
4196
|
maxFileBytes: config.maxFileBytes ?? 5242880,
|
|
4172
4197
|
embedding: {
|
|
4173
4198
|
dims,
|
|
4199
|
+
provider,
|
|
4174
4200
|
command: command === void 0 ? void 0 : command.trim(),
|
|
4201
|
+
ollamaUrl,
|
|
4202
|
+
ollamaModel,
|
|
4175
4203
|
timeoutMs: embedding.timeoutMs ?? 3e4,
|
|
4176
4204
|
graceMs: embedding.graceMs ?? 1e3,
|
|
4177
4205
|
maxOutputBytes: embedding.maxOutputBytes ?? 1048576,
|
|
@@ -4418,6 +4446,177 @@ async function embedWithCommand(subprocess, argv, cwd, texts, dims, caps) {
|
|
|
4418
4446
|
function splitCommandLine(command) {
|
|
4419
4447
|
return command.trim().split(/\s+/u).filter((part) => part.length > 0);
|
|
4420
4448
|
}
|
|
4449
|
+
/** Built-in deterministic hash embedder: zero downloads, pure local hashing. */
|
|
4450
|
+
var HashEmbedder = class {
|
|
4451
|
+
dims;
|
|
4452
|
+
name = "hash";
|
|
4453
|
+
/**
|
|
4454
|
+
* @param dims - vector dimensionality (≥ 1).
|
|
4455
|
+
*/
|
|
4456
|
+
constructor(dims) {
|
|
4457
|
+
this.dims = dims;
|
|
4458
|
+
}
|
|
4459
|
+
async embed(texts) {
|
|
4460
|
+
return texts.map((text) => embedHash(text, this.dims));
|
|
4461
|
+
}
|
|
4462
|
+
};
|
|
4463
|
+
/** External command embedder over the JSON-lines subprocess protocol. */
|
|
4464
|
+
var CommandEmbedder = class {
|
|
4465
|
+
subprocess;
|
|
4466
|
+
argv;
|
|
4467
|
+
cwd;
|
|
4468
|
+
dims;
|
|
4469
|
+
caps;
|
|
4470
|
+
maxBatchItems;
|
|
4471
|
+
name = "command";
|
|
4472
|
+
/**
|
|
4473
|
+
* @param subprocess - the mounted subprocess seam.
|
|
4474
|
+
* @param argv - the command line split into argv (argv[0] = program).
|
|
4475
|
+
* @param cwd - working directory for the child.
|
|
4476
|
+
* @param dims - expected vector dimensionality.
|
|
4477
|
+
* @param caps - timeout, grace, and output caps.
|
|
4478
|
+
* @param maxBatchItems - texts per subprocess invocation; larger batches split.
|
|
4479
|
+
*/
|
|
4480
|
+
constructor(subprocess, argv, cwd, dims, caps, maxBatchItems) {
|
|
4481
|
+
this.subprocess = subprocess;
|
|
4482
|
+
this.argv = argv;
|
|
4483
|
+
this.cwd = cwd;
|
|
4484
|
+
this.dims = dims;
|
|
4485
|
+
this.caps = caps;
|
|
4486
|
+
this.maxBatchItems = maxBatchItems;
|
|
4487
|
+
}
|
|
4488
|
+
async embed(texts) {
|
|
4489
|
+
const out = [];
|
|
4490
|
+
for (let index = 0; index < texts.length; index += this.maxBatchItems) out.push(...await embedWithCommand(this.subprocess, this.argv, this.cwd, texts.slice(index, index + this.maxBatchItems), this.dims, this.caps));
|
|
4491
|
+
return out;
|
|
4492
|
+
}
|
|
4493
|
+
};
|
|
4494
|
+
/**
|
|
4495
|
+
* Local Ollama embedder over the `/api/embed` endpoint. Zero cloud: the model
|
|
4496
|
+
* runs on a localhost Ollama server. A non-2xx response or a malformed vector
|
|
4497
|
+
* fails the batch (misconfiguration must surface, not silently degrade
|
|
4498
|
+
* retrieval); availability is probed up front by {@link probeOllama}.
|
|
4499
|
+
*/
|
|
4500
|
+
var OllamaEmbedder = class {
|
|
4501
|
+
url;
|
|
4502
|
+
model;
|
|
4503
|
+
dims;
|
|
4504
|
+
timeoutMs;
|
|
4505
|
+
maxBatchItems;
|
|
4506
|
+
name = "ollama";
|
|
4507
|
+
/**
|
|
4508
|
+
* @param url - the Ollama base URL (e.g. `http://127.0.0.1:11434`).
|
|
4509
|
+
* @param model - the embedding model name (e.g. `nomic-embed-text`).
|
|
4510
|
+
* @param dims - the probed embedding dimensionality.
|
|
4511
|
+
* @param timeoutMs - per-request timeout.
|
|
4512
|
+
* @param maxBatchItems - texts per `/api/embed` call; larger batches split.
|
|
4513
|
+
*/
|
|
4514
|
+
constructor(url, model, dims, timeoutMs, maxBatchItems) {
|
|
4515
|
+
this.url = url;
|
|
4516
|
+
this.model = model;
|
|
4517
|
+
this.dims = dims;
|
|
4518
|
+
this.timeoutMs = timeoutMs;
|
|
4519
|
+
this.maxBatchItems = maxBatchItems;
|
|
4520
|
+
}
|
|
4521
|
+
async embed(texts) {
|
|
4522
|
+
const out = [];
|
|
4523
|
+
for (let index = 0; index < texts.length; index += this.maxBatchItems) out.push(...await this.embedBatch(texts.slice(index, index + this.maxBatchItems)));
|
|
4524
|
+
return out;
|
|
4525
|
+
}
|
|
4526
|
+
/** POST one batch to `/api/embed` and validate the returned vectors. */
|
|
4527
|
+
async embedBatch(texts) {
|
|
4528
|
+
let response;
|
|
4529
|
+
try {
|
|
4530
|
+
response = await fetch(`${this.url}/api/embed`, {
|
|
4531
|
+
method: "POST",
|
|
4532
|
+
headers: { "content-type": "application/json" },
|
|
4533
|
+
body: JSON.stringify({
|
|
4534
|
+
model: this.model,
|
|
4535
|
+
input: [...texts]
|
|
4536
|
+
}),
|
|
4537
|
+
signal: AbortSignal.timeout(this.timeoutMs)
|
|
4538
|
+
});
|
|
4539
|
+
} catch (error) {
|
|
4540
|
+
throw new Error(`dsh-library: ollama embed request failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
4541
|
+
}
|
|
4542
|
+
if (!response.ok) throw new Error(`dsh-library: ollama embed failed (HTTP ${response.status})`);
|
|
4543
|
+
const embeddings = (await response.json())["embeddings"];
|
|
4544
|
+
if (!Array.isArray(embeddings) || embeddings.length !== texts.length) throw new Error(`dsh-library: ollama answered ${Array.isArray(embeddings) ? embeddings.length : 0} of ${texts.length} inputs`);
|
|
4545
|
+
return embeddings.map((vector) => {
|
|
4546
|
+
if (!Array.isArray(vector) || vector.length !== this.dims || vector.some((value) => typeof value !== "number" || !Number.isFinite(value))) throw new Error("dsh-library: ollama returned a malformed embedding vector");
|
|
4547
|
+
return normalizeVector(Float64Array.from(vector));
|
|
4548
|
+
});
|
|
4549
|
+
}
|
|
4550
|
+
};
|
|
4551
|
+
/**
|
|
4552
|
+
* Probe a local Ollama server for one embedding model. Sends a single-word
|
|
4553
|
+
* probe through `/api/embed` and returns the model's vector dimensionality, or
|
|
4554
|
+
* undefined when the server is unreachable, the model is unknown, or the
|
|
4555
|
+
* response is malformed. Zero cloud — only the configured local URL is ever
|
|
4556
|
+
* contacted.
|
|
4557
|
+
* @param url - the Ollama base URL.
|
|
4558
|
+
* @param model - the embedding model name.
|
|
4559
|
+
* @param timeoutMs - probe timeout.
|
|
4560
|
+
* @returns the model's dimensionality, or undefined when unavailable.
|
|
4561
|
+
*/
|
|
4562
|
+
async function probeOllama(url, model, timeoutMs) {
|
|
4563
|
+
try {
|
|
4564
|
+
const response = await fetch(`${url}/api/embed`, {
|
|
4565
|
+
method: "POST",
|
|
4566
|
+
headers: { "content-type": "application/json" },
|
|
4567
|
+
body: JSON.stringify({
|
|
4568
|
+
model,
|
|
4569
|
+
input: ["probe"]
|
|
4570
|
+
}),
|
|
4571
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
4572
|
+
});
|
|
4573
|
+
if (!response.ok) return void 0;
|
|
4574
|
+
const embeddings = (await response.json())["embeddings"];
|
|
4575
|
+
if (!Array.isArray(embeddings) || embeddings.length !== 1 || !Array.isArray(embeddings[0])) return void 0;
|
|
4576
|
+
return embeddings[0].length;
|
|
4577
|
+
} catch {
|
|
4578
|
+
return;
|
|
4579
|
+
}
|
|
4580
|
+
}
|
|
4581
|
+
/**
|
|
4582
|
+
* Resolve the embedder backend from the resolved config: the configured
|
|
4583
|
+
* command wins (fail loud without a subprocess seam), the `ollama` provider is
|
|
4584
|
+
* probed and degrades gracefully to the built-in hash embedder when the server
|
|
4585
|
+
* is unreachable, and the built-in hash embedder is the zero-download default.
|
|
4586
|
+
* @param config - the resolved embedding config.
|
|
4587
|
+
* @param subprocess - the optional subprocess seam (command provider only).
|
|
4588
|
+
* @param cwd - working directory for the command provider.
|
|
4589
|
+
* @returns the backend plus its degradation note.
|
|
4590
|
+
*/
|
|
4591
|
+
async function resolveEmbedder(config, subprocess, cwd) {
|
|
4592
|
+
if (config.provider === "command") {
|
|
4593
|
+
if (subprocess === void 0) throw new Error("dsh-library: embedding.provider=command but ctx.subprocess is not mounted — the external embedder cannot run");
|
|
4594
|
+
return {
|
|
4595
|
+
embedder: new CommandEmbedder(subprocess, splitCommandLine(config.command), cwd, config.dims, {
|
|
4596
|
+
timeoutMs: config.timeoutMs,
|
|
4597
|
+
graceMs: config.graceMs,
|
|
4598
|
+
maxOutputBytes: config.maxOutputBytes
|
|
4599
|
+
}, config.maxBatchItems),
|
|
4600
|
+
degraded: false
|
|
4601
|
+
};
|
|
4602
|
+
}
|
|
4603
|
+
if (config.provider === "ollama") {
|
|
4604
|
+
const dims = await probeOllama(config.ollamaUrl, config.ollamaModel, config.timeoutMs);
|
|
4605
|
+
if (dims === void 0) return {
|
|
4606
|
+
embedder: new HashEmbedder(config.dims),
|
|
4607
|
+
degraded: true,
|
|
4608
|
+
reason: `ollama unavailable at ${config.ollamaUrl} (model ${config.ollamaModel}); degraded to the built-in hash embedder`
|
|
4609
|
+
};
|
|
4610
|
+
return {
|
|
4611
|
+
embedder: new OllamaEmbedder(config.ollamaUrl, config.ollamaModel, dims, config.timeoutMs, config.maxBatchItems),
|
|
4612
|
+
degraded: false
|
|
4613
|
+
};
|
|
4614
|
+
}
|
|
4615
|
+
return {
|
|
4616
|
+
embedder: new HashEmbedder(config.dims),
|
|
4617
|
+
degraded: false
|
|
4618
|
+
};
|
|
4619
|
+
}
|
|
4421
4620
|
//#endregion
|
|
4422
4621
|
//#region src/ids.ts
|
|
4423
4622
|
/**
|
|
@@ -5173,6 +5372,48 @@ function verifyPurge(remaining, removedContent, options = {}) {
|
|
|
5173
5372
|
};
|
|
5174
5373
|
}
|
|
5175
5374
|
//#endregion
|
|
5375
|
+
//#region src/events.ts
|
|
5376
|
+
/**
|
|
5377
|
+
* Session audit events for dsh-library (declaration merging into the harness's
|
|
5378
|
+
* `SessionEventMap`) and the adaptive append gate. Both events are log-only;
|
|
5379
|
+
* tool arguments and rendered results are already logged by the tool runtime
|
|
5380
|
+
* as `tool/call` + `tool/result`, and these events carry the audit facts that
|
|
5381
|
+
* exist outside them: the inject id linking the injected marker text back to
|
|
5382
|
+
* the search that produced it, and each purge-verification verdict.
|
|
5383
|
+
*
|
|
5384
|
+
* The gate appends only when the host can carry the events safely:
|
|
5385
|
+
* - hosts whose known-type set covers the vocabulary append plainly;
|
|
5386
|
+
* - hosts with an `ignorable` append option (pre-0.1.2 master builds) append
|
|
5387
|
+
* with the marker, so builds that do not know the type skip it on restore;
|
|
5388
|
+
* - envelope-less hosts (0.1.0-rc.6/rc.8, 0.1.1-rc.2, and 0.1.2-alpha.1,
|
|
5389
|
+
* which removed the envelope and fails closed on unknown types at read)
|
|
5390
|
+
* get no append — the tool results remain the reconstructable audit trail.
|
|
5391
|
+
*
|
|
5392
|
+
* @module dsh-library/events
|
|
5393
|
+
*/
|
|
5394
|
+
/** The injection audit event type. */
|
|
5395
|
+
const INJECT_EVENT = "library/inject";
|
|
5396
|
+
/** The purge audit event type. */
|
|
5397
|
+
const PURGE_EVENT = "library/purge";
|
|
5398
|
+
/**
|
|
5399
|
+
* Append one dsh-library audit event when the host can carry it safely; skip
|
|
5400
|
+
* silently otherwise (the `tool/call` + `tool/result` events remain the
|
|
5401
|
+
* model-visible log, so nothing model-visible is lost). See the module doc
|
|
5402
|
+
* for the three host classes.
|
|
5403
|
+
* @param session - the calling session.
|
|
5404
|
+
* @param type - the audit event type.
|
|
5405
|
+
* @param data - the audit payload.
|
|
5406
|
+
*/
|
|
5407
|
+
function appendAuditEvent(session, type, data) {
|
|
5408
|
+
if (KNOWN_SESSION_EVENT_TYPES.has(type)) {
|
|
5409
|
+
if (type === "library/inject") session.append(type, data);
|
|
5410
|
+
else session.append(type, data);
|
|
5411
|
+
return;
|
|
5412
|
+
}
|
|
5413
|
+
const append = session.append;
|
|
5414
|
+
if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
|
|
5415
|
+
}
|
|
5416
|
+
//#endregion
|
|
5176
5417
|
//#region src/quality/few-shot.ts
|
|
5177
5418
|
/**
|
|
5178
5419
|
* Port of Few-Shot-Selector (upstream/PerryLink, Apache-2.0): the
|
|
@@ -5310,16 +5551,9 @@ var LibraryStore = class {
|
|
|
5310
5551
|
this.chunks = domain.table("chunks");
|
|
5311
5552
|
this.purges = domain.table("purges");
|
|
5312
5553
|
}
|
|
5313
|
-
/** Embed a text batch with the
|
|
5554
|
+
/** Embed a text batch with the resolved embedder backend. */
|
|
5314
5555
|
async embed(texts) {
|
|
5315
|
-
|
|
5316
|
-
if (command === void 0) return texts.map((text) => embedHash(text, this.config.embedding.dims));
|
|
5317
|
-
if (this.deps.subprocess === void 0) throw new Error("embedding.command is configured but ctx.subprocess is not mounted — the external embedder cannot run");
|
|
5318
|
-
return embedWithCommand(this.deps.subprocess, splitCommandLine(command), process.cwd(), texts, this.config.embedding.dims, {
|
|
5319
|
-
timeoutMs: this.config.embedding.timeoutMs,
|
|
5320
|
-
graceMs: this.config.embedding.graceMs,
|
|
5321
|
-
maxOutputBytes: this.config.embedding.maxOutputBytes
|
|
5322
|
-
});
|
|
5556
|
+
return this.deps.embedder.embed(texts);
|
|
5323
5557
|
}
|
|
5324
5558
|
/**
|
|
5325
5559
|
* Import one document: chunk it, embed the chunks, and store the records.
|
|
@@ -5551,16 +5785,19 @@ var LibraryStore = class {
|
|
|
5551
5785
|
};
|
|
5552
5786
|
}
|
|
5553
5787
|
};
|
|
5554
|
-
/**
|
|
5555
|
-
function storeDepsOf(ctx) {
|
|
5556
|
-
|
|
5557
|
-
|
|
5558
|
-
|
|
5788
|
+
/** Resolve the embedder backend at mount; a degraded Ollama falls back to hash with a logged reason. */
|
|
5789
|
+
async function storeDepsOf(ctx, config) {
|
|
5790
|
+
const subprocess = ctx.get("subprocess");
|
|
5791
|
+
const resolution = await resolveEmbedder(config.embedding, subprocess, process.cwd());
|
|
5792
|
+
if (resolution.degraded) ctx.logger("dsh-library").warn(`embedder: ${resolution.reason}`);
|
|
5793
|
+
return { embedder: resolution.embedder };
|
|
5794
|
+
}
|
|
5795
|
+
/** Append one audit event through the adaptive host gate; a failed append never changes the outcome. */
|
|
5559
5796
|
function audit(exec, type, event) {
|
|
5560
5797
|
const session = exec.agent?.session;
|
|
5561
5798
|
if (session === void 0) return;
|
|
5562
5799
|
try {
|
|
5563
|
-
session
|
|
5800
|
+
appendAuditEvent(session, type, event);
|
|
5564
5801
|
} catch {}
|
|
5565
5802
|
}
|
|
5566
5803
|
/**
|
|
@@ -5574,7 +5811,7 @@ async function apply(ctx, config = {}) {
|
|
|
5574
5811
|
const resolved = resolveConfig(config);
|
|
5575
5812
|
const domain = await ctx.storageDomain.open(libraryDomainSpec);
|
|
5576
5813
|
ctx.effect(() => () => domain.close(), "dsh-library: storage domain");
|
|
5577
|
-
const store = new LibraryStore(domain, resolved, storeDepsOf(ctx));
|
|
5814
|
+
const store = new LibraryStore(domain, resolved, await storeDepsOf(ctx, resolved));
|
|
5578
5815
|
const services = {
|
|
5579
5816
|
ctx,
|
|
5580
5817
|
config: resolved,
|
|
@@ -6071,4 +6308,4 @@ function libraryDiagnoseTool(services) {
|
|
|
6071
6308
|
});
|
|
6072
6309
|
}
|
|
6073
6310
|
//#endregion
|
|
6074
|
-
export { CHUNK_SIZE_BUCKETS, Config, LIBRARY_NAME, LibraryStore, PROBE_PREFIX, allTools, apply, avoidLostMiddle, buildProbePrompt, checkDiversity, chunkHash, chunkSizeHistogram, chunkText, cosine, embedHash, extractCitationNumbers, extractCitations, extractContext, extractSentenceWithCitation, filterDocuments, findDuplicateChunks, formatFewShotPrompt, fuzzyPartialRatio, inject, insertProbe, libraryDomainSpec, makeProbe, maximalMarginalRelevance, middlePenalty, name, positionBins, resolveConfig, sampleSignatures, scoreBatch, scoreRelevance, selectFewShot, splitCommandLine, successByPosition, validateCitation, validateCitations, validateQaPair, verifyPurge, verifyReferences };
|
|
6311
|
+
export { CHUNK_SIZE_BUCKETS, CommandEmbedder, Config, HashEmbedder, INJECT_EVENT, LIBRARY_NAME, LibraryStore, OllamaEmbedder, PROBE_PREFIX, PURGE_EVENT, allTools, appendAuditEvent, apply, avoidLostMiddle, buildProbePrompt, checkDiversity, chunkHash, chunkSizeHistogram, chunkText, cosine, embedHash, extractCitationNumbers, extractCitations, extractContext, extractSentenceWithCitation, filterDocuments, findDuplicateChunks, formatFewShotPrompt, fuzzyPartialRatio, inject, insertProbe, libraryDomainSpec, makeProbe, maximalMarginalRelevance, middlePenalty, name, positionBins, probeOllama, resolveConfig, resolveEmbedder, sampleSignatures, scoreBatch, scoreRelevance, selectFewShot, splitCommandLine, successByPosition, validateCitation, validateCitations, validateQaPair, verifyPurge, verifyReferences };
|
package/lib/types/config.d.ts
CHANGED
|
@@ -17,6 +17,10 @@ export declare const DEFAULT_MAX_FILE_BYTES: number;
|
|
|
17
17
|
export declare const MAX_CHUNK_CHARS = 4000;
|
|
18
18
|
/** Hash-embedding dimensionality used by the zero-download local embedder. */
|
|
19
19
|
export declare const DEFAULT_EMBEDDING_DIMS = 256;
|
|
20
|
+
/** Local Ollama base URL (zero cloud — only this localhost endpoint is ever contacted). */
|
|
21
|
+
export declare const DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434";
|
|
22
|
+
/** Default Ollama embedding model. */
|
|
23
|
+
export declare const DEFAULT_OLLAMA_MODEL = "nomic-embed-text";
|
|
20
24
|
/** Embedder subprocess budget. */
|
|
21
25
|
export declare const DEFAULT_EMBEDDING_TIMEOUT_MS = 30000;
|
|
22
26
|
export declare const DEFAULT_EMBEDDING_GRACE_MS = 1000;
|
|
@@ -44,10 +48,17 @@ export declare const DEFAULT_PURGE_MAX_PROBES = 24;
|
|
|
44
48
|
export declare const DEFAULT_DIAGNOSE_MAX_DUPLICATE_PAIRS = 24;
|
|
45
49
|
export declare const DEFAULT_DIAGNOSE_SAMPLE_CAP = 200;
|
|
46
50
|
export declare const DEFAULT_DIAGNOSE_POSITION_BINS = 5;
|
|
47
|
-
/** Embedder selection: built-in hash embedding,
|
|
51
|
+
/** Embedder selection: built-in hash embedding, an external command, or a local Ollama server. */
|
|
48
52
|
export interface EmbeddingConfig {
|
|
49
53
|
/** Hash-embedding dimensionality (built-in embedder only). Must be ≥ 8. */
|
|
50
54
|
dims?: number;
|
|
55
|
+
/**
|
|
56
|
+
* Which embedder backend to use: `hash` (built-in, zero downloads), `command`
|
|
57
|
+
* (the external subprocess protocol, requires `command`), or `ollama` (a
|
|
58
|
+
* local Ollama server, probed and degraded to `hash` when unavailable).
|
|
59
|
+
* When `command` is set, the command provider wins regardless of this field.
|
|
60
|
+
*/
|
|
61
|
+
provider?: 'hash' | 'command' | 'ollama';
|
|
51
62
|
/**
|
|
52
63
|
* Optional external embedder command line (space-separated, no shell
|
|
53
64
|
* interpretation; executed through `ctx.subprocess`). The command must read
|
|
@@ -58,6 +69,10 @@ export interface EmbeddingConfig {
|
|
|
58
69
|
* embedder runs with zero downloads.
|
|
59
70
|
*/
|
|
60
71
|
command?: string;
|
|
72
|
+
/** Local Ollama base URL for the `ollama` provider (default `http://127.0.0.1:11434`). */
|
|
73
|
+
ollamaUrl?: string;
|
|
74
|
+
/** Ollama embedding model name (default `nomic-embed-text`). */
|
|
75
|
+
ollamaModel?: string;
|
|
61
76
|
/** Cooperative timeout for one embedder invocation (ms). */
|
|
62
77
|
timeoutMs?: number;
|
|
63
78
|
/** Terminate-escalation grace handed to the subprocess seam (ms). */
|
|
@@ -143,7 +158,10 @@ export interface Config {
|
|
|
143
158
|
/** Resolved embedding config: defaults applied, `command` explicitly optional. */
|
|
144
159
|
export interface ResolvedEmbeddingConfig {
|
|
145
160
|
readonly dims: number;
|
|
161
|
+
readonly provider: 'hash' | 'command' | 'ollama';
|
|
146
162
|
readonly command: string | undefined;
|
|
163
|
+
readonly ollamaUrl: string;
|
|
164
|
+
readonly ollamaModel: string;
|
|
147
165
|
readonly timeoutMs: number;
|
|
148
166
|
readonly graceMs: number;
|
|
149
167
|
readonly maxOutputBytes: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,0EAA0E;AAC1E,eAAO,MAAM,YAAY,QAA8B,CAAA;AAEvD,iEAAiE;AACjE,eAAO,MAAM,kBAAkB,MAAM,CAAA;AACrC,eAAO,MAAM,qBAAqB,MAAM,CAAA;AACxC,yFAAyF;AACzF,eAAO,MAAM,sBAAsB,QAAkB,CAAA;AACrD,gFAAgF;AAChF,eAAO,MAAM,eAAe,OAAO,CAAA;AACnC,8EAA8E;AAC9E,eAAO,MAAM,sBAAsB,MAAM,CAAA;AACzC,kCAAkC;AAClC,eAAO,MAAM,4BAA4B,QAAS,CAAA;AAClD,eAAO,MAAM,0BAA0B,OAAO,CAAA;AAC9C,eAAO,MAAM,kCAAkC,QAAc,CAAA;AAC7D,eAAO,MAAM,2BAA2B,KAAK,CAAA;AAC7C,uBAAuB;AACvB,eAAO,MAAM,mBAAmB,IAAI,CAAA;AACpC,eAAO,MAAM,4BAA4B,MAAM,CAAA;AAC/C,eAAO,MAAM,4BAA4B,OAAO,CAAA;AAChD,eAAO,MAAM,+BAA+B,MAAM,CAAA;AAClD,eAAO,MAAM,+BAA+B,QAAS,CAAA;AACrD,+EAA+E;AAC/E,eAAO,MAAM,wBAAwB,IAAI,CAAA;AACzC,eAAO,MAAM,wBAAwB,IAAI,CAAA;AACzC,0BAA0B;AAC1B,eAAO,MAAM,wBAAwB,QAAS,CAAA;AAC9C,yBAAyB;AACzB,eAAO,MAAM,6BAA6B,MAAM,CAAA;AAChD,eAAO,MAAM,0BAA0B,KAAK,CAAA;AAC5C,eAAO,MAAM,6BAA6B,MAAM,CAAA;AAChD,mCAAmC;AACnC,eAAO,MAAM,8BAA8B,IAAI,CAAA;AAC/C,eAAO,MAAM,wBAAwB,KAAK,CAAA;AAC1C,yBAAyB;AACzB,eAAO,MAAM,oCAAoC,KAAK,CAAA;AACtD,eAAO,MAAM,2BAA2B,MAAM,CAAA;AAC9C,eAAO,MAAM,8BAA8B,IAAI,CAAA;AAE/C,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,0EAA0E;AAC1E,eAAO,MAAM,YAAY,QAA8B,CAAA;AAEvD,iEAAiE;AACjE,eAAO,MAAM,kBAAkB,MAAM,CAAA;AACrC,eAAO,MAAM,qBAAqB,MAAM,CAAA;AACxC,yFAAyF;AACzF,eAAO,MAAM,sBAAsB,QAAkB,CAAA;AACrD,gFAAgF;AAChF,eAAO,MAAM,eAAe,OAAO,CAAA;AACnC,8EAA8E;AAC9E,eAAO,MAAM,sBAAsB,MAAM,CAAA;AACzC,2FAA2F;AAC3F,eAAO,MAAM,kBAAkB,2BAA2B,CAAA;AAC1D,sCAAsC;AACtC,eAAO,MAAM,oBAAoB,qBAAqB,CAAA;AACtD,kCAAkC;AAClC,eAAO,MAAM,4BAA4B,QAAS,CAAA;AAClD,eAAO,MAAM,0BAA0B,OAAO,CAAA;AAC9C,eAAO,MAAM,kCAAkC,QAAc,CAAA;AAC7D,eAAO,MAAM,2BAA2B,KAAK,CAAA;AAC7C,uBAAuB;AACvB,eAAO,MAAM,mBAAmB,IAAI,CAAA;AACpC,eAAO,MAAM,4BAA4B,MAAM,CAAA;AAC/C,eAAO,MAAM,4BAA4B,OAAO,CAAA;AAChD,eAAO,MAAM,+BAA+B,MAAM,CAAA;AAClD,eAAO,MAAM,+BAA+B,QAAS,CAAA;AACrD,+EAA+E;AAC/E,eAAO,MAAM,wBAAwB,IAAI,CAAA;AACzC,eAAO,MAAM,wBAAwB,IAAI,CAAA;AACzC,0BAA0B;AAC1B,eAAO,MAAM,wBAAwB,QAAS,CAAA;AAC9C,yBAAyB;AACzB,eAAO,MAAM,6BAA6B,MAAM,CAAA;AAChD,eAAO,MAAM,0BAA0B,KAAK,CAAA;AAC5C,eAAO,MAAM,6BAA6B,MAAM,CAAA;AAChD,mCAAmC;AACnC,eAAO,MAAM,8BAA8B,IAAI,CAAA;AAC/C,eAAO,MAAM,wBAAwB,KAAK,CAAA;AAC1C,yBAAyB;AACzB,eAAO,MAAM,oCAAoC,KAAK,CAAA;AACtD,eAAO,MAAM,2BAA2B,MAAM,CAAA;AAC9C,eAAO,MAAM,8BAA8B,IAAI,CAAA;AAE/C,kGAAkG;AAClG,MAAM,WAAW,eAAe;IAC9B,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAA;IACxC;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,0FAA0F;IAC1F,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,gEAAgE;IAChE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,0GAA0G;AAC1G,MAAM,WAAW,YAAY;IAC3B,kEAAkE;IAClE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,6GAA6G;IAC7G,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,8EAA8E;IAC9E,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,yFAAyF;IACzF,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mGAAmG;IACnG,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,eAAe;IAC9B,mFAAmF;IACnF,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,6DAA6D;AAC7D,MAAM,WAAW,cAAc;IAC7B,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,oFAAoF;IACpF,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,6FAA6F;IAC7F,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,sFAAsF;AACtF,MAAM,WAAW,MAAM;IACrB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,2FAA2F;IAC3F,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAA;IAC3B,0DAA0D;IAC1D,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAA;IAC3B,qDAAqD;IACrD,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,WAAW,CAAA;IACnB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,cAAc,CAAA;CAC1B;AAED,kFAAkF;AAClF,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAA;IAChD,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC/B;AAED,+CAA+C;AAC/C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;CAChC;AAED,kDAAkD;AAClD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAC1B;AAED,iDAAiD;AACjD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAED,8CAA8C;AAC9C,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;CAC3B;AAED,iDAAiD;AACjD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAC9B;AAED,yFAAyF;AACzF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,SAAS,EAAE,uBAAuB,CAAA;IAC3C,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAA;IACrC,QAAQ,CAAC,SAAS,EAAE,uBAAuB,CAAA;IAC3C,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAA;IACzC,QAAQ,CAAC,KAAK,EAAE,mBAAmB,CAAA;IACnC,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAA;CAC1C;AAED,kFAAkF;AAClF,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,MAAM,CAoE3B,CAAA;AAgBF;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,MAAM,GAAE,MAAW,GAAG,cAAc,CA6IjE"}
|
package/lib/types/embedding.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* @module dsh-library/embedding
|
|
8
8
|
*/
|
|
9
9
|
import type { SubprocessRuntime } from '@deepseek-ai/dsh-subprocess';
|
|
10
|
+
import type { ResolvedEmbeddingConfig } from './config.js';
|
|
10
11
|
/**
|
|
11
12
|
* Embed one text with the built-in hash embedder. Each word token and each
|
|
12
13
|
* character tri-gram hashes (FNV-1a) to a signed bucket: the index is the
|
|
@@ -53,4 +54,113 @@ export declare function embedWithCommand(subprocess: SubprocessRuntime, argv: re
|
|
|
53
54
|
* @returns the argv vector; an empty string yields an empty vector.
|
|
54
55
|
*/
|
|
55
56
|
export declare function splitCommandLine(command: string): string[];
|
|
57
|
+
/**
|
|
58
|
+
* The embedder provider seam: one pluggable text→vector backend. The built-in
|
|
59
|
+
* {@link HashEmbedder} is the zero-download default; {@link CommandEmbedder}
|
|
60
|
+
* and {@link OllamaEmbedder} are optional backends (probe → use → degrade for
|
|
61
|
+
* Ollama; fail loud for a configured-but-absent command).
|
|
62
|
+
*/
|
|
63
|
+
export interface Embedder {
|
|
64
|
+
/** Provider name recorded alongside the index (hash | command | ollama). */
|
|
65
|
+
readonly name: 'hash' | 'command' | 'ollama';
|
|
66
|
+
/** Vector dimensionality this backend emits. */
|
|
67
|
+
readonly dims: number;
|
|
68
|
+
/**
|
|
69
|
+
* Embed one batch of texts, returning one vector per input in the same order.
|
|
70
|
+
* @param texts - the texts to embed.
|
|
71
|
+
* @returns the L2-normalized embeddings.
|
|
72
|
+
*/
|
|
73
|
+
embed(texts: readonly string[]): Promise<number[][]>;
|
|
74
|
+
}
|
|
75
|
+
/** Built-in deterministic hash embedder: zero downloads, pure local hashing. */
|
|
76
|
+
export declare class HashEmbedder implements Embedder {
|
|
77
|
+
readonly dims: number;
|
|
78
|
+
readonly name: "hash";
|
|
79
|
+
/**
|
|
80
|
+
* @param dims - vector dimensionality (≥ 1).
|
|
81
|
+
*/
|
|
82
|
+
constructor(dims: number);
|
|
83
|
+
embed(texts: readonly string[]): Promise<number[][]>;
|
|
84
|
+
}
|
|
85
|
+
/** External command embedder over the JSON-lines subprocess protocol. */
|
|
86
|
+
export declare class CommandEmbedder implements Embedder {
|
|
87
|
+
private readonly subprocess;
|
|
88
|
+
private readonly argv;
|
|
89
|
+
private readonly cwd;
|
|
90
|
+
readonly dims: number;
|
|
91
|
+
private readonly caps;
|
|
92
|
+
private readonly maxBatchItems;
|
|
93
|
+
readonly name: "command";
|
|
94
|
+
/**
|
|
95
|
+
* @param subprocess - the mounted subprocess seam.
|
|
96
|
+
* @param argv - the command line split into argv (argv[0] = program).
|
|
97
|
+
* @param cwd - working directory for the child.
|
|
98
|
+
* @param dims - expected vector dimensionality.
|
|
99
|
+
* @param caps - timeout, grace, and output caps.
|
|
100
|
+
* @param maxBatchItems - texts per subprocess invocation; larger batches split.
|
|
101
|
+
*/
|
|
102
|
+
constructor(subprocess: SubprocessRuntime, argv: readonly string[], cwd: string, dims: number, caps: {
|
|
103
|
+
timeoutMs: number;
|
|
104
|
+
graceMs: number;
|
|
105
|
+
maxOutputBytes: number;
|
|
106
|
+
}, maxBatchItems: number);
|
|
107
|
+
embed(texts: readonly string[]): Promise<number[][]>;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Local Ollama embedder over the `/api/embed` endpoint. Zero cloud: the model
|
|
111
|
+
* runs on a localhost Ollama server. A non-2xx response or a malformed vector
|
|
112
|
+
* fails the batch (misconfiguration must surface, not silently degrade
|
|
113
|
+
* retrieval); availability is probed up front by {@link probeOllama}.
|
|
114
|
+
*/
|
|
115
|
+
export declare class OllamaEmbedder implements Embedder {
|
|
116
|
+
private readonly url;
|
|
117
|
+
private readonly model;
|
|
118
|
+
readonly dims: number;
|
|
119
|
+
private readonly timeoutMs;
|
|
120
|
+
private readonly maxBatchItems;
|
|
121
|
+
readonly name: "ollama";
|
|
122
|
+
/**
|
|
123
|
+
* @param url - the Ollama base URL (e.g. `http://127.0.0.1:11434`).
|
|
124
|
+
* @param model - the embedding model name (e.g. `nomic-embed-text`).
|
|
125
|
+
* @param dims - the probed embedding dimensionality.
|
|
126
|
+
* @param timeoutMs - per-request timeout.
|
|
127
|
+
* @param maxBatchItems - texts per `/api/embed` call; larger batches split.
|
|
128
|
+
*/
|
|
129
|
+
constructor(url: string, model: string, dims: number, timeoutMs: number, maxBatchItems: number);
|
|
130
|
+
embed(texts: readonly string[]): Promise<number[][]>;
|
|
131
|
+
/** POST one batch to `/api/embed` and validate the returned vectors. */
|
|
132
|
+
private embedBatch;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Probe a local Ollama server for one embedding model. Sends a single-word
|
|
136
|
+
* probe through `/api/embed` and returns the model's vector dimensionality, or
|
|
137
|
+
* undefined when the server is unreachable, the model is unknown, or the
|
|
138
|
+
* response is malformed. Zero cloud — only the configured local URL is ever
|
|
139
|
+
* contacted.
|
|
140
|
+
* @param url - the Ollama base URL.
|
|
141
|
+
* @param model - the embedding model name.
|
|
142
|
+
* @param timeoutMs - probe timeout.
|
|
143
|
+
* @returns the model's dimensionality, or undefined when unavailable.
|
|
144
|
+
*/
|
|
145
|
+
export declare function probeOllama(url: string, model: string, timeoutMs: number): Promise<number | undefined>;
|
|
146
|
+
/** The outcome of {@link resolveEmbedder}: the backend plus its degradation note. */
|
|
147
|
+
export interface EmbedderResolution {
|
|
148
|
+
/** The resolved backend. */
|
|
149
|
+
readonly embedder: Embedder;
|
|
150
|
+
/** True when the configured backend was unavailable and a default was substituted. */
|
|
151
|
+
readonly degraded: boolean;
|
|
152
|
+
/** Human-readable degradation reason (present only when {@link EmbedderResolution.degraded}). */
|
|
153
|
+
readonly reason?: string;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Resolve the embedder backend from the resolved config: the configured
|
|
157
|
+
* command wins (fail loud without a subprocess seam), the `ollama` provider is
|
|
158
|
+
* probed and degrades gracefully to the built-in hash embedder when the server
|
|
159
|
+
* is unreachable, and the built-in hash embedder is the zero-download default.
|
|
160
|
+
* @param config - the resolved embedding config.
|
|
161
|
+
* @param subprocess - the optional subprocess seam (command provider only).
|
|
162
|
+
* @param cwd - working directory for the command provider.
|
|
163
|
+
* @returns the backend plus its degradation note.
|
|
164
|
+
*/
|
|
165
|
+
export declare function resolveEmbedder(config: ResolvedEmbeddingConfig, subprocess: SubprocessRuntime | undefined, cwd: string): Promise<EmbedderResolution>;
|
|
56
166
|
//# sourceMappingURL=embedding.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/embedding.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAA;
|
|
1
|
+
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/embedding.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAA;AAEpE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAA;AAG1D;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAU9D;AAcD;;;;;;GAMG;AACH,wBAAgB,MAAM,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAYzE;AAqBD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GACnE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CA0CrB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAE1D;AAID;;;;;GAKG;AACH,MAAM,WAAW,QAAQ;IACvB,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAA;IAC5C,gDAAgD;IAChD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB;;;;OAIG;IACH,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;CACrD;AAED,gFAAgF;AAChF,qBAAa,YAAa,YAAW,QAAQ;IAM/B,QAAQ,CAAC,IAAI,EAAE,MAAM;IALjC,QAAQ,CAAC,IAAI,EAAG,MAAM,CAAS;IAE/B;;OAEG;gBACkB,IAAI,EAAE,MAAM;IAE3B,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAG3D;AAED,yEAAyE;AACzE,qBAAa,eAAgB,YAAW,QAAQ;IAY5C,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAhBhC,QAAQ,CAAC,IAAI,EAAG,SAAS,CAAS;IAElC;;;;;;;OAOG;gBAEgB,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,GAAG,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,EACJ,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,EACpE,aAAa,EAAE,MAAM;IAGlC,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAc3D;AAED;;;;;GAKG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAW3C,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAdhC,QAAQ,CAAC,IAAI,EAAG,QAAQ,CAAS;IAEjC;;;;;;OAMG;gBAEgB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACrB,IAAI,EAAE,MAAM,EACJ,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM;IAGlC,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAQ1D,wEAAwE;YAC1D,UAAU;CA2BzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAgB5G;AAED,qFAAqF;AACrF,MAAM,WAAW,kBAAkB;IACjC,4BAA4B;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,sFAAsF;IACtF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;IAC1B,iGAAiG;IACjG,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,uBAAuB,EAC/B,UAAU,EAAE,iBAAiB,GAAG,SAAS,EACzC,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,kBAAkB,CAAC,CAgC7B"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session audit events for dsh-library (declaration merging into the harness's
|
|
3
|
+
* `SessionEventMap`) and the adaptive append gate. Both events are log-only;
|
|
4
|
+
* tool arguments and rendered results are already logged by the tool runtime
|
|
5
|
+
* as `tool/call` + `tool/result`, and these events carry the audit facts that
|
|
6
|
+
* exist outside them: the inject id linking the injected marker text back to
|
|
7
|
+
* the search that produced it, and each purge-verification verdict.
|
|
8
|
+
*
|
|
9
|
+
* The gate appends only when the host can carry the events safely:
|
|
10
|
+
* - hosts whose known-type set covers the vocabulary append plainly;
|
|
11
|
+
* - hosts with an `ignorable` append option (pre-0.1.2 master builds) append
|
|
12
|
+
* with the marker, so builds that do not know the type skip it on restore;
|
|
13
|
+
* - envelope-less hosts (0.1.0-rc.6/rc.8, 0.1.1-rc.2, and 0.1.2-alpha.1,
|
|
14
|
+
* which removed the envelope and fails closed on unknown types at read)
|
|
15
|
+
* get no append — the tool results remain the reconstructable audit trail.
|
|
16
|
+
*
|
|
17
|
+
* @module dsh-library/events
|
|
18
|
+
*/
|
|
19
|
+
import { type Session } from '@deepseek-ai/dsh-session';
|
|
20
|
+
declare module '@deepseek-ai/dsh-session/types' {
|
|
21
|
+
interface SessionEventMap {
|
|
22
|
+
/** One `library_search` injection: id links the injected marker text back to this event. */
|
|
23
|
+
'library/inject': LibraryInjectEvent;
|
|
24
|
+
/** One `library_remove` purge verification outcome. */
|
|
25
|
+
'library/purge': LibraryPurgeEvent;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** The `library_search` injection audit payload. */
|
|
29
|
+
export interface LibraryInjectEvent {
|
|
30
|
+
/** Inject id carried by the injected marker text. */
|
|
31
|
+
injectId: string;
|
|
32
|
+
/** Library the search ran against. */
|
|
33
|
+
library: string;
|
|
34
|
+
/** The search query. */
|
|
35
|
+
query: string;
|
|
36
|
+
/** Chunk ids of the injected result page. */
|
|
37
|
+
chunks: string[];
|
|
38
|
+
/** Injected page length in characters (after the budget cap). */
|
|
39
|
+
chars: number;
|
|
40
|
+
}
|
|
41
|
+
/** The `library_remove` purge-verification audit payload. */
|
|
42
|
+
export interface LibraryPurgeEvent {
|
|
43
|
+
/** Purge probe run id. */
|
|
44
|
+
purgeId: string;
|
|
45
|
+
/** Library the document was removed from. */
|
|
46
|
+
library: string;
|
|
47
|
+
/** Removed document id. */
|
|
48
|
+
documentId: string;
|
|
49
|
+
/** Whether the purge probes found no residue. */
|
|
50
|
+
passed: boolean;
|
|
51
|
+
/** Residue hits found by the purge probes. */
|
|
52
|
+
totalFound: number;
|
|
53
|
+
}
|
|
54
|
+
/** The injection audit event type. */
|
|
55
|
+
export declare const INJECT_EVENT: "library/inject";
|
|
56
|
+
/** The purge audit event type. */
|
|
57
|
+
export declare const PURGE_EVENT: "library/purge";
|
|
58
|
+
/**
|
|
59
|
+
* Append one dsh-library audit event when the host can carry it safely; skip
|
|
60
|
+
* silently otherwise (the `tool/call` + `tool/result` events remain the
|
|
61
|
+
* model-visible log, so nothing model-visible is lost). See the module doc
|
|
62
|
+
* for the three host classes.
|
|
63
|
+
* @param session - the calling session.
|
|
64
|
+
* @param type - the audit event type.
|
|
65
|
+
* @param data - the audit payload.
|
|
66
|
+
*/
|
|
67
|
+
export declare function appendAuditEvent(session: Session, type: typeof INJECT_EVENT | typeof PURGE_EVENT, data: LibraryInjectEvent | LibraryPurgeEvent): void;
|
|
68
|
+
//# sourceMappingURL=events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/events.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAA6B,KAAK,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAElF,OAAO,QAAQ,gCAAgC,CAAC;IAC9C,UAAU,eAAe;QACvB,4FAA4F;QAC5F,gBAAgB,EAAE,kBAAkB,CAAA;QACpC,uDAAuD;QACvD,eAAe,EAAE,iBAAiB,CAAA;KACnC;CACF;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAkB;IACjC,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAA;IAChB,sCAAsC;IACtC,OAAO,EAAE,MAAM,CAAA;IACf,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,6CAA6C;IAC7C,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAA;CACd;AAED,6DAA6D;AAC7D,MAAM,WAAW,iBAAiB;IAChC,0BAA0B;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAA;IACf,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,iDAAiD;IACjD,MAAM,EAAE,OAAO,CAAA;IACf,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,sCAAsC;AACtC,eAAO,MAAM,YAAY,EAAG,gBAAyB,CAAA;AAErD,kCAAkC;AAClC,eAAO,MAAM,WAAW,EAAG,eAAwB,CAAA;AAKnD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,OAAO,YAAY,GAAG,OAAO,WAAW,EAC9C,IAAI,EAAE,kBAAkB,GAAG,iBAAiB,GAC3C,IAAI,CAUN"}
|