@crewhaus/tool-retrieve 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +142 -3
- package/dist/index.js +217 -22
- package/package.json +7 -6
package/dist/index.d.ts
CHANGED
|
@@ -8,11 +8,20 @@
|
|
|
8
8
|
* Configuration is registered at boot via
|
|
9
9
|
* `registerRetrieveConfig({ embedder, vectorStore, defaultK? })` so
|
|
10
10
|
* the codegen layer can wire the tool with whichever backends the
|
|
11
|
-
* spec declares.
|
|
11
|
+
* spec declares (the pipeline shape's path).
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
13
|
+
* The agent-shape RAG path (`knowledge:` on cli/channel/managed, Batch E
|
|
14
|
+
* item 3/G22) is served by {@link knowledgeRetrieve}: it ingests declared
|
|
15
|
+
* `sources` (path/glob/url) through the SAME chunker → embedder →
|
|
16
|
+
* vector-store engine target-pipeline emits inline, then hands back a
|
|
17
|
+
* self-contained `Retrieve` tool bound to its own config (no module
|
|
18
|
+
* singleton). {@link resolveKnowledgeEmbedder} enforces the G76 embedder
|
|
19
|
+
* resolution order the emitters share.
|
|
20
|
+
*
|
|
21
|
+
* Layer R12. Pairs with `embedder`, `vector-store`, `chunker`,
|
|
22
|
+
* `pipeline-engine`, `target-pipeline`, `target-cli`/`-channel-bot`/`-managed`.
|
|
15
23
|
*/
|
|
24
|
+
import { type ChunkStrategy, type Document } from "@crewhaus/chunker";
|
|
16
25
|
import { type Embedder } from "@crewhaus/embedder";
|
|
17
26
|
import { CrewhausError } from "@crewhaus/errors";
|
|
18
27
|
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
@@ -52,3 +61,133 @@ export declare function getRetrieveConfig(): RetrieveConfig | undefined;
|
|
|
52
61
|
/** Test-only — clear cached config. */
|
|
53
62
|
export declare function _resetRetrieveConfig(): void;
|
|
54
63
|
export declare const retrieve: RegisteredTool;
|
|
64
|
+
/** The target's default embedder model when nothing else resolves (G76). A
|
|
65
|
+
* vector store needs embeddings, so knowledge RAG never degrades to BM25 —
|
|
66
|
+
* this is the floor {@link resolveKnowledgeEmbedder} lands on. Mirrors the
|
|
67
|
+
* egress-classifier default model the serving emitters already carry. */
|
|
68
|
+
export declare const DEFAULT_KNOWLEDGE_EMBEDDER_MODEL = "openai/text-embedding-3-small";
|
|
69
|
+
/** Resolved knowledge-RAG defaults — kept in lockstep with the pipeline
|
|
70
|
+
* retrieve engine (`in-memory` / k=5 / 400-char chunks / no overlap) so a
|
|
71
|
+
* `knowledge:` block and a `retrieve:` block index identically. */
|
|
72
|
+
export declare const DEFAULT_KNOWLEDGE_K = 5;
|
|
73
|
+
export declare const DEFAULT_KNOWLEDGE_CHUNK_SIZE = 400;
|
|
74
|
+
export declare const DEFAULT_KNOWLEDGE_CHUNK_OVERLAP = 0;
|
|
75
|
+
/** Knowledge sources are arbitrary docs, so the general-purpose fixed-window
|
|
76
|
+
* strategy is the safe default (pipeline lets the spec pick; `knowledge:`
|
|
77
|
+
* carries only size/overlap, so the strategy is fixed here). */
|
|
78
|
+
export declare const DEFAULT_KNOWLEDGE_CHUNK_STRATEGY: ChunkStrategy;
|
|
79
|
+
/**
|
|
80
|
+
* One declared knowledge corpus entry — EXACTLY one of path/glob/url, mirroring
|
|
81
|
+
* the `knowledge.sources[]` spec grammar and `IrKnowledgeSource`. Paths and
|
|
82
|
+
* globs resolve against the ingest `cwd`; a url is fetched verbatim at boot.
|
|
83
|
+
*/
|
|
84
|
+
export type KnowledgeSource = {
|
|
85
|
+
readonly kind: "path";
|
|
86
|
+
readonly path: string;
|
|
87
|
+
} | {
|
|
88
|
+
readonly kind: "glob";
|
|
89
|
+
readonly glob: string;
|
|
90
|
+
} | {
|
|
91
|
+
readonly kind: "url";
|
|
92
|
+
readonly url: string;
|
|
93
|
+
};
|
|
94
|
+
/**
|
|
95
|
+
* The four candidate embedder-model strings for a knowledge block, in the
|
|
96
|
+
* G76 resolution order. Every field is the raw declared string the IR carries
|
|
97
|
+
* (or absent); {@link resolveKnowledgeEmbedder} folds them.
|
|
98
|
+
*/
|
|
99
|
+
export type KnowledgeEmbedderInputs = {
|
|
100
|
+
/** `knowledge.embedder` — highest precedence. */
|
|
101
|
+
readonly knowledgeEmbedder?: string;
|
|
102
|
+
/** `memory.embedder` — the fact-store/curator embedder. */
|
|
103
|
+
readonly memoryEmbedder?: string;
|
|
104
|
+
/** `memory.wiki.embedder` — the wiki-tier embedder. */
|
|
105
|
+
readonly wikiEmbedder?: string;
|
|
106
|
+
/** The emitting target's own default (defaults to
|
|
107
|
+
* {@link DEFAULT_KNOWLEDGE_EMBEDDER_MODEL} when omitted). */
|
|
108
|
+
readonly targetDefault?: string;
|
|
109
|
+
};
|
|
110
|
+
/**
|
|
111
|
+
* G76 embedder resolution for agent-shape RAG, the one place the order is
|
|
112
|
+
* enforced so cli/channel/managed emitters cannot drift:
|
|
113
|
+
*
|
|
114
|
+
* `knowledge.embedder → memory.embedder → memory.wiki.embedder → target default`
|
|
115
|
+
*
|
|
116
|
+
* Unlike fact-store recall + the curator (which degrade to BM25 when nothing
|
|
117
|
+
* resolves), a vector store REQUIRES an embedding model, so this always
|
|
118
|
+
* returns a non-empty string — falling through to
|
|
119
|
+
* {@link DEFAULT_KNOWLEDGE_EMBEDDER_MODEL} rather than ever yielding
|
|
120
|
+
* "BM25-only". (Contrast the wiki tier, which flips memory/wiki precedence,
|
|
121
|
+
* and the fact store, which stops at BM25.)
|
|
122
|
+
*/
|
|
123
|
+
export declare function resolveKnowledgeEmbedder(inputs: KnowledgeEmbedderInputs): string;
|
|
124
|
+
/** The subset of the WHATWG `fetch` reply {@link knowledgeRetrieve} reads from
|
|
125
|
+
* a `url` source. Kept structural so tests inject a stub without a live net. */
|
|
126
|
+
export type KnowledgeFetchResponse = {
|
|
127
|
+
readonly ok: boolean;
|
|
128
|
+
readonly status: number;
|
|
129
|
+
text(): Promise<string>;
|
|
130
|
+
};
|
|
131
|
+
export type KnowledgeFetch = (url: string) => Promise<KnowledgeFetchResponse>;
|
|
132
|
+
/**
|
|
133
|
+
* The config {@link knowledgeRetrieve} ingests. The embedder + vector store
|
|
134
|
+
* arrive EITHER as a ready instance (tests, or a caller that already built
|
|
135
|
+
* them) OR as the primitives to construct — exactly the two-way shape
|
|
136
|
+
* {@link RetrieveConfigInput} accepts, so the pipeline and knowledge paths
|
|
137
|
+
* build their backends identically. `embedderModel` is expected to already
|
|
138
|
+
* be the {@link resolveKnowledgeEmbedder} output.
|
|
139
|
+
*/
|
|
140
|
+
export type KnowledgeRetrieveConfig = {
|
|
141
|
+
readonly sources: readonly KnowledgeSource[];
|
|
142
|
+
/** Prebuilt embedder (wins over `embedderModel`). */
|
|
143
|
+
readonly embedder?: Embedder;
|
|
144
|
+
/** Resolved embedder model string (built when no `embedder` instance). */
|
|
145
|
+
readonly embedderModel?: string;
|
|
146
|
+
readonly embedderApiKey?: string;
|
|
147
|
+
/** Prebuilt vector store (wins over `vectorBackend` + connection fields). */
|
|
148
|
+
readonly vectorStore?: VectorStore;
|
|
149
|
+
readonly vectorBackend?: VectorBackendId;
|
|
150
|
+
/** Vector-store connection (HTTP backends need url + collection; lance reads
|
|
151
|
+
* url as its on-disk path) — the vector store's key, not the embedder's. */
|
|
152
|
+
readonly url?: string;
|
|
153
|
+
readonly collection?: string;
|
|
154
|
+
readonly apiKey?: string;
|
|
155
|
+
readonly defaultK?: number;
|
|
156
|
+
readonly chunkSize?: number;
|
|
157
|
+
readonly chunkOverlap?: number;
|
|
158
|
+
readonly chunkStrategy?: ChunkStrategy;
|
|
159
|
+
/** Base directory for relative path/glob sources. Default `process.cwd()`. */
|
|
160
|
+
readonly cwd?: string;
|
|
161
|
+
/** `url`-source loader. Default: the global `fetch`. */
|
|
162
|
+
readonly fetch?: KnowledgeFetch;
|
|
163
|
+
/** Boot status-line sink (`[knowledge] …`). Compiled bundles stay silent. */
|
|
164
|
+
readonly log?: (line: string) => void;
|
|
165
|
+
};
|
|
166
|
+
/**
|
|
167
|
+
* Read every declared source into a `chunker` {@link Document}. `path` reads
|
|
168
|
+
* one file; `glob` scans the cwd for matches (a zero-match glob contributes
|
|
169
|
+
* nothing — it is a wildcard, not an assertion); `url` fetches the body.
|
|
170
|
+
* A missing explicit `path`/`url`, or an errored fetch, throws with the
|
|
171
|
+
* offending source named — a boot-time ingest fails loudly rather than
|
|
172
|
+
* silently indexing an empty corpus.
|
|
173
|
+
*/
|
|
174
|
+
export declare function loadKnowledgeSources(sources: readonly KnowledgeSource[], opts?: {
|
|
175
|
+
readonly cwd?: string;
|
|
176
|
+
readonly fetch?: KnowledgeFetch;
|
|
177
|
+
}): Promise<Document[]>;
|
|
178
|
+
/**
|
|
179
|
+
* Item 3 (G22) — the one wiring helper all three serving emitters
|
|
180
|
+
* (cli/channel/managed) call to turn a `knowledge:` block into a live,
|
|
181
|
+
* citation-bearing `Retrieve` tool. It ingests `config.sources` through the
|
|
182
|
+
* SAME chunk → embed → upsert engine target-pipeline emits inline (reusing
|
|
183
|
+
* `@crewhaus/chunker`, `@crewhaus/embedder`, `@crewhaus/vector-store`, and
|
|
184
|
+
* this package's `formatHits` citations — no fork), then returns a
|
|
185
|
+
* self-contained tool bound to its own already-populated store. It does NOT
|
|
186
|
+
* touch the pipeline `activeConfig` singleton, so a bundle may register it
|
|
187
|
+
* alongside anything else without global cross-talk.
|
|
188
|
+
*
|
|
189
|
+
* Failure is loud: an unconstructable embedder, or zero documents across all
|
|
190
|
+
* sources, throws {@link RetrieveConfigError} at ingest rather than shipping a
|
|
191
|
+
* `Retrieve` tool that silently answers "no hits" forever.
|
|
192
|
+
*/
|
|
193
|
+
export declare function knowledgeRetrieve(config: KnowledgeRetrieveConfig): Promise<RegisteredTool>;
|
package/dist/index.js
CHANGED
|
@@ -8,11 +8,23 @@
|
|
|
8
8
|
* Configuration is registered at boot via
|
|
9
9
|
* `registerRetrieveConfig({ embedder, vectorStore, defaultK? })` so
|
|
10
10
|
* the codegen layer can wire the tool with whichever backends the
|
|
11
|
-
* spec declares.
|
|
11
|
+
* spec declares (the pipeline shape's path).
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
13
|
+
* The agent-shape RAG path (`knowledge:` on cli/channel/managed, Batch E
|
|
14
|
+
* item 3/G22) is served by {@link knowledgeRetrieve}: it ingests declared
|
|
15
|
+
* `sources` (path/glob/url) through the SAME chunker → embedder →
|
|
16
|
+
* vector-store engine target-pipeline emits inline, then hands back a
|
|
17
|
+
* self-contained `Retrieve` tool bound to its own config (no module
|
|
18
|
+
* singleton). {@link resolveKnowledgeEmbedder} enforces the G76 embedder
|
|
19
|
+
* resolution order the emitters share.
|
|
20
|
+
*
|
|
21
|
+
* Layer R12. Pairs with `embedder`, `vector-store`, `chunker`,
|
|
22
|
+
* `pipeline-engine`, `target-pipeline`, `target-cli`/`-channel-bot`/`-managed`.
|
|
15
23
|
*/
|
|
24
|
+
import { existsSync } from "node:fs";
|
|
25
|
+
import { readFile } from "node:fs/promises";
|
|
26
|
+
import { isAbsolute, resolve as resolvePath } from "node:path";
|
|
27
|
+
import { chunk } from "@crewhaus/chunker";
|
|
16
28
|
import { createEmbedder } from "@crewhaus/embedder";
|
|
17
29
|
import { CrewhausError } from "@crewhaus/errors";
|
|
18
30
|
import { buildTool } from "@crewhaus/tool-builder";
|
|
@@ -77,24 +89,207 @@ function formatHits(hits) {
|
|
|
77
89
|
});
|
|
78
90
|
return lines.join("\n\n");
|
|
79
91
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
92
|
+
/**
|
|
93
|
+
* Build a `Retrieve` tool whose backends come from `getConfig` at call time.
|
|
94
|
+
* Shared by the singleton `retrieve` export (pipeline shape — reads
|
|
95
|
+
* `activeConfig`) and every {@link knowledgeRetrieve} tool (agent shape —
|
|
96
|
+
* reads its own captured, already-ingested config), so the schema,
|
|
97
|
+
* citation formatting, and safety flags never fork between the two paths.
|
|
98
|
+
*/
|
|
99
|
+
function makeRetrieveTool(getConfig) {
|
|
100
|
+
return buildTool({
|
|
101
|
+
name: "Retrieve",
|
|
102
|
+
description: "Retrieve top-k chunks from the configured vector store for a natural-language query. Pass `query` (required), optional `k` (default 5), and optional `filter` (metadata predicates). Returns a numbered list of hits with citations.",
|
|
103
|
+
inputSchema: retrieveSchema,
|
|
104
|
+
readOnly: true,
|
|
105
|
+
concurrencySafe: true,
|
|
106
|
+
execute: async (input) => {
|
|
107
|
+
const cfg = getConfig();
|
|
108
|
+
if (cfg === undefined) {
|
|
109
|
+
throw new RetrieveConfigError("Retrieve tool is unconfigured — call registerRetrieveConfig at boot");
|
|
110
|
+
}
|
|
111
|
+
const inp = input;
|
|
112
|
+
const k = inp.k ?? cfg.defaultK ?? 5;
|
|
113
|
+
const [vec] = await cfg.embedder.embed([inp.query]);
|
|
114
|
+
if (vec === undefined) {
|
|
115
|
+
throw new RetrieveConfigError("embedder returned 0 vectors for the query");
|
|
116
|
+
}
|
|
117
|
+
const hits = await cfg.vectorStore.query(vec, k, inp.filter);
|
|
118
|
+
return formatHits(hits);
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
export const retrieve = makeRetrieveTool(() => activeConfig);
|
|
123
|
+
// ===========================================================================
|
|
124
|
+
// Agent-shape RAG — `knowledge:` on cli/channel/managed (Batch E item 3, G22)
|
|
125
|
+
// ===========================================================================
|
|
126
|
+
/** The target's default embedder model when nothing else resolves (G76). A
|
|
127
|
+
* vector store needs embeddings, so knowledge RAG never degrades to BM25 —
|
|
128
|
+
* this is the floor {@link resolveKnowledgeEmbedder} lands on. Mirrors the
|
|
129
|
+
* egress-classifier default model the serving emitters already carry. */
|
|
130
|
+
export const DEFAULT_KNOWLEDGE_EMBEDDER_MODEL = "openai/text-embedding-3-small";
|
|
131
|
+
/** Resolved knowledge-RAG defaults — kept in lockstep with the pipeline
|
|
132
|
+
* retrieve engine (`in-memory` / k=5 / 400-char chunks / no overlap) so a
|
|
133
|
+
* `knowledge:` block and a `retrieve:` block index identically. */
|
|
134
|
+
export const DEFAULT_KNOWLEDGE_K = 5;
|
|
135
|
+
export const DEFAULT_KNOWLEDGE_CHUNK_SIZE = 400;
|
|
136
|
+
export const DEFAULT_KNOWLEDGE_CHUNK_OVERLAP = 0;
|
|
137
|
+
/** Knowledge sources are arbitrary docs, so the general-purpose fixed-window
|
|
138
|
+
* strategy is the safe default (pipeline lets the spec pick; `knowledge:`
|
|
139
|
+
* carries only size/overlap, so the strategy is fixed here). */
|
|
140
|
+
export const DEFAULT_KNOWLEDGE_CHUNK_STRATEGY = "fixed";
|
|
141
|
+
function firstNonEmpty(...values) {
|
|
142
|
+
for (const v of values) {
|
|
143
|
+
if (typeof v === "string" && v.trim() !== "")
|
|
144
|
+
return v;
|
|
145
|
+
}
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* G76 embedder resolution for agent-shape RAG, the one place the order is
|
|
150
|
+
* enforced so cli/channel/managed emitters cannot drift:
|
|
151
|
+
*
|
|
152
|
+
* `knowledge.embedder → memory.embedder → memory.wiki.embedder → target default`
|
|
153
|
+
*
|
|
154
|
+
* Unlike fact-store recall + the curator (which degrade to BM25 when nothing
|
|
155
|
+
* resolves), a vector store REQUIRES an embedding model, so this always
|
|
156
|
+
* returns a non-empty string — falling through to
|
|
157
|
+
* {@link DEFAULT_KNOWLEDGE_EMBEDDER_MODEL} rather than ever yielding
|
|
158
|
+
* "BM25-only". (Contrast the wiki tier, which flips memory/wiki precedence,
|
|
159
|
+
* and the fact store, which stops at BM25.)
|
|
160
|
+
*/
|
|
161
|
+
export function resolveKnowledgeEmbedder(inputs) {
|
|
162
|
+
return (firstNonEmpty(inputs.knowledgeEmbedder, inputs.memoryEmbedder, inputs.wikiEmbedder, inputs.targetDefault) ?? DEFAULT_KNOWLEDGE_EMBEDDER_MODEL);
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Read every declared source into a `chunker` {@link Document}. `path` reads
|
|
166
|
+
* one file; `glob` scans the cwd for matches (a zero-match glob contributes
|
|
167
|
+
* nothing — it is a wildcard, not an assertion); `url` fetches the body.
|
|
168
|
+
* A missing explicit `path`/`url`, or an errored fetch, throws with the
|
|
169
|
+
* offending source named — a boot-time ingest fails loudly rather than
|
|
170
|
+
* silently indexing an empty corpus.
|
|
171
|
+
*/
|
|
172
|
+
export async function loadKnowledgeSources(sources, opts = {}) {
|
|
173
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
174
|
+
const fetchImpl = opts.fetch ?? globalThis.fetch;
|
|
175
|
+
const docs = [];
|
|
176
|
+
for (const src of sources) {
|
|
177
|
+
if (src.kind === "path") {
|
|
178
|
+
const abs = isAbsolute(src.path) ? src.path : resolvePath(cwd, src.path);
|
|
179
|
+
if (!existsSync(abs)) {
|
|
180
|
+
throw new RetrieveConfigError(`knowledge source path not found: ${src.path}`);
|
|
181
|
+
}
|
|
182
|
+
let text;
|
|
183
|
+
try {
|
|
184
|
+
text = await readFile(abs, "utf-8");
|
|
185
|
+
}
|
|
186
|
+
catch (cause) {
|
|
187
|
+
throw new RetrieveConfigError(`knowledge source path unreadable: ${src.path}`, cause);
|
|
188
|
+
}
|
|
189
|
+
docs.push({ id: src.path, text, metadata: { docId: src.path, source: src.path } });
|
|
90
190
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
191
|
+
else if (src.kind === "glob") {
|
|
192
|
+
// Bun.Glob — the same matcher tool-fs's Glob tool uses.
|
|
193
|
+
const matcher = new Bun.Glob(src.glob);
|
|
194
|
+
const rels = [];
|
|
195
|
+
for await (const rel of matcher.scan({ cwd, onlyFiles: true }))
|
|
196
|
+
rels.push(rel);
|
|
197
|
+
rels.sort(); // deterministic chunk ids across boots
|
|
198
|
+
for (const rel of rels) {
|
|
199
|
+
const abs = resolvePath(cwd, rel);
|
|
200
|
+
let text;
|
|
201
|
+
try {
|
|
202
|
+
text = await readFile(abs, "utf-8");
|
|
203
|
+
}
|
|
204
|
+
catch (cause) {
|
|
205
|
+
throw new RetrieveConfigError(`knowledge glob match unreadable: ${rel}`, cause);
|
|
206
|
+
}
|
|
207
|
+
docs.push({ id: rel, text, metadata: { docId: rel, source: rel } });
|
|
208
|
+
}
|
|
96
209
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
}
|
|
210
|
+
else {
|
|
211
|
+
if (fetchImpl === undefined) {
|
|
212
|
+
throw new RetrieveConfigError(`knowledge url source needs a fetch implementation: ${src.url}`);
|
|
213
|
+
}
|
|
214
|
+
let res;
|
|
215
|
+
try {
|
|
216
|
+
res = await fetchImpl(src.url);
|
|
217
|
+
}
|
|
218
|
+
catch (cause) {
|
|
219
|
+
throw new RetrieveConfigError(`knowledge url fetch failed: ${src.url}`, cause);
|
|
220
|
+
}
|
|
221
|
+
if (!res.ok) {
|
|
222
|
+
throw new RetrieveConfigError(`knowledge url fetch failed (${res.status}): ${src.url}`);
|
|
223
|
+
}
|
|
224
|
+
const text = await res.text();
|
|
225
|
+
docs.push({ id: src.url, text, metadata: { docId: src.url, source: src.url } });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return docs;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Item 3 (G22) — the one wiring helper all three serving emitters
|
|
232
|
+
* (cli/channel/managed) call to turn a `knowledge:` block into a live,
|
|
233
|
+
* citation-bearing `Retrieve` tool. It ingests `config.sources` through the
|
|
234
|
+
* SAME chunk → embed → upsert engine target-pipeline emits inline (reusing
|
|
235
|
+
* `@crewhaus/chunker`, `@crewhaus/embedder`, `@crewhaus/vector-store`, and
|
|
236
|
+
* this package's `formatHits` citations — no fork), then returns a
|
|
237
|
+
* self-contained tool bound to its own already-populated store. It does NOT
|
|
238
|
+
* touch the pipeline `activeConfig` singleton, so a bundle may register it
|
|
239
|
+
* alongside anything else without global cross-talk.
|
|
240
|
+
*
|
|
241
|
+
* Failure is loud: an unconstructable embedder, or zero documents across all
|
|
242
|
+
* sources, throws {@link RetrieveConfigError} at ingest rather than shipping a
|
|
243
|
+
* `Retrieve` tool that silently answers "no hits" forever.
|
|
244
|
+
*/
|
|
245
|
+
export async function knowledgeRetrieve(config) {
|
|
246
|
+
if (config.sources.length === 0) {
|
|
247
|
+
throw new RetrieveConfigError("knowledgeRetrieve requires at least one source");
|
|
248
|
+
}
|
|
249
|
+
let embedder = config.embedder;
|
|
250
|
+
if (embedder === undefined) {
|
|
251
|
+
if (config.embedderModel === undefined || config.embedderModel.trim() === "") {
|
|
252
|
+
throw new RetrieveConfigError("knowledgeRetrieve requires either an `embedder` instance or a resolved `embedderModel`");
|
|
253
|
+
}
|
|
254
|
+
embedder = createEmbedder({
|
|
255
|
+
model: config.embedderModel,
|
|
256
|
+
...(config.embedderApiKey !== undefined ? { apiKey: config.embedderApiKey } : {}),
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
const vectorStore = config.vectorStore ??
|
|
260
|
+
createVectorStore({
|
|
261
|
+
backend: config.vectorBackend ?? "in-memory",
|
|
262
|
+
...(config.url !== undefined ? { url: config.url } : {}),
|
|
263
|
+
...(config.apiKey !== undefined ? { apiKey: config.apiKey } : {}),
|
|
264
|
+
...(config.collection !== undefined ? { collection: config.collection } : {}),
|
|
265
|
+
});
|
|
266
|
+
const docs = await loadKnowledgeSources(config.sources, {
|
|
267
|
+
...(config.cwd !== undefined ? { cwd: config.cwd } : {}),
|
|
268
|
+
...(config.fetch !== undefined ? { fetch: config.fetch } : {}),
|
|
269
|
+
});
|
|
270
|
+
if (docs.length === 0) {
|
|
271
|
+
throw new RetrieveConfigError("knowledgeRetrieve indexed zero documents — every source was empty or matched nothing");
|
|
272
|
+
}
|
|
273
|
+
const chunks = docs.flatMap((doc) => chunk(doc, {
|
|
274
|
+
strategy: config.chunkStrategy ?? DEFAULT_KNOWLEDGE_CHUNK_STRATEGY,
|
|
275
|
+
size: config.chunkSize ?? DEFAULT_KNOWLEDGE_CHUNK_SIZE,
|
|
276
|
+
overlap: config.chunkOverlap ?? DEFAULT_KNOWLEDGE_CHUNK_OVERLAP,
|
|
277
|
+
}));
|
|
278
|
+
if (chunks.length > 0) {
|
|
279
|
+
const vectors = await embedder.embed(chunks.map((c) => c.text));
|
|
280
|
+
for (let i = 0; i < chunks.length; i += 1) {
|
|
281
|
+
const c = chunks[i];
|
|
282
|
+
const v = vectors[i];
|
|
283
|
+
if (c === undefined || v === undefined)
|
|
284
|
+
continue;
|
|
285
|
+
await vectorStore.upsert(c.id, v, { docId: c.docId, text: c.text });
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
config.log?.(`[knowledge] indexed ${await vectorStore.count()} chunks from ${docs.length} source doc(s)\n`);
|
|
289
|
+
const bound = {
|
|
290
|
+
embedder,
|
|
291
|
+
vectorStore,
|
|
292
|
+
defaultK: config.defaultK ?? DEFAULT_KNOWLEDGE_K,
|
|
293
|
+
};
|
|
294
|
+
return makeRetrieveTool(() => bound);
|
|
295
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/tool-retrieve",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Retrieve(query, k?, filter?) tool — composes embedder + vector-store",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -15,11 +15,12 @@
|
|
|
15
15
|
"test": "bun test src"
|
|
16
16
|
},
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@crewhaus/
|
|
19
|
-
"@crewhaus/
|
|
20
|
-
"@crewhaus/
|
|
21
|
-
"@crewhaus/tool-
|
|
22
|
-
"@crewhaus/
|
|
18
|
+
"@crewhaus/chunker": "0.4.0",
|
|
19
|
+
"@crewhaus/embedder": "0.4.0",
|
|
20
|
+
"@crewhaus/errors": "0.4.0",
|
|
21
|
+
"@crewhaus/tool-builder": "0.4.0",
|
|
22
|
+
"@crewhaus/tool-catalog": "0.4.0",
|
|
23
|
+
"@crewhaus/vector-store": "0.4.0",
|
|
23
24
|
"zod": "^3.23.8"
|
|
24
25
|
},
|
|
25
26
|
"license": "Apache-2.0",
|