@hasna/knowledge 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/bin/open-knowledge-mcp.js +2275 -1643
- package/bin/open-knowledge.js +105 -79
- package/docs/architecture/ai-native-knowledge-base.md +11 -2
- package/docs/architecture/hybrid-semantic-search.md +12 -2
- package/package.json +2 -1
- package/src/agent.ts +367 -0
- package/src/cli.ts +58 -7
- package/src/mcp.js +36 -0
- package/src/providers.ts +1 -1
- package/src/retrieval.ts +326 -0
- package/src/service.ts +20 -0
|
@@ -223,8 +223,11 @@ generated metadata rows, not raw source bytes, and pins each row to `open-files`
|
|
|
223
223
|
provenance: source ref/URI, revision/hash, chunk offsets, token count, provider,
|
|
224
224
|
model, dimensions, status, and timestamps. The structured `search` contract
|
|
225
225
|
merges keyword FTS, wiki/index catalog hits, generated wiki chunks, and optional
|
|
226
|
-
vector results
|
|
227
|
-
|
|
226
|
+
vector results. `open-knowledge search --context` and MCP `knowledge_search`
|
|
227
|
+
turn those rows into reranked citation context packs with selected excerpts,
|
|
228
|
+
freshness and permission notes, graph evidence, and final rerank scores. The
|
|
229
|
+
local SQLite index can later move to pgvector or a managed hosted vector store
|
|
230
|
+
without changing CLI/MCP result shape.
|
|
228
231
|
|
|
229
232
|
## Agent Workflow
|
|
230
233
|
|
|
@@ -246,6 +249,12 @@ The command should:
|
|
|
246
249
|
7. Record a run ledger with tool calls, sources, costs, outputs, and generated
|
|
247
250
|
records.
|
|
248
251
|
|
|
252
|
+
The first implementation exposes this as `open-knowledge ask|build <prompt>`
|
|
253
|
+
and the installed `knowledge <prompt>` bin alias. It retrieves read-only context,
|
|
254
|
+
returns a local citation draft by default, optionally calls AI SDK generation via
|
|
255
|
+
`--generate`, records `runs`, `run_events`, and `provider_usage`, and only
|
|
256
|
+
proposes durable wiki updates until the wiki compile/write task owns writes.
|
|
257
|
+
|
|
249
258
|
## Provider Registry
|
|
250
259
|
|
|
251
260
|
AI provider setup is BYOK and AI SDK v6 based. The local provider layer tracks:
|
|
@@ -49,14 +49,15 @@ The current local command surface is:
|
|
|
49
49
|
```bash
|
|
50
50
|
open-knowledge search "company wiki policy" --scope project --json
|
|
51
51
|
open-knowledge search "company wiki policy" --scope project --semantic --json
|
|
52
|
+
open-knowledge search "company wiki policy" --scope project --context --json
|
|
52
53
|
open-knowledge embeddings index --scope project --model openai:text-embedding-3-small
|
|
53
54
|
open-knowledge embeddings search "company wiki policy" --scope project --json
|
|
54
55
|
```
|
|
55
56
|
|
|
56
57
|
`search` is the structured hybrid layer for agents. `embeddings search` is the
|
|
57
58
|
lower-level vector-only command. MCP exposes the same capability through
|
|
58
|
-
`ok_search`, `ok_embeddings_status`, `ok_embeddings_index`,
|
|
59
|
-
`ok_semantic_search`. Deterministic `--fake` embeddings exist for tests and
|
|
59
|
+
`ok_search`, `knowledge_search`, `ok_embeddings_status`, `ok_embeddings_index`,
|
|
60
|
+
and `ok_semantic_search`. Deterministic `--fake` embeddings exist for tests and
|
|
60
61
|
offline verification only.
|
|
61
62
|
|
|
62
63
|
## Hosted Indexes
|
|
@@ -132,6 +133,15 @@ They should receive context packs:
|
|
|
132
133
|
|
|
133
134
|
This keeps agent prompts stable while the retrieval internals evolve.
|
|
134
135
|
|
|
136
|
+
The local context-pack implementation is available through
|
|
137
|
+
`open-knowledge search --context` and MCP `knowledge_search`. It reranks merged
|
|
138
|
+
search rows using exact-term coverage, citation availability, source freshness,
|
|
139
|
+
and source/wiki authority, then emits excerpts and citation objects that preserve
|
|
140
|
+
source refs, artifact URIs, revision/hash metadata, offsets, and provenance.
|
|
141
|
+
`open-knowledge ask|build <prompt>`, the installed `knowledge <prompt>` alias,
|
|
142
|
+
and MCP `knowledge_ask` wrap this context pack in a run ledger and return a
|
|
143
|
+
citation draft or explicit AI SDK generated answer.
|
|
144
|
+
|
|
135
145
|
## Reindexing
|
|
136
146
|
|
|
137
147
|
Reindexing is driven by source revisions:
|
package/package.json
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hasna/knowledge",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.17",
|
|
4
4
|
"description": "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
|
+
"knowledge": "bin/open-knowledge.js",
|
|
7
8
|
"open-knowledge": "bin/open-knowledge.js",
|
|
8
9
|
"open-knowledge-mcp": "bin/open-knowledge-mcp.js"
|
|
9
10
|
},
|
package/src/agent.ts
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
|
|
3
|
+
import { languageModelFor, normalizeAiSdkUsage, parseModelRef, recordProviderUsage, resolveModelRef } from './providers';
|
|
4
|
+
import { retrieveKnowledgeContext, type KnowledgeContextPack, type RetrievalOptions } from './retrieval';
|
|
5
|
+
import type { KnowledgeConfig } from './workspace';
|
|
6
|
+
|
|
7
|
+
export interface KnowledgePromptOptions extends Omit<RetrievalOptions, 'query'> {
|
|
8
|
+
prompt: string;
|
|
9
|
+
generate?: boolean;
|
|
10
|
+
approveWrite?: boolean;
|
|
11
|
+
now?: Date;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface KnowledgePromptResult {
|
|
15
|
+
run_id: string;
|
|
16
|
+
prompt: string;
|
|
17
|
+
generated: boolean;
|
|
18
|
+
provider: string;
|
|
19
|
+
model: string;
|
|
20
|
+
answer: string;
|
|
21
|
+
context: KnowledgeContextPack;
|
|
22
|
+
citations: KnowledgeContextPack['citations'];
|
|
23
|
+
proposed_wiki_updates: Array<{
|
|
24
|
+
kind: 'answer_note';
|
|
25
|
+
title: string;
|
|
26
|
+
citations: string[];
|
|
27
|
+
requires_approval: boolean;
|
|
28
|
+
}>;
|
|
29
|
+
write_policy: {
|
|
30
|
+
approved: boolean;
|
|
31
|
+
durable_writes_performed: false;
|
|
32
|
+
reason: string;
|
|
33
|
+
};
|
|
34
|
+
usage: {
|
|
35
|
+
input_tokens: number;
|
|
36
|
+
output_tokens: number;
|
|
37
|
+
cost_usd: number;
|
|
38
|
+
};
|
|
39
|
+
warnings: string[];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function estimateTokens(text: string): number {
|
|
43
|
+
const words = text.trim().split(/\s+/).filter(Boolean).length;
|
|
44
|
+
return Math.max(1, Math.ceil(words * 1.25));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function citationLabel(index: number): string {
|
|
48
|
+
return `C${index + 1}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function localAnswer(prompt: string, context: KnowledgeContextPack): string {
|
|
52
|
+
if (context.excerpts.length === 0) {
|
|
53
|
+
return `No indexed knowledge matched the prompt: ${prompt}`;
|
|
54
|
+
}
|
|
55
|
+
const lines = [
|
|
56
|
+
`Found ${context.excerpts.length} relevant knowledge excerpt(s) for: ${prompt}`,
|
|
57
|
+
'',
|
|
58
|
+
...context.excerpts.slice(0, 5).map((excerpt, index) => {
|
|
59
|
+
const citation = context.citations.find((entry) => entry.id === excerpt.citation_id);
|
|
60
|
+
const ref = citation?.source_ref ?? citation?.source_uri ?? citation?.artifact_path ?? citation?.artifact_uri ?? 'unknown source';
|
|
61
|
+
return `[${citationLabel(index)}] ${excerpt.text} (${ref})`;
|
|
62
|
+
}),
|
|
63
|
+
];
|
|
64
|
+
return lines.join('\n');
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function promptForModel(prompt: string, context: KnowledgeContextPack): string {
|
|
68
|
+
const citations = context.citations.map((citation, index) => ({
|
|
69
|
+
id: citationLabel(index),
|
|
70
|
+
source_ref: citation.source_ref,
|
|
71
|
+
source_uri: citation.source_uri,
|
|
72
|
+
artifact_path: citation.artifact_path,
|
|
73
|
+
revision: citation.revision,
|
|
74
|
+
hash: citation.hash,
|
|
75
|
+
quote: citation.quote,
|
|
76
|
+
}));
|
|
77
|
+
const excerpts = context.excerpts.map((excerpt, index) => ({
|
|
78
|
+
id: citationLabel(index),
|
|
79
|
+
kind: excerpt.kind,
|
|
80
|
+
text: excerpt.text,
|
|
81
|
+
score: excerpt.score,
|
|
82
|
+
}));
|
|
83
|
+
return [
|
|
84
|
+
`Prompt: ${prompt}`,
|
|
85
|
+
'',
|
|
86
|
+
'Use only the provided context. Cite claims with citation ids like [C1]. If context is insufficient, say what is missing.',
|
|
87
|
+
'',
|
|
88
|
+
`Context excerpts:\n${JSON.stringify(excerpts, null, 2)}`,
|
|
89
|
+
'',
|
|
90
|
+
`Citations:\n${JSON.stringify(citations, null, 2)}`,
|
|
91
|
+
].join('\n');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function proposedUpdates(prompt: string, context: KnowledgeContextPack): KnowledgePromptResult['proposed_wiki_updates'] {
|
|
95
|
+
if (context.citations.length === 0) return [];
|
|
96
|
+
return [{
|
|
97
|
+
kind: 'answer_note',
|
|
98
|
+
title: prompt.length > 80 ? `${prompt.slice(0, 77)}...` : prompt,
|
|
99
|
+
citations: context.citations.map((citation) => citation.id),
|
|
100
|
+
requires_approval: true,
|
|
101
|
+
}];
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function insertRun(dbPath: string, input: {
|
|
105
|
+
runId: string;
|
|
106
|
+
prompt: string;
|
|
107
|
+
status: string;
|
|
108
|
+
provider: string;
|
|
109
|
+
model: string;
|
|
110
|
+
metadata: Record<string, unknown>;
|
|
111
|
+
now: string;
|
|
112
|
+
}): void {
|
|
113
|
+
const db = openKnowledgeDb(dbPath);
|
|
114
|
+
try {
|
|
115
|
+
db.run(
|
|
116
|
+
`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
117
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
118
|
+
[
|
|
119
|
+
input.runId,
|
|
120
|
+
'knowledge-prompt',
|
|
121
|
+
input.prompt,
|
|
122
|
+
input.status,
|
|
123
|
+
input.provider,
|
|
124
|
+
input.model,
|
|
125
|
+
JSON.stringify(input.metadata),
|
|
126
|
+
input.now,
|
|
127
|
+
input.now,
|
|
128
|
+
],
|
|
129
|
+
);
|
|
130
|
+
} finally {
|
|
131
|
+
db.close();
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function addRunEvent(dbPath: string, input: {
|
|
136
|
+
runId: string;
|
|
137
|
+
level: 'info' | 'warn' | 'error';
|
|
138
|
+
event: string;
|
|
139
|
+
metadata: Record<string, unknown>;
|
|
140
|
+
now: string;
|
|
141
|
+
}): void {
|
|
142
|
+
const db = openKnowledgeDb(dbPath);
|
|
143
|
+
try {
|
|
144
|
+
db.run(
|
|
145
|
+
`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
146
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
147
|
+
[
|
|
148
|
+
`evt_${randomUUID()}`,
|
|
149
|
+
input.runId,
|
|
150
|
+
input.level,
|
|
151
|
+
input.event,
|
|
152
|
+
JSON.stringify(input.metadata),
|
|
153
|
+
input.now,
|
|
154
|
+
],
|
|
155
|
+
);
|
|
156
|
+
} finally {
|
|
157
|
+
db.close();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function updateRun(dbPath: string, input: {
|
|
162
|
+
runId: string;
|
|
163
|
+
status: string;
|
|
164
|
+
provider: string;
|
|
165
|
+
model: string;
|
|
166
|
+
metadata: Record<string, unknown>;
|
|
167
|
+
now: string;
|
|
168
|
+
}): void {
|
|
169
|
+
const db = openKnowledgeDb(dbPath);
|
|
170
|
+
try {
|
|
171
|
+
db.run(
|
|
172
|
+
`UPDATE runs
|
|
173
|
+
SET status = ?, provider = ?, model = ?, metadata_json = ?, updated_at = ?
|
|
174
|
+
WHERE id = ?`,
|
|
175
|
+
[
|
|
176
|
+
input.status,
|
|
177
|
+
input.provider,
|
|
178
|
+
input.model,
|
|
179
|
+
JSON.stringify(input.metadata),
|
|
180
|
+
input.now,
|
|
181
|
+
input.runId,
|
|
182
|
+
],
|
|
183
|
+
);
|
|
184
|
+
} finally {
|
|
185
|
+
db.close();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function recordUsage(dbPath: string, runId: string, usage: KnowledgePromptResult['usage'], provider: string, model: string, now: string, metadata: Record<string, unknown> = {}): void {
|
|
190
|
+
const db = openKnowledgeDb(dbPath);
|
|
191
|
+
try {
|
|
192
|
+
recordProviderUsage(db, {
|
|
193
|
+
run_id: runId,
|
|
194
|
+
provider,
|
|
195
|
+
model,
|
|
196
|
+
input_tokens: usage.input_tokens,
|
|
197
|
+
output_tokens: usage.output_tokens,
|
|
198
|
+
cost_usd: usage.cost_usd,
|
|
199
|
+
metadata,
|
|
200
|
+
created_at: now,
|
|
201
|
+
});
|
|
202
|
+
} finally {
|
|
203
|
+
db.close();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export async function runKnowledgePrompt(options: KnowledgePromptOptions): Promise<KnowledgePromptResult> {
|
|
208
|
+
const prompt = options.prompt.trim();
|
|
209
|
+
if (!prompt) throw new Error('Knowledge prompt is required.');
|
|
210
|
+
const now = (options.now ?? new Date()).toISOString();
|
|
211
|
+
const runId = `run_${randomUUID()}`;
|
|
212
|
+
const modelRef = resolveModelRef(options.modelRef ?? 'default', options.config);
|
|
213
|
+
const parsed = parseModelRef(modelRef);
|
|
214
|
+
|
|
215
|
+
migrateKnowledgeDb(options.dbPath);
|
|
216
|
+
insertRun(options.dbPath, {
|
|
217
|
+
runId,
|
|
218
|
+
prompt,
|
|
219
|
+
status: options.generate ? 'running' : 'dry_run',
|
|
220
|
+
provider: options.generate ? parsed.provider : 'local',
|
|
221
|
+
model: options.generate ? parsed.model : 'context-draft',
|
|
222
|
+
metadata: {
|
|
223
|
+
semantic: options.semantic === true || options.fake === true || Boolean(options.modelRef),
|
|
224
|
+
approve_write: options.approveWrite === true,
|
|
225
|
+
generated: options.generate === true,
|
|
226
|
+
},
|
|
227
|
+
now,
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const { prompt: _prompt, generate: _generate, approveWrite: _approveWrite, now: _now, ...retrievalOptions } = options;
|
|
231
|
+
const context = await retrieveKnowledgeContext({
|
|
232
|
+
...retrievalOptions,
|
|
233
|
+
query: prompt,
|
|
234
|
+
});
|
|
235
|
+
addRunEvent(options.dbPath, {
|
|
236
|
+
runId,
|
|
237
|
+
level: 'info',
|
|
238
|
+
event: 'context_retrieved',
|
|
239
|
+
metadata: {
|
|
240
|
+
results: context.results.length,
|
|
241
|
+
citations: context.citations.length,
|
|
242
|
+
warnings: context.warnings,
|
|
243
|
+
},
|
|
244
|
+
now,
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
let answer = localAnswer(prompt, context);
|
|
248
|
+
let generated = false;
|
|
249
|
+
let provider = 'local';
|
|
250
|
+
let model = 'context-draft';
|
|
251
|
+
let usage = {
|
|
252
|
+
input_tokens: estimateTokens(prompt) + context.excerpts.reduce((sum, excerpt) => sum + estimateTokens(excerpt.text), 0),
|
|
253
|
+
output_tokens: estimateTokens(answer),
|
|
254
|
+
cost_usd: 0,
|
|
255
|
+
};
|
|
256
|
+
const warnings = [...context.warnings];
|
|
257
|
+
|
|
258
|
+
if (options.generate) {
|
|
259
|
+
try {
|
|
260
|
+
if (options.fake) {
|
|
261
|
+
generated = true;
|
|
262
|
+
provider = parsed.provider;
|
|
263
|
+
model = parsed.model;
|
|
264
|
+
answer = `Fake generated answer for: ${prompt}\n\n${answer}`;
|
|
265
|
+
} else {
|
|
266
|
+
const { generateText } = await import('ai');
|
|
267
|
+
const languageModel = await languageModelFor(modelRef, {
|
|
268
|
+
config: options.config,
|
|
269
|
+
env: options.env,
|
|
270
|
+
});
|
|
271
|
+
const result = await generateText({
|
|
272
|
+
model: languageModel as never,
|
|
273
|
+
system: 'You answer company knowledge-base prompts using only provided context and citation ids.',
|
|
274
|
+
prompt: promptForModel(prompt, context),
|
|
275
|
+
});
|
|
276
|
+
generated = true;
|
|
277
|
+
provider = parsed.provider;
|
|
278
|
+
model = parsed.model;
|
|
279
|
+
answer = result.text;
|
|
280
|
+
const normalized = normalizeAiSdkUsage({
|
|
281
|
+
provider,
|
|
282
|
+
model,
|
|
283
|
+
usage: result.usage as Record<string, unknown> | undefined,
|
|
284
|
+
providerMetadata: result.providerMetadata as Record<string, unknown> | undefined,
|
|
285
|
+
});
|
|
286
|
+
usage = {
|
|
287
|
+
input_tokens: normalized.input_tokens,
|
|
288
|
+
output_tokens: normalized.output_tokens,
|
|
289
|
+
cost_usd: normalized.cost_usd,
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
} catch (error) {
|
|
293
|
+
addRunEvent(options.dbPath, {
|
|
294
|
+
runId,
|
|
295
|
+
level: 'error',
|
|
296
|
+
event: 'answer_generation_failed',
|
|
297
|
+
metadata: { message: error instanceof Error ? error.message : String(error) },
|
|
298
|
+
now,
|
|
299
|
+
});
|
|
300
|
+
updateRun(options.dbPath, {
|
|
301
|
+
runId,
|
|
302
|
+
status: 'failed',
|
|
303
|
+
provider: parsed.provider,
|
|
304
|
+
model: parsed.model,
|
|
305
|
+
metadata: {
|
|
306
|
+
generated: false,
|
|
307
|
+
error: error instanceof Error ? error.message : String(error),
|
|
308
|
+
},
|
|
309
|
+
now,
|
|
310
|
+
});
|
|
311
|
+
throw error;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const updates = proposedUpdates(prompt, context);
|
|
316
|
+
const writePolicy = {
|
|
317
|
+
approved: options.approveWrite === true,
|
|
318
|
+
durable_writes_performed: false as const,
|
|
319
|
+
reason: options.approveWrite
|
|
320
|
+
? 'Approval flag recorded; durable wiki writing is deferred to the wiki compile task.'
|
|
321
|
+
: 'Dry-run mode: proposed wiki updates require approval before durable writes.',
|
|
322
|
+
};
|
|
323
|
+
addRunEvent(options.dbPath, {
|
|
324
|
+
runId,
|
|
325
|
+
level: 'info',
|
|
326
|
+
event: generated ? 'answer_generated' : 'answer_drafted',
|
|
327
|
+
metadata: {
|
|
328
|
+
provider,
|
|
329
|
+
model,
|
|
330
|
+
proposed_updates: updates.length,
|
|
331
|
+
durable_writes_performed: false,
|
|
332
|
+
},
|
|
333
|
+
now,
|
|
334
|
+
});
|
|
335
|
+
recordUsage(options.dbPath, runId, usage, provider, model, now, {
|
|
336
|
+
generated,
|
|
337
|
+
citations: context.citations.length,
|
|
338
|
+
});
|
|
339
|
+
updateRun(options.dbPath, {
|
|
340
|
+
runId,
|
|
341
|
+
status: generated ? 'completed' : 'dry_run',
|
|
342
|
+
provider,
|
|
343
|
+
model,
|
|
344
|
+
metadata: {
|
|
345
|
+
generated,
|
|
346
|
+
citations: context.citations.length,
|
|
347
|
+
proposed_updates: updates.length,
|
|
348
|
+
approve_write: options.approveWrite === true,
|
|
349
|
+
},
|
|
350
|
+
now,
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
return {
|
|
354
|
+
run_id: runId,
|
|
355
|
+
prompt,
|
|
356
|
+
generated,
|
|
357
|
+
provider,
|
|
358
|
+
model,
|
|
359
|
+
answer,
|
|
360
|
+
context,
|
|
361
|
+
citations: context.citations,
|
|
362
|
+
proposed_wiki_updates: updates,
|
|
363
|
+
write_policy: writePolicy,
|
|
364
|
+
usage,
|
|
365
|
+
warnings,
|
|
366
|
+
};
|
|
367
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { openKnowledgeDb } from './knowledge-db';
|
|
|
9
9
|
import { createKnowledgeService } from './service';
|
|
10
10
|
import { assertProviderCredentials, parseModelRef, resolveModelRef, type AiProviderId } from './providers';
|
|
11
11
|
import { approvalStatus, assertS3ReadAllowed, assertWebSearchAllowed, createApprovalGate, recordAuditEvent, recordRedactionFindings, redactSecrets } from './safety';
|
|
12
|
+
import { basename } from 'node:path';
|
|
12
13
|
import pkg from '../package.json' with { type: 'json' };
|
|
13
14
|
|
|
14
15
|
type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
@@ -50,6 +51,9 @@ interface Flags {
|
|
|
50
51
|
model?: string;
|
|
51
52
|
dimensions?: number;
|
|
52
53
|
semantic?: boolean;
|
|
54
|
+
context?: boolean;
|
|
55
|
+
generate?: boolean;
|
|
56
|
+
approveWrite?: boolean;
|
|
53
57
|
noColor?: boolean;
|
|
54
58
|
scope?: string;
|
|
55
59
|
olderThan?: number;
|
|
@@ -64,12 +68,13 @@ interface ParseResult {
|
|
|
64
68
|
flags: Flags;
|
|
65
69
|
}
|
|
66
70
|
|
|
67
|
-
const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'search', 'embeddings', 'providers', 'safety', 'help'];
|
|
71
|
+
const COMMANDS = ['add', 'list', 'get', 'delete', 'update', 'archive', 'restore', 'upsert', 'untag', 'export', 'prune', 'dedupe', 'stats', 'paths', 'storage', 'db', 'wiki', 'source', 'ingest', 'reindex', 'search', 'ask', 'build', 'embeddings', 'providers', 'safety', 'help'];
|
|
68
72
|
const COMMAND_ALIASES: Record<string, string> = {
|
|
69
73
|
ls: 'list',
|
|
70
74
|
rm: 'delete',
|
|
71
75
|
edit: 'update',
|
|
72
76
|
unarchive: 'restore',
|
|
77
|
+
knowledge: 'ask',
|
|
73
78
|
};
|
|
74
79
|
|
|
75
80
|
function parseArgs(argv: string[]): ParseResult {
|
|
@@ -103,6 +108,9 @@ function parseArgs(argv: string[]): ParseResult {
|
|
|
103
108
|
case '--model': flags.model = argv[i + 1]; i += 1; break;
|
|
104
109
|
case '--dimensions': flags.dimensions = Number(argv[i + 1]); i += 1; break;
|
|
105
110
|
case '--semantic': flags.semantic = true; break;
|
|
111
|
+
case '--context': flags.context = true; break;
|
|
112
|
+
case '--generate': flags.generate = true; break;
|
|
113
|
+
case '--approve-write': flags.approveWrite = true; break;
|
|
106
114
|
case '--fake': flags.fake = true; break;
|
|
107
115
|
case '--no-color': flags.noColor = true; break;
|
|
108
116
|
case '--scope': flags.scope = argv[i + 1]; i += 1; break;
|
|
@@ -149,6 +157,10 @@ function suggestCommand(input: string): string {
|
|
|
149
157
|
return bestScore <= 3 ? best : '';
|
|
150
158
|
}
|
|
151
159
|
|
|
160
|
+
function invokedAsKnowledge(): boolean {
|
|
161
|
+
return basename(process.argv[1] ?? '') === 'knowledge';
|
|
162
|
+
}
|
|
163
|
+
|
|
152
164
|
function printGlobalHelp(): void {
|
|
153
165
|
console.log(`open-knowledge - local agent knowledge store
|
|
154
166
|
|
|
@@ -177,7 +189,8 @@ Commands:
|
|
|
177
189
|
ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
|
|
178
190
|
ingest source <source-ref> Ingest a read-only source ref into knowledge.db
|
|
179
191
|
reindex outbox <file|s3://> Consume open-files change events and invalidate chunks
|
|
180
|
-
search <query> Hybrid search sources, wiki pages,
|
|
192
|
+
search <query> Hybrid search sources, wiki pages, indexes, or context
|
|
193
|
+
ask|build <prompt> Build a read-only citation answer/context pack
|
|
181
194
|
embeddings status|index|search Build/query local vector embeddings
|
|
182
195
|
providers status|models|check Inspect AI SDK provider config and credentials
|
|
183
196
|
safety status|check|approve|audit|redact
|
|
@@ -190,6 +203,9 @@ Global Options:
|
|
|
190
203
|
--model <provider:model> AI/embedding model ref
|
|
191
204
|
--dimensions <n> Embedding dimensions for local/fake providers
|
|
192
205
|
--semantic Include vector semantic results in search
|
|
206
|
+
--context Return a reranked citation context pack for search
|
|
207
|
+
--generate Call AI SDK text generation for ask/build
|
|
208
|
+
--approve-write Record approval intent for future durable wiki writes
|
|
193
209
|
--fake Use deterministic fake embeddings for local tests
|
|
194
210
|
--scope local|global|project Store scope (default: global ~/.hasna/apps/knowledge/)
|
|
195
211
|
--no-color Disable color output
|
|
@@ -251,7 +267,8 @@ function printCommandHelp(command: string): void {
|
|
|
251
267
|
if (command === 'source') { console.log('Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]'); return; }
|
|
252
268
|
if (command === 'ingest') { console.log('Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]'); return; }
|
|
253
269
|
if (command === 'reindex') { console.log('Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]'); return; }
|
|
254
|
-
if (command === 'search') { console.log('Usage: open-knowledge search <query> [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]'); return; }
|
|
270
|
+
if (command === 'search') { console.log('Usage: open-knowledge search <query> [--context] [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]'); return; }
|
|
271
|
+
if (command === 'ask' || command === 'build' || command === 'knowledge') { console.log('Usage: open-knowledge ask|build <prompt> [--generate] [--semantic] [--model default|provider:model] [--approve-write] [--scope local|global|project] [--json]'); return; }
|
|
255
272
|
if (command === 'embeddings') { console.log('Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]'); return; }
|
|
256
273
|
if (command === 'providers') { console.log('Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]'); return; }
|
|
257
274
|
if (command === 'safety') { console.log('Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]'); return; }
|
|
@@ -299,18 +316,23 @@ async function run(argv: string[]): Promise<void> {
|
|
|
299
316
|
if (flags.completions) {
|
|
300
317
|
const shell = flags.completions;
|
|
301
318
|
if (shell === 'bash') {
|
|
302
|
-
console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
|
|
319
|
+
console.log(`_open_knowledge() { local cur; cur="${"$"}{COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --context --generate --approve-write --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge`);
|
|
303
320
|
} else if (shell === 'zsh') {
|
|
304
|
-
console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
|
|
321
|
+
console.log(`#compdef open-knowledge\n_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"\{created,title\}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"\{local,global,project\}:" }; _open_knowledge`);
|
|
305
322
|
} else if (shell === 'fish') {
|
|
306
|
-
console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
|
|
323
|
+
console.log(`complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"`);
|
|
307
324
|
} else {
|
|
308
325
|
throw new Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");
|
|
309
326
|
}
|
|
310
327
|
return;
|
|
311
328
|
}
|
|
312
329
|
|
|
313
|
-
|
|
330
|
+
let command = resolveCommand(positional[0]);
|
|
331
|
+
let commandArgOffset = 1;
|
|
332
|
+
if (invokedAsKnowledge() && command && !COMMANDS.includes(command)) {
|
|
333
|
+
command = 'ask';
|
|
334
|
+
commandArgOffset = 0;
|
|
335
|
+
}
|
|
314
336
|
|
|
315
337
|
if (!command || flags.help || command === 'help') { printCommandHelp(positional[1]); return; }
|
|
316
338
|
|
|
@@ -586,6 +608,18 @@ async function run(argv: string[]): Promise<void> {
|
|
|
586
608
|
if (command === 'search') {
|
|
587
609
|
const query = positional.slice(1).join(' ');
|
|
588
610
|
if (!query) throw new Error('Usage: open-knowledge search <query>');
|
|
611
|
+
if (flags.context) {
|
|
612
|
+
const context = await service.retrieveContext({
|
|
613
|
+
query,
|
|
614
|
+
limit: flags.limit,
|
|
615
|
+
semantic: flags.semantic,
|
|
616
|
+
modelRef: flags.model,
|
|
617
|
+
dimensions: flags.dimensions,
|
|
618
|
+
fake: flags.fake,
|
|
619
|
+
});
|
|
620
|
+
output({ ok: true, ...context, message: `${context.excerpts.length} context excerpt(s)` }, flags.json);
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
589
623
|
const result = await service.search({
|
|
590
624
|
query,
|
|
591
625
|
limit: flags.limit,
|
|
@@ -598,6 +632,23 @@ async function run(argv: string[]): Promise<void> {
|
|
|
598
632
|
return;
|
|
599
633
|
}
|
|
600
634
|
|
|
635
|
+
if (command === 'ask' || command === 'build') {
|
|
636
|
+
const prompt = positional.slice(commandArgOffset).join(' ');
|
|
637
|
+
if (!prompt) throw new Error('Usage: open-knowledge ask <prompt>');
|
|
638
|
+
const result = await service.runPrompt({
|
|
639
|
+
prompt,
|
|
640
|
+
limit: flags.limit,
|
|
641
|
+
semantic: flags.semantic,
|
|
642
|
+
modelRef: flags.model,
|
|
643
|
+
dimensions: flags.dimensions,
|
|
644
|
+
fake: flags.fake,
|
|
645
|
+
generate: flags.generate,
|
|
646
|
+
approveWrite: flags.approveWrite,
|
|
647
|
+
});
|
|
648
|
+
output({ ok: true, ...result, message: result.generated ? 'Generated answer with citations' : 'Prepared citation context draft' }, flags.json);
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
|
|
601
652
|
if (command === 'providers') {
|
|
602
653
|
const action = positional[1] ?? 'status';
|
|
603
654
|
if (action === 'status') {
|
package/src/mcp.js
CHANGED
|
@@ -186,6 +186,42 @@ export function buildServer() {
|
|
|
186
186
|
}
|
|
187
187
|
});
|
|
188
188
|
|
|
189
|
+
registerTool(server, 'knowledge_search', 'Knowledge context search', 'Return a reranked citation context pack for agent prompts', {
|
|
190
|
+
scope: scopeField,
|
|
191
|
+
query: z.string().describe('Search query or prompt'),
|
|
192
|
+
limit: z.number().optional().describe('Maximum context results'),
|
|
193
|
+
semantic: z.boolean().optional().describe('Include vector semantic results'),
|
|
194
|
+
model: z.string().optional().describe('Embedding model ref, default openai:text-embedding-3-small'),
|
|
195
|
+
dimensions: z.number().optional().describe('Embedding dimensions for deterministic fake mode'),
|
|
196
|
+
fake: z.boolean().optional().describe('Use deterministic fake embeddings for local tests'),
|
|
197
|
+
}, async ({ scope, query, limit, semantic, model, dimensions, fake }) => {
|
|
198
|
+
const service = createKnowledgeService({ scope });
|
|
199
|
+
try {
|
|
200
|
+
return jsonText({ ok: true, ...await service.retrieveContext({ query, limit, semantic, modelRef: model, dimensions, fake }) });
|
|
201
|
+
} catch (error) {
|
|
202
|
+
return errorText(error instanceof Error ? error.message : String(error));
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
registerTool(server, 'knowledge_ask', 'Knowledge prompt answer', 'Answer a prompt using read-only knowledge context and optional AI SDK generation', {
|
|
207
|
+
scope: scopeField,
|
|
208
|
+
prompt: z.string().describe('Prompt to answer with the knowledge base'),
|
|
209
|
+
limit: z.number().optional().describe('Maximum context results'),
|
|
210
|
+
semantic: z.boolean().optional().describe('Include vector semantic results'),
|
|
211
|
+
generate: z.boolean().optional().describe('Call AI SDK text generation; omitted returns a local citation draft'),
|
|
212
|
+
approve_write: z.boolean().optional().describe('Record approval intent for future durable wiki writes'),
|
|
213
|
+
model: z.string().optional().describe('Model alias/ref, default configured provider default'),
|
|
214
|
+
dimensions: z.number().optional().describe('Embedding dimensions for deterministic fake mode'),
|
|
215
|
+
fake: z.boolean().optional().describe('Use deterministic fake embeddings/generation for local tests'),
|
|
216
|
+
}, async ({ scope, prompt, limit, semantic, generate, approve_write, model, dimensions, fake }) => {
|
|
217
|
+
const service = createKnowledgeService({ scope });
|
|
218
|
+
try {
|
|
219
|
+
return jsonText({ ok: true, ...await service.runPrompt({ prompt, limit, semantic, generate, approveWrite: approve_write, modelRef: model, dimensions, fake }) });
|
|
220
|
+
} catch (error) {
|
|
221
|
+
return errorText(error instanceof Error ? error.message : String(error));
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
|
|
189
225
|
registerTool(server, 'ok_add', 'Add a knowledge item', 'Add a new item to the knowledge store', {
|
|
190
226
|
title: z.string().describe('Item title'),
|
|
191
227
|
content: z.string().describe('Item content/body'),
|
package/src/providers.ts
CHANGED
|
@@ -128,7 +128,7 @@ const BUILTIN_ALIASES: Record<string, string> = {
|
|
|
128
128
|
};
|
|
129
129
|
|
|
130
130
|
function providerConfig(config?: KnowledgeConfig): AiProvidersConfig {
|
|
131
|
-
return (config as KnowledgeConfig & { providers?: AiProvidersConfig })
|
|
131
|
+
return (config as KnowledgeConfig & { providers?: AiProvidersConfig } | undefined)?.providers ?? {};
|
|
132
132
|
}
|
|
133
133
|
|
|
134
134
|
export function providerSettings(config: KnowledgeConfig | undefined, provider: AiProviderId): AiProviderSettings {
|