@kibhq/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -0
- package/package.json +13 -1
- package/src/compile/backlinks.ts +1 -3
- package/src/compile/compiler.test.ts +2 -2
- package/src/compile/compiler.ts +2 -4
- package/src/compile/index-manager.test.ts +1 -1
- package/src/compile/index-manager.ts +1 -1
- package/src/ingest/extractors/github.ts +11 -3
- package/src/ingest/extractors/pdf.ts +40 -12
- package/src/ingest/extractors/youtube.ts +5 -4
- package/src/ingest/ingest.ts +1 -2
- package/src/lint/lint.test.ts +4 -5
- package/src/lint/lint.ts +1 -1
- package/src/lint/rules.ts +2 -2
- package/src/providers/anthropic.ts +27 -7
- package/src/providers/ollama.ts +11 -2
- package/src/providers/openai.ts +18 -3
- package/src/query/query.test.ts +8 -2
- package/src/query/query.ts +1 -1
- package/src/schemas.ts +1 -1
- package/src/search/engine.ts +10 -10
- package/src/skills/runner.ts +1 -9
- package/src/skills/skills.test.ts +10 -4
- package/src/vault.ts +1 -1
package/README.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# @kibhq/core
|
|
2
|
+
|
|
3
|
+
Core engine for [kib](https://github.com/keeganthomp/kib) — the headless knowledge compiler.
|
|
4
|
+
|
|
5
|
+
This package provides the vault operations, LLM providers, ingest extractors, compile engine, BM25 search, and RAG query engine that power the `kib` CLI.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm i @kibhq/core
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
> Most users should install the CLI instead: `npm i -g @kibhq/cli`
|
|
14
|
+
|
|
15
|
+
## What's inside
|
|
16
|
+
|
|
17
|
+
| Module | Description |
|
|
18
|
+
|--------|-------------|
|
|
19
|
+
| **Vault** | Filesystem operations, manifest tracking, config management |
|
|
20
|
+
| **Ingest** | Extractors for web pages, PDFs, YouTube, GitHub repos, and local files |
|
|
21
|
+
| **Compile** | LLM-powered compilation from raw sources into structured wiki articles |
|
|
22
|
+
| **Search** | BM25 full-text search with English stemming |
|
|
23
|
+
| **Query** | RAG engine — retrieves relevant articles and generates cited answers |
|
|
24
|
+
| **Lint** | 5 health-check rules (orphan articles, broken links, stale sources, etc.) |
|
|
25
|
+
| **Skills** | Skill loader and runner for extensible vault operations |
|
|
26
|
+
| **Providers** | LLM adapters for Anthropic Claude, OpenAI, and Ollama |
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
```typescript
|
|
31
|
+
import {
|
|
32
|
+
resolveVaultRoot,
|
|
33
|
+
loadManifest,
|
|
34
|
+
loadConfig,
|
|
35
|
+
SearchIndex,
|
|
36
|
+
queryVault,
|
|
37
|
+
ingestSource,
|
|
38
|
+
compileVault,
|
|
39
|
+
createProvider,
|
|
40
|
+
} from "@kibhq/core";
|
|
41
|
+
|
|
42
|
+
// Find the vault
|
|
43
|
+
const root = resolveVaultRoot();
|
|
44
|
+
const config = await loadConfig(root);
|
|
45
|
+
const manifest = await loadManifest(root);
|
|
46
|
+
|
|
47
|
+
// Search
|
|
48
|
+
const index = new SearchIndex();
|
|
49
|
+
await index.build(root, "all");
|
|
50
|
+
const results = index.search("attention mechanisms", { limit: 10 });
|
|
51
|
+
|
|
52
|
+
// RAG query
|
|
53
|
+
const provider = await createProvider(config.provider.default, config.provider.model);
|
|
54
|
+
const answer = await queryVault(root, "how does self-attention work?", provider);
|
|
55
|
+
console.log(answer.answer);
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## LLM Providers
|
|
59
|
+
|
|
60
|
+
| Provider | Env Variable | Default Model |
|
|
61
|
+
|----------|-------------|---------------|
|
|
62
|
+
| Anthropic | `ANTHROPIC_API_KEY` | claude-sonnet-4-20250514 |
|
|
63
|
+
| OpenAI | `OPENAI_API_KEY` | gpt-4o |
|
|
64
|
+
| Ollama | (auto-detect localhost:11434) | llama3 |
|
|
65
|
+
|
|
66
|
+
## License
|
|
67
|
+
|
|
68
|
+
MIT
|
package/package.json
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kibhq/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Core engine for kib — vault operations, LLM providers, ingest, compile, search, and query",
|
|
4
5
|
"type": "module",
|
|
5
6
|
"exports": {
|
|
6
7
|
".": "./src/index.ts"
|
|
7
8
|
},
|
|
8
9
|
"files": [
|
|
9
10
|
"src",
|
|
11
|
+
"README.md",
|
|
10
12
|
"package.json"
|
|
11
13
|
],
|
|
12
14
|
"repository": {
|
|
@@ -15,6 +17,16 @@
|
|
|
15
17
|
"directory": "packages/core"
|
|
16
18
|
},
|
|
17
19
|
"license": "MIT",
|
|
20
|
+
"keywords": [
|
|
21
|
+
"knowledge-base",
|
|
22
|
+
"wiki",
|
|
23
|
+
"llm",
|
|
24
|
+
"compiler",
|
|
25
|
+
"rag",
|
|
26
|
+
"ai",
|
|
27
|
+
"search",
|
|
28
|
+
"bm25"
|
|
29
|
+
],
|
|
18
30
|
"publishConfig": {
|
|
19
31
|
"access": "public"
|
|
20
32
|
},
|
package/src/compile/backlinks.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
import { WIKI_DIR } from "../constants.js";
|
|
4
2
|
import { listWiki } from "../vault.js";
|
|
5
3
|
import { extractWikilinks, parseFrontmatter } from "./diff.js";
|
|
6
4
|
|
|
@@ -76,5 +74,5 @@ export function generateGraphMd(graph: LinkGraph): string {
|
|
|
76
74
|
lines.push("(no connections yet)");
|
|
77
75
|
}
|
|
78
76
|
|
|
79
|
-
return lines.join("\n")
|
|
77
|
+
return `${lines.join("\n")}\n`;
|
|
80
78
|
}
|
|
@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
|
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { ingestSource } from "../ingest/ingest.js";
|
|
7
7
|
import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
|
|
8
|
-
import { initVault,
|
|
8
|
+
import { initVault, loadManifest, readWiki } from "../vault.js";
|
|
9
9
|
import { compileVault } from "./compiler.js";
|
|
10
10
|
|
|
11
11
|
let tempDir: string;
|
|
@@ -27,7 +27,7 @@ function createMockProvider(responses: string[]): LLMProvider {
|
|
|
27
27
|
let callIndex = 0;
|
|
28
28
|
return {
|
|
29
29
|
name: "mock",
|
|
30
|
-
async complete(
|
|
30
|
+
async complete(_params: CompletionParams): Promise<CompletionResult> {
|
|
31
31
|
const content = responses[callIndex] ?? "[]";
|
|
32
32
|
callIndex++;
|
|
33
33
|
return {
|
package/src/compile/compiler.ts
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { GRAPH_FILE, INDEX_FILE, WIKI_DIR } from "../constants.js";
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { GRAPH_FILE, INDEX_FILE } from "../constants.js";
|
|
4
3
|
import { hash } from "../hash.js";
|
|
5
4
|
import { countWords } from "../ingest/normalize.js";
|
|
6
5
|
import type { CompileResult, LLMProvider, Manifest, VaultConfig } from "../types.js";
|
|
7
6
|
import {
|
|
8
7
|
deleteFile,
|
|
9
|
-
listWiki,
|
|
10
8
|
loadManifest,
|
|
11
9
|
readIndex,
|
|
12
10
|
readRaw,
|
|
@@ -139,7 +139,7 @@ describe("generateIndexMd", () => {
|
|
|
139
139
|
);
|
|
140
140
|
|
|
141
141
|
const index = await generateIndexMd(root);
|
|
142
|
-
const
|
|
142
|
+
const _conceptsIdx = index.indexOf("## Concepts");
|
|
143
143
|
const alphaIdx = index.indexOf("Alpha Concept");
|
|
144
144
|
const midIdx = index.indexOf("Mid Concept");
|
|
145
145
|
const zebraIdx = index.indexOf("Zebra Concept");
|
|
@@ -30,7 +30,13 @@ export function createGithubExtractor(): Extractor {
|
|
|
30
30
|
`Failed to fetch repo info: ${repoResponse.status} ${repoResponse.statusText}`,
|
|
31
31
|
);
|
|
32
32
|
}
|
|
33
|
-
const repoData = (await repoResponse.json()) as
|
|
33
|
+
const repoData = (await repoResponse.json()) as {
|
|
34
|
+
full_name?: string;
|
|
35
|
+
description?: string;
|
|
36
|
+
stargazers_count?: number;
|
|
37
|
+
language?: string;
|
|
38
|
+
default_branch?: string;
|
|
39
|
+
};
|
|
34
40
|
|
|
35
41
|
// Fetch README
|
|
36
42
|
let readme = "";
|
|
@@ -51,9 +57,11 @@ export function createGithubExtractor(): Extractor {
|
|
|
51
57
|
const ref = branch ?? repoData.default_branch ?? "main";
|
|
52
58
|
const treeResponse = await fetch(`${apiBase}/git/trees/${ref}`, { headers });
|
|
53
59
|
if (treeResponse.ok) {
|
|
54
|
-
const treeData = (await treeResponse.json()) as
|
|
60
|
+
const treeData = (await treeResponse.json()) as {
|
|
61
|
+
tree?: { type: string; path: string }[];
|
|
62
|
+
};
|
|
55
63
|
const files = (treeData.tree ?? [])
|
|
56
|
-
.map((f
|
|
64
|
+
.map((f) => `${f.type === "tree" ? "📁" : "📄"} ${f.path}`)
|
|
57
65
|
.slice(0, 50); // Cap at 50 entries
|
|
58
66
|
fileTree = files.join("\n");
|
|
59
67
|
}
|
|
@@ -2,14 +2,39 @@ import { readFile } from "node:fs/promises";
|
|
|
2
2
|
import { basename, extname } from "node:path";
|
|
3
3
|
import type { ExtractOptions, Extractor, ExtractResult } from "./interface.js";
|
|
4
4
|
|
|
5
|
+
type PdfParseFn = (buffer: Buffer) => Promise<{
|
|
6
|
+
text: string;
|
|
7
|
+
numpages: number;
|
|
8
|
+
info?: { Title?: string; Author?: string };
|
|
9
|
+
date?: string;
|
|
10
|
+
}>;
|
|
11
|
+
|
|
5
12
|
// Lazy-load pdf-parse (it's heavy)
|
|
6
|
-
let pdfParse:
|
|
13
|
+
let pdfParse: PdfParseFn | null = null;
|
|
7
14
|
|
|
8
|
-
async function getPdfParse() {
|
|
15
|
+
async function getPdfParse(): Promise<PdfParseFn> {
|
|
9
16
|
if (!pdfParse) {
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
17
|
+
const { PDFParse } = await import("pdf-parse");
|
|
18
|
+
pdfParse = async (buffer: Buffer) => {
|
|
19
|
+
const parser = new PDFParse({ data: buffer });
|
|
20
|
+
|
|
21
|
+
try {
|
|
22
|
+
const textResult = await parser.getText();
|
|
23
|
+
const infoResult = await parser.getInfo();
|
|
24
|
+
|
|
25
|
+
return {
|
|
26
|
+
text: textResult.text,
|
|
27
|
+
numpages: textResult.total,
|
|
28
|
+
info: {
|
|
29
|
+
Title: getOptionalString(infoResult.info?.Title),
|
|
30
|
+
Author: getOptionalString(infoResult.info?.Author),
|
|
31
|
+
},
|
|
32
|
+
date: formatPdfDate(infoResult.getDateNode().CreationDate),
|
|
33
|
+
};
|
|
34
|
+
} finally {
|
|
35
|
+
await parser.destroy();
|
|
36
|
+
}
|
|
37
|
+
};
|
|
13
38
|
}
|
|
14
39
|
return pdfParse;
|
|
15
40
|
}
|
|
@@ -50,7 +75,7 @@ export function createPdfExtractor(): Extractor {
|
|
|
50
75
|
formatFilename(input);
|
|
51
76
|
|
|
52
77
|
const author = data.info?.Author ?? undefined;
|
|
53
|
-
const date = data.
|
|
78
|
+
const date = data.date;
|
|
54
79
|
|
|
55
80
|
// Clean up the extracted text into readable markdown
|
|
56
81
|
const content = formatPdfText(data.text, title);
|
|
@@ -119,12 +144,15 @@ function formatPdfText(text: string, title: string): string {
|
|
|
119
144
|
}
|
|
120
145
|
|
|
121
146
|
/**
|
|
122
|
-
*
|
|
147
|
+
* Normalize parsed PDF dates to YYYY-MM-DD.
|
|
123
148
|
*/
|
|
124
|
-
function
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
return `${match[1]}-${match[2]}-${match[3]}`;
|
|
149
|
+
function formatPdfDate(date: Date | null | undefined): string | undefined {
|
|
150
|
+
if (!(date instanceof Date) || Number.isNaN(date.getTime())) {
|
|
151
|
+
return undefined;
|
|
128
152
|
}
|
|
129
|
-
return
|
|
153
|
+
return date.toISOString().slice(0, 10);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function getOptionalString(value: unknown): string | undefined {
|
|
157
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
130
158
|
}
|
|
@@ -82,7 +82,10 @@ async function fetchVideoPage(videoId: string): Promise<VideoPageData> {
|
|
|
82
82
|
`https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v=${videoId}&format=json`,
|
|
83
83
|
);
|
|
84
84
|
if (response.ok) {
|
|
85
|
-
const data = (await response.json()) as
|
|
85
|
+
const data = (await response.json()) as {
|
|
86
|
+
title?: string;
|
|
87
|
+
author_name?: string;
|
|
88
|
+
};
|
|
86
89
|
return {
|
|
87
90
|
title: data.title ?? null,
|
|
88
91
|
description: null, // oembed doesn't include description
|
|
@@ -139,9 +142,7 @@ async function fetchTranscript(videoId: string): Promise<string> {
|
|
|
139
142
|
function parseTranscriptXml(xml: string): string {
|
|
140
143
|
const lines: string[] = [];
|
|
141
144
|
const textRegex = /<text[^>]*>([\s\S]*?)<\/text>/g;
|
|
142
|
-
let match
|
|
143
|
-
|
|
144
|
-
while ((match = textRegex.exec(xml)) !== null) {
|
|
145
|
+
for (let match = textRegex.exec(xml); match !== null; match = textRegex.exec(xml)) {
|
|
145
146
|
const text = match[1]!
|
|
146
147
|
.replace(/&/g, "&")
|
|
147
148
|
.replace(/</g, "<")
|
package/src/ingest/ingest.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { basename, extname, resolve } from "node:path";
|
|
2
1
|
import { hash } from "../hash.js";
|
|
3
2
|
import type { IngestResult, Manifest, SourceEntry, SourceType } from "../types.js";
|
|
4
3
|
import { loadManifest, saveManifest, writeRaw } from "../vault.js";
|
|
@@ -161,7 +160,7 @@ function categoryForType(sourceType: SourceType): string {
|
|
|
161
160
|
|
|
162
161
|
function findExistingSource(
|
|
163
162
|
manifest: Manifest,
|
|
164
|
-
|
|
163
|
+
_uri: string,
|
|
165
164
|
contentHash: string,
|
|
166
165
|
): { id: string; path: string } | null {
|
|
167
166
|
for (const [id, source] of Object.entries(manifest.sources)) {
|
package/src/lint/lint.test.ts
CHANGED
|
@@ -3,7 +3,6 @@ import { writeFile as fsWriteFile, mkdtemp, rm } from "node:fs/promises";
|
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { ingestSource } from "../ingest/ingest.js";
|
|
6
|
-
import type { Manifest } from "../types.js";
|
|
7
6
|
import { initVault, loadManifest, saveManifest, writeWiki } from "../vault.js";
|
|
8
7
|
import { lintVault } from "./lint.js";
|
|
9
8
|
|
|
@@ -61,7 +60,7 @@ describe("lint engine", () => {
|
|
|
61
60
|
|
|
62
61
|
// Update manifest to include articles with backlinks
|
|
63
62
|
const manifest = await loadManifest(root);
|
|
64
|
-
manifest.articles
|
|
63
|
+
manifest.articles.alpha = {
|
|
65
64
|
hash: "a",
|
|
66
65
|
createdAt: new Date().toISOString(),
|
|
67
66
|
lastUpdated: new Date().toISOString(),
|
|
@@ -73,7 +72,7 @@ describe("lint engine", () => {
|
|
|
73
72
|
wordCount: 10,
|
|
74
73
|
category: "concept",
|
|
75
74
|
};
|
|
76
|
-
manifest.articles
|
|
75
|
+
manifest.articles.beta = {
|
|
77
76
|
hash: "b",
|
|
78
77
|
createdAt: new Date().toISOString(),
|
|
79
78
|
lastUpdated: new Date().toISOString(),
|
|
@@ -103,7 +102,7 @@ describe("lint engine", () => {
|
|
|
103
102
|
|
|
104
103
|
// Add to manifest with no backlinks
|
|
105
104
|
const manifest = await loadManifest(root);
|
|
106
|
-
manifest.articles
|
|
105
|
+
manifest.articles.orphan = {
|
|
107
106
|
hash: "o",
|
|
108
107
|
createdAt: new Date().toISOString(),
|
|
109
108
|
lastUpdated: new Date().toISOString(),
|
|
@@ -137,7 +136,7 @@ describe("lint engine", () => {
|
|
|
137
136
|
);
|
|
138
137
|
|
|
139
138
|
const manifest = await loadManifest(root);
|
|
140
|
-
manifest.articles
|
|
139
|
+
manifest.articles.test = {
|
|
141
140
|
hash: "t",
|
|
142
141
|
createdAt: new Date().toISOString(),
|
|
143
142
|
lastUpdated: new Date().toISOString(),
|
package/src/lint/lint.ts
CHANGED
package/src/lint/rules.ts
CHANGED
|
@@ -10,7 +10,7 @@ type LintRuleFn = (root: string, manifest: Manifest) => Promise<LintDiagnostic[]
|
|
|
10
10
|
/**
|
|
11
11
|
* Find articles with no backlinks from other articles (orphans).
|
|
12
12
|
*/
|
|
13
|
-
export const orphanRule: LintRuleFn = async (
|
|
13
|
+
export const orphanRule: LintRuleFn = async (_root, manifest) => {
|
|
14
14
|
const diagnostics: LintDiagnostic[] = [];
|
|
15
15
|
|
|
16
16
|
for (const [slug, article] of Object.entries(manifest.articles)) {
|
|
@@ -132,7 +132,7 @@ export const frontmatterRule: LintRuleFn = async (root) => {
|
|
|
132
132
|
*/
|
|
133
133
|
export const missingRule: LintRuleFn = async (root, manifest) => {
|
|
134
134
|
const diagnostics: LintDiagnostic[] = [];
|
|
135
|
-
const
|
|
135
|
+
const _wikiDir = `${root}/${WIKI_DIR}`;
|
|
136
136
|
const files = await listWiki(root);
|
|
137
137
|
const articleFiles = files.filter((f) => !f.endsWith("INDEX.md") && !f.endsWith("GRAPH.md"));
|
|
138
138
|
|
|
@@ -1,12 +1,32 @@
|
|
|
1
1
|
import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
|
|
2
2
|
|
|
3
|
+
interface ContentBlock {
|
|
4
|
+
type: string;
|
|
5
|
+
text?: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
3
8
|
// Lazy-loaded SDK
|
|
4
|
-
let AnthropicClass:
|
|
9
|
+
let AnthropicClass: (new () => AnthropicClient) | null = null;
|
|
10
|
+
|
|
11
|
+
interface AnthropicClient {
|
|
12
|
+
messages: {
|
|
13
|
+
create(params: Record<string, unknown>): Promise<{
|
|
14
|
+
content: ContentBlock[];
|
|
15
|
+
usage: { input_tokens: number; output_tokens: number };
|
|
16
|
+
stop_reason: string;
|
|
17
|
+
}>;
|
|
18
|
+
stream(params: Record<string, unknown>): AsyncIterable<{
|
|
19
|
+
type: string;
|
|
20
|
+
delta: { type: string; text: string };
|
|
21
|
+
usage?: { input_tokens: number; output_tokens: number };
|
|
22
|
+
}>;
|
|
23
|
+
};
|
|
24
|
+
}
|
|
5
25
|
|
|
6
|
-
async function getClient() {
|
|
26
|
+
async function getClient(): Promise<AnthropicClient> {
|
|
7
27
|
if (!AnthropicClass) {
|
|
8
28
|
const mod = await import("@anthropic-ai/sdk");
|
|
9
|
-
AnthropicClass = mod.default;
|
|
29
|
+
AnthropicClass = mod.default as unknown as new () => AnthropicClient;
|
|
10
30
|
}
|
|
11
31
|
return new AnthropicClass();
|
|
12
32
|
}
|
|
@@ -29,8 +49,8 @@ export function createAnthropicProvider(model: string): LLMProvider {
|
|
|
29
49
|
});
|
|
30
50
|
|
|
31
51
|
const content = response.content
|
|
32
|
-
.filter((b
|
|
33
|
-
.map((b
|
|
52
|
+
.filter((b) => b.type === "text")
|
|
53
|
+
.map((b) => b.text ?? "")
|
|
34
54
|
.join("");
|
|
35
55
|
|
|
36
56
|
return {
|
|
@@ -99,8 +119,8 @@ export function createAnthropicProvider(model: string): LLMProvider {
|
|
|
99
119
|
});
|
|
100
120
|
|
|
101
121
|
return response.content
|
|
102
|
-
.filter((b
|
|
103
|
-
.map((b
|
|
122
|
+
.filter((b) => b.type === "text")
|
|
123
|
+
.map((b) => b.text ?? "")
|
|
104
124
|
.join("");
|
|
105
125
|
},
|
|
106
126
|
};
|
package/src/providers/ollama.ts
CHANGED
|
@@ -30,7 +30,11 @@ export function createOllamaProvider(model: string): LLMProvider {
|
|
|
30
30
|
throw new Error(`Ollama request failed: ${response.status} ${response.statusText}`);
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
const data = (await response.json()) as
|
|
33
|
+
const data = (await response.json()) as {
|
|
34
|
+
message?: { content: string };
|
|
35
|
+
prompt_eval_count?: number;
|
|
36
|
+
eval_count?: number;
|
|
37
|
+
};
|
|
34
38
|
return {
|
|
35
39
|
content: data.message?.content ?? "",
|
|
36
40
|
usage: {
|
|
@@ -81,7 +85,12 @@ export function createOllamaProvider(model: string): LLMProvider {
|
|
|
81
85
|
|
|
82
86
|
for (const line of lines) {
|
|
83
87
|
if (!line.trim()) continue;
|
|
84
|
-
const data = JSON.parse(line) as
|
|
88
|
+
const data = JSON.parse(line) as {
|
|
89
|
+
message?: { content: string };
|
|
90
|
+
done?: boolean;
|
|
91
|
+
prompt_eval_count?: number;
|
|
92
|
+
eval_count?: number;
|
|
93
|
+
};
|
|
85
94
|
if (data.message?.content) {
|
|
86
95
|
yield { type: "text", text: data.message.content };
|
|
87
96
|
}
|
package/src/providers/openai.ts
CHANGED
|
@@ -1,12 +1,27 @@
|
|
|
1
1
|
import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
|
|
2
2
|
|
|
3
|
+
interface OpenAIClient {
|
|
4
|
+
chat: {
|
|
5
|
+
completions: {
|
|
6
|
+
create(params: Record<string, unknown>): Promise<{
|
|
7
|
+
choices: {
|
|
8
|
+
message?: { content: string };
|
|
9
|
+
finish_reason: string;
|
|
10
|
+
delta?: { content?: string };
|
|
11
|
+
}[];
|
|
12
|
+
usage?: { prompt_tokens: number; completion_tokens: number };
|
|
13
|
+
}>;
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
3
18
|
// Lazy-loaded SDK
|
|
4
|
-
let OpenAIClass:
|
|
19
|
+
let OpenAIClass: (new () => OpenAIClient) | null = null;
|
|
5
20
|
|
|
6
|
-
async function getClient() {
|
|
21
|
+
async function getClient(): Promise<OpenAIClient> {
|
|
7
22
|
if (!OpenAIClass) {
|
|
8
23
|
const mod = await import("openai");
|
|
9
|
-
OpenAIClass = mod.default;
|
|
24
|
+
OpenAIClass = mod.default as unknown as new () => OpenAIClient;
|
|
10
25
|
}
|
|
11
26
|
return new OpenAIClass();
|
|
12
27
|
}
|
package/src/query/query.test.ts
CHANGED
|
@@ -3,7 +3,13 @@ import { mkdtemp, rm } from "node:fs/promises";
|
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { SearchIndex } from "../search/engine.js";
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
CompletionParams,
|
|
8
|
+
CompletionResult,
|
|
9
|
+
LLMProvider,
|
|
10
|
+
Message,
|
|
11
|
+
StreamChunk,
|
|
12
|
+
} from "../types.js";
|
|
7
13
|
import { initVault, writeWiki } from "../vault.js";
|
|
8
14
|
import { queryVault } from "./query.js";
|
|
9
15
|
|
|
@@ -133,7 +139,7 @@ describe("queryVault", () => {
|
|
|
133
139
|
await index.save(root);
|
|
134
140
|
|
|
135
141
|
// Track what gets sent to the provider
|
|
136
|
-
let receivedMessages:
|
|
142
|
+
let receivedMessages: Message[] = [];
|
|
137
143
|
const provider: LLMProvider = {
|
|
138
144
|
name: "mock",
|
|
139
145
|
async complete(params: CompletionParams): Promise<CompletionResult> {
|
package/src/query/query.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
|
|
|
2
2
|
import { parseFrontmatter } from "../compile/diff.js";
|
|
3
3
|
import { SearchIndex } from "../search/engine.js";
|
|
4
4
|
import type { CompletionResult, LLMProvider, Message } from "../types.js";
|
|
5
|
-
import {
|
|
5
|
+
import { readIndex } from "../vault.js";
|
|
6
6
|
|
|
7
7
|
export interface QueryOptions {
|
|
8
8
|
/** Maximum articles to include as context */
|
package/src/schemas.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { DEFAULT_CATEGORIES, DEFAULTS, MANIFEST_VERSION
|
|
2
|
+
import { DEFAULT_CATEGORIES, DEFAULTS, MANIFEST_VERSION } from "./constants.js";
|
|
3
3
|
|
|
4
4
|
// ─── Source Types ────────────────────────────────────────────────
|
|
5
5
|
|
package/src/search/engine.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
|
|
|
2
2
|
import { readFile } from "node:fs/promises";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { parseFrontmatter } from "../compile/diff.js";
|
|
5
|
-
import { CACHE_DIR, VAULT_DIR
|
|
5
|
+
import { CACHE_DIR, VAULT_DIR } from "../constants.js";
|
|
6
6
|
import type { SearchResult } from "../types.js";
|
|
7
7
|
import { listRaw, listWiki } from "../vault.js";
|
|
8
8
|
|
|
@@ -133,18 +133,18 @@ function tokenize(text: string): string[] {
|
|
|
133
133
|
function stem(word: string): string {
|
|
134
134
|
if (word.length < 4) return word;
|
|
135
135
|
// Order matters: try longest suffixes first
|
|
136
|
-
if (word.endsWith("ization")) return word.slice(0, -7)
|
|
137
|
-
if (word.endsWith("ational")) return word.slice(0, -7)
|
|
138
|
-
if (word.endsWith("iveness")) return word.slice(0, -7)
|
|
139
|
-
if (word.endsWith("fulness")) return word.slice(0, -7)
|
|
140
|
-
if (word.endsWith("ousli")) return word.slice(0, -5)
|
|
141
|
-
if (word.endsWith("ation")) return word.slice(0, -5)
|
|
136
|
+
if (word.endsWith("ization")) return `${word.slice(0, -7)}ize`;
|
|
137
|
+
if (word.endsWith("ational")) return `${word.slice(0, -7)}ate`;
|
|
138
|
+
if (word.endsWith("iveness")) return `${word.slice(0, -7)}ive`;
|
|
139
|
+
if (word.endsWith("fulness")) return `${word.slice(0, -7)}ful`;
|
|
140
|
+
if (word.endsWith("ousli")) return `${word.slice(0, -5)}ous`;
|
|
141
|
+
if (word.endsWith("ation")) return `${word.slice(0, -5)}ate`;
|
|
142
142
|
if (word.endsWith("ness")) return word.slice(0, -4);
|
|
143
143
|
if (word.endsWith("ment")) return word.slice(0, -4);
|
|
144
|
-
if (word.endsWith("ting")) return word.slice(0, -3)
|
|
144
|
+
if (word.endsWith("ting")) return `${word.slice(0, -3)}e`;
|
|
145
145
|
if (word.endsWith("ing") && word.length > 5) return word.slice(0, -3);
|
|
146
|
-
if (word.endsWith("ies") && word.length > 4) return word.slice(0, -3)
|
|
147
|
-
if (word.endsWith("ied")) return word.slice(0, -3)
|
|
146
|
+
if (word.endsWith("ies") && word.length > 4) return `${word.slice(0, -3)}y`;
|
|
147
|
+
if (word.endsWith("ied")) return `${word.slice(0, -3)}y`;
|
|
148
148
|
if (word.endsWith("ous")) return word.slice(0, -3);
|
|
149
149
|
if (word.endsWith("ful")) return word.slice(0, -3);
|
|
150
150
|
if (word.endsWith("ers")) return word.slice(0, -3);
|
package/src/skills/runner.ts
CHANGED
|
@@ -10,15 +10,7 @@ import type {
|
|
|
10
10
|
SkillDefinition,
|
|
11
11
|
VaultConfig,
|
|
12
12
|
} from "../types.js";
|
|
13
|
-
import {
|
|
14
|
-
listRaw,
|
|
15
|
-
listWiki,
|
|
16
|
-
loadConfig,
|
|
17
|
-
loadManifest,
|
|
18
|
-
readIndex,
|
|
19
|
-
readWiki,
|
|
20
|
-
writeWiki,
|
|
21
|
-
} from "../vault.js";
|
|
13
|
+
import { listRaw, listWiki, loadConfig, loadManifest, readIndex, writeWiki } from "../vault.js";
|
|
22
14
|
|
|
23
15
|
export interface RunSkillOptions {
|
|
24
16
|
/** Additional CLI args */
|
|
@@ -2,7 +2,13 @@ import { afterEach, describe, expect, test } from "bun:test";
|
|
|
2
2
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
|
-
import type {
|
|
5
|
+
import type {
|
|
6
|
+
CompletionResult,
|
|
7
|
+
LLMProvider,
|
|
8
|
+
SkillContext,
|
|
9
|
+
SkillDefinition,
|
|
10
|
+
StreamChunk,
|
|
11
|
+
} from "../types.js";
|
|
6
12
|
import { initVault, writeWiki } from "../vault.js";
|
|
7
13
|
import { findSkill, loadSkills } from "./loader.js";
|
|
8
14
|
import { runSkill } from "./runner.js";
|
|
@@ -119,16 +125,16 @@ describe("skill runner", () => {
|
|
|
119
125
|
description: "Test skill context",
|
|
120
126
|
input: "wiki" as const,
|
|
121
127
|
output: "report" as const,
|
|
122
|
-
async run(ctx:
|
|
128
|
+
async run(ctx: SkillContext) {
|
|
123
129
|
const articles = await ctx.vault.readWiki();
|
|
124
130
|
const index = await ctx.vault.readIndex();
|
|
125
131
|
return {
|
|
126
132
|
content: `Found ${articles.length} articles. Index length: ${index.length}`,
|
|
127
133
|
};
|
|
128
134
|
},
|
|
129
|
-
};
|
|
135
|
+
} satisfies SkillDefinition;
|
|
130
136
|
|
|
131
|
-
const result = await runSkill(root, testSkill
|
|
137
|
+
const result = await runSkill(root, testSkill);
|
|
132
138
|
expect(result.content).toContain("Found 2 articles");
|
|
133
139
|
});
|
|
134
140
|
});
|
package/src/vault.ts
CHANGED
|
@@ -152,7 +152,7 @@ export async function loadConfig(root: string): Promise<VaultConfig> {
|
|
|
152
152
|
export async function saveConfig(root: string, config: VaultConfig): Promise<void> {
|
|
153
153
|
const path = join(root, VAULT_DIR, CONFIG_FILE);
|
|
154
154
|
const tmp = `${path}.tmp`;
|
|
155
|
-
await writeFile(tmp, TOML.stringify(config as
|
|
155
|
+
await writeFile(tmp, TOML.stringify(config as unknown as TOML.JsonMap), "utf-8");
|
|
156
156
|
await rename(tmp, path);
|
|
157
157
|
}
|
|
158
158
|
|