@draht/knowledge 2026.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/client-config.d.ts +24 -0
- package/dist/client-config.d.ts.map +1 -0
- package/dist/client-config.js +73 -0
- package/dist/client-config.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/knowledge-manager.d.ts +38 -0
- package/dist/knowledge-manager.d.ts.map +1 -0
- package/dist/knowledge-manager.js +140 -0
- package/dist/knowledge-manager.js.map +1 -0
- package/dist/vector-store.d.ts +35 -0
- package/dist/vector-store.d.ts.map +1 -0
- package/dist/vector-store.js +184 -0
- package/dist/vector-store.js.map +1 -0
- package/package.json +37 -0
package/README.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# @draht/knowledge
|
|
2
|
+
|
|
3
|
+
Client knowledge base with vector search for the Draht coding agent.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Vector Store** — Chunk and embed documents for semantic search
|
|
8
|
+
- **Knowledge Manager** — Manage project-specific knowledge bases
|
|
9
|
+
- **Extension** — Registers as a Draht coding agent extension
|
|
10
|
+
- **Client Config** — Per-project knowledge configuration
|
|
11
|
+
|
|
12
|
+
## Usage
|
|
13
|
+
|
|
14
|
+
```typescript
|
|
15
|
+
import { KnowledgeManager, VectorStore, knowledgeExtension } from "@draht/knowledge";
|
|
16
|
+
|
|
17
|
+
// As a coding agent extension
|
|
18
|
+
export default knowledgeExtension;
|
|
19
|
+
|
|
20
|
+
// Direct usage
|
|
21
|
+
const store = new VectorStore(dbPath);
|
|
22
|
+
const results = await store.search(query, { limit: 5 });
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Exports
|
|
26
|
+
|
|
27
|
+
| Export | Description |
|
|
28
|
+
|--------|-------------|
|
|
29
|
+
| `KnowledgeManager` | Manages knowledge base lifecycle |
|
|
30
|
+
| `VectorStore` | Vector storage and semantic search |
|
|
31
|
+
| `knowledgeExtension` | Draht coding agent extension |
|
|
32
|
+
| `loadClientConfig` | Load per-project config |
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client configuration and namespace management.
|
|
3
|
+
* Each client gets an isolated knowledge directory under ~/.draht/knowledge/{slug}/.
|
|
4
|
+
*/
|
|
5
|
+
export interface ClientConfig {
|
|
6
|
+
slug: string;
|
|
7
|
+
name: string;
|
|
8
|
+
agentsMdPath?: string;
|
|
9
|
+
watchPaths?: string[];
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Load client config from cwd.
|
|
13
|
+
* Checks .draht/client.json first, then falls back to AGENTS.md extraction.
|
|
14
|
+
*/
|
|
15
|
+
export declare function loadClientConfig(cwd: string): ClientConfig | null;
|
|
16
|
+
/**
|
|
17
|
+
* Get the knowledge store directory for a client.
|
|
18
|
+
*/
|
|
19
|
+
export declare function getKnowledgePath(slug: string): string;
|
|
20
|
+
/**
|
|
21
|
+
* Get the database file path for a client's knowledge store.
|
|
22
|
+
*/
|
|
23
|
+
export declare function getKnowledgeDbPath(slug: string): string;
|
|
24
|
+
//# sourceMappingURL=client-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client-config.d.ts","sourceRoot":"","sources":["../src/client-config.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAgCjE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client configuration and namespace management.
|
|
3
|
+
* Each client gets an isolated knowledge directory under ~/.draht/knowledge/{slug}/.
|
|
4
|
+
*/
|
|
5
|
+
import * as fs from "node:fs";
|
|
6
|
+
import * as os from "node:os";
|
|
7
|
+
import * as path from "node:path";
|
|
8
|
+
/**
|
|
9
|
+
* Load client config from cwd.
|
|
10
|
+
* Checks .draht/client.json first, then falls back to AGENTS.md extraction.
|
|
11
|
+
*/
|
|
12
|
+
export function loadClientConfig(cwd) {
|
|
13
|
+
// Try .draht/client.json
|
|
14
|
+
const configPath = path.join(cwd, ".draht", "client.json");
|
|
15
|
+
if (fs.existsSync(configPath)) {
|
|
16
|
+
try {
|
|
17
|
+
const raw = fs.readFileSync(configPath, "utf-8");
|
|
18
|
+
const config = JSON.parse(raw);
|
|
19
|
+
if (config.slug && config.name) {
|
|
20
|
+
return {
|
|
21
|
+
slug: config.slug,
|
|
22
|
+
name: config.name,
|
|
23
|
+
agentsMdPath: config.agentsMdPath ?? findAgentsMd(cwd),
|
|
24
|
+
watchPaths: config.watchPaths,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// Invalid JSON, fall through
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
// Try AGENTS.md extraction
|
|
33
|
+
const agentsMdPath = findAgentsMd(cwd);
|
|
34
|
+
if (agentsMdPath) {
|
|
35
|
+
const slug = slugifyPath(cwd);
|
|
36
|
+
return {
|
|
37
|
+
slug,
|
|
38
|
+
name: path.basename(cwd),
|
|
39
|
+
agentsMdPath,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Get the knowledge store directory for a client.
|
|
46
|
+
*/
|
|
47
|
+
export function getKnowledgePath(slug) {
|
|
48
|
+
return path.join(os.homedir(), ".draht", "knowledge", slug);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Get the database file path for a client's knowledge store.
|
|
52
|
+
*/
|
|
53
|
+
export function getKnowledgeDbPath(slug) {
|
|
54
|
+
return path.join(getKnowledgePath(slug), "vectors.db");
|
|
55
|
+
}
|
|
56
|
+
function findAgentsMd(cwd) {
|
|
57
|
+
const candidates = ["AGENTS.md", ".agents/AGENTS.md", "docs/AGENTS.md"];
|
|
58
|
+
for (const candidate of candidates) {
|
|
59
|
+
const full = path.join(cwd, candidate);
|
|
60
|
+
if (fs.existsSync(full))
|
|
61
|
+
return full;
|
|
62
|
+
}
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
function slugifyPath(cwd) {
|
|
66
|
+
return path
|
|
67
|
+
.basename(cwd)
|
|
68
|
+
.toLowerCase()
|
|
69
|
+
.replace(/[^a-z0-9-]/g, "-")
|
|
70
|
+
.replace(/-+/g, "-")
|
|
71
|
+
.replace(/^-|-$/g, "");
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=client-config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client-config.js","sourceRoot":"","sources":["../src/client-config.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AASlC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC3C,yBAAyB;IACzB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA0B,CAAC;YACxD,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChC,OAAO;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,YAAY,CAAC,GAAG,CAAC;oBACtD,UAAU,EAAE,MAAM,CAAC,UAAU;iBAC7B,CAAC;YACH,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,6BAA6B;QAC9B,CAAC;IACF,CAAC;IAED,2BAA2B;IAC3B,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO;YACN,IAAI;YACJ,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACxB,YAAY;SACZ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAChC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,mBAAmB,EAAE,gBAAgB,CAAC,CAAC;IACxE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;IACtC,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC/B,OAAO,IAAI;SACT,QAAQ,CAAC,GAAG,CAAC;SACb,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACzB,CAAC","sourcesContent":["/**\n * Client configuration and namespace management.\n * Each client gets an isolated knowledge directory under ~/.draht/knowledge/{slug}/.\n */\n\nimport * as fs from \"node:fs\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\n\nexport interface ClientConfig {\n\tslug: string;\n\tname: string;\n\tagentsMdPath?: string;\n\twatchPaths?: string[];\n}\n\n/**\n * Load client config from cwd.\n * Checks .draht/client.json first, then falls back to AGENTS.md extraction.\n */\nexport function loadClientConfig(cwd: string): ClientConfig | null {\n\t// Try .draht/client.json\n\tconst configPath = path.join(cwd, \".draht\", \"client.json\");\n\tif (fs.existsSync(configPath)) {\n\t\ttry {\n\t\t\tconst raw = fs.readFileSync(configPath, \"utf-8\");\n\t\t\tconst config = JSON.parse(raw) as Partial<ClientConfig>;\n\t\t\tif (config.slug && config.name) {\n\t\t\t\treturn {\n\t\t\t\t\tslug: config.slug,\n\t\t\t\t\tname: config.name,\n\t\t\t\t\tagentsMdPath: config.agentsMdPath ?? findAgentsMd(cwd),\n\t\t\t\t\twatchPaths: config.watchPaths,\n\t\t\t\t};\n\t\t\t}\n\t\t} catch {\n\t\t\t// Invalid JSON, fall through\n\t\t}\n\t}\n\n\t// Try AGENTS.md extraction\n\tconst agentsMdPath = findAgentsMd(cwd);\n\tif (agentsMdPath) {\n\t\tconst slug = slugifyPath(cwd);\n\t\treturn {\n\t\t\tslug,\n\t\t\tname: path.basename(cwd),\n\t\t\tagentsMdPath,\n\t\t};\n\t}\n\n\treturn null;\n}\n\n/**\n * Get the knowledge store directory for a client.\n */\nexport function getKnowledgePath(slug: string): string {\n\treturn path.join(os.homedir(), \".draht\", \"knowledge\", slug);\n}\n\n/**\n * Get the database file path for a client's knowledge store.\n */\nexport function getKnowledgeDbPath(slug: string): string {\n\treturn path.join(getKnowledgePath(slug), \"vectors.db\");\n}\n\nfunction findAgentsMd(cwd: string): string | undefined {\n\tconst candidates = [\"AGENTS.md\", \".agents/AGENTS.md\", \"docs/AGENTS.md\"];\n\tfor (const candidate of candidates) {\n\t\tconst full = path.join(cwd, candidate);\n\t\tif (fs.existsSync(full)) return full;\n\t}\n\treturn undefined;\n}\n\nfunction slugifyPath(cwd: string): string {\n\treturn path\n\t\t.basename(cwd)\n\t\t.toLowerCase()\n\t\t.replace(/[^a-z0-9-]/g, \"-\")\n\t\t.replace(/-+/g, \"-\")\n\t\t.replace(/^-|-$/g, \"\");\n}\n"]}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { type ClientConfig, getKnowledgeDbPath, getKnowledgePath, loadClientConfig } from "./client-config.js";
|
|
2
|
+
export { KnowledgeManager, type SearchMode } from "./knowledge-manager.js";
|
|
3
|
+
export { type Chunk, type ChunkMetadata, type ChunkType, type SearchResult, VectorStore } from "./vector-store.js";
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC/G,OAAO,EAAE,gBAAgB,EAAE,KAAK,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAE,KAAK,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,kBAAkB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC/G,OAAO,EAAE,gBAAgB,EAAmB,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAqE,WAAW,EAAE,MAAM,mBAAmB,CAAC","sourcesContent":["export { type ClientConfig, getKnowledgeDbPath, getKnowledgePath, loadClientConfig } from \"./client-config.js\";\nexport { KnowledgeManager, type SearchMode } from \"./knowledge-manager.js\";\nexport { type Chunk, type ChunkMetadata, type ChunkType, type SearchResult, VectorStore } from \"./vector-store.js\";\n"]}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Knowledge manager — high-level API for client knowledge operations.
|
|
3
|
+
* Wraps VectorStore with chunking, file indexing, and search modes.
|
|
4
|
+
*/
|
|
5
|
+
import { type SearchResult } from "./vector-store.js";
|
|
6
|
+
export type SearchMode = "decide" | "connect" | "fuzzy" | "general";
|
|
7
|
+
export declare class KnowledgeManager {
|
|
8
|
+
private store;
|
|
9
|
+
private clientSlug;
|
|
10
|
+
constructor(clientSlug: string, openaiApiKey?: string);
|
|
11
|
+
/**
|
|
12
|
+
* Index a file into the knowledge base.
|
|
13
|
+
* Chunks by headings/paragraphs, deduplicates via content hash.
|
|
14
|
+
*/
|
|
15
|
+
indexFile(filePath: string): Promise<number>;
|
|
16
|
+
/**
|
|
17
|
+
* Index a directory recursively (markdown files only).
|
|
18
|
+
*/
|
|
19
|
+
indexDirectory(dirPath: string): Promise<number>;
|
|
20
|
+
/**
|
|
21
|
+
* Search knowledge with optional mode for query refinement.
|
|
22
|
+
*/
|
|
23
|
+
searchKnowledge(query: string, mode?: SearchMode, limit?: number): Promise<SearchResult[]>;
|
|
24
|
+
/**
|
|
25
|
+
* Remove all knowledge from a specific source file.
|
|
26
|
+
*/
|
|
27
|
+
forget(sourceFile: string): Promise<number>;
|
|
28
|
+
/**
|
|
29
|
+
* Get knowledge base statistics.
|
|
30
|
+
*/
|
|
31
|
+
getStats(): {
|
|
32
|
+
totalChunks: number;
|
|
33
|
+
sources: string[];
|
|
34
|
+
clientSlug: string;
|
|
35
|
+
};
|
|
36
|
+
close(): void;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=knowledge-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-manager.d.ts","sourceRoot":"","sources":["../src/knowledge-manager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAkD,KAAK,YAAY,EAAe,MAAM,mBAAmB,CAAC;AAEnH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;AAEpE,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,UAAU,CAAS;IAE3B,YAAY,UAAU,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAOpD;IAED;;;OAGG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAejD;IAED;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiBrD;IAED;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,UAAsB,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAGrG;IAED;;OAEG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEhD;IAED;;OAEG;IACH,QAAQ,IAAI;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAGzE;IAED,KAAK,IAAI,IAAI,CAEZ;CACD"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Knowledge manager — high-level API for client knowledge operations.
|
|
3
|
+
* Wraps VectorStore with chunking, file indexing, and search modes.
|
|
4
|
+
*/
|
|
5
|
+
import * as fs from "node:fs";
|
|
6
|
+
import * as path from "node:path";
|
|
7
|
+
import { getKnowledgeDbPath } from "./client-config.js";
|
|
8
|
+
import { VectorStore } from "./vector-store.js";
|
|
9
|
+
export class KnowledgeManager {
|
|
10
|
+
constructor(clientSlug, openaiApiKey) {
|
|
11
|
+
const apiKey = openaiApiKey ?? process.env.OPENAI_API_KEY;
|
|
12
|
+
if (!apiKey) {
|
|
13
|
+
throw new Error("OPENAI_API_KEY required for knowledge base embeddings");
|
|
14
|
+
}
|
|
15
|
+
this.clientSlug = clientSlug;
|
|
16
|
+
this.store = new VectorStore(getKnowledgeDbPath(clientSlug), apiKey);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Index a file into the knowledge base.
|
|
20
|
+
* Chunks by headings/paragraphs, deduplicates via content hash.
|
|
21
|
+
*/
|
|
22
|
+
async indexFile(filePath) {
|
|
23
|
+
const absPath = path.resolve(filePath);
|
|
24
|
+
if (!fs.existsSync(absPath)) {
|
|
25
|
+
throw new Error(`File not found: ${absPath}`);
|
|
26
|
+
}
|
|
27
|
+
const content = fs.readFileSync(absPath, "utf-8");
|
|
28
|
+
const chunks = chunkContent(content, {
|
|
29
|
+
source: absPath,
|
|
30
|
+
client: this.clientSlug,
|
|
31
|
+
timestamp: Date.now(),
|
|
32
|
+
type: inferChunkType(absPath),
|
|
33
|
+
});
|
|
34
|
+
return this.store.index(chunks);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Index a directory recursively (markdown files only).
|
|
38
|
+
*/
|
|
39
|
+
async indexDirectory(dirPath) {
|
|
40
|
+
const absDir = path.resolve(dirPath);
|
|
41
|
+
if (!fs.existsSync(absDir)) {
|
|
42
|
+
throw new Error(`Directory not found: ${absDir}`);
|
|
43
|
+
}
|
|
44
|
+
let total = 0;
|
|
45
|
+
const entries = fs.readdirSync(absDir, { withFileTypes: true });
|
|
46
|
+
for (const entry of entries) {
|
|
47
|
+
const full = path.join(absDir, entry.name);
|
|
48
|
+
if (entry.isDirectory() && !entry.name.startsWith(".") && entry.name !== "node_modules") {
|
|
49
|
+
total += await this.indexDirectory(full);
|
|
50
|
+
}
|
|
51
|
+
else if (entry.isFile() && /\.(md|txt)$/i.test(entry.name)) {
|
|
52
|
+
total += await this.indexFile(full);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return total;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Search knowledge with optional mode for query refinement.
|
|
59
|
+
*/
|
|
60
|
+
async searchKnowledge(query, mode = "general", limit = 5) {
|
|
61
|
+
const refinedQuery = refineQueryByMode(query, mode);
|
|
62
|
+
return this.store.search(refinedQuery, limit);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Remove all knowledge from a specific source file.
|
|
66
|
+
*/
|
|
67
|
+
async forget(sourceFile) {
|
|
68
|
+
return this.store.delete(path.resolve(sourceFile));
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Get knowledge base statistics.
|
|
72
|
+
*/
|
|
73
|
+
getStats() {
|
|
74
|
+
const stats = this.store.getStats();
|
|
75
|
+
return { ...stats, clientSlug: this.clientSlug };
|
|
76
|
+
}
|
|
77
|
+
close() {
|
|
78
|
+
this.store.close();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// ============================================================================
|
|
82
|
+
// Chunking
|
|
83
|
+
// ============================================================================
|
|
84
|
+
const MAX_CHUNK_CHARS = 2000; // ~500 tokens
|
|
85
|
+
function chunkContent(content, baseMetadata) {
|
|
86
|
+
const chunks = [];
|
|
87
|
+
// Split by markdown headings
|
|
88
|
+
const sections = content.split(/^(?=#{1,3}\s)/m);
|
|
89
|
+
for (const section of sections) {
|
|
90
|
+
const trimmed = section.trim();
|
|
91
|
+
if (!trimmed)
|
|
92
|
+
continue;
|
|
93
|
+
if (trimmed.length <= MAX_CHUNK_CHARS) {
|
|
94
|
+
chunks.push({ content: trimmed, metadata: { ...baseMetadata } });
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
// Split long sections by paragraphs
|
|
98
|
+
const paragraphs = trimmed.split(/\n\n+/);
|
|
99
|
+
let current = "";
|
|
100
|
+
for (const para of paragraphs) {
|
|
101
|
+
if (current.length + para.length + 2 > MAX_CHUNK_CHARS) {
|
|
102
|
+
if (current.trim()) {
|
|
103
|
+
chunks.push({ content: current.trim(), metadata: { ...baseMetadata } });
|
|
104
|
+
}
|
|
105
|
+
current = para;
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
current = current ? `${current}\n\n${para}` : para;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (current.trim()) {
|
|
112
|
+
chunks.push({ content: current.trim(), metadata: { ...baseMetadata } });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return chunks;
|
|
117
|
+
}
|
|
118
|
+
function inferChunkType(filePath) {
|
|
119
|
+
const basename = path.basename(filePath).toLowerCase();
|
|
120
|
+
if (basename.includes("agents") || basename.includes("convention"))
|
|
121
|
+
return "convention";
|
|
122
|
+
if (basename.includes("decision") || basename.includes("adr"))
|
|
123
|
+
return "decision";
|
|
124
|
+
if (basename.includes("pattern"))
|
|
125
|
+
return "pattern";
|
|
126
|
+
return "general";
|
|
127
|
+
}
|
|
128
|
+
function refineQueryByMode(query, mode) {
|
|
129
|
+
switch (mode) {
|
|
130
|
+
case "decide":
|
|
131
|
+
return `decisions and choices about: ${query}`;
|
|
132
|
+
case "connect":
|
|
133
|
+
return `relationships and connections related to: ${query}`;
|
|
134
|
+
case "fuzzy":
|
|
135
|
+
return query; // Use as-is for fuzzy/vague queries
|
|
136
|
+
default:
|
|
137
|
+
return query;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=knowledge-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-manager.js","sourceRoot":"","sources":["../src/knowledge-manager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAqE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAInH,MAAM,OAAO,gBAAgB;IAI5B,YAAY,UAAkB,EAAE,YAAqB;QACpD,MAAM,MAAM,GAAG,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,WAAW,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;IACtE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,QAAgB;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE;YACpC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,cAAc,CAAC,OAAO,CAAC;SAC7B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAe;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBACzF,KAAK,IAAI,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9D,KAAK,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,IAAI,GAAe,SAAS,EAAE,KAAK,GAAG,CAAC;QAC3E,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,QAAQ;QACP,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACpC,OAAO,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IAClD,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACD;AAED,+EAA+E;AAC/E,WAAW;AACX,+EAA+E;AAE/E,MAAM,eAAe,GAAG,IAAI,CAAC,CAAC,cAAc;AAE5C,SAAS,YAAY,CAAC,OAAe,EAAE,YAA2B;IACjE,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,YAAY,EAAE,EAAE,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACP,oCAAoC;YACpC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,eAAe,EAAE,CAAC;oBACxD,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;wBACpB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,YAAY,EAAE,EAAE,CAAC,CAAC;oBACzE,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACP,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,YAAY,EAAE,EAAE,CAAC,CAAC;YACzE,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACvD,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,YAAY,CAAC;IACxF,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC;IACjF,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACnD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,IAAgB;IACzD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,QAAQ;YACZ,OAAO,gCAAgC,KAAK,EAAE,CAAC;QAChD,KAAK,SAAS;YACb,OAAO,6CAA6C,KAAK,EAAE,CAAC;QAC7D,KAAK,OAAO;YACX,OAAO,KAAK,CAAC,CAAC,oCAAoC;QACnD;YACC,OAAO,KAAK,CAAC;IACf,CAAC;AACF,CAAC","sourcesContent":["/**\n * Knowledge manager — high-level API for client knowledge operations.\n * Wraps VectorStore with chunking, file indexing, and search modes.\n */\n\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport { getKnowledgeDbPath } from \"./client-config.js\";\nimport { type Chunk, type ChunkMetadata, type ChunkType, type SearchResult, VectorStore } from \"./vector-store.js\";\n\nexport type SearchMode = \"decide\" | \"connect\" | \"fuzzy\" | \"general\";\n\nexport class KnowledgeManager {\n\tprivate store: VectorStore;\n\tprivate clientSlug: string;\n\n\tconstructor(clientSlug: string, openaiApiKey?: string) {\n\t\tconst apiKey = openaiApiKey ?? process.env.OPENAI_API_KEY;\n\t\tif (!apiKey) {\n\t\t\tthrow new Error(\"OPENAI_API_KEY required for knowledge base embeddings\");\n\t\t}\n\t\tthis.clientSlug = clientSlug;\n\t\tthis.store = new VectorStore(getKnowledgeDbPath(clientSlug), apiKey);\n\t}\n\n\t/**\n\t * Index a file into the knowledge base.\n\t * Chunks by headings/paragraphs, deduplicates via content hash.\n\t */\n\tasync indexFile(filePath: string): Promise<number> {\n\t\tconst absPath = path.resolve(filePath);\n\t\tif (!fs.existsSync(absPath)) {\n\t\t\tthrow new Error(`File not found: ${absPath}`);\n\t\t}\n\n\t\tconst content = fs.readFileSync(absPath, \"utf-8\");\n\t\tconst chunks = chunkContent(content, {\n\t\t\tsource: absPath,\n\t\t\tclient: this.clientSlug,\n\t\t\ttimestamp: Date.now(),\n\t\t\ttype: inferChunkType(absPath),\n\t\t});\n\n\t\treturn this.store.index(chunks);\n\t}\n\n\t/**\n\t * Index a directory recursively (markdown files only).\n\t */\n\tasync indexDirectory(dirPath: string): Promise<number> {\n\t\tconst absDir = path.resolve(dirPath);\n\t\tif (!fs.existsSync(absDir)) {\n\t\t\tthrow new Error(`Directory not found: ${absDir}`);\n\t\t}\n\n\t\tlet total = 0;\n\t\tconst entries = fs.readdirSync(absDir, { withFileTypes: true });\n\t\tfor (const entry of entries) {\n\t\t\tconst full = path.join(absDir, entry.name);\n\t\t\tif (entry.isDirectory() && !entry.name.startsWith(\".\") && entry.name !== \"node_modules\") {\n\t\t\t\ttotal += await this.indexDirectory(full);\n\t\t\t} else if (entry.isFile() && /\\.(md|txt)$/i.test(entry.name)) {\n\t\t\t\ttotal += await this.indexFile(full);\n\t\t\t}\n\t\t}\n\t\treturn total;\n\t}\n\n\t/**\n\t * Search knowledge with optional mode for query refinement.\n\t */\n\tasync searchKnowledge(query: string, mode: SearchMode = \"general\", limit = 5): Promise<SearchResult[]> {\n\t\tconst refinedQuery = refineQueryByMode(query, mode);\n\t\treturn this.store.search(refinedQuery, limit);\n\t}\n\n\t/**\n\t * Remove all knowledge from a specific source file.\n\t */\n\tasync forget(sourceFile: string): Promise<number> {\n\t\treturn this.store.delete(path.resolve(sourceFile));\n\t}\n\n\t/**\n\t * Get knowledge base statistics.\n\t */\n\tgetStats(): { totalChunks: number; sources: string[]; clientSlug: string } {\n\t\tconst stats = this.store.getStats();\n\t\treturn { ...stats, clientSlug: this.clientSlug };\n\t}\n\n\tclose(): void {\n\t\tthis.store.close();\n\t}\n}\n\n// ============================================================================\n// Chunking\n// ============================================================================\n\nconst MAX_CHUNK_CHARS = 2000; // ~500 tokens\n\nfunction chunkContent(content: string, baseMetadata: ChunkMetadata): Chunk[] {\n\tconst chunks: Chunk[] = [];\n\n\t// Split by markdown headings\n\tconst sections = content.split(/^(?=#{1,3}\\s)/m);\n\n\tfor (const section of sections) {\n\t\tconst trimmed = section.trim();\n\t\tif (!trimmed) continue;\n\n\t\tif (trimmed.length <= MAX_CHUNK_CHARS) {\n\t\t\tchunks.push({ content: trimmed, metadata: { ...baseMetadata } });\n\t\t} else {\n\t\t\t// Split long sections by paragraphs\n\t\t\tconst paragraphs = trimmed.split(/\\n\\n+/);\n\t\t\tlet current = \"\";\n\t\t\tfor (const para of paragraphs) {\n\t\t\t\tif (current.length + para.length + 2 > MAX_CHUNK_CHARS) {\n\t\t\t\t\tif (current.trim()) {\n\t\t\t\t\t\tchunks.push({ content: current.trim(), metadata: { ...baseMetadata } });\n\t\t\t\t\t}\n\t\t\t\t\tcurrent = para;\n\t\t\t\t} else {\n\t\t\t\t\tcurrent = current ? `${current}\\n\\n${para}` : para;\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (current.trim()) {\n\t\t\t\tchunks.push({ content: current.trim(), metadata: { ...baseMetadata } });\n\t\t\t}\n\t\t}\n\t}\n\n\treturn chunks;\n}\n\nfunction inferChunkType(filePath: string): ChunkType {\n\tconst basename = path.basename(filePath).toLowerCase();\n\tif (basename.includes(\"agents\") || basename.includes(\"convention\")) return \"convention\";\n\tif (basename.includes(\"decision\") || basename.includes(\"adr\")) return \"decision\";\n\tif (basename.includes(\"pattern\")) return \"pattern\";\n\treturn \"general\";\n}\n\nfunction refineQueryByMode(query: string, mode: SearchMode): string {\n\tswitch (mode) {\n\t\tcase \"decide\":\n\t\t\treturn `decisions and choices about: ${query}`;\n\t\tcase \"connect\":\n\t\t\treturn `relationships and connections related to: ${query}`;\n\t\tcase \"fuzzy\":\n\t\t\treturn query; // Use as-is for fuzzy/vague queries\n\t\tdefault:\n\t\t\treturn query;\n\t}\n}\n"]}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector store backed by SQLite with OpenAI embeddings.
|
|
3
|
+
* Provides local-first semantic search per client namespace.
|
|
4
|
+
*/
|
|
5
|
+
export type ChunkType = "decision" | "convention" | "pattern" | "general";
|
|
6
|
+
export interface ChunkMetadata {
|
|
7
|
+
source: string;
|
|
8
|
+
client: string;
|
|
9
|
+
timestamp: number;
|
|
10
|
+
type: ChunkType;
|
|
11
|
+
}
|
|
12
|
+
export interface Chunk {
|
|
13
|
+
content: string;
|
|
14
|
+
metadata: ChunkMetadata;
|
|
15
|
+
}
|
|
16
|
+
export interface SearchResult {
|
|
17
|
+
content: string;
|
|
18
|
+
metadata: ChunkMetadata;
|
|
19
|
+
score: number;
|
|
20
|
+
}
|
|
21
|
+
export declare class VectorStore {
|
|
22
|
+
private db;
|
|
23
|
+
private apiKey;
|
|
24
|
+
constructor(dbPath: string, openaiApiKey: string);
|
|
25
|
+
private initSchema;
|
|
26
|
+
index(chunks: Chunk[]): Promise<number>;
|
|
27
|
+
search(query: string, limit?: number): Promise<SearchResult[]>;
|
|
28
|
+
delete(sourceFile: string): Promise<number>;
|
|
29
|
+
getStats(): {
|
|
30
|
+
totalChunks: number;
|
|
31
|
+
sources: string[];
|
|
32
|
+
};
|
|
33
|
+
close(): void;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=vector-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,YAAY,GAAG,SAAS,GAAG,SAAS,CAAC;AAE1E,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;CAChB;AAED,MAAM,WAAW,KAAK;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;CACd;AAwFD,qBAAa,WAAW;IACvB,OAAO,CAAC,EAAE,CAAoB;IAC9B,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAY/C;IAED,OAAO,CAAC,UAAU;IAkBZ,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAoD5C;IAEK,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA0B9D;IAEK,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAGhD;IAED,QAAQ,IAAI;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CASrD;IAED,KAAK,IAAI,IAAI,CAEZ;CACD"}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector store backed by SQLite with OpenAI embeddings.
|
|
3
|
+
* Provides local-first semantic search per client namespace.
|
|
4
|
+
*/
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import * as fs from "node:fs";
|
|
7
|
+
import * as path from "node:path";
|
|
8
|
+
import Database from "better-sqlite3";
|
|
9
|
+
// ============================================================================
|
|
10
|
+
// Embedding
|
|
11
|
+
// ============================================================================
|
|
12
|
+
async function getEmbedding(text, apiKey) {
|
|
13
|
+
const response = await fetch("https://api.openai.com/v1/embeddings", {
|
|
14
|
+
method: "POST",
|
|
15
|
+
headers: {
|
|
16
|
+
Authorization: `Bearer ${apiKey}`,
|
|
17
|
+
"Content-Type": "application/json",
|
|
18
|
+
},
|
|
19
|
+
body: JSON.stringify({
|
|
20
|
+
model: "text-embedding-3-small",
|
|
21
|
+
input: text,
|
|
22
|
+
}),
|
|
23
|
+
});
|
|
24
|
+
if (!response.ok) {
|
|
25
|
+
throw new Error(`Embedding API error: ${response.status} ${response.statusText}`);
|
|
26
|
+
}
|
|
27
|
+
const data = (await response.json());
|
|
28
|
+
return data.data[0].embedding;
|
|
29
|
+
}
|
|
30
|
+
async function getEmbeddingsBatch(texts, apiKey) {
|
|
31
|
+
if (texts.length === 0)
|
|
32
|
+
return [];
|
|
33
|
+
if (texts.length === 1)
|
|
34
|
+
return [await getEmbedding(texts[0], apiKey)];
|
|
35
|
+
const response = await fetch("https://api.openai.com/v1/embeddings", {
|
|
36
|
+
method: "POST",
|
|
37
|
+
headers: {
|
|
38
|
+
Authorization: `Bearer ${apiKey}`,
|
|
39
|
+
"Content-Type": "application/json",
|
|
40
|
+
},
|
|
41
|
+
body: JSON.stringify({
|
|
42
|
+
model: "text-embedding-3-small",
|
|
43
|
+
input: texts,
|
|
44
|
+
}),
|
|
45
|
+
});
|
|
46
|
+
if (!response.ok) {
|
|
47
|
+
throw new Error(`Embedding API error: ${response.status} ${response.statusText}`);
|
|
48
|
+
}
|
|
49
|
+
const data = (await response.json());
|
|
50
|
+
// Sort by index to maintain order
|
|
51
|
+
return data.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
|
|
52
|
+
}
|
|
53
|
+
function cosineSimilarity(a, b) {
|
|
54
|
+
let dot = 0;
|
|
55
|
+
let normA = 0;
|
|
56
|
+
let normB = 0;
|
|
57
|
+
for (let i = 0; i < a.length; i++) {
|
|
58
|
+
dot += a[i] * b[i];
|
|
59
|
+
normA += a[i] * a[i];
|
|
60
|
+
normB += b[i] * b[i];
|
|
61
|
+
}
|
|
62
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
63
|
+
}
|
|
64
|
+
function contentHash(content) {
|
|
65
|
+
return createHash("sha256").update(content).digest("hex");
|
|
66
|
+
}
|
|
67
|
+
function embeddingToBuffer(embedding) {
|
|
68
|
+
const buf = Buffer.alloc(embedding.length * 4);
|
|
69
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
70
|
+
buf.writeFloatLE(embedding[i], i * 4);
|
|
71
|
+
}
|
|
72
|
+
return buf;
|
|
73
|
+
}
|
|
74
|
+
function bufferToEmbedding(buf) {
|
|
75
|
+
const result = [];
|
|
76
|
+
for (let i = 0; i < buf.length; i += 4) {
|
|
77
|
+
result.push(buf.readFloatLE(i));
|
|
78
|
+
}
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
// ============================================================================
|
|
82
|
+
// VectorStore
|
|
83
|
+
// ============================================================================
|
|
84
|
+
export class VectorStore {
|
|
85
|
+
constructor(dbPath, openaiApiKey) {
|
|
86
|
+
this.apiKey = openaiApiKey;
|
|
87
|
+
// Ensure directory exists
|
|
88
|
+
const dir = path.dirname(dbPath);
|
|
89
|
+
if (!fs.existsSync(dir)) {
|
|
90
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
91
|
+
}
|
|
92
|
+
this.db = new Database(dbPath);
|
|
93
|
+
this.db.pragma("journal_mode = WAL");
|
|
94
|
+
this.initSchema();
|
|
95
|
+
}
|
|
96
|
+
initSchema() {
|
|
97
|
+
this.db.exec(`
|
|
98
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
99
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
100
|
+
content TEXT NOT NULL,
|
|
101
|
+
embedding BLOB NOT NULL,
|
|
102
|
+
source TEXT NOT NULL,
|
|
103
|
+
client TEXT NOT NULL,
|
|
104
|
+
timestamp INTEGER NOT NULL,
|
|
105
|
+
type TEXT NOT NULL DEFAULT 'general',
|
|
106
|
+
content_hash TEXT NOT NULL UNIQUE
|
|
107
|
+
);
|
|
108
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);
|
|
109
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_client ON chunks(client);
|
|
110
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(content_hash);
|
|
111
|
+
`);
|
|
112
|
+
}
|
|
113
|
+
async index(chunks) {
|
|
114
|
+
if (chunks.length === 0)
|
|
115
|
+
return 0;
|
|
116
|
+
// Filter out duplicates by content hash
|
|
117
|
+
const existingHashes = new Set();
|
|
118
|
+
const checkStmt = this.db.prepare("SELECT 1 FROM chunks WHERE content_hash = ?");
|
|
119
|
+
const newChunks = [];
|
|
120
|
+
for (const chunk of chunks) {
|
|
121
|
+
const hash = contentHash(chunk.content);
|
|
122
|
+
if (existingHashes.has(hash))
|
|
123
|
+
continue;
|
|
124
|
+
const exists = checkStmt.get(hash);
|
|
125
|
+
if (exists) {
|
|
126
|
+
existingHashes.add(hash);
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
newChunks.push({ ...chunk, hash });
|
|
130
|
+
existingHashes.add(hash);
|
|
131
|
+
}
|
|
132
|
+
if (newChunks.length === 0)
|
|
133
|
+
return 0;
|
|
134
|
+
// Batch embed
|
|
135
|
+
const embeddings = await getEmbeddingsBatch(newChunks.map((c) => c.content), this.apiKey);
|
|
136
|
+
// Insert
|
|
137
|
+
const insertStmt = this.db.prepare("INSERT OR IGNORE INTO chunks (content, embedding, source, client, timestamp, type, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?)");
|
|
138
|
+
const insertMany = this.db.transaction(() => {
|
|
139
|
+
let inserted = 0;
|
|
140
|
+
for (let i = 0; i < newChunks.length; i++) {
|
|
141
|
+
const chunk = newChunks[i];
|
|
142
|
+
const result = insertStmt.run(chunk.content, embeddingToBuffer(embeddings[i]), chunk.metadata.source, chunk.metadata.client, chunk.metadata.timestamp, chunk.metadata.type, chunk.hash);
|
|
143
|
+
if (result.changes > 0)
|
|
144
|
+
inserted++;
|
|
145
|
+
}
|
|
146
|
+
return inserted;
|
|
147
|
+
});
|
|
148
|
+
return insertMany();
|
|
149
|
+
}
|
|
150
|
+
async search(query, limit = 5) {
|
|
151
|
+
const queryEmbedding = await getEmbedding(query, this.apiKey);
|
|
152
|
+
const rows = this.db
|
|
153
|
+
.prepare("SELECT content, embedding, source, client, timestamp, type FROM chunks")
|
|
154
|
+
.all();
|
|
155
|
+
const scored = rows.map((row) => ({
|
|
156
|
+
content: row.content,
|
|
157
|
+
metadata: {
|
|
158
|
+
source: row.source,
|
|
159
|
+
client: row.client,
|
|
160
|
+
timestamp: row.timestamp,
|
|
161
|
+
type: row.type,
|
|
162
|
+
},
|
|
163
|
+
score: cosineSimilarity(queryEmbedding, bufferToEmbedding(row.embedding)),
|
|
164
|
+
}));
|
|
165
|
+
scored.sort((a, b) => b.score - a.score);
|
|
166
|
+
return scored.slice(0, limit);
|
|
167
|
+
}
|
|
168
|
+
async delete(sourceFile) {
|
|
169
|
+
const result = this.db.prepare("DELETE FROM chunks WHERE source = ?").run(sourceFile);
|
|
170
|
+
return result.changes;
|
|
171
|
+
}
|
|
172
|
+
getStats() {
|
|
173
|
+
const count = this.db.prepare("SELECT COUNT(*) as count FROM chunks").get();
|
|
174
|
+
const sources = this.db.prepare("SELECT DISTINCT source FROM chunks ORDER BY source").all();
|
|
175
|
+
return {
|
|
176
|
+
totalChunks: count.count,
|
|
177
|
+
sources: sources.map((s) => s.source),
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
close() {
|
|
181
|
+
this.db.close();
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=vector-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store.js","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AA0BtC,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,MAAc;IACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,sCAAsC,EAAE;QACpE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACR,aAAa,EAAE,UAAU,MAAM,EAAE;YACjC,cAAc,EAAE,kBAAkB;SAClC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACpB,KAAK,EAAE,wBAAwB;YAC/B,KAAK,EAAE,IAAI;SACX,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA6C,CAAC;IACjF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAe,EAAE,MAAc;IAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAEtE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,sCAAsC,EAAE;QACpE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACR,aAAa,EAAE,UAAU,MAAM,EAAE;YACjC,cAAc,EAAE,kBAAkB;SAClC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACpB,KAAK,EAAE,wBAAwB;YAC/B,KAAK,EAAE,KAAK;SACZ,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA4D,CAAC;IAChG,kCAAkC;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACjD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IACnC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAmB;IAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,cAAc;AACd,+EAA+E;AAE/E,MAAM,OAAO,WAAW;IAIvB,YAAY,MAAc,EAAE,YAAoB;QAC/C,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC;QAE3B,0BAA0B;QAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,UAAU,EAAE,CAAC;IACnB,CAAC;IAEO,UAAU;QACjB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;GAcZ,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAe;QAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAElC,wCAAwC;QACxC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC;QACjF,MAAM,SAAS,GAAoC,EAAE,CAAC;QAEtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACxC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YACvC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACZ,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzB,SAAS;YACV,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACnC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAErC,cAAc;QACd,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAC1C,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAC/B,IAAI,CAAC,MAAM,CACX,CAAC;QAEF,SAAS;QACT,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACjC,+HAA+H,CAC/H,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAC5B,KAAK,CAAC,OAAO,EACb,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAChC,KAAK,CAAC,QAAQ,CAAC,MAAM,EACrB,KAAK,CAAC,QAAQ,CAAC,MAAM,EACrB,KAAK,CAAC,QAAQ,CAAC,SAAS,EACxB,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,IAAI,CACV,CAAC;gBACF,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC;oBAAE,QAAQ,EAAE,CAAC;YACpC,CAAC;YACD,OAAO,QAAQ,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,OAAO,UAAU,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAK,GAAG,CAAC;QACpC,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aAClB,OAAO,CAAC,wEAAwE,CAAC;aACjF,GAAG,EAOH,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACjC,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE;gBACT,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,IAAI,EAAE,GAAG,CAAC,IAAI;aACd;YACD,KAAK,EAAE,gBAAgB,CAAC,cAAc,EAAE,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;SACzE,CAAC,CAAC,CAAC;QAEJ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtF,OAAO,MAAM,CAAC,OAAO,CAAC;IACvB,CAAC;IAED,QAAQ;QACP,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAuB,CAAC;QACjG,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,EAEvF,CAAC;QACH,OAAO;YACN,WAAW,EAAE,KAAK,CAAC,KAAK;YACxB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SACrC,CAAC;IACH,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;CACD","sourcesContent":["/**\n * Vector store backed by SQLite with OpenAI embeddings.\n * Provides local-first semantic search per client namespace.\n */\n\nimport { createHash } from \"node:crypto\";\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport Database from \"better-sqlite3\";\n\n// ============================================================================\n// Types\n// ============================================================================\n\nexport type ChunkType = \"decision\" | \"convention\" | \"pattern\" | \"general\";\n\nexport interface ChunkMetadata {\n\tsource: string;\n\tclient: string;\n\ttimestamp: number;\n\ttype: ChunkType;\n}\n\nexport interface Chunk {\n\tcontent: string;\n\tmetadata: ChunkMetadata;\n}\n\nexport interface SearchResult {\n\tcontent: string;\n\tmetadata: ChunkMetadata;\n\tscore: number;\n}\n\n// ============================================================================\n// Embedding\n// ============================================================================\n\nasync function getEmbedding(text: string, apiKey: string): Promise<number[]> {\n\tconst response = await fetch(\"https://api.openai.com/v1/embeddings\", {\n\t\tmethod: \"POST\",\n\t\theaders: {\n\t\t\tAuthorization: `Bearer ${apiKey}`,\n\t\t\t\"Content-Type\": \"application/json\",\n\t\t},\n\t\tbody: JSON.stringify({\n\t\t\tmodel: \"text-embedding-3-small\",\n\t\t\tinput: text,\n\t\t}),\n\t});\n\n\tif (!response.ok) {\n\t\tthrow new Error(`Embedding API error: ${response.status} ${response.statusText}`);\n\t}\n\n\tconst data = (await response.json()) as { data: Array<{ embedding: number[] }> };\n\treturn data.data[0].embedding;\n}\n\nasync function getEmbeddingsBatch(texts: string[], apiKey: string): Promise<number[][]> {\n\tif (texts.length === 0) return [];\n\tif (texts.length === 1) return [await getEmbedding(texts[0], apiKey)];\n\n\tconst response = await fetch(\"https://api.openai.com/v1/embeddings\", {\n\t\tmethod: \"POST\",\n\t\theaders: {\n\t\t\tAuthorization: `Bearer ${apiKey}`,\n\t\t\t\"Content-Type\": \"application/json\",\n\t\t},\n\t\tbody: JSON.stringify({\n\t\t\tmodel: \"text-embedding-3-small\",\n\t\t\tinput: texts,\n\t\t}),\n\t});\n\n\tif (!response.ok) {\n\t\tthrow new Error(`Embedding API error: ${response.status} ${response.statusText}`);\n\t}\n\n\tconst data = (await response.json()) as { data: Array<{ embedding: number[]; index: number }> };\n\t// Sort by index to maintain order\n\treturn data.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n\tlet dot = 0;\n\tlet normA = 0;\n\tlet normB = 0;\n\tfor (let i = 0; i < a.length; i++) {\n\t\tdot += a[i] * b[i];\n\t\tnormA += a[i] * a[i];\n\t\tnormB += b[i] * b[i];\n\t}\n\treturn dot / (Math.sqrt(normA) * Math.sqrt(normB));\n}\n\nfunction contentHash(content: string): string {\n\treturn createHash(\"sha256\").update(content).digest(\"hex\");\n}\n\nfunction embeddingToBuffer(embedding: number[]): Buffer {\n\tconst buf = Buffer.alloc(embedding.length * 4);\n\tfor (let i = 0; i < embedding.length; i++) {\n\t\tbuf.writeFloatLE(embedding[i], i * 4);\n\t}\n\treturn buf;\n}\n\nfunction bufferToEmbedding(buf: Buffer): number[] {\n\tconst result: number[] = [];\n\tfor (let i = 0; i < buf.length; i += 4) {\n\t\tresult.push(buf.readFloatLE(i));\n\t}\n\treturn result;\n}\n\n// ============================================================================\n// VectorStore\n// ============================================================================\n\nexport class VectorStore {\n\tprivate db: Database.Database;\n\tprivate apiKey: string;\n\n\tconstructor(dbPath: string, openaiApiKey: string) {\n\t\tthis.apiKey = openaiApiKey;\n\n\t\t// Ensure directory exists\n\t\tconst dir = path.dirname(dbPath);\n\t\tif (!fs.existsSync(dir)) {\n\t\t\tfs.mkdirSync(dir, { recursive: true });\n\t\t}\n\n\t\tthis.db = new Database(dbPath);\n\t\tthis.db.pragma(\"journal_mode = WAL\");\n\t\tthis.initSchema();\n\t}\n\n\tprivate initSchema(): void {\n\t\tthis.db.exec(`\n\t\t\tCREATE TABLE IF NOT EXISTS chunks (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tcontent TEXT NOT NULL,\n\t\t\t\tembedding BLOB NOT NULL,\n\t\t\t\tsource TEXT NOT NULL,\n\t\t\t\tclient TEXT NOT NULL,\n\t\t\t\ttimestamp INTEGER NOT NULL,\n\t\t\t\ttype TEXT NOT NULL DEFAULT 'general',\n\t\t\t\tcontent_hash TEXT NOT NULL UNIQUE\n\t\t\t);\n\t\t\tCREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);\n\t\t\tCREATE INDEX IF NOT EXISTS idx_chunks_client ON chunks(client);\n\t\t\tCREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(content_hash);\n\t\t`);\n\t}\n\n\tasync index(chunks: Chunk[]): Promise<number> {\n\t\tif (chunks.length === 0) return 0;\n\n\t\t// Filter out duplicates by content hash\n\t\tconst existingHashes = new Set<string>();\n\t\tconst checkStmt = this.db.prepare(\"SELECT 1 FROM chunks WHERE content_hash = ?\");\n\t\tconst newChunks: Array<Chunk & { hash: string }> = [];\n\n\t\tfor (const chunk of chunks) {\n\t\t\tconst hash = contentHash(chunk.content);\n\t\t\tif (existingHashes.has(hash)) continue;\n\t\t\tconst exists = checkStmt.get(hash);\n\t\t\tif (exists) {\n\t\t\t\texistingHashes.add(hash);\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tnewChunks.push({ ...chunk, hash });\n\t\t\texistingHashes.add(hash);\n\t\t}\n\n\t\tif (newChunks.length === 0) return 0;\n\n\t\t// Batch embed\n\t\tconst embeddings = await getEmbeddingsBatch(\n\t\t\tnewChunks.map((c) => c.content),\n\t\t\tthis.apiKey,\n\t\t);\n\n\t\t// Insert\n\t\tconst insertStmt = this.db.prepare(\n\t\t\t\"INSERT OR IGNORE INTO chunks (content, embedding, source, client, timestamp, type, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?)\",\n\t\t);\n\n\t\tconst insertMany = this.db.transaction(() => {\n\t\t\tlet inserted = 0;\n\t\t\tfor (let i = 0; i < newChunks.length; i++) {\n\t\t\t\tconst chunk = newChunks[i];\n\t\t\t\tconst result = insertStmt.run(\n\t\t\t\t\tchunk.content,\n\t\t\t\t\tembeddingToBuffer(embeddings[i]),\n\t\t\t\t\tchunk.metadata.source,\n\t\t\t\t\tchunk.metadata.client,\n\t\t\t\t\tchunk.metadata.timestamp,\n\t\t\t\t\tchunk.metadata.type,\n\t\t\t\t\tchunk.hash,\n\t\t\t\t);\n\t\t\t\tif (result.changes > 0) inserted++;\n\t\t\t}\n\t\t\treturn inserted;\n\t\t});\n\n\t\treturn insertMany();\n\t}\n\n\tasync search(query: string, limit = 5): Promise<SearchResult[]> {\n\t\tconst queryEmbedding = await getEmbedding(query, this.apiKey);\n\t\tconst rows = this.db\n\t\t\t.prepare(\"SELECT content, embedding, source, client, timestamp, type FROM chunks\")\n\t\t\t.all() as Array<{\n\t\t\tcontent: string;\n\t\t\tembedding: Buffer;\n\t\t\tsource: string;\n\t\t\tclient: string;\n\t\t\ttimestamp: number;\n\t\t\ttype: ChunkType;\n\t\t}>;\n\n\t\tconst scored = rows.map((row) => ({\n\t\t\tcontent: row.content,\n\t\t\tmetadata: {\n\t\t\t\tsource: row.source,\n\t\t\t\tclient: row.client,\n\t\t\t\ttimestamp: row.timestamp,\n\t\t\t\ttype: row.type,\n\t\t\t},\n\t\t\tscore: cosineSimilarity(queryEmbedding, bufferToEmbedding(row.embedding)),\n\t\t}));\n\n\t\tscored.sort((a, b) => b.score - a.score);\n\t\treturn scored.slice(0, limit);\n\t}\n\n\tasync delete(sourceFile: string): Promise<number> {\n\t\tconst result = this.db.prepare(\"DELETE FROM chunks WHERE source = ?\").run(sourceFile);\n\t\treturn result.changes;\n\t}\n\n\tgetStats(): { totalChunks: number; sources: string[] } {\n\t\tconst count = this.db.prepare(\"SELECT COUNT(*) as count FROM chunks\").get() as { count: number };\n\t\tconst sources = this.db.prepare(\"SELECT DISTINCT source FROM chunks ORDER BY source\").all() as Array<{\n\t\t\tsource: string;\n\t\t}>;\n\t\treturn {\n\t\t\ttotalChunks: count.count,\n\t\t\tsources: sources.map((s) => s.source),\n\t\t};\n\t}\n\n\tclose(): void {\n\t\tthis.db.close();\n\t}\n}\n"]}
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@draht/knowledge",
|
|
3
|
+
"version": "2026.7.13",
|
|
4
|
+
"description": "Client knowledge base — per-client vector DB for knowledge persistence across projects",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"dist",
|
|
10
|
+
"README.md"
|
|
11
|
+
],
|
|
12
|
+
"exports": {
|
|
13
|
+
".": {
|
|
14
|
+
"types": "./dist/index.d.ts",
|
|
15
|
+
"import": "./dist/index.js"
|
|
16
|
+
},
|
|
17
|
+
"./extension": {
|
|
18
|
+
"types": "./src/extension.ts",
|
|
19
|
+
"import": "./src/extension.ts"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"scripts": {
|
|
23
|
+
"build": "tsc -p tsconfig.build.json",
|
|
24
|
+
"test": "vitest run",
|
|
25
|
+
"clean": "rm -rf dist"
|
|
26
|
+
},
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"better-sqlite3": "^11.7.0"
|
|
29
|
+
},
|
|
30
|
+
"devDependencies": {
|
|
31
|
+
"vitest": "^3.2.4",
|
|
32
|
+
"@draht/coding-agent": "2026.7.13",
|
|
33
|
+
"@types/better-sqlite3": "^7.6.12",
|
|
34
|
+
"@types/node": "^22.10.5",
|
|
35
|
+
"typescript": "7.0.2"
|
|
36
|
+
}
|
|
37
|
+
}
|