@jcode.labs/mimir 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -0
- package/dist/chunking.d.ts +3 -0
- package/dist/chunking.d.ts.map +1 -0
- package/dist/chunking.js +55 -0
- package/dist/chunking.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +120 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +89 -0
- package/dist/config.js.map +1 -0
- package/dist/embeddings.d.ts +4 -0
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +23 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/files.d.ts +4 -0
- package/dist/files.d.ts.map +1 -0
- package/dist/files.js +70 -0
- package/dist/files.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest.d.ts +4 -0
- package/dist/ingest.d.ts.map +1 -0
- package/dist/ingest.js +81 -0
- package/dist/ingest.js.map +1 -0
- package/dist/init.d.ts +2 -0
- package/dist/init.d.ts.map +1 -0
- package/dist/init.js +49 -0
- package/dist/init.js.map +1 -0
- package/dist/parsing.d.ts +3 -0
- package/dist/parsing.d.ts.map +1 -0
- package/dist/parsing.js +46 -0
- package/dist/parsing.js.map +1 -0
- package/dist/query.d.ts +4 -0
- package/dist/query.d.ts.map +1 -0
- package/dist/query.js +53 -0
- package/dist/query.js.map +1 -0
- package/dist/store.d.ts +6 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +29 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +79 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +58 -0
package/README.md
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# JCode Mimir
|
|
2
|
+
|
|
3
|
+
Local-first memory and retrieval for private project knowledge.
|
|
4
|
+
|
|
5
|
+
JCode Mimir provides a TypeScript CLI and library that can be installed in any Node.js
|
|
6
|
+
repository. It indexes files from the target repository, stores vectors locally with LanceDB,
|
|
7
|
+
and uses Ollama for local embeddings and answers.
|
|
8
|
+
|
|
9
|
+
## Status
|
|
10
|
+
|
|
11
|
+
Early public package. The repository is public, but no open-source license is granted yet.
|
|
12
|
+
|
|
13
|
+
## Requirements
|
|
14
|
+
|
|
15
|
+
- Node.js 20+
|
|
16
|
+
- pnpm, npm, yarn or bun
|
|
17
|
+
- Ollama running locally
|
|
18
|
+
- Embedding model installed once:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
ollama pull nomic-embed-text
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Optional answer model:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
ollama pull gemma4
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Install From npm
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pnpm add -D @jcode.labs/mimir
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Install From Git
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pnpm add -D git+ssh://git@github.com/jcode-works/jcode-mimir.git
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
For local development:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pnpm add -D file:../jcode-mimir
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Before creating an npm tarball later, run:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pnpm build
|
|
52
|
+
pnpm pack
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Use In Any Repository
|
|
56
|
+
|
|
57
|
+
Initialize the local project config:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pnpm kb init
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Add private documents under `private/`, then run:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pnpm kb ingest
|
|
67
|
+
pnpm kb search "vendor invoice status"
|
|
68
|
+
pnpm kb ask "What do the documents prove?"
|
|
69
|
+
pnpm kb audit
|
|
70
|
+
pnpm kb status
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Data Boundary
|
|
74
|
+
|
|
75
|
+
The package code lives in `node_modules` or in this repository. Project data stays in the
|
|
76
|
+
repository where you run the CLI:
|
|
77
|
+
|
|
78
|
+
```plain text
|
|
79
|
+
your-project/
|
|
80
|
+
private/ # raw documents to ingest
|
|
81
|
+
.kb/config.json # local config
|
|
82
|
+
.kb/sources.txt # optional extra source paths
|
|
83
|
+
.kb/storage/ # generated LanceDB index
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The package never ships project documents. `kb init` adds gitignore entries for `.kb/storage/`
|
|
87
|
+
and `private/**`.
|
|
88
|
+
|
|
89
|
+
## Supported Files
|
|
90
|
+
|
|
91
|
+
- Markdown: `.md`, `.mdx`
|
|
92
|
+
- Text: `.txt`, `.text`
|
|
93
|
+
- JSON: `.json`
|
|
94
|
+
- YAML: `.yaml`, `.yml`
|
|
95
|
+
- CSV/TSV: `.csv`, `.tsv`
|
|
96
|
+
- HTML: `.html`, `.htm`
|
|
97
|
+
- PDF: `.pdf`
|
|
98
|
+
|
|
99
|
+
## Config
|
|
100
|
+
|
|
101
|
+
`.kb/config.json`:
|
|
102
|
+
|
|
103
|
+
```json
|
|
104
|
+
{
|
|
105
|
+
"rawDir": "private",
|
|
106
|
+
"storageDir": ".kb/storage",
|
|
107
|
+
"sourcesFile": ".kb/sources.txt",
|
|
108
|
+
"tableName": "chunks",
|
|
109
|
+
"ollamaHost": "http://localhost:11434",
|
|
110
|
+
"embedModel": "nomic-embed-text",
|
|
111
|
+
"llmModel": "gemma4:latest",
|
|
112
|
+
"topK": 5,
|
|
113
|
+
"chunkSize": 1200,
|
|
114
|
+
"chunkOverlap": 150
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Environment overrides:
|
|
119
|
+
|
|
120
|
+
- `KB_RAW_DIR`
|
|
121
|
+
- `KB_STORAGE_DIR`
|
|
122
|
+
- `KB_SOURCES_FILE`
|
|
123
|
+
- `KB_OLLAMA_HOST`
|
|
124
|
+
- `KB_EMBED_MODEL`
|
|
125
|
+
- `KB_LLM_MODEL`
|
|
126
|
+
- `KB_TOP_K`
|
|
127
|
+
- `KB_CHUNK_SIZE`
|
|
128
|
+
- `KB_CHUNK_OVERLAP`
|
|
129
|
+
|
|
130
|
+
## Library API
|
|
131
|
+
|
|
132
|
+
```ts
|
|
133
|
+
import { ingest, search, ask } from "@jcode.labs/mimir"
|
|
134
|
+
|
|
135
|
+
await ingest({ rebuild: true })
|
|
136
|
+
const results = await search("vendor invoice status")
|
|
137
|
+
const answer = await ask("What documents support the project timeline?")
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Privacy
|
|
141
|
+
|
|
142
|
+
- Embeddings and answers use local Ollama by default.
|
|
143
|
+
- The vector index is stored locally.
|
|
144
|
+
- Raw private documents should stay in the target repository's ignored `private/` folder.
|
|
145
|
+
- Do not put secrets or scans inside this package repository.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5D,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAqCb"}
|
package/dist/chunking.js
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
3
|
+
if (!document.text) {
|
|
4
|
+
return [];
|
|
5
|
+
}
|
|
6
|
+
const chunks = [];
|
|
7
|
+
let cursor = 0;
|
|
8
|
+
let chunkIndex = 0;
|
|
9
|
+
while (cursor < document.text.length) {
|
|
10
|
+
const end = chooseChunkEnd(document.text, cursor, chunkSize);
|
|
11
|
+
const text = document.text.slice(cursor, end).trim();
|
|
12
|
+
if (text) {
|
|
13
|
+
const id = createHash("sha256")
|
|
14
|
+
.update(`${document.file.relativePath}:${chunkIndex}:${text}`)
|
|
15
|
+
.digest("hex");
|
|
16
|
+
chunks.push({
|
|
17
|
+
id,
|
|
18
|
+
source: document.file.source,
|
|
19
|
+
relativePath: document.file.relativePath,
|
|
20
|
+
chunkIndex,
|
|
21
|
+
text,
|
|
22
|
+
checksum: document.file.checksum,
|
|
23
|
+
bytes: document.file.bytes,
|
|
24
|
+
mtimeMs: document.file.mtimeMs,
|
|
25
|
+
});
|
|
26
|
+
chunkIndex += 1;
|
|
27
|
+
}
|
|
28
|
+
if (end >= document.text.length) {
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
cursor = Math.max(end - chunkOverlap, cursor + 1);
|
|
32
|
+
}
|
|
33
|
+
return chunks;
|
|
34
|
+
}
|
|
35
|
+
function chooseChunkEnd(text, cursor, chunkSize) {
|
|
36
|
+
const hardEnd = Math.min(cursor + chunkSize, text.length);
|
|
37
|
+
if (hardEnd === text.length) {
|
|
38
|
+
return hardEnd;
|
|
39
|
+
}
|
|
40
|
+
const window = text.slice(cursor, hardEnd);
|
|
41
|
+
const paragraphBreak = window.lastIndexOf("\n\n");
|
|
42
|
+
if (paragraphBreak > chunkSize * 0.45) {
|
|
43
|
+
return cursor + paragraphBreak;
|
|
44
|
+
}
|
|
45
|
+
const sentenceBreak = Math.max(window.lastIndexOf(". "), window.lastIndexOf("? "), window.lastIndexOf("! "));
|
|
46
|
+
if (sentenceBreak > chunkSize * 0.55) {
|
|
47
|
+
return cursor + sentenceBreak + 1;
|
|
48
|
+
}
|
|
49
|
+
const whitespace = window.lastIndexOf(" ");
|
|
50
|
+
if (whitespace > chunkSize * 0.75) {
|
|
51
|
+
return cursor + whitespace;
|
|
52
|
+
}
|
|
53
|
+
return hardEnd;
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=chunking.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,UAAU,aAAa,CAC3B,QAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAErD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC;iBAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;iBAC7D,MAAM,CAAC,KAAK,CAAC,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAC5B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY;gBACxC,UAAU;gBACV,IAAI;gBACJ,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ;gBAChC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;gBAC1B,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;aAC/B,CAAC,CAAC;YACH,UAAU,IAAI,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM;QACR,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,MAAc,EAAE,SAAiB;IACrE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAClD,IAAI,cAAc,GAAG,SAAS,GAAG,IAAI,EAAE,CAAC;QACtC,OAAO,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7G,IAAI,aAAa,GAAG,SAAS,GAAG,IAAI,EAAE,CAAC;QACrC,OAAO,MAAM,GAAG,aAAa,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,UAAU,GAAG,SAAS,GAAG,IAAI,EAAE,CAAC;QAClC,OAAO,MAAM,GAAG,UAAU,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from "commander";
|
|
3
|
+
import pc from "picocolors";
|
|
4
|
+
import { audit, ingest } from "./ingest.js";
|
|
5
|
+
import { initProject } from "./init.js";
|
|
6
|
+
import { ask, search } from "./query.js";
|
|
7
|
+
import { countRows } from "./store.js";
|
|
8
|
+
import { loadConfig } from "./config.js";
|
|
9
|
+
const program = new Command();
|
|
10
|
+
program
|
|
11
|
+
.name("kb")
|
|
12
|
+
.description("Local-first RAG knowledge base for private project documents.")
|
|
13
|
+
.version("0.1.0");
|
|
14
|
+
program
|
|
15
|
+
.command("init")
|
|
16
|
+
.description("Create .kb config files and private/ document folder in the current repository.")
|
|
17
|
+
.action(async () => {
|
|
18
|
+
const created = await initProject(process.cwd());
|
|
19
|
+
if (created.length === 0) {
|
|
20
|
+
console.log(pc.green("Already initialized."));
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
console.log(pc.green("Created:"));
|
|
24
|
+
for (const file of created) {
|
|
25
|
+
console.log(` - ${file}`);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
program
|
|
29
|
+
.command("ingest")
|
|
30
|
+
.description("Parse documents, create chunks, embed them locally, and rebuild the LanceDB index.")
|
|
31
|
+
.option("--rebuild", "Accepted for compatibility; ingest always rebuilds the local index.")
|
|
32
|
+
.action(async () => {
|
|
33
|
+
const result = await ingest({ cwd: process.cwd(), rebuild: true });
|
|
34
|
+
console.log(pc.green(`Done. indexedFiles=${result.indexedFiles} chunks=${result.chunks} skippedFiles=${result.skippedFiles} errors=${result.errors.length}`));
|
|
35
|
+
for (const error of result.errors) {
|
|
36
|
+
console.error(pc.red(` - ${error.path}: ${error.message}`));
|
|
37
|
+
}
|
|
38
|
+
if (result.errors.length > 0) {
|
|
39
|
+
process.exitCode = 1;
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
program
|
|
43
|
+
.command("search")
|
|
44
|
+
.description("Retrieve the most relevant passages without calling an LLM.")
|
|
45
|
+
.argument("<query>", "Search query.")
|
|
46
|
+
.option("-k, --top-k <number>", "Number of passages to return.", parsePositiveInt)
|
|
47
|
+
.action(async (query, options) => {
|
|
48
|
+
const results = await search(query, withTopK(options.topK));
|
|
49
|
+
if (results.length === 0) {
|
|
50
|
+
console.error(pc.yellow("No results. Run `kb ingest` first, or add documents."));
|
|
51
|
+
process.exitCode = 1;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
for (const [index, result] of results.entries()) {
|
|
55
|
+
const distance = result.distance === null ? "n/a" : result.distance.toFixed(4);
|
|
56
|
+
console.log(`\n${pc.cyan(`[${index + 1}] ${result.relativePath}`)} chunk=${result.chunkIndex} distance=${distance}`);
|
|
57
|
+
console.log(result.text.slice(0, 900));
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
program
|
|
61
|
+
.command("ask")
|
|
62
|
+
.description("Answer a question using retrieved passages and a local Ollama model.")
|
|
63
|
+
.argument("<query>", "Question to answer.")
|
|
64
|
+
.option("-k, --top-k <number>", "Number of passages to use.", parsePositiveInt)
|
|
65
|
+
.action(async (query, options) => {
|
|
66
|
+
const result = await ask(query, withTopK(options.topK));
|
|
67
|
+
console.log(`\n${result.answer}\n`);
|
|
68
|
+
if (result.sources.length > 0) {
|
|
69
|
+
console.log(pc.dim("Sources:"));
|
|
70
|
+
for (const [index, source] of result.sources.entries()) {
|
|
71
|
+
console.log(` [${index + 1}] ${source.relativePath} chunk=${source.chunkIndex}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
program
|
|
76
|
+
.command("audit")
|
|
77
|
+
.description("Compare supported files on disk with the current vector index.")
|
|
78
|
+
.action(async () => {
|
|
79
|
+
const report = await audit(process.cwd());
|
|
80
|
+
console.log(`supportedFiles=${report.supportedFiles.length}`);
|
|
81
|
+
console.log(`indexedFiles=${report.indexedFiles.length}`);
|
|
82
|
+
console.log(`totalChunks=${report.totalChunks}`);
|
|
83
|
+
console.log(`missingFromIndex=${report.missingFromIndex.length}`);
|
|
84
|
+
console.log(`staleInIndex=${report.staleInIndex.length}`);
|
|
85
|
+
for (const file of report.missingFromIndex) {
|
|
86
|
+
console.log(pc.yellow(`missing: ${file}`));
|
|
87
|
+
}
|
|
88
|
+
for (const file of report.staleInIndex) {
|
|
89
|
+
console.log(pc.red(`stale: ${file}`));
|
|
90
|
+
}
|
|
91
|
+
if (report.missingFromIndex.length > 0 || report.staleInIndex.length > 0) {
|
|
92
|
+
process.exitCode = 1;
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
program
|
|
96
|
+
.command("status")
|
|
97
|
+
.description("Show active configuration and index row count.")
|
|
98
|
+
.action(async () => {
|
|
99
|
+
const config = await loadConfig(process.cwd());
|
|
100
|
+
const rows = await countRows(config);
|
|
101
|
+
console.log(`projectRoot=${config.projectRoot}`);
|
|
102
|
+
console.log(`rawDir=${config.rawDir}`);
|
|
103
|
+
console.log(`storageDir=${config.storageDir}`);
|
|
104
|
+
console.log(`sourcesFile=${config.sourcesFile}`);
|
|
105
|
+
console.log(`embedModel=${config.embedModel}`);
|
|
106
|
+
console.log(`llmModel=${config.llmModel}`);
|
|
107
|
+
console.log(`chunksIndexed=${rows}`);
|
|
108
|
+
});
|
|
109
|
+
await program.parseAsync(process.argv);
|
|
110
|
+
function parsePositiveInt(value) {
|
|
111
|
+
const parsed = Number.parseInt(value, 10);
|
|
112
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
113
|
+
throw new Error("Expected a positive integer.");
|
|
114
|
+
}
|
|
115
|
+
return parsed;
|
|
116
|
+
}
|
|
117
|
+
function withTopK(topK) {
|
|
118
|
+
return topK === undefined ? { cwd: process.cwd() } : { cwd: process.cwd(), topK };
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,IAAI,CAAC;KACV,WAAW,CAAC,+DAA+D,CAAC;KAC5E,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,iFAAiF,CAAC;KAC9F,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,oFAAoF,CAAC;KACjG,MAAM,CAAC,WAAW,EAAE,qEAAqE,CAAC;KAC1F,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CACT,EAAE,CAAC,KAAK,CACN,sBAAsB,MAAM,CAAC,YAAY,WAAW,MAAM,CAAC,MAAM,iBAAiB,MAAM,CAAC,YAAY,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CACvI,CACF,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6DAA6D,CAAC;KAC1E,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;KACpC,MAAM,CAAC,sBAAsB,EAAE,+BAA+B,EAAE,gBAAgB,CAAC;KACjF,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAE;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,sDAAsD,CAAC,CAAC,CAAC;QACjF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,EAAE,CAAC,UAAU,MAAM,CAAC,UAAU,aAAa,QAAQ,EAAE,CAAC,CAAC;QACrH,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,sEAAsE,CAAC;KACnF,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAC;KAC1C,MAAM,CAAC,sBAAsB,EAAE,4BAA4B,EAAE,gBAAgB,CAAC;KAC9E,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;QAChC,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,gEAAgE,CAAC;KAC7E,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IAE1D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AAEL,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAEvC,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAwB;IACxC,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC;AACpF,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAmBzC,wBAAgB,eAAe,CAAC,KAAK,SAAgB,GAAG,MAAM,CAc7D;AAED,wBAAsB,UAAU,CAAC,KAAK,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CA2BvE"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
const rawConfigSchema = z.object({
|
|
6
|
+
rawDir: z.string().default("private"),
|
|
7
|
+
storageDir: z.string().default(".kb/storage"),
|
|
8
|
+
sourcesFile: z.string().default(".kb/sources.txt"),
|
|
9
|
+
tableName: z.string().default("chunks"),
|
|
10
|
+
ollamaHost: z.string().default("http://localhost:11434"),
|
|
11
|
+
embedModel: z.string().default("nomic-embed-text"),
|
|
12
|
+
llmModel: z.string().default("gemma4:latest"),
|
|
13
|
+
topK: z.number().int().positive().default(5),
|
|
14
|
+
chunkSize: z.number().int().positive().default(1200),
|
|
15
|
+
chunkOverlap: z.number().int().nonnegative().default(150),
|
|
16
|
+
});
|
|
17
|
+
const CONFIG_PATH = ".kb/config.json";
|
|
18
|
+
export function findProjectRoot(start = process.cwd()) {
|
|
19
|
+
let current = path.resolve(start);
|
|
20
|
+
while (true) {
|
|
21
|
+
if (existsSync(path.join(current, CONFIG_PATH))) {
|
|
22
|
+
return current;
|
|
23
|
+
}
|
|
24
|
+
const parent = path.dirname(current);
|
|
25
|
+
if (parent === current) {
|
|
26
|
+
return path.resolve(start);
|
|
27
|
+
}
|
|
28
|
+
current = parent;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export async function loadConfig(start = process.cwd()) {
|
|
32
|
+
const projectRoot = findProjectRoot(start);
|
|
33
|
+
const configFile = path.join(projectRoot, CONFIG_PATH);
|
|
34
|
+
const raw = existsSync(configFile)
|
|
35
|
+
? JSON.parse(await readFile(configFile, "utf8"))
|
|
36
|
+
: {};
|
|
37
|
+
const parsed = rawConfigSchema.parse(raw);
|
|
38
|
+
const withEnv = applyEnv(parsed);
|
|
39
|
+
if (withEnv.chunkOverlap >= withEnv.chunkSize) {
|
|
40
|
+
throw new Error("chunkOverlap must be lower than chunkSize.");
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
projectRoot,
|
|
44
|
+
rawDir: resolveFromRoot(projectRoot, withEnv.rawDir),
|
|
45
|
+
storageDir: resolveFromRoot(projectRoot, withEnv.storageDir),
|
|
46
|
+
sourcesFile: resolveFromRoot(projectRoot, withEnv.sourcesFile),
|
|
47
|
+
tableName: withEnv.tableName,
|
|
48
|
+
ollamaHost: withEnv.ollamaHost,
|
|
49
|
+
embedModel: withEnv.embedModel,
|
|
50
|
+
llmModel: withEnv.llmModel,
|
|
51
|
+
topK: withEnv.topK,
|
|
52
|
+
chunkSize: withEnv.chunkSize,
|
|
53
|
+
chunkOverlap: withEnv.chunkOverlap,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function resolveFromRoot(projectRoot, input) {
|
|
57
|
+
return path.isAbsolute(input) ? input : path.resolve(projectRoot, input);
|
|
58
|
+
}
|
|
59
|
+
function applyEnv(config) {
|
|
60
|
+
return {
|
|
61
|
+
...config,
|
|
62
|
+
rawDir: process.env.KB_RAW_DIR ?? config.rawDir,
|
|
63
|
+
storageDir: process.env.KB_STORAGE_DIR ?? config.storageDir,
|
|
64
|
+
sourcesFile: process.env.KB_SOURCES_FILE ?? config.sourcesFile,
|
|
65
|
+
ollamaHost: process.env.KB_OLLAMA_HOST ?? config.ollamaHost,
|
|
66
|
+
embedModel: process.env.KB_EMBED_MODEL ?? config.embedModel,
|
|
67
|
+
llmModel: process.env.KB_LLM_MODEL ?? config.llmModel,
|
|
68
|
+
topK: readPositiveIntEnv("KB_TOP_K", config.topK),
|
|
69
|
+
chunkSize: readPositiveIntEnv("KB_CHUNK_SIZE", config.chunkSize),
|
|
70
|
+
chunkOverlap: readNonNegativeIntEnv("KB_CHUNK_OVERLAP", config.chunkOverlap),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
function readPositiveIntEnv(name, fallback) {
|
|
74
|
+
const raw = process.env[name];
|
|
75
|
+
if (!raw) {
|
|
76
|
+
return fallback;
|
|
77
|
+
}
|
|
78
|
+
const value = Number.parseInt(raw, 10);
|
|
79
|
+
return Number.isInteger(value) && value > 0 ? value : fallback;
|
|
80
|
+
}
|
|
81
|
+
function readNonNegativeIntEnv(name, fallback) {
|
|
82
|
+
const raw = process.env[name];
|
|
83
|
+
if (!raw) {
|
|
84
|
+
return fallback;
|
|
85
|
+
}
|
|
86
|
+
const value = Number.parseInt(raw, 10);
|
|
87
|
+
return Number.isInteger(value) && value >= 0 ? value : fallback;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC;IAC7C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAClD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;IACvC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC;IACxD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACpD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;CAC1D,CAAC,CAAC;AAIH,MAAM,WAAW,GAAG,iBAAiB,CAAC;AAEtC,MAAM,UAAU,eAAe,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAElC,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,GAAG,MAAM,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE;IACpD,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC;QAChC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAY;QAC3D,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEjC,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,OAAO;QACL,WAAW;QACX,MAAM,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC;QACpD,UAAU,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC;QAC5D,WAAW,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;KACnC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,WAAmB,EAAE,KAAa;IACzD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,QAAQ,CAAC,MAAiB;IACjC,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM;QAC/C,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC,WAAW;QAC9D,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC,QAAQ;QACrD,IAAI,EAAE,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC;QACjD,SAAS,EAAE,kBAAkB,CAAC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC;QAChE,YAAY,EAAE,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC;KAC7E,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,QAAgB;IACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AACjE,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,QAAgB;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAgBrF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAM/E"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Ollama } from "ollama";
|
|
2
|
+
export async function embedTexts(texts, config) {
|
|
3
|
+
if (texts.length === 0) {
|
|
4
|
+
return [];
|
|
5
|
+
}
|
|
6
|
+
const client = new Ollama({ host: config.ollamaHost });
|
|
7
|
+
const response = await client.embed({
|
|
8
|
+
model: config.embedModel,
|
|
9
|
+
input: texts,
|
|
10
|
+
});
|
|
11
|
+
if (!response.embeddings || response.embeddings.length !== texts.length) {
|
|
12
|
+
throw new Error(`Expected ${texts.length} embeddings, received ${response.embeddings?.length ?? 0}.`);
|
|
13
|
+
}
|
|
14
|
+
return response.embeddings;
|
|
15
|
+
}
|
|
16
|
+
export async function embedText(text, config) {
|
|
17
|
+
const [embedding] = await embedTexts([text], config);
|
|
18
|
+
if (!embedding) {
|
|
19
|
+
throw new Error("No embedding returned for query.");
|
|
20
|
+
}
|
|
21
|
+
return embedding;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=embeddings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAe,EAAE,MAAc;IAC9D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;QAClC,KAAK,EAAE,MAAM,CAAC,UAAU;QACxB,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,MAAM,yBAAyB,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,QAAQ,CAAC,UAAU,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc;IAC1D,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACrD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
package/dist/files.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAErD,eAAO,MAAM,oBAAoB,aAa/B,CAAC;AAEH,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuC3E"}
|
package/dist/files.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { readFile, stat } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import fg from "fast-glob";
|
|
6
|
+
export const SUPPORTED_EXTENSIONS = new Set([
|
|
7
|
+
".csv",
|
|
8
|
+
".htm",
|
|
9
|
+
".html",
|
|
10
|
+
".json",
|
|
11
|
+
".md",
|
|
12
|
+
".mdx",
|
|
13
|
+
".pdf",
|
|
14
|
+
".text",
|
|
15
|
+
".tsv",
|
|
16
|
+
".txt",
|
|
17
|
+
".yaml",
|
|
18
|
+
".yml",
|
|
19
|
+
]);
|
|
20
|
+
export async function listSourceFiles(config) {
|
|
21
|
+
const roots = await sourceRoots(config);
|
|
22
|
+
const files = new Map();
|
|
23
|
+
for (const root of roots) {
|
|
24
|
+
if (!existsSync(root)) {
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
const entries = await fg("**/*", {
|
|
28
|
+
cwd: root,
|
|
29
|
+
absolute: true,
|
|
30
|
+
onlyFiles: true,
|
|
31
|
+
dot: false,
|
|
32
|
+
followSymbolicLinks: false,
|
|
33
|
+
ignore: ["**/.git/**", "**/node_modules/**", "**/.kb/storage/**"],
|
|
34
|
+
});
|
|
35
|
+
for (const absolutePath of entries) {
|
|
36
|
+
const extension = path.extname(absolutePath).toLowerCase();
|
|
37
|
+
if (!SUPPORTED_EXTENSIONS.has(extension)) {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
const info = await stat(absolutePath);
|
|
41
|
+
const buffer = await readFile(absolutePath);
|
|
42
|
+
files.set(absolutePath, {
|
|
43
|
+
absolutePath,
|
|
44
|
+
relativePath: path.relative(config.projectRoot, absolutePath),
|
|
45
|
+
source: path.relative(root, absolutePath) || path.basename(absolutePath),
|
|
46
|
+
extension,
|
|
47
|
+
bytes: info.size,
|
|
48
|
+
mtimeMs: info.mtimeMs,
|
|
49
|
+
checksum: createHash("sha256").update(buffer).digest("hex"),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return [...files.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath));
|
|
54
|
+
}
|
|
55
|
+
async function sourceRoots(config) {
|
|
56
|
+
const roots = [config.rawDir];
|
|
57
|
+
if (!existsSync(config.sourcesFile)) {
|
|
58
|
+
return roots;
|
|
59
|
+
}
|
|
60
|
+
const content = await readFile(config.sourcesFile, "utf8");
|
|
61
|
+
for (const line of content.split(/\r?\n/u)) {
|
|
62
|
+
const trimmed = line.trim();
|
|
63
|
+
if (!trimmed || trimmed.startsWith("#")) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
roots.push(path.isAbsolute(trimmed) ? trimmed : path.resolve(config.projectRoot, trimmed));
|
|
67
|
+
}
|
|
68
|
+
return roots;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.js","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,WAAW,CAAC;AAG3B,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IAC1C,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;CACP,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc;IAClD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE;YAC/B,GAAG,EAAE,IAAI;YACT,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,KAAK;YAC1B,MAAM,EAAE,CAAC,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,CAAC;SAClE,CAAC,CAAC;QAEH,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;YAC3D,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;YACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC5C,KAAK,CAAC,GAAG,CAAC,YAAY,EAAE;gBACtB,YAAY;gBACZ,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC;gBAC7D,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;gBACxE,SAAS;gBACT,KAAK,EAAE,IAAI,CAAC,IAAI;gBAChB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC5D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;AAC1F,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAc;IACvC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC3D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { ask, search } from "./query.js";
|
|
2
|
+
export { audit, ingest } from "./ingest.js";
|
|
3
|
+
export { initProject } from "./init.js";
|
|
4
|
+
export { loadConfig } from "./config.js";
|
|
5
|
+
export type { AskResult, AuditReport, Config, IngestResult, SearchResult, } from "./types.js";
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,YAAY,EACV,SAAS,EACT,WAAW,EACX,MAAM,EACN,YAAY,EACZ,YAAY,GACb,MAAM,YAAY,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/ingest.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAwB,MAAM,YAAY,CAAC;AAIjG,wBAAsB,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CA4C/E;AAED,wBAAsB,KAAK,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAkCrE"}
|
package/dist/ingest.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { loadConfig } from "./config.js";
|
|
2
|
+
import { listSourceFiles } from "./files.js";
|
|
3
|
+
import { chunkDocument } from "./chunking.js";
|
|
4
|
+
import { embedTexts } from "./embeddings.js";
|
|
5
|
+
import { parseFile } from "./parsing.js";
|
|
6
|
+
import { openRowsTable, writeRows } from "./store.js";
|
|
7
|
+
const EMBED_BATCH_SIZE = 32;
|
|
8
|
+
export async function ingest(options = {}) {
|
|
9
|
+
const config = await loadConfig(String(options.cwd ?? process.cwd()));
|
|
10
|
+
const files = await listSourceFiles(config);
|
|
11
|
+
const allChunks = [];
|
|
12
|
+
const errors = [];
|
|
13
|
+
let skippedFiles = 0;
|
|
14
|
+
for (const file of files) {
|
|
15
|
+
try {
|
|
16
|
+
const parsed = await parseFile(file);
|
|
17
|
+
const chunks = chunkDocument(parsed, config.chunkSize, config.chunkOverlap);
|
|
18
|
+
if (chunks.length === 0) {
|
|
19
|
+
skippedFiles += 1;
|
|
20
|
+
}
|
|
21
|
+
allChunks.push(...chunks);
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
errors.push({
|
|
25
|
+
path: file.relativePath,
|
|
26
|
+
message: error instanceof Error ? error.message : String(error),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const rows = [];
|
|
31
|
+
for (let i = 0; i < allChunks.length; i += EMBED_BATCH_SIZE) {
|
|
32
|
+
const batch = allChunks.slice(i, i + EMBED_BATCH_SIZE);
|
|
33
|
+
const embeddings = await embedTexts(batch.map((chunk) => chunk.text), config);
|
|
34
|
+
for (const [index, chunk] of batch.entries()) {
|
|
35
|
+
const vector = embeddings[index];
|
|
36
|
+
if (!vector) {
|
|
37
|
+
throw new Error(`Missing embedding for chunk ${chunk.relativePath}#${chunk.chunkIndex}.`);
|
|
38
|
+
}
|
|
39
|
+
rows.push({ ...chunk, vector });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
await writeRows(rows, config);
|
|
43
|
+
return {
|
|
44
|
+
indexedFiles: new Set(rows.map((row) => row.relativePath)).size,
|
|
45
|
+
chunks: rows.length,
|
|
46
|
+
skippedFiles,
|
|
47
|
+
errors,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export async function audit(cwd = process.cwd()) {
|
|
51
|
+
const config = await loadConfig(cwd);
|
|
52
|
+
const files = await listSourceFiles(config);
|
|
53
|
+
const supportedFiles = files.map((file) => file.relativePath);
|
|
54
|
+
const table = await openRowsTable(config);
|
|
55
|
+
if (!table) {
|
|
56
|
+
return {
|
|
57
|
+
indexedFiles: [],
|
|
58
|
+
supportedFiles,
|
|
59
|
+
missingFromIndex: supportedFiles,
|
|
60
|
+
staleInIndex: [],
|
|
61
|
+
totalChunks: 0,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const rows = await table.query().limit(100000).toArray();
|
|
65
|
+
const counts = new Map();
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
counts.set(row.relativePath, (counts.get(row.relativePath) ?? 0) + 1);
|
|
68
|
+
}
|
|
69
|
+
const supportedSet = new Set(supportedFiles);
|
|
70
|
+
const indexedSet = new Set(counts.keys());
|
|
71
|
+
return {
|
|
72
|
+
indexedFiles: [...counts.entries()]
|
|
73
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
74
|
+
.map(([source, chunks]) => ({ source, chunks })),
|
|
75
|
+
supportedFiles,
|
|
76
|
+
missingFromIndex: supportedFiles.filter((file) => !indexedSet.has(file)),
|
|
77
|
+
staleInIndex: [...indexedSet].filter((file) => !supportedSet.has(file)).sort(),
|
|
78
|
+
totalChunks: rows.length,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=ingest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGtD,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAgB,EAAE,CAAC;IAClC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;YAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,YAAY,IAAI,CAAC,CAAC;YACpB,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAChE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QAC9E,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;YAC5F,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE9B,OAAO;QACL,YAAY,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;QAC/D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,YAAY;QACZ,MAAM;KACP,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAE1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,YAAY,EAAE,EAAE;YAChB,cAAc;YACd,gBAAgB,EAAE,cAAc;YAChC,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,CAAC;SACf,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,EAAqC,CAAC;IAC5F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAE1C,OAAO;QACL,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,cAAc;QACd,gBAAgB,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxE,YAAY,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9E,WAAW,EAAE,IAAI,CAAC,MAAM;KACzB,CAAC;AACJ,CAAC"}
|
package/dist/init.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAmBA,wBAAsB,WAAW,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA6CxE"}
|
package/dist/init.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
const DEFAULT_CONFIG = {
|
|
5
|
+
rawDir: "private",
|
|
6
|
+
storageDir: ".kb/storage",
|
|
7
|
+
sourcesFile: ".kb/sources.txt",
|
|
8
|
+
tableName: "chunks",
|
|
9
|
+
ollamaHost: "http://localhost:11434",
|
|
10
|
+
embedModel: "nomic-embed-text",
|
|
11
|
+
llmModel: "gemma4:latest",
|
|
12
|
+
topK: 5,
|
|
13
|
+
chunkSize: 1200,
|
|
14
|
+
chunkOverlap: 150,
|
|
15
|
+
};
|
|
16
|
+
const GITIGNORE_BLOCK = `\n# JCode Mimir\n.kb/storage/\n.kb/cache/\n.kb/*.local.json\nprivate/**\n!private/\n!private/README.md\n!private/**/\n!private/**/.gitkeep\n`;
|
|
17
|
+
export async function initProject(cwd = process.cwd()) {
|
|
18
|
+
const root = path.resolve(cwd);
|
|
19
|
+
const kbDir = path.join(root, ".kb");
|
|
20
|
+
const privateDir = path.join(root, "private");
|
|
21
|
+
const created = [];
|
|
22
|
+
await mkdir(kbDir, { recursive: true });
|
|
23
|
+
await mkdir(privateDir, { recursive: true });
|
|
24
|
+
const configPath = path.join(kbDir, "config.json");
|
|
25
|
+
if (!existsSync(configPath)) {
|
|
26
|
+
await writeFile(configPath, `${JSON.stringify(DEFAULT_CONFIG, null, 2)}\n`, "utf8");
|
|
27
|
+
created.push(path.relative(root, configPath));
|
|
28
|
+
}
|
|
29
|
+
const sourcesPath = path.join(kbDir, "sources.txt");
|
|
30
|
+
if (!existsSync(sourcesPath)) {
|
|
31
|
+
await writeFile(sourcesPath, "# Optional extra source paths, one per line. Relative paths resolve from the project root.\n", "utf8");
|
|
32
|
+
created.push(path.relative(root, sourcesPath));
|
|
33
|
+
}
|
|
34
|
+
const readmePath = path.join(privateDir, "README.md");
|
|
35
|
+
if (!existsSync(readmePath)) {
|
|
36
|
+
await writeFile(readmePath, "# Private documents\n\nPut raw documents to ingest here. Keep this folder ignored by Git.\n", "utf8");
|
|
37
|
+
created.push(path.relative(root, readmePath));
|
|
38
|
+
}
|
|
39
|
+
const gitignorePath = path.join(root, ".gitignore");
|
|
40
|
+
const currentGitignore = existsSync(gitignorePath)
|
|
41
|
+
? await readFile(gitignorePath, "utf8")
|
|
42
|
+
: "";
|
|
43
|
+
if (!currentGitignore.includes("# JCode Mimir")) {
|
|
44
|
+
await writeFile(gitignorePath, `${currentGitignore.trimEnd()}${GITIGNORE_BLOCK}`, "utf8");
|
|
45
|
+
created.push(path.relative(root, gitignorePath));
|
|
46
|
+
}
|
|
47
|
+
return created;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=init.js.map
|
package/dist/init.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,cAAc,GAAG;IACrB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,aAAa;IACzB,WAAW,EAAE,iBAAiB;IAC9B,SAAS,EAAE,QAAQ;IACnB,UAAU,EAAE,wBAAwB;IACpC,UAAU,EAAE,kBAAkB;IAC9B,QAAQ,EAAE,eAAe;IACzB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,GAAG;CAClB,CAAC;AAEF,MAAM,eAAe,GAAG,8IAA8I,CAAC;AAEvK,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IACnD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IACpD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,SAAS,CACb,WAAW,EACX,8FAA8F,EAC9F,MAAM,CACP,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CACb,UAAU,EACV,6FAA6F,EAC7F,MAAM,CACP,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACpD,MAAM,gBAAgB,GAAG,UAAU,CAAC,aAAa,CAAC;QAChD,CAAC,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;QACvC,CAAC,CAAC,EAAE,CAAC;IACP,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAChD,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,gBAAgB,CAAC,OAAO,EAAE,GAAG,eAAe,EAAE,EAAE,MAAM,CAAC,CAAC;QAC1F,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsing.d.ts","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7D,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,CA6BzE"}
|
package/dist/parsing.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { htmlToText } from "html-to-text";
|
|
3
|
+
import { extractText, getDocumentProxy } from "unpdf";
|
|
4
|
+
import YAML from "yaml";
|
|
5
|
+
export async function parseFile(file) {
|
|
6
|
+
let text;
|
|
7
|
+
switch (file.extension) {
|
|
8
|
+
case ".pdf":
|
|
9
|
+
text = await parsePdf(file.absolutePath);
|
|
10
|
+
break;
|
|
11
|
+
case ".html":
|
|
12
|
+
case ".htm":
|
|
13
|
+
text = htmlToText(await readFile(file.absolutePath, "utf8"), {
|
|
14
|
+
wordwrap: false,
|
|
15
|
+
selectors: [
|
|
16
|
+
{ selector: "a", options: { ignoreHref: true } },
|
|
17
|
+
{ selector: "img", format: "skip" },
|
|
18
|
+
],
|
|
19
|
+
});
|
|
20
|
+
break;
|
|
21
|
+
case ".json":
|
|
22
|
+
text = JSON.stringify(JSON.parse(await readFile(file.absolutePath, "utf8")), null, 2);
|
|
23
|
+
break;
|
|
24
|
+
case ".yaml":
|
|
25
|
+
case ".yml":
|
|
26
|
+
text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8")));
|
|
27
|
+
break;
|
|
28
|
+
default:
|
|
29
|
+
text = await readFile(file.absolutePath, "utf8");
|
|
30
|
+
}
|
|
31
|
+
return { file, text: normalizeText(text) };
|
|
32
|
+
}
|
|
33
|
+
async function parsePdf(filePath) {
|
|
34
|
+
const buffer = await readFile(filePath);
|
|
35
|
+
const pdf = await getDocumentProxy(new Uint8Array(buffer));
|
|
36
|
+
const result = await extractText(pdf, { mergePages: true });
|
|
37
|
+
return result.text;
|
|
38
|
+
}
|
|
39
|
+
function normalizeText(input) {
|
|
40
|
+
return input
|
|
41
|
+
.replace(/\r\n/g, "\n")
|
|
42
|
+
.replace(/[ \t]+\n/g, "\n")
|
|
43
|
+
.replace(/\n{4,}/g, "\n\n\n")
|
|
44
|
+
.trim();
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=parsing.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsing.js","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAC;AACtD,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,IAAI,IAAY,CAAC;IAEjB,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACzC,MAAM;QACR,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE;gBAC3D,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE;oBACT,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;oBAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACpC;aACF,CAAC,CAAC;YACH,MAAM;QACR,KAAK,OAAO;YACV,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACtF,MAAM;QACR,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7E,MAAM;QACR;YACE,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;AAC7C,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;SAC5B,IAAI,EAAE,CAAC;AACZ,CAAC"}
|
package/dist/query.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { AskResult, SearchOptions, SearchResult } from "./types.js";
|
|
2
|
+
export declare function search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
|
|
3
|
+
export declare function ask(query: string, options?: SearchOptions): Promise<AskResult>;
|
|
4
|
+
//# sourceMappingURL=query.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAUzE,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAiBhG;AAED,wBAAsB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC,CAoCxF"}
|
package/dist/query.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { Ollama } from "ollama";
|
|
2
|
+
import { loadConfig } from "./config.js";
|
|
3
|
+
import { embedText } from "./embeddings.js";
|
|
4
|
+
import { openRowsTable } from "./store.js";
|
|
5
|
+
export async function search(query, options = {}) {
|
|
6
|
+
const config = await loadConfig(String(options.cwd ?? process.cwd()));
|
|
7
|
+
const table = await openRowsTable(config);
|
|
8
|
+
if (!table) {
|
|
9
|
+
return [];
|
|
10
|
+
}
|
|
11
|
+
const vector = await embedText(query, config);
|
|
12
|
+
const rows = await table.vectorSearch(vector).limit(options.topK ?? config.topK).toArray();
|
|
13
|
+
return rows.map((row) => ({
|
|
14
|
+
source: row.source,
|
|
15
|
+
relativePath: row.relativePath,
|
|
16
|
+
chunkIndex: row.chunkIndex,
|
|
17
|
+
text: row.text,
|
|
18
|
+
distance: typeof row._distance === "number" ? row._distance : null,
|
|
19
|
+
}));
|
|
20
|
+
}
|
|
21
|
+
export async function ask(query, options = {}) {
|
|
22
|
+
const config = await loadConfig(String(options.cwd ?? process.cwd()));
|
|
23
|
+
const sources = await search(query, options);
|
|
24
|
+
if (sources.length === 0) {
|
|
25
|
+
return {
|
|
26
|
+
answer: "No relevant passages were found. Add documents and run `kb ingest` first.",
|
|
27
|
+
sources,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const context = sources
|
|
31
|
+
.map((source, index) => `[${index + 1}] ${source.relativePath}#${source.chunkIndex}\n${source.text}`)
|
|
32
|
+
.join("\n\n---\n\n");
|
|
33
|
+
const client = new Ollama({ host: config.ollamaHost });
|
|
34
|
+
const response = await client.chat({
|
|
35
|
+
model: config.llmModel,
|
|
36
|
+
messages: [
|
|
37
|
+
{
|
|
38
|
+
role: "system",
|
|
39
|
+
content: "Answer only from the provided context. If the context is insufficient, say what is missing. Cite sources with [1], [2], etc.",
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
role: "user",
|
|
43
|
+
content: `Question:\n${query}\n\nContext:\n${context}`,
|
|
44
|
+
},
|
|
45
|
+
],
|
|
46
|
+
stream: false,
|
|
47
|
+
});
|
|
48
|
+
return {
|
|
49
|
+
answer: response.message.content,
|
|
50
|
+
sources,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=query.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query.js","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAW3C,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;IACrE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAiB,CAAC;IAE1G,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE;IAClE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE7C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,2EAA2E;YACnF,OAAO;SACR,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,OAAO;SACpB,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;SACpG,IAAI,CAAC,aAAa,CAAC,CAAC;IAEvB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,QAAQ;QACtB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EACL,8HAA8H;aACjI;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,cAAc,KAAK,iBAAiB,OAAO,EAAE;aACvD;SACF;QACD,MAAM,EAAE,KAAK;KACd,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO;QAChC,OAAO;KACR,CAAC;AACJ,CAAC"}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import * as lancedb from "@lancedb/lancedb";
|
|
2
|
+
import type { Config, VectorRow } from "./types.js";
|
|
3
|
+
export declare function writeRows(rows: VectorRow[], config: Config): Promise<void>;
|
|
4
|
+
export declare function openRowsTable(config: Config): Promise<lancedb.Table | null>;
|
|
5
|
+
export declare function countRows(config: Config): Promise<number>;
|
|
6
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEpD,wBAAsB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAehF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAOjF;AAED,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/D"}
|
package/dist/store.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import * as lancedb from "@lancedb/lancedb";
|
|
3
|
+
export async function writeRows(rows, config) {
|
|
4
|
+
await mkdir(config.storageDir, { recursive: true });
|
|
5
|
+
const db = await lancedb.connect(config.storageDir);
|
|
6
|
+
if (rows.length === 0) {
|
|
7
|
+
const tableNames = await db.tableNames();
|
|
8
|
+
if (tableNames.includes(config.tableName)) {
|
|
9
|
+
await db.dropTable(config.tableName);
|
|
10
|
+
}
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
await db.createTable(config.tableName, rows, {
|
|
14
|
+
mode: "overwrite",
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
export async function openRowsTable(config) {
|
|
18
|
+
const db = await lancedb.connect(config.storageDir);
|
|
19
|
+
const tableNames = await db.tableNames();
|
|
20
|
+
if (!tableNames.includes(config.tableName)) {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
return db.openTable(config.tableName);
|
|
24
|
+
}
|
|
25
|
+
export async function countRows(config) {
|
|
26
|
+
const table = await openRowsTable(config);
|
|
27
|
+
return table ? table.countRows() : 0;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAG5C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAiB,EAAE,MAAc;IAC/D,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,IAA4C,EAAE;QACnF,IAAI,EAAE,WAAW;KAClB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;IACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACvC,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { PathLike } from "node:fs";
|
|
2
|
+
export interface Config {
|
|
3
|
+
projectRoot: string;
|
|
4
|
+
rawDir: string;
|
|
5
|
+
storageDir: string;
|
|
6
|
+
sourcesFile: string;
|
|
7
|
+
tableName: string;
|
|
8
|
+
ollamaHost: string;
|
|
9
|
+
embedModel: string;
|
|
10
|
+
llmModel: string;
|
|
11
|
+
topK: number;
|
|
12
|
+
chunkSize: number;
|
|
13
|
+
chunkOverlap: number;
|
|
14
|
+
}
|
|
15
|
+
export interface SourceFile {
|
|
16
|
+
absolutePath: string;
|
|
17
|
+
relativePath: string;
|
|
18
|
+
source: string;
|
|
19
|
+
extension: string;
|
|
20
|
+
bytes: number;
|
|
21
|
+
mtimeMs: number;
|
|
22
|
+
checksum: string;
|
|
23
|
+
}
|
|
24
|
+
export interface ParsedDocument {
|
|
25
|
+
file: SourceFile;
|
|
26
|
+
text: string;
|
|
27
|
+
}
|
|
28
|
+
export interface TextChunk {
|
|
29
|
+
id: string;
|
|
30
|
+
source: string;
|
|
31
|
+
relativePath: string;
|
|
32
|
+
chunkIndex: number;
|
|
33
|
+
text: string;
|
|
34
|
+
checksum: string;
|
|
35
|
+
bytes: number;
|
|
36
|
+
mtimeMs: number;
|
|
37
|
+
}
|
|
38
|
+
export interface VectorRow extends TextChunk {
|
|
39
|
+
vector: number[];
|
|
40
|
+
}
|
|
41
|
+
export interface IngestOptions {
|
|
42
|
+
cwd?: PathLike;
|
|
43
|
+
rebuild?: boolean;
|
|
44
|
+
}
|
|
45
|
+
export interface IngestResult {
|
|
46
|
+
indexedFiles: number;
|
|
47
|
+
chunks: number;
|
|
48
|
+
skippedFiles: number;
|
|
49
|
+
errors: Array<{
|
|
50
|
+
path: string;
|
|
51
|
+
message: string;
|
|
52
|
+
}>;
|
|
53
|
+
}
|
|
54
|
+
export interface SearchOptions {
|
|
55
|
+
cwd?: PathLike;
|
|
56
|
+
topK?: number;
|
|
57
|
+
}
|
|
58
|
+
export interface SearchResult {
|
|
59
|
+
source: string;
|
|
60
|
+
relativePath: string;
|
|
61
|
+
chunkIndex: number;
|
|
62
|
+
text: string;
|
|
63
|
+
distance: number | null;
|
|
64
|
+
}
|
|
65
|
+
export interface AskResult {
|
|
66
|
+
answer: string;
|
|
67
|
+
sources: SearchResult[];
|
|
68
|
+
}
|
|
69
|
+
export interface AuditReport {
|
|
70
|
+
indexedFiles: Array<{
|
|
71
|
+
source: string;
|
|
72
|
+
chunks: number;
|
|
73
|
+
}>;
|
|
74
|
+
supportedFiles: string[];
|
|
75
|
+
missingFromIndex: string[];
|
|
76
|
+
staleInIndex: string[];
|
|
77
|
+
totalChunks: number;
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,MAAM;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxD,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;CACrB"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@jcode.labs/mimir",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Local-first memory and retrieval for private project knowledge.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "UNLICENSED",
|
|
7
|
+
"author": "Jean-Baptiste Thery",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/jcode-works/jcode-mimir.git"
|
|
11
|
+
},
|
|
12
|
+
"homepage": "https://github.com/jcode-works/jcode-mimir#readme",
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/jcode-works/jcode-mimir/issues"
|
|
15
|
+
},
|
|
16
|
+
"engines": {
|
|
17
|
+
"node": ">=20"
|
|
18
|
+
},
|
|
19
|
+
"exports": {
|
|
20
|
+
".": {
|
|
21
|
+
"types": "./dist/index.d.ts",
|
|
22
|
+
"import": "./dist/index.js"
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"bin": {
|
|
26
|
+
"kb": "dist/cli.js"
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"dist",
|
|
30
|
+
"README.md"
|
|
31
|
+
],
|
|
32
|
+
"publishConfig": {
|
|
33
|
+
"access": "public"
|
|
34
|
+
},
|
|
35
|
+
"scripts": {
|
|
36
|
+
"build": "tsc -p tsconfig.json",
|
|
37
|
+
"check": "tsc -p tsconfig.json --noEmit",
|
|
38
|
+
"test": "vitest run"
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"@lancedb/lancedb": "^0.22.1",
|
|
42
|
+
"commander": "^14.0.2",
|
|
43
|
+
"fast-glob": "^3.3.3",
|
|
44
|
+
"html-to-text": "^9.0.5",
|
|
45
|
+
"ollama": "^0.5.16",
|
|
46
|
+
"picocolors": "^1.1.1",
|
|
47
|
+
"unpdf": "^1.4.0",
|
|
48
|
+
"yaml": "^2.8.1",
|
|
49
|
+
"zod": "^4.1.13"
|
|
50
|
+
},
|
|
51
|
+
"devDependencies": {
|
|
52
|
+
"@types/html-to-text": "^9.0.4",
|
|
53
|
+
"@types/node": "^24.10.1",
|
|
54
|
+
"typescript": "^5.9.3",
|
|
55
|
+
"vitest": "^4.0.15"
|
|
56
|
+
},
|
|
57
|
+
"packageManager": "pnpm@11.9.0"
|
|
58
|
+
}
|