ko-qmd 2.8.3-ko.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1376 -0
- package/LICENSE +21 -0
- package/README.ko.md +86 -0
- package/README.md +1307 -0
- package/bin/qmd +192 -0
- package/dist/ast.d.ts +65 -0
- package/dist/ast.js +334 -0
- package/dist/bench/bench.d.ts +35 -0
- package/dist/bench/bench.js +327 -0
- package/dist/bench/score.d.ts +33 -0
- package/dist/bench/score.js +88 -0
- package/dist/bench/types.d.ts +80 -0
- package/dist/bench/types.js +8 -0
- package/dist/cli/build-info.json +4 -0
- package/dist/cli/embed-lock.d.ts +24 -0
- package/dist/cli/embed-lock.js +94 -0
- package/dist/cli/formatter.d.ts +120 -0
- package/dist/cli/formatter.js +355 -0
- package/dist/cli/mcp-pid.d.ts +25 -0
- package/dist/cli/mcp-pid.js +86 -0
- package/dist/cli/qmd.d.ts +43 -0
- package/dist/cli/qmd.js +4424 -0
- package/dist/cli/version.d.ts +42 -0
- package/dist/cli/version.js +80 -0
- package/dist/collections.d.ts +167 -0
- package/dist/collections.js +410 -0
- package/dist/db.d.ts +63 -0
- package/dist/db.js +130 -0
- package/dist/hangul.d.ts +54 -0
- package/dist/hangul.js +140 -0
- package/dist/index.d.ts +237 -0
- package/dist/index.js +244 -0
- package/dist/llm.d.ts +543 -0
- package/dist/llm.js +1747 -0
- package/dist/maintenance.d.ts +33 -0
- package/dist/maintenance.js +52 -0
- package/dist/mcp/origin-guard.d.ts +67 -0
- package/dist/mcp/origin-guard.js +137 -0
- package/dist/mcp/server.d.ts +116 -0
- package/dist/mcp/server.js +883 -0
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +4 -0
- package/dist/store.d.ts +1100 -0
- package/dist/store.js +4965 -0
- package/dist/trust.d.ts +152 -0
- package/dist/trust.js +249 -0
- package/package.json +114 -0
- package/scripts/build.mjs +72 -0
- package/scripts/check-package-grammars.mjs +29 -0
- package/scripts/package-smoke.mjs +65 -0
- package/scripts/test-all.mjs +38 -0
- package/skills/qmd/SKILL.md +316 -0
- package/skills/qmd/references/mcp-setup.md +119 -0
- package/skills/release/SKILL.md +141 -0
- package/skills/release/scripts/install-hooks.sh +38 -0
- package/skills/release/scripts/release-context.sh +129 -0
package/dist/db.d.ts
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* db.ts - Cross-runtime SQLite compatibility layer
|
|
3
|
+
*
|
|
4
|
+
* Provides a unified Database export that works under both Bun (bun:sqlite)
|
|
5
|
+
* and Node.js (better-sqlite3). The APIs are nearly identical — the main
|
|
6
|
+
* difference is the import path.
|
|
7
|
+
*
|
|
8
|
+
* On macOS, Apple's system SQLite is compiled with SQLITE_OMIT_LOAD_EXTENSION,
|
|
9
|
+
* which prevents loading native extensions like sqlite-vec. When running under
|
|
10
|
+
* Bun we call Database.setCustomSQLite() to swap in Homebrew's full-featured
|
|
11
|
+
* SQLite build before creating any database instances.
|
|
12
|
+
*/
|
|
13
|
+
export declare const isBun: boolean;
|
|
14
|
+
export type SQLiteValue = string | number | bigint | Buffer | Uint8Array | Float32Array | null;
|
|
15
|
+
export type SQLiteParams = readonly SQLiteValue[];
|
|
16
|
+
/**
|
|
17
|
+
* Open a SQLite database. Works with both bun:sqlite and better-sqlite3.
|
|
18
|
+
*
|
|
19
|
+
* `bun:sqlite` and `better-sqlite3` both default `busy_timeout` to 0, so
|
|
20
|
+
* concurrent writers throw `SQLITE_BUSY` instead of waiting. WAL improves
|
|
21
|
+
* read-while-write concurrency but does not serialise writers. Setting the
|
|
22
|
+
* timeout at connection open makes parallel processes (e.g. an `update` or
|
|
23
|
+
* `query` racing a long `embed`, or a first-open schema migration racing any
|
|
24
|
+
* routine command) queue at batch boundaries instead of failing on contact.
|
|
25
|
+
*
|
|
26
|
+
* WAL is enabled here too (with a bounded retry) so connection-level pragmas
|
|
27
|
+
* live in one place and the cold-database journal migration survives concurrent
|
|
28
|
+
* opens.
|
|
29
|
+
*
|
|
30
|
+
* Default 120_000 ms outlasts the worst-case batch commit on a multi-GB
|
|
31
|
+
* index. Override with `QMD_SQLITE_BUSY_TIMEOUT` (value in milliseconds; `0`
|
|
32
|
+
* restores the upstream fail-fast behaviour). See
|
|
33
|
+
* https://bun.sh/docs/api/sqlite#busy-timeout.
|
|
34
|
+
*/
|
|
35
|
+
export declare function openDatabase(path: string): Database;
|
|
36
|
+
/**
|
|
37
|
+
* Common subset of the Database interface used throughout QMD.
|
|
38
|
+
*/
|
|
39
|
+
export interface Database {
|
|
40
|
+
exec(sql: string): void;
|
|
41
|
+
prepare(sql: string): Statement;
|
|
42
|
+
loadExtension(path: string): void;
|
|
43
|
+
transaction<T extends (...args: SQLiteValue[]) => unknown>(fn: T): T & {
|
|
44
|
+
immediate: T;
|
|
45
|
+
};
|
|
46
|
+
close(): void;
|
|
47
|
+
}
|
|
48
|
+
export interface Statement {
|
|
49
|
+
run(...params: SQLiteValue[]): {
|
|
50
|
+
changes: number;
|
|
51
|
+
lastInsertRowid: number | bigint;
|
|
52
|
+
};
|
|
53
|
+
get<T = unknown>(...params: SQLiteValue[]): T | undefined;
|
|
54
|
+
all<T = unknown>(...params: SQLiteValue[]): T[];
|
|
55
|
+
iterate<T = unknown>(...params: SQLiteValue[]): IterableIterator<T>;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Load the sqlite-vec extension into a database.
|
|
59
|
+
*
|
|
60
|
+
* Throws with platform-specific fix instructions when the extension is
|
|
61
|
+
* unavailable.
|
|
62
|
+
*/
|
|
63
|
+
export declare function loadSqliteVec(db: Database): void;
|
package/dist/db.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* db.ts - Cross-runtime SQLite compatibility layer
|
|
3
|
+
*
|
|
4
|
+
* Provides a unified Database export that works under both Bun (bun:sqlite)
|
|
5
|
+
* and Node.js (better-sqlite3). The APIs are nearly identical — the main
|
|
6
|
+
* difference is the import path.
|
|
7
|
+
*
|
|
8
|
+
* On macOS, Apple's system SQLite is compiled with SQLITE_OMIT_LOAD_EXTENSION,
|
|
9
|
+
* which prevents loading native extensions like sqlite-vec. When running under
|
|
10
|
+
* Bun we call Database.setCustomSQLite() to swap in Homebrew's full-featured
|
|
11
|
+
* SQLite build before creating any database instances.
|
|
12
|
+
*/
|
|
13
|
+
export const isBun = "Bun" in globalThis;
|
|
14
|
+
let _Database;
|
|
15
|
+
let _sqliteVecLoad;
|
|
16
|
+
if (isBun) {
|
|
17
|
+
// Dynamic string prevents tsc from resolving bun:sqlite on Node.js builds
|
|
18
|
+
const bunSqlite = "bun:" + "sqlite";
|
|
19
|
+
const BunDatabase = (await import(/* @vite-ignore */ bunSqlite)).Database;
|
|
20
|
+
// See: https://bun.com/docs/runtime/sqlite#setcustomsqlite
|
|
21
|
+
if (process.platform === "darwin") {
|
|
22
|
+
const homebrewPaths = [
|
|
23
|
+
"/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib", // Apple Silicon
|
|
24
|
+
"/usr/local/opt/sqlite/lib/libsqlite3.dylib", // Intel
|
|
25
|
+
];
|
|
26
|
+
for (const p of homebrewPaths) {
|
|
27
|
+
try {
|
|
28
|
+
BunDatabase.setCustomSQLite(p);
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
catch { }
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
_Database = BunDatabase;
|
|
35
|
+
// setCustomSQLite may have silently failed — test that extensions actually work.
|
|
36
|
+
try {
|
|
37
|
+
const { getLoadablePath } = await import("sqlite-vec");
|
|
38
|
+
const vecPath = getLoadablePath();
|
|
39
|
+
const testDb = new BunDatabase(":memory:");
|
|
40
|
+
testDb.loadExtension(vecPath);
|
|
41
|
+
testDb.close();
|
|
42
|
+
_sqliteVecLoad = (db) => db.loadExtension(vecPath);
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// Vector search won't work, but BM25 and other operations are unaffected.
|
|
46
|
+
_sqliteVecLoad = null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
_Database = (await import("better-sqlite3")).default;
|
|
51
|
+
const sqliteVec = await import("sqlite-vec");
|
|
52
|
+
_sqliteVecLoad = (db) => sqliteVec.load(db);
|
|
53
|
+
}
|
|
54
|
+
function isBusyError(err) {
|
|
55
|
+
if (typeof err !== "object" || err === null)
|
|
56
|
+
return false;
|
|
57
|
+
const code = err.code;
|
|
58
|
+
if (code === "SQLITE_BUSY" || code === "SQLITE_BUSY_SNAPSHOT")
|
|
59
|
+
return true;
|
|
60
|
+
const message = err.message;
|
|
61
|
+
return typeof message === "string" && /database is locked|database is busy|SQLITE_BUSY/i.test(message);
|
|
62
|
+
}
|
|
63
|
+
function sleepSync(ms) {
|
|
64
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Switch a connection to WAL, retrying on `SQLITE_BUSY` within the busy-timeout
|
|
68
|
+
* budget. Unlike ordinary writes, migrating the journal needs a brief exclusive
|
|
69
|
+
* lock and does NOT invoke the busy handler, so concurrent first-time opens of a
|
|
70
|
+
* cold database throw "database is locked" even with `busy_timeout` set. Once the
|
|
71
|
+
* database is already WAL the pragma is a cheap no-op that does not contend.
|
|
72
|
+
*/
|
|
73
|
+
function enableWal(db, budgetMs) {
|
|
74
|
+
const deadline = Date.now() + Math.max(budgetMs, 0);
|
|
75
|
+
for (let attempt = 0;; attempt++) {
|
|
76
|
+
try {
|
|
77
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
if (!isBusyError(err) || Date.now() >= deadline)
|
|
82
|
+
throw err;
|
|
83
|
+
sleepSync(Math.min(5 + attempt, 25));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Open a SQLite database. Works with both bun:sqlite and better-sqlite3.
|
|
89
|
+
*
|
|
90
|
+
* `bun:sqlite` and `better-sqlite3` both default `busy_timeout` to 0, so
|
|
91
|
+
* concurrent writers throw `SQLITE_BUSY` instead of waiting. WAL improves
|
|
92
|
+
* read-while-write concurrency but does not serialise writers. Setting the
|
|
93
|
+
* timeout at connection open makes parallel processes (e.g. an `update` or
|
|
94
|
+
* `query` racing a long `embed`, or a first-open schema migration racing any
|
|
95
|
+
* routine command) queue at batch boundaries instead of failing on contact.
|
|
96
|
+
*
|
|
97
|
+
* WAL is enabled here too (with a bounded retry) so connection-level pragmas
|
|
98
|
+
* live in one place and the cold-database journal migration survives concurrent
|
|
99
|
+
* opens.
|
|
100
|
+
*
|
|
101
|
+
* Default 120_000 ms outlasts the worst-case batch commit on a multi-GB
|
|
102
|
+
* index. Override with `QMD_SQLITE_BUSY_TIMEOUT` (value in milliseconds; `0`
|
|
103
|
+
* restores the upstream fail-fast behaviour). See
|
|
104
|
+
* https://bun.sh/docs/api/sqlite#busy-timeout.
|
|
105
|
+
*/
|
|
106
|
+
export function openDatabase(path) {
|
|
107
|
+
const db = new _Database(path);
|
|
108
|
+
const raw = process.env.QMD_SQLITE_BUSY_TIMEOUT;
|
|
109
|
+
const parsed = raw !== undefined && raw !== "" ? Number(raw) : Number.NaN;
|
|
110
|
+
const busyTimeoutMs = Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 120_000;
|
|
111
|
+
db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`);
|
|
112
|
+
enableWal(db, busyTimeoutMs);
|
|
113
|
+
return db;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Load the sqlite-vec extension into a database.
|
|
117
|
+
*
|
|
118
|
+
* Throws with platform-specific fix instructions when the extension is
|
|
119
|
+
* unavailable.
|
|
120
|
+
*/
|
|
121
|
+
export function loadSqliteVec(db) {
|
|
122
|
+
if (!_sqliteVecLoad) {
|
|
123
|
+
const hint = isBun && process.platform === "darwin"
|
|
124
|
+
? "On macOS with Bun, install Homebrew SQLite: brew install sqlite\n" +
|
|
125
|
+
"Or install qmd with npm instead: npm install -g ko-qmd"
|
|
126
|
+
: "Ensure the sqlite-vec native module is installed correctly.";
|
|
127
|
+
throw new Error(`sqlite-vec extension is unavailable. ${hint}`);
|
|
128
|
+
}
|
|
129
|
+
_sqliteVecLoad(db);
|
|
130
|
+
}
|
package/dist/hangul.d.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hangul.ts - Korean (Hangul) handling for FTS5 indexing and queries.
|
|
3
|
+
*
|
|
4
|
+
* ko-qmd patch stack: all Hangul-specific logic lives here so store.ts keeps
|
|
5
|
+
* at most two call sites. Han and kana text never reaches these functions.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Syllable bigrams of every Hangul run in `text`, space-joined in run order
|
|
9
|
+
* (`검색을 개선` → `검색 색을 개선`). Appended after the character-spaced field
|
|
10
|
+
* text at index time: the character tokens keep their original adjacency, so
|
|
11
|
+
* existing phrase queries match exactly as before, and bigram phrases from
|
|
12
|
+
* one run stay contiguous.
|
|
13
|
+
*/
|
|
14
|
+
export declare function hangulBigramTail(text: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* Strip one trailing particle (or 기) from a pure-Hangul word when the stem
|
|
17
|
+
* keeps at least two syllables (`검색을` → `검색`, `책을` stays). Returns null
|
|
18
|
+
* when nothing is stripped.
|
|
19
|
+
*/
|
|
20
|
+
export declare function stripHangulParticle(word: string): string | null;
|
|
21
|
+
/**
|
|
22
|
+
* Strip one trailing verbal/nominalizing ending (`토큰화하는` → `토큰화`,
|
|
23
|
+
* `검색하기` → `검색`). Returns null when nothing is stripped. Endings are tried
|
|
24
|
+
* before particles so `하기` wins over the bare `기` that would leave `검색하`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function stripHangulEnding(word: string): string | null;
|
|
27
|
+
/**
|
|
28
|
+
* Stems worth querying for one Hangul word, longest first and without the word
|
|
29
|
+
* itself: an ending (`토큰화하는` → `토큰화`), then particles, twice, so a chain
|
|
30
|
+
* like `청킹에서의` reaches `청킹`. Two rounds cover the chains Korean actually
|
|
31
|
+
* stacks in queries (`에서 + 의`, `으로 + 는`) without unwinding real syllables.
|
|
32
|
+
*/
|
|
33
|
+
export declare function hangulStems(word: string): string[];
|
|
34
|
+
/**
|
|
35
|
+
* FTS5 expression for a plain (unquoted) Hangul query term: for the word and
|
|
36
|
+
* each of its stems, a bigram phrase (from hangulBigramTail) OR the character
|
|
37
|
+
* phrase (`검색` → `("검색" OR "검 색")`). The character phrase keeps spacing
|
|
38
|
+
* variants matching (`간격반복` over `간격 반복`), which bigrams cannot span; the
|
|
39
|
+
* bigram phrase adds its BM25 weight. Returns null for single syllables and
|
|
40
|
+
* non-Hangul or mixed-script terms, so the caller keeps its default character
|
|
41
|
+
* phrase.
|
|
42
|
+
*/
|
|
43
|
+
export declare function hangulTermQuery(term: string): string | null;
|
|
44
|
+
/**
|
|
45
|
+
* FTS5 expression for a term that glues Hangul to Latin or digits
|
|
46
|
+
* (`SKILL.md계약의핵심`, `hook이유일한hardboundary다`, `auto-dream의발화조건`).
|
|
47
|
+
* The index already splits scripts — hangulBigramTail only sees Hangul runs and the
|
|
48
|
+
* porter tokenizer splits the rest — so the query has to split too, or the glued term
|
|
49
|
+
* is searched as one phrase that appears nowhere. Each Hangul run goes through
|
|
50
|
+
* hangulTermQuery, each Latin/digit run becomes a prefix term, and the runs are ANDed.
|
|
51
|
+
* Separators (`.`, `-`, `/`, `+`) are dropped with the rest of the non-alphanumerics.
|
|
52
|
+
* Returns null when the term is not mixed script, so callers keep their own handling.
|
|
53
|
+
*/
|
|
54
|
+
export declare function hangulMixedQuery(term: string): string | null;
|
package/dist/hangul.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hangul.ts - Korean (Hangul) handling for FTS5 indexing and queries.
|
|
3
|
+
*
|
|
4
|
+
* ko-qmd patch stack: all Hangul-specific logic lives here so store.ts keeps
|
|
5
|
+
* at most two call sites. Han and kana text never reaches these functions.
|
|
6
|
+
*/
|
|
7
|
+
const HANGUL_WORD_PATTERN = /^\p{Script=Hangul}+$/u;
|
|
8
|
+
const HANGUL_RUN_PATTERN = /\p{Script=Hangul}+/gu;
|
|
9
|
+
// Particles (design §4) plus the nominalizing ending 기 (`나누기` → `나누`).
|
|
10
|
+
// Longest first, so 에서 wins over 에 and 으로 over 로.
|
|
11
|
+
const SUFFIXES = [
|
|
12
|
+
"에게", "에서", "으로", "까지", "부터", "처럼",
|
|
13
|
+
"은", "는", "이", "가", "을", "를", "에", "로", "의", "도", "와", "과", "기",
|
|
14
|
+
];
|
|
15
|
+
// Verbal and nominalizing endings on 하다/되다 stems (`토큰화하는` → `토큰화`).
|
|
16
|
+
// Longest first, and always tried before SUFFIXES so `하기` wins over bare `기`.
|
|
17
|
+
const ENDINGS = [
|
|
18
|
+
"하려는", "되려는", "하면서", "되면서",
|
|
19
|
+
"하는", "하기", "한다", "했다", "하며", "하고", "하여", "해서", "하지", "하다",
|
|
20
|
+
"되는", "되기", "된다", "됐다", "되며", "되고", "되어", "돼서", "되다",
|
|
21
|
+
"한", "된",
|
|
22
|
+
];
|
|
23
|
+
const MIN_STEM_SYLLABLES = 2;
|
|
24
|
+
function syllableBigrams(word) {
|
|
25
|
+
const syllables = Array.from(word);
|
|
26
|
+
return syllables.slice(1).map((s, i) => syllables[i] + s);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Syllable bigrams of every Hangul run in `text`, space-joined in run order
|
|
30
|
+
* (`검색을 개선` → `검색 색을 개선`). Appended after the character-spaced field
|
|
31
|
+
* text at index time: the character tokens keep their original adjacency, so
|
|
32
|
+
* existing phrase queries match exactly as before, and bigram phrases from
|
|
33
|
+
* one run stay contiguous.
|
|
34
|
+
*/
|
|
35
|
+
export function hangulBigramTail(text) {
|
|
36
|
+
const runs = text.match(HANGUL_RUN_PATTERN) ?? [];
|
|
37
|
+
return runs.flatMap(syllableBigrams).join(" ");
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Strip one trailing particle (or 기) from a pure-Hangul word when the stem
|
|
41
|
+
* keeps at least two syllables (`검색을` → `검색`, `책을` stays). Returns null
|
|
42
|
+
* when nothing is stripped.
|
|
43
|
+
*/
|
|
44
|
+
export function stripHangulParticle(word) {
|
|
45
|
+
return stripOnce(word, SUFFIXES);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Strip one trailing verbal/nominalizing ending (`토큰화하는` → `토큰화`,
|
|
49
|
+
* `검색하기` → `검색`). Returns null when nothing is stripped. Endings are tried
|
|
50
|
+
* before particles so `하기` wins over the bare `기` that would leave `검색하`.
|
|
51
|
+
*/
|
|
52
|
+
export function stripHangulEnding(word) {
|
|
53
|
+
return stripOnce(word, ENDINGS);
|
|
54
|
+
}
|
|
55
|
+
function stripOnce(word, suffixes) {
|
|
56
|
+
if (!HANGUL_WORD_PATTERN.test(word))
|
|
57
|
+
return null;
|
|
58
|
+
const syllables = Array.from(word);
|
|
59
|
+
for (const suffix of suffixes) {
|
|
60
|
+
const stemLength = syllables.length - Array.from(suffix).length;
|
|
61
|
+
if (stemLength < MIN_STEM_SYLLABLES)
|
|
62
|
+
continue;
|
|
63
|
+
if (word.endsWith(suffix))
|
|
64
|
+
return syllables.slice(0, stemLength).join("");
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Stems worth querying for one Hangul word, longest first and without the word
|
|
70
|
+
* itself: an ending (`토큰화하는` → `토큰화`), then particles, twice, so a chain
|
|
71
|
+
* like `청킹에서의` reaches `청킹`. Two rounds cover the chains Korean actually
|
|
72
|
+
* stacks in queries (`에서 + 의`, `으로 + 는`) without unwinding real syllables.
|
|
73
|
+
*/
|
|
74
|
+
export function hangulStems(word) {
|
|
75
|
+
const stems = [];
|
|
76
|
+
let current = word;
|
|
77
|
+
for (let round = 0; round < 2; round++) {
|
|
78
|
+
const next = stripHangulEnding(current) ?? stripHangulParticle(current);
|
|
79
|
+
if (next === null || next === current)
|
|
80
|
+
break;
|
|
81
|
+
stems.push(next);
|
|
82
|
+
current = next;
|
|
83
|
+
}
|
|
84
|
+
return [...stems, ...stems.flatMap(contractions)];
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* 하 ↔ 해 siblings of a stem (`더하` → `더해`). 하다 verbs contract in running
|
|
88
|
+
* text (`더하여` is written `더해`), so a stem stripped back to 하 would never
|
|
89
|
+
* meet the contracted spelling the document actually uses. One syllable swap,
|
|
90
|
+
* no new stripping.
|
|
91
|
+
*/
|
|
92
|
+
function contractions(stem) {
|
|
93
|
+
if (stem.endsWith("하"))
|
|
94
|
+
return [`${stem.slice(0, -1)}해`];
|
|
95
|
+
if (stem.endsWith("해"))
|
|
96
|
+
return [`${stem.slice(0, -1)}하`];
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* FTS5 expression for a plain (unquoted) Hangul query term: for the word and
|
|
101
|
+
* each of its stems, a bigram phrase (from hangulBigramTail) OR the character
|
|
102
|
+
* phrase (`검색` → `("검색" OR "검 색")`). The character phrase keeps spacing
|
|
103
|
+
* variants matching (`간격반복` over `간격 반복`), which bigrams cannot span; the
|
|
104
|
+
* bigram phrase adds its BM25 weight. Returns null for single syllables and
|
|
105
|
+
* non-Hangul or mixed-script terms, so the caller keeps its default character
|
|
106
|
+
* phrase.
|
|
107
|
+
*/
|
|
108
|
+
export function hangulTermQuery(term) {
|
|
109
|
+
if (!HANGUL_WORD_PATTERN.test(term) || Array.from(term).length < 2)
|
|
110
|
+
return null;
|
|
111
|
+
const phrases = (s) => [`"${syllableBigrams(s).join(" ")}"`, `"${Array.from(s).join(" ")}"`];
|
|
112
|
+
const stems = hangulStems(term);
|
|
113
|
+
return `(${[...stems.flatMap(phrases), ...phrases(term)].join(" OR ")})`;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* FTS5 expression for a term that glues Hangul to Latin or digits
|
|
117
|
+
* (`SKILL.md계약의핵심`, `hook이유일한hardboundary다`, `auto-dream의발화조건`).
|
|
118
|
+
* The index already splits scripts — hangulBigramTail only sees Hangul runs and the
|
|
119
|
+
* porter tokenizer splits the rest — so the query has to split too, or the glued term
|
|
120
|
+
* is searched as one phrase that appears nowhere. Each Hangul run goes through
|
|
121
|
+
* hangulTermQuery, each Latin/digit run becomes a prefix term, and the runs are ANDed.
|
|
122
|
+
* Separators (`.`, `-`, `/`, `+`) are dropped with the rest of the non-alphanumerics.
|
|
123
|
+
* Returns null when the term is not mixed script, so callers keep their own handling.
|
|
124
|
+
*/
|
|
125
|
+
export function hangulMixedQuery(term) {
|
|
126
|
+
// Set subtraction (v flag) keeps a non-Hangul run from swallowing the Hangul next to it —
|
|
127
|
+
// plain alternation matches `[\p{L}\p{N}]+` across the boundary once it starts on Latin.
|
|
128
|
+
const runs = term.match(/\p{Script=Hangul}+|[[\p{L}\p{N}]--[\p{Script=Hangul}]]+/gv) ?? [];
|
|
129
|
+
const hangulRuns = runs.filter((r) => HANGUL_WORD_PATTERN.test(r));
|
|
130
|
+
if (hangulRuns.length === 0 || hangulRuns.length === runs.length)
|
|
131
|
+
return null;
|
|
132
|
+
const parts = runs.flatMap((run) => {
|
|
133
|
+
if (HANGUL_WORD_PATTERN.test(run)) {
|
|
134
|
+
return [hangulTermQuery(run) ?? `"${Array.from(run).join(" ")}"`];
|
|
135
|
+
}
|
|
136
|
+
const latin = run.replace(/[^\p{L}\p{N}'_]/gu, "").toLowerCase();
|
|
137
|
+
return latin ? [`"${latin}"*`] : [];
|
|
138
|
+
});
|
|
139
|
+
return parts.length > 0 ? `(${parts.join(" AND ")})` : null;
|
|
140
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD SDK - Library mode for programmatic access to QMD search and indexing.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* import { createStore } from '@tobilu/qmd'
|
|
6
|
+
*
|
|
7
|
+
* const store = await createStore({
|
|
8
|
+
* dbPath: './my-index.sqlite',
|
|
9
|
+
* config: {
|
|
10
|
+
* collections: {
|
|
11
|
+
* docs: { path: '/path/to/docs', pattern: '**\/*.md' }
|
|
12
|
+
* }
|
|
13
|
+
* }
|
|
14
|
+
* })
|
|
15
|
+
*
|
|
16
|
+
* const results = await store.search({ query: "how does auth work?" })
|
|
17
|
+
* await store.close()
|
|
18
|
+
*/
|
|
19
|
+
import { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES, type Store as InternalStore, type DocumentResult, type DocumentNotFound, type DocumentExcludedByIgnore, type DocumentLookupError, type SearchResult, type HybridQueryResult, type HybridQueryOptions, type HybridQueryExplain, type ExpandedQuery, type StructuredSearchOptions, type MultiGetResult, type IndexStatus, type IndexHealthInfo, type SearchHooks, type ReindexProgress, type ReindexResult, type EmbedProgress, type EmbedResult, type ChunkStrategy } from "./store.js";
|
|
20
|
+
import { type Collection, type CollectionConfig, type NamedCollection, type ContextMap } from "./collections.js";
|
|
21
|
+
export type { DocumentResult, DocumentNotFound, DocumentExcludedByIgnore, DocumentLookupError, SearchResult, HybridQueryResult, HybridQueryOptions, HybridQueryExplain, ExpandedQuery, StructuredSearchOptions, MultiGetResult, IndexStatus, IndexHealthInfo, SearchHooks, ReindexProgress, ReindexResult, EmbedProgress, EmbedResult, Collection, CollectionConfig, NamedCollection, ContextMap, };
|
|
22
|
+
export type { InternalStore };
|
|
23
|
+
export { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES };
|
|
24
|
+
export type { ChunkStrategy } from "./store.js";
|
|
25
|
+
export { getDefaultDbPath } from "./store.js";
|
|
26
|
+
export { Maintenance } from "./maintenance.js";
|
|
27
|
+
/**
|
|
28
|
+
* Progress info emitted during update() for each file processed.
|
|
29
|
+
*/
|
|
30
|
+
export type UpdateProgress = {
|
|
31
|
+
collection: string;
|
|
32
|
+
file: string;
|
|
33
|
+
current: number;
|
|
34
|
+
total: number;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Aggregated result from update() across all collections.
|
|
38
|
+
*/
|
|
39
|
+
export type UpdateResult = {
|
|
40
|
+
collections: number;
|
|
41
|
+
indexed: number;
|
|
42
|
+
updated: number;
|
|
43
|
+
unchanged: number;
|
|
44
|
+
removed: number;
|
|
45
|
+
skipped: number;
|
|
46
|
+
needsEmbedding: number;
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Options for the unified search() method.
|
|
50
|
+
*/
|
|
51
|
+
export interface SearchOptions {
|
|
52
|
+
/** Simple query string — will be auto-expanded via LLM */
|
|
53
|
+
query?: string;
|
|
54
|
+
/** Pre-expanded queries (from expandQuery) — skips auto-expansion */
|
|
55
|
+
queries?: ExpandedQuery[];
|
|
56
|
+
/** Domain intent hint — steers reranking and snippet/chunk selection */
|
|
57
|
+
intent?: string;
|
|
58
|
+
/** Rerank results using LLM (default: true) */
|
|
59
|
+
rerank?: boolean;
|
|
60
|
+
/** Filter to a specific collection */
|
|
61
|
+
collection?: string;
|
|
62
|
+
/** Filter to specific collections */
|
|
63
|
+
collections?: string[];
|
|
64
|
+
/** Max results (default: 10) */
|
|
65
|
+
limit?: number;
|
|
66
|
+
/** Max candidates to rerank (default: 40) */
|
|
67
|
+
candidateLimit?: number;
|
|
68
|
+
/** Minimum score threshold */
|
|
69
|
+
minScore?: number;
|
|
70
|
+
/** Include explain traces */
|
|
71
|
+
explain?: boolean;
|
|
72
|
+
/** Chunk strategy: "auto" (default, uses AST for code files) or "regex" (legacy) */
|
|
73
|
+
chunkStrategy?: ChunkStrategy;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Options for searchLex() — BM25 keyword search.
|
|
77
|
+
*/
|
|
78
|
+
export interface LexSearchOptions {
|
|
79
|
+
limit?: number;
|
|
80
|
+
collection?: string | string[];
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Options for searchVector() — vector similarity search.
|
|
84
|
+
*/
|
|
85
|
+
export interface VectorSearchOptions {
|
|
86
|
+
limit?: number;
|
|
87
|
+
collection?: string | string[];
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Options for expandQuery() — manual query expansion.
|
|
91
|
+
*/
|
|
92
|
+
export interface ExpandQueryOptions {
|
|
93
|
+
/**
|
|
94
|
+
* @deprecated Ignored. Intent no longer feeds the expansion model — caller
|
|
95
|
+
* intent is meta-language the model reproduced verbatim as sub-queries.
|
|
96
|
+
* Pass intent via SearchOptions instead, where it shapes reranking and
|
|
97
|
+
* snippet selection.
|
|
98
|
+
*/
|
|
99
|
+
intent?: string;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Options for creating a QMD store.
|
|
103
|
+
*
|
|
104
|
+
* Provide `dbPath` and optionally `configPath` (YAML file) or `config` (inline).
|
|
105
|
+
* If neither configPath nor config is provided, the store reads from existing
|
|
106
|
+
* DB state (useful for reopening a previously-configured store).
|
|
107
|
+
*/
|
|
108
|
+
export interface StoreOptions {
|
|
109
|
+
/** Path to the SQLite database file */
|
|
110
|
+
dbPath: string;
|
|
111
|
+
/** Path to a YAML config file (mutually exclusive with `config`) */
|
|
112
|
+
configPath?: string;
|
|
113
|
+
/** Inline collection config (mutually exclusive with `configPath`) */
|
|
114
|
+
config?: CollectionConfig;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* The QMD SDK store — provides search, retrieval, collection management,
|
|
118
|
+
* context management, and indexing operations.
|
|
119
|
+
*
|
|
120
|
+
* All methods are async. The store manages its own LlamaCpp instance
|
|
121
|
+
* (lazy-loaded, auto-unloaded after inactivity) — no global singletons.
|
|
122
|
+
*/
|
|
123
|
+
export interface QMDStore {
|
|
124
|
+
/** The underlying internal store (for advanced use) */
|
|
125
|
+
readonly internal: InternalStore;
|
|
126
|
+
/** Path to the SQLite database */
|
|
127
|
+
readonly dbPath: string;
|
|
128
|
+
/** Full search: query expansion + multi-signal retrieval + LLM reranking */
|
|
129
|
+
search(options: SearchOptions): Promise<HybridQueryResult[]>;
|
|
130
|
+
/** BM25 keyword search (fast, no LLM) */
|
|
131
|
+
searchLex(query: string, options?: LexSearchOptions): Promise<SearchResult[]>;
|
|
132
|
+
/** Vector similarity search (embedding model, no reranking) */
|
|
133
|
+
searchVector(query: string, options?: VectorSearchOptions): Promise<SearchResult[]>;
|
|
134
|
+
/** Expand a query into typed sub-searches (lex/vec/hyde) for manual control */
|
|
135
|
+
expandQuery(query: string, options?: ExpandQueryOptions): Promise<ExpandedQuery[]>;
|
|
136
|
+
/** Get a single document by path or docid */
|
|
137
|
+
get(pathOrDocid: string, options?: {
|
|
138
|
+
includeBody?: boolean;
|
|
139
|
+
}): Promise<DocumentResult | DocumentLookupError>;
|
|
140
|
+
/** Get the body content of a document, optionally sliced by line range */
|
|
141
|
+
getDocumentBody(pathOrDocid: string, opts?: {
|
|
142
|
+
fromLine?: number;
|
|
143
|
+
maxLines?: number;
|
|
144
|
+
}): Promise<string | null>;
|
|
145
|
+
/** Get multiple documents by glob pattern or comma-separated list */
|
|
146
|
+
multiGet(pattern: string, options?: {
|
|
147
|
+
includeBody?: boolean;
|
|
148
|
+
maxBytes?: number;
|
|
149
|
+
}): Promise<{
|
|
150
|
+
docs: MultiGetResult[];
|
|
151
|
+
errors: string[];
|
|
152
|
+
}>;
|
|
153
|
+
/** Add or update a collection */
|
|
154
|
+
addCollection(name: string, opts: {
|
|
155
|
+
path: string;
|
|
156
|
+
pattern?: string;
|
|
157
|
+
ignore?: string[];
|
|
158
|
+
}): Promise<void>;
|
|
159
|
+
/** Remove a collection */
|
|
160
|
+
removeCollection(name: string): Promise<boolean>;
|
|
161
|
+
/** Rename a collection */
|
|
162
|
+
renameCollection(oldName: string, newName: string): Promise<boolean>;
|
|
163
|
+
/** List all collections with document stats */
|
|
164
|
+
listCollections(): Promise<{
|
|
165
|
+
name: string;
|
|
166
|
+
pwd: string;
|
|
167
|
+
glob_pattern: string;
|
|
168
|
+
doc_count: number;
|
|
169
|
+
active_count: number;
|
|
170
|
+
last_modified: string | null;
|
|
171
|
+
includeByDefault: boolean;
|
|
172
|
+
}[]>;
|
|
173
|
+
/** Get names of collections included by default in queries */
|
|
174
|
+
getDefaultCollectionNames(): Promise<string[]>;
|
|
175
|
+
/** Add context for a path within a collection */
|
|
176
|
+
addContext(collectionName: string, pathPrefix: string, contextText: string): Promise<boolean>;
|
|
177
|
+
/** Remove context from a collection path */
|
|
178
|
+
removeContext(collectionName: string, pathPrefix: string): Promise<boolean>;
|
|
179
|
+
/** Set global context (applies to all collections) */
|
|
180
|
+
setGlobalContext(context: string | undefined): Promise<void>;
|
|
181
|
+
/** Get global context */
|
|
182
|
+
getGlobalContext(): Promise<string | undefined>;
|
|
183
|
+
/** List all contexts across all collections */
|
|
184
|
+
listContexts(): Promise<Array<{
|
|
185
|
+
collection: string;
|
|
186
|
+
path: string;
|
|
187
|
+
context: string;
|
|
188
|
+
}>>;
|
|
189
|
+
/** Re-index collections by scanning the filesystem */
|
|
190
|
+
update(options?: {
|
|
191
|
+
collections?: string[];
|
|
192
|
+
onProgress?: (info: UpdateProgress) => void;
|
|
193
|
+
}): Promise<UpdateResult>;
|
|
194
|
+
/** Generate vector embeddings for documents that need them */
|
|
195
|
+
embed(options?: {
|
|
196
|
+
force?: boolean;
|
|
197
|
+
model?: string;
|
|
198
|
+
/** Restrict embedding to documents in one collection. */
|
|
199
|
+
collection?: string;
|
|
200
|
+
maxDocsPerBatch?: number;
|
|
201
|
+
maxBatchBytes?: number;
|
|
202
|
+
chunkStrategy?: ChunkStrategy;
|
|
203
|
+
onProgress?: (info: EmbedProgress) => void;
|
|
204
|
+
}): Promise<EmbedResult>;
|
|
205
|
+
/** Get index status (document counts, collections, embedding state) */
|
|
206
|
+
getStatus(): Promise<IndexStatus>;
|
|
207
|
+
/** Get index health info (stale embeddings, etc.) */
|
|
208
|
+
getIndexHealth(): Promise<IndexHealthInfo>;
|
|
209
|
+
/** Close the store and release all resources (LLM models, DB connection) */
|
|
210
|
+
close(): Promise<void>;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Create a QMD store for programmatic access to search and indexing.
|
|
214
|
+
*
|
|
215
|
+
* @example
|
|
216
|
+
* ```typescript
|
|
217
|
+
* // With a YAML config file
|
|
218
|
+
* const store = await createStore({
|
|
219
|
+
* dbPath: './index.sqlite',
|
|
220
|
+
* configPath: './qmd.yml',
|
|
221
|
+
* })
|
|
222
|
+
*
|
|
223
|
+
* // With inline config (no files needed besides the DB)
|
|
224
|
+
* const store = await createStore({
|
|
225
|
+
* dbPath: './index.sqlite',
|
|
226
|
+
* config: {
|
|
227
|
+
* collections: {
|
|
228
|
+
* docs: { path: '/path/to/docs', pattern: '**\/*.md' }
|
|
229
|
+
* }
|
|
230
|
+
* }
|
|
231
|
+
* })
|
|
232
|
+
*
|
|
233
|
+
* const results = await store.search({ query: "authentication flow" })
|
|
234
|
+
* await store.close()
|
|
235
|
+
* ```
|
|
236
|
+
*/
|
|
237
|
+
export declare function createStore(options: StoreOptions): Promise<QMDStore>;
|