@rekal/mem 0.0.0 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{db-BMh1OP4b.mjs → db-CHpq7OOi.mjs} +46 -15
- package/dist/db-CHpq7OOi.mjs.map +1 -0
- package/dist/doc-DnYN4jAU.mjs +2 -0
- package/dist/doc-DnYN4jAU.mjs.map +1 -0
- package/dist/{embed-rUMZxqed.mjs → embed-CZI5Dz1q.mjs} +3 -1
- package/dist/embed-CZI5Dz1q.mjs.map +1 -0
- package/dist/frecency-CiaqPIOy.mjs +30 -0
- package/dist/frecency-CiaqPIOy.mjs.map +1 -0
- package/dist/fs-DMp26Byo.mjs +2 -0
- package/dist/fs-DMp26Byo.mjs.map +1 -0
- package/dist/glob.d.mts +2 -1
- package/dist/glob.d.mts.map +1 -0
- package/dist/glob.mjs +2 -0
- package/dist/glob.mjs.map +1 -0
- package/dist/index.d.mts +21 -11
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +7 -5
- package/dist/index.mjs.map +1 -0
- package/dist/{llama-CT3dc9Cn.mjs → llama-CpNV7Lh9.mjs} +3 -1
- package/dist/llama-CpNV7Lh9.mjs.map +1 -0
- package/dist/{models-DFQSgBNr.mjs → models-Bo6czhQe.mjs} +5 -3
- package/dist/models-Bo6czhQe.mjs.map +1 -0
- package/dist/{openai-j2_2GM4J.mjs → openai-ALl6_YhI.mjs} +3 -1
- package/dist/openai-ALl6_YhI.mjs.map +1 -0
- package/dist/progress-B1JdNapX.mjs +2 -0
- package/dist/progress-B1JdNapX.mjs.map +1 -0
- package/dist/query-VFSpErTB.mjs +2 -0
- package/dist/query-VFSpErTB.mjs.map +1 -0
- package/dist/runtime.node-DlQPaGrV.mjs +2 -0
- package/dist/runtime.node-DlQPaGrV.mjs.map +1 -0
- package/dist/{search-BllHWtZF.mjs → search-DsVjB-9f.mjs} +2 -0
- package/dist/search-DsVjB-9f.mjs.map +1 -0
- package/dist/{store-DE7S35SS.mjs → store-I5nVEYxK.mjs} +10 -6
- package/dist/store-I5nVEYxK.mjs.map +1 -0
- package/dist/{transformers-CJ3QA2PK.mjs → transformers-Df56Nq9G.mjs} +3 -1
- package/dist/transformers-Df56Nq9G.mjs.map +1 -0
- package/dist/uri-CehXVDGB.mjs +2 -0
- package/dist/uri-CehXVDGB.mjs.map +1 -0
- package/dist/util-DNyrmcA3.mjs +2 -0
- package/dist/util-DNyrmcA3.mjs.map +1 -0
- package/dist/{vfs-CNQbkhsf.mjs → vfs-QUP1rnSI.mjs} +2 -0
- package/dist/vfs-QUP1rnSI.mjs.map +1 -0
- package/package.json +25 -25
- package/src/db.ts +73 -23
- package/src/frecency.ts +29 -46
- package/src/store.ts +13 -7
- package/foo.ts +0 -3
- package/foo2.ts +0 -20
- package/test/doc.test.ts +0 -61
- package/test/fixtures/ignore-test/keep.md +0 -0
- package/test/fixtures/ignore-test/skip.log +0 -0
- package/test/fixtures/ignore-test/sub/keep.md +0 -0
- package/test/fixtures/store/agent/index.md +0 -9
- package/test/fixtures/store/agent/lessons.md +0 -21
- package/test/fixtures/store/agent/soul.md +0 -28
- package/test/fixtures/store/agent/tools.md +0 -25
- package/test/fixtures/store/concepts/frecency.md +0 -30
- package/test/fixtures/store/concepts/index.md +0 -9
- package/test/fixtures/store/concepts/memory-coherence.md +0 -33
- package/test/fixtures/store/concepts/rag.md +0 -27
- package/test/fixtures/store/index.md +0 -9
- package/test/fixtures/store/projects/index.md +0 -9
- package/test/fixtures/store/projects/rekall-inc/architecture.md +0 -41
- package/test/fixtures/store/projects/rekall-inc/decisions/index.md +0 -9
- package/test/fixtures/store/projects/rekall-inc/decisions/no-military.md +0 -20
- package/test/fixtures/store/projects/rekall-inc/index.md +0 -28
- package/test/fixtures/store/user/family.md +0 -13
- package/test/fixtures/store/user/index.md +0 -9
- package/test/fixtures/store/user/preferences.md +0 -29
- package/test/fixtures/store/user/profile.md +0 -29
- package/test/fs.test.ts +0 -15
- package/test/glob.test.ts +0 -190
- package/test/md.test.ts +0 -177
- package/test/query.test.ts +0 -105
- package/test/uri.test.ts +0 -46
- package/test/util.test.ts +0 -62
- package/test/vfs.test.ts +0 -164
- package/tsconfig.json +0 -3
- package/tsdown.config.ts +0 -8
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
import { t as openDatabase } from "./runtime.node-DlQPaGrV.mjs";
|
|
2
|
+
import { n as toDeadline, r as toScore, t as addVisit } from "./frecency-CiaqPIOy.mjs";
|
|
2
3
|
//#region src/db.ts
|
|
4
|
+
function splitCsv(s) {
|
|
5
|
+
return s ? s.split(",") : [];
|
|
6
|
+
}
|
|
7
|
+
function toDocRow(rows) {
|
|
8
|
+
return rows.map((row) => Object.assign(row, {
|
|
9
|
+
entities: splitCsv(row.entities),
|
|
10
|
+
frecency: toScore(row.deadline),
|
|
11
|
+
synced_at: row.synced_at ? new Date(row.synced_at) : void 0,
|
|
12
|
+
tags: splitCsv(row.tags),
|
|
13
|
+
updated_at: new Date(row.updated_at)
|
|
14
|
+
}));
|
|
15
|
+
}
|
|
3
16
|
const SEARCH_LIMIT = 20;
|
|
4
17
|
const STOPWORD_THRESHOLD = .3;
|
|
5
18
|
const STOPWORD_MIN_DOCS = 10;
|
|
@@ -37,7 +50,7 @@ var Db = class Db {
|
|
|
37
50
|
entities TEXT NOT NULL DEFAULT '',
|
|
38
51
|
updated_at TEXT NOT NULL,
|
|
39
52
|
synced_at TEXT,
|
|
40
|
-
deadline REAL
|
|
53
|
+
deadline REAL NOT NULL DEFAULT 0
|
|
41
54
|
)
|
|
42
55
|
`);
|
|
43
56
|
this.#db.run(`CREATE INDEX IF NOT EXISTS idx_docs_path ON docs(path)`);
|
|
@@ -122,21 +135,30 @@ var Db = class Db {
|
|
|
122
135
|
}
|
|
123
136
|
getDoc(from) {
|
|
124
137
|
const field = typeof from === "number" ? "id" : "path";
|
|
125
|
-
|
|
138
|
+
const row = this.#db.query(`SELECT * FROM docs WHERE ${field} = ?`).get(from);
|
|
139
|
+
return row ? toDocRow([row])[0] : void 0;
|
|
126
140
|
}
|
|
127
141
|
getDocs(from) {
|
|
128
142
|
let ret;
|
|
129
|
-
if (!from) ret = this.#db.query(`SELECT * FROM docs`).all();
|
|
143
|
+
if (!from) ret = toDocRow(this.#db.query(`SELECT * FROM docs`).all());
|
|
130
144
|
else {
|
|
131
145
|
const field = typeof from[0] === "number" ? "id" : "path";
|
|
132
146
|
const placeholders = from.map(() => "?").join(",");
|
|
133
|
-
ret = this.#db.query(`SELECT * FROM docs WHERE ${field} IN (${placeholders})`).all(...from);
|
|
147
|
+
ret = toDocRow(this.#db.query(`SELECT * FROM docs WHERE ${field} IN (${placeholders})`).all(...from));
|
|
134
148
|
}
|
|
135
149
|
return new Map(ret.map((row) => [row.id, row]));
|
|
136
150
|
}
|
|
137
151
|
addDoc(row) {
|
|
138
|
-
|
|
139
|
-
|
|
152
|
+
const raw = {
|
|
153
|
+
...row,
|
|
154
|
+
deadline: row.deadline,
|
|
155
|
+
entities: row.entities.join(","),
|
|
156
|
+
synced_at: row.synced_at?.toISOString() ?? null,
|
|
157
|
+
tags: row.tags.join(","),
|
|
158
|
+
updated_at: row.updated_at.toISOString()
|
|
159
|
+
};
|
|
160
|
+
return this.#db.query(`INSERT INTO docs (path, hash, body, description, title, tags, entities, updated_at, synced_at, deadline)
|
|
161
|
+
VALUES($path, $hash, $body, $description, $title, $tags, $entities, $updated_at, $synced_at, $deadline)
|
|
140
162
|
ON CONFLICT(path) DO UPDATE SET
|
|
141
163
|
hash = excluded.hash,
|
|
142
164
|
body = excluded.body,
|
|
@@ -145,8 +167,9 @@ var Db = class Db {
|
|
|
145
167
|
tags = excluded.tags,
|
|
146
168
|
entities = excluded.entities,
|
|
147
169
|
updated_at = excluded.updated_at,
|
|
148
|
-
synced_at = excluded.synced_at
|
|
149
|
-
|
|
170
|
+
synced_at = excluded.synced_at,
|
|
171
|
+
deadline = excluded.deadline
|
|
172
|
+
RETURNING id`).get(raw).id;
|
|
150
173
|
}
|
|
151
174
|
deleteDoc(id, tables = {}) {
|
|
152
175
|
if (tables.vec) this.deleteEmbeddings(id);
|
|
@@ -180,12 +203,12 @@ var Db = class Db {
|
|
|
180
203
|
return this.#db.transaction(fn);
|
|
181
204
|
}
|
|
182
205
|
getUnembeddedDocs() {
|
|
183
|
-
return this.#db.query(`SELECT * FROM docs
|
|
206
|
+
return toDocRow(this.#db.query(`SELECT * FROM docs
|
|
184
207
|
WHERE vec_hash IS NULL OR vec_hash != hash
|
|
185
|
-
ORDER BY path`).all();
|
|
208
|
+
ORDER BY path`).all());
|
|
186
209
|
}
|
|
187
|
-
touchDoc(id) {
|
|
188
|
-
this.#db.query(`UPDATE docs SET synced_at = ? WHERE id = ?`).run(
|
|
210
|
+
touchDoc(id, syncedAt = /* @__PURE__ */ new Date()) {
|
|
211
|
+
this.#db.query(`UPDATE docs SET synced_at = ? WHERE id = ?`).run(syncedAt.toISOString(), id);
|
|
189
212
|
}
|
|
190
213
|
markEmbedded(id, docHash) {
|
|
191
214
|
this.#db.query(`UPDATE docs SET vec_hash = ? WHERE id = ?`).run(docHash, id);
|
|
@@ -193,7 +216,7 @@ var Db = class Db {
|
|
|
193
216
|
/** Delete docs not seen since the given sync timestamp, optionally scoped to a path prefix. */
|
|
194
217
|
deleteStaleDocs(syncedBefore, prefix) {
|
|
195
218
|
let query = `SELECT id FROM docs WHERE synced_at IS NULL OR synced_at < ?`;
|
|
196
|
-
const params = [syncedBefore];
|
|
219
|
+
const params = [syncedBefore.toISOString()];
|
|
197
220
|
if (prefix) {
|
|
198
221
|
query += ` AND path LIKE ? || '%'`;
|
|
199
222
|
params.push(prefix);
|
|
@@ -264,8 +287,14 @@ var Db = class Db {
|
|
|
264
287
|
AND k = ?
|
|
265
288
|
ORDER BY distance`).all(JSON.stringify(embedding), limit);
|
|
266
289
|
}
|
|
267
|
-
|
|
268
|
-
|
|
290
|
+
visit(doc, value) {
|
|
291
|
+
if (typeof doc === "number") {
|
|
292
|
+
const row = this.getDoc(doc);
|
|
293
|
+
if (!row) return;
|
|
294
|
+
doc = row;
|
|
295
|
+
}
|
|
296
|
+
const frecency = addVisit(doc.frecency, value);
|
|
297
|
+
this.#db.query(`UPDATE docs SET deadline = ? WHERE id = ?`).run(toDeadline(frecency), doc.id);
|
|
269
298
|
}
|
|
270
299
|
getMeta(key) {
|
|
271
300
|
return this.#db.query(`SELECT value FROM meta WHERE key = ?`).get(key)?.value;
|
|
@@ -292,3 +321,5 @@ var Db = class Db {
|
|
|
292
321
|
};
|
|
293
322
|
//#endregion
|
|
294
323
|
export { Db };
|
|
324
|
+
|
|
325
|
+
//# sourceMappingURL=db-CHpq7OOi.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db-CHpq7OOi.mjs","names":["#db","#vec"],"sources":["../src/db.ts"],"sourcesContent":["import type { EmbedderChunk } from \"./embed/index.ts\"\nimport type { FrecencyScore } from \"./frecency.ts\"\nimport type { Database } from \"./sqlite.ts\"\nimport type { StoreChunk } from \"./store.ts\"\n\nimport { addVisit, toDeadline, toScore } from \"./frecency.ts\"\nimport { openDatabase } from \"./sqlite.ts\"\n\nexport type { Database }\n\nexport type DocRow = {\n id: number\n path: string\n hash: string\n body: string\n vec_hash?: string\n description: string\n title: string\n tags: string[]\n entities: string[]\n updated_at: Date\n synced_at?: Date\n deadline: number\n frecency: number\n}\n\ntype RawDocRow = Omit<DocRow, \"updated_at\" | \"synced_at\" | \"tags\" | \"entities\" | \"frecency\"> & {\n updated_at: string\n synced_at?: string\n tags: string\n entities: string\n}\n\nfunction splitCsv(s: string): string[] {\n return s ? s.split(\",\") : []\n}\n\nfunction toDocRow(rows: RawDocRow[]): DocRow[] {\n return rows.map(\n (row) =>\n Object.assign(row, {\n entities: splitCsv(row.entities),\n frecency: toScore(row.deadline),\n synced_at: row.synced_at ? new Date(row.synced_at) : undefined,\n tags: splitCsv(row.tags),\n updated_at: new Date(row.updated_at),\n }) as unknown as DocRow\n )\n}\n\nexport type VecResult = {\n doc_id: number\n path: string\n seq: number\n distance: number\n score: number\n rank?: number\n}\n\nexport type FTSResult = {\n rowid: number\n score: number\n rank?: number\n}\n\nexport type DbSearchOptions = {\n limit?: number\n scope?: string[] // path prefixes to limit search to (e.g. [\"folder1/\", \"folder2/sub\"])\n}\n\nconst SEARCH_LIMIT = 20\nconst STOPWORD_THRESHOLD = 0.3 // terms in >50% of docs are candidates\nconst STOPWORD_MIN_DOCS = 10 // terms must be in at least 5 docs to be considered stop words\nconst STOPWORD_LIMIT = 1000 // max number of stop words to return\n\nexport function hasEmbedding<T extends EmbedderChunk>(c: T): c is T & { embedding: number[] } {\n return Array.isArray(c.embedding)\n}\n\nexport function assertEmbeddings<T extends EmbedderChunk>(\n chunks: T[]\n): asserts chunks is (T & { embedding: number[] })[] {\n for (const c of chunks) {\n if (!hasEmbedding(c)) throw new Error(`Chunk is missing embedding: ${JSON.stringify(c)}`)\n }\n}\n\nexport class Db {\n #db: Database\n #vec?: { exists: boolean; dims?: number; init?: boolean }\n\n private constructor(db: Database) {\n this.#db = db\n this.init()\n }\n\n static async load(dbPath: string) {\n return new Db(await openDatabase(dbPath))\n }\n\n private init() {\n this.#db.run(\"PRAGMA journal_mode = WAL\")\n this.#db.run(\"PRAGMA foreign_keys = ON\")\n this.#db.run(\"PRAGMA busy_timeout = 5000\")\n\n this.#db.run(`\n CREATE TABLE IF NOT EXISTS docs (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n path TEXT NOT NULL UNIQUE,\n hash TEXT NOT NULL,\n vec_hash TEXT,\n body TEXT NOT NULL DEFAULT '',\n description TEXT NOT NULL DEFAULT '',\n title TEXT NOT NULL DEFAULT '',\n tags TEXT NOT NULL DEFAULT '',\n entities TEXT NOT NULL DEFAULT '',\n updated_at TEXT NOT NULL,\n synced_at TEXT,\n deadline REAL NOT NULL DEFAULT 0\n )\n `)\n\n this.#db.run(`CREATE INDEX IF NOT EXISTS idx_docs_path ON docs(path)`)\n this.#db.run(`CREATE INDEX IF NOT EXISTS idx_docs_hash ON docs(hash)`)\n\n // Content-synced FTS5: reads content from docs table, no duplication.\n // Fields ordered by BM25 weight: entities(10), tags(8), description(5), title(3), body(1)\n this.#db.run(`\n CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(\n entities, tags, description, title, body,\n content='docs',\n content_rowid='id',\n tokenize='porter unicode61'\n )\n `)\n\n // Triggers to keep FTS in sync with docs table\n this.#db.run(`\n CREATE TRIGGER IF NOT EXISTS docs_fts_insert AFTER INSERT ON docs BEGIN\n INSERT INTO docs_fts(rowid, entities, tags, description, title, body)\n VALUES (new.id, new.entities, new.tags, new.description, new.title, new.body);\n END\n `)\n\n this.#db.run(`\n CREATE TRIGGER IF NOT EXISTS docs_fts_delete AFTER DELETE ON docs BEGIN\n INSERT INTO docs_fts(docs_fts, rowid, entities, tags, description, title, body)\n VALUES ('delete', old.id, old.entities, old.tags, old.description, old.title, old.body);\n END\n `)\n\n this.#db.run(`\n CREATE TRIGGER IF NOT EXISTS docs_fts_update AFTER UPDATE ON docs\n WHEN old.body != new.body\n OR old.title != new.title\n OR old.description != new.description\n OR old.tags != new.tags\n OR old.entities != new.entities\n BEGIN\n INSERT INTO docs_fts(docs_fts, rowid, entities, tags, description, title, body)\n VALUES ('delete', old.id, old.entities, old.tags, old.description, old.title, old.body);\n INSERT INTO docs_fts(rowid, entities, tags, description, title, body)\n VALUES (new.id, new.entities, new.tags, new.description, new.title, new.body);\n END\n `)\n\n // FTS5 vocabulary table for IDF-based term weighting\n this.#db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab USING fts5vocab('docs_fts', 'row')`)\n\n this.#db.run(`\n CREATE TABLE IF NOT EXISTS meta (\n key TEXT PRIMARY KEY,\n value TEXT\n )\n `)\n\n this.#db.run(`\n CREATE TABLE IF NOT EXISTS cache (\n key TEXT PRIMARY KEY,\n value TEXT NOT NULL,\n accessed_at TEXT NOT NULL\n )\n `)\n }\n\n reset() {\n // Drop triggers first, then FTS (which references docs), then docs\n this.#db.run(`DROP TRIGGER IF EXISTS docs_fts_insert`)\n this.#db.run(`DROP TRIGGER IF EXISTS docs_fts_delete`)\n this.#db.run(`DROP TRIGGER IF EXISTS docs_fts_update`)\n this.#db.run(`DROP TABLE IF EXISTS docs_fts`)\n this.#db.run(`DROP TABLE IF EXISTS vec`)\n this.#db.run(`DROP TABLE IF EXISTS cache`)\n this.#db.run(`DROP TABLE IF EXISTS docs`)\n this.#db.run(`DROP TABLE IF EXISTS meta`)\n this.#db.run(`VACUUM`)\n this.#vec = { exists: false }\n this.init()\n }\n\n private initVec(dims: number) {\n if (this.vec.init) return\n const existingDims = this.vec.dims\n if (existingDims && existingDims !== dims)\n throw new Error(\n `Vector dimension mismatch: existing **vec** has \\`${existingDims}\\` dims, but got \\`${dims}\\`.\\n` +\n `Run \\`rekal reset\\` and \\`rekal sync\\` to recreate with the correct dimensions.`\n )\n this.#db.run(\n `CREATE VIRTUAL TABLE IF NOT EXISTS vec USING vec0(\n doc_id INTEGER NOT NULL,\n seq INTEGER NOT NULL,\n +path TEXT NOT NULL,\n embedding float[${dims}] distance_metric=cosine\n )`\n )\n this.#vec = { dims, exists: true, init: true }\n }\n\n // --- Docs ---\n\n getDoc(from: string | number) {\n const field = typeof from === \"number\" ? \"id\" : \"path\"\n const row = this.#db.query(`SELECT * FROM docs WHERE ${field} = ?`).get(from) as\n | RawDocRow\n | undefined\n return row ? toDocRow([row])[0] : undefined\n }\n\n getDocs(from?: (string | number)[]) {\n let ret: DocRow[]\n\n if (!from) ret = toDocRow(this.#db.query(`SELECT * FROM docs`).all() as RawDocRow[])\n else {\n const field = typeof from[0] === \"number\" ? \"id\" : \"path\"\n const placeholders = from.map(() => \"?\").join(\",\")\n ret = toDocRow(\n this.#db\n .query(`SELECT * FROM docs WHERE ${field} IN (${placeholders})`)\n .all(...from) as RawDocRow[]\n )\n }\n return new Map(ret.map((row) => [row.id, row]))\n }\n\n addDoc(row: Omit<DocRow, \"id\" | \"frecency\">) {\n const raw = {\n ...row,\n deadline: row.deadline,\n entities: row.entities.join(\",\"),\n // oxlint-disable-next-line unicorn/no-null\n synced_at: row.synced_at?.toISOString() ?? null,\n tags: row.tags.join(\",\"),\n updated_at: row.updated_at.toISOString(),\n }\n const result = this.#db\n .query(\n `INSERT INTO docs (path, hash, body, description, title, tags, entities, updated_at, synced_at, deadline)\n VALUES($path, $hash, $body, $description, $title, $tags, $entities, $updated_at, $synced_at, $deadline)\n ON CONFLICT(path) DO UPDATE SET\n hash = excluded.hash,\n body = excluded.body,\n description = excluded.description,\n title = excluded.title,\n tags = excluded.tags,\n entities = excluded.entities,\n updated_at = excluded.updated_at,\n synced_at = excluded.synced_at,\n deadline = excluded.deadline\n RETURNING id`\n )\n .get(raw) as { id: number }\n return result.id\n }\n\n deleteDoc(id: number, tables: { docs?: boolean; vec?: boolean } = {}) {\n // FTS is auto-synced via triggers when docs are deleted/updated\n if (tables.vec) this.deleteEmbeddings(id)\n if (tables.docs) this.#db.query(`DELETE FROM docs WHERE id = ?`).run(id)\n }\n\n get vec() {\n if (this.#vec) return this.#vec\n const row = this.#db\n .query(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'vec'`)\n .get() as { sql: string } | undefined\n const match = row?.sql.match(/embedding float\\[(\\d+)\\]/)\n this.#vec = { dims: match ? parseInt(match[1]) : undefined, exists: !!row?.sql }\n return this.#vec\n }\n\n getStatus() {\n const count = (sql: string) => (this.#db.query(sql).get() as { n: number }).n\n return {\n cache: count(`SELECT count(*) as n FROM cache`),\n dbSize: (\n this.#db\n .query(`SELECT page_count * page_size as n FROM pragma_page_count, pragma_page_size`)\n .get() as { n: number }\n ).n,\n docs: count(`SELECT count(*) as n FROM docs`),\n docsWithDescription: count(`SELECT count(*) as n FROM docs WHERE description != ''`),\n lastSync: (\n this.#db.query(`SELECT max(synced_at) as t FROM docs`).get() as { t: string | null }\n ).t,\n unembedded: count(\n `SELECT count(*) as n FROM docs WHERE vec_hash IS NULL OR vec_hash != hash`\n ),\n vecDims: this.vec.dims,\n vecs: this.vec.exists ? count(`SELECT count(*) as n FROM vec`) : 0,\n vocabTerms: count(`SELECT count(DISTINCT term) as n FROM docs_vocab`),\n }\n }\n\n transaction<A extends any[], T>(fn: (...args: A) => T) {\n return this.#db.transaction(fn)\n }\n\n getUnembeddedDocs(): DocRow[] {\n return toDocRow(\n this.#db\n .query(`SELECT * FROM docs\n WHERE vec_hash IS NULL OR vec_hash != hash\n ORDER BY path`)\n .all() as RawDocRow[]\n )\n }\n\n touchDoc(id: number, syncedAt = new Date()) {\n this.#db.query(`UPDATE docs SET synced_at = ? WHERE id = ?`).run(syncedAt.toISOString(), id)\n }\n\n markEmbedded(id: number, docHash: string) {\n this.#db.query(`UPDATE docs SET vec_hash = ? WHERE id = ?`).run(docHash, id)\n }\n\n /** Delete docs not seen since the given sync timestamp, optionally scoped to a path prefix. */\n deleteStaleDocs(syncedBefore: Date, prefix?: string): number {\n let query = `SELECT id FROM docs WHERE synced_at IS NULL OR synced_at < ?`\n const params = [syncedBefore.toISOString()]\n if (prefix) {\n query += ` AND path LIKE ? || '%'`\n params.push(prefix)\n }\n const stale = this.#db.query(query).all(...params) as { id: number }[]\n for (const { id } of stale) {\n this.deleteDoc(id, { docs: true, vec: true })\n }\n return stale.length\n }\n\n // --- FTS ---\n // FTS is auto-synced via triggers on the docs table.\n\n /** Scoped FTS search: only match docs whose path starts with one of the given prefixes */\n searchFts(query: string, opts?: DbSearchOptions): FTSResult[] {\n if (opts?.scope?.length === 0) return [] // empty scope means no results\n const scope = opts?.scope ?? []\n const scopeQuery =\n scope.length === 0 ? \"\" : `AND (${scope.map(() => `d.path LIKE ? || '%'`).join(\" OR \")})`\n return this.#db\n .query(\n `SELECT f.rowid, bm25(docs_fts, 10, 8, 5, 3, 1) as score\n FROM docs_fts f\n ${scope.length > 0 ? \"JOIN docs d ON d.id = f.rowid\" : \"\"}\n WHERE docs_fts MATCH ?\n ${scopeQuery}\n ORDER BY score\n LIMIT ?`\n )\n .all(query, ...scope, opts?.limit ?? SEARCH_LIMIT) as FTSResult[]\n }\n\n /** * Gets weights for high-frequency terms.\n * Note: Truly common words will result in an IDF of 0 or less.\n */\n getStopWords(): Map<string, number> {\n // 1. Get total doc count (N) first\n const totalDocs =\n (this.#db.query(\"SELECT count(*) as n FROM docs\").get() as { n: number } | undefined)?.n ?? 0\n\n if (totalDocs === 0) return new Map()\n\n // 2. Fetch the high-frequency terms\n const rows = this.#db\n .query(\n `SELECT v.term, v.doc \n FROM docs_vocab v\n WHERE v.doc > ? AND v.doc > ?\n ORDER BY v.doc DESC\n LIMIT ?`\n )\n .all(totalDocs * STOPWORD_THRESHOLD, STOPWORD_MIN_DOCS, STOPWORD_LIMIT) as {\n term: string\n doc: number\n }[]\n\n return new Map(\n rows.map((r) => {\n // Calculate IDF\n const idf = Math.log((totalDocs - r.doc + 0.5) / (r.doc + 0.5))\n\n // For stop words, we usually want to clamp at 0.\n // If a word is in >50% of docs, the formula goes negative.\n return [r.term, Math.max(0, idf)]\n })\n )\n }\n\n getWeights(terms: string[]): number[] {\n if (terms.length === 0) return []\n const total = (this.#db.query(`SELECT count(*) as n FROM docs`).get() as { n: number }).n\n const placeholders = terms.map(() => \"?\").join(\",\")\n const rows = this.#db\n .query(`SELECT term, doc FROM docs_vocab WHERE term IN (${placeholders})`)\n .all(...terms) as { term: string; doc: number }[]\n const df = new Map(rows.map((r) => [r.term, r.doc]))\n return terms.map((t) => Math.log((total - (df.get(t) ?? 0) + 0.5) / ((df.get(t) ?? 0) + 0.5)))\n }\n\n // --- Vector ---\n\n /** Insert embeddings into the vec table */\n insertEmbeddings(chunks: StoreChunk[]) {\n assertEmbeddings(chunks)\n if (chunks.length === 0) return\n this.initVec(chunks[0].embedding.length)\n const stmt = this.#db.query(`INSERT INTO vec(doc_id, seq, path, embedding) VALUES (?, ?, ?, ?)`)\n for (const chunk of chunks) {\n stmt.run(chunk.doc_id, chunk.seq, chunk.doc.path, JSON.stringify(chunk.embedding))\n }\n }\n\n /** Delete all vec entries for a doc */\n deleteEmbeddings(docId: number) {\n if (this.vec.exists) this.#db.query(`DELETE FROM vec WHERE doc_id = ?`).run(docId)\n }\n\n /** Global KNN search, returns top results across all docs */\n searchVec(embedding: number[], opts?: DbSearchOptions): VecResult[] {\n if (!this.vec.exists) return []\n const limit = opts?.limit ?? SEARCH_LIMIT\n return this.#db\n .query(\n `SELECT doc_id, seq, path, distance, (1 - distance/2) as score\n FROM vec\n WHERE embedding MATCH ?\n AND k = ?\n ORDER BY distance`\n )\n .all(JSON.stringify(embedding), limit) as VecResult[]\n }\n\n // --- Frecency ---\n\n visit(doc: DocRow | number, value?: FrecencyScore | number) {\n if (typeof doc === \"number\") {\n const row = this.getDoc(doc)\n if (!row) return\n doc = row\n }\n const frecency = addVisit(doc.frecency, value)\n this.#db.query(`UPDATE docs SET deadline = ? WHERE id = ?`).run(toDeadline(frecency), doc.id)\n }\n\n // --- Meta ---\n\n getMeta(key: string) {\n return (\n this.#db.query(`SELECT value FROM meta WHERE key = ?`).get(key) as\n | { value: string }\n | undefined\n )?.value\n }\n\n setMeta(key: string, value: string) {\n this.#db\n .query(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?`)\n .run(key, value, value)\n }\n\n // --- Cache ---\n\n cacheGet<T>(key: string): T | undefined {\n const row = this.#db.query(`SELECT value FROM cache WHERE key = ?`).get(key) as\n | { value: string }\n | undefined\n if (!row) return\n this.#db\n .query(`UPDATE cache SET accessed_at = ? WHERE key = ?`)\n .run(new Date().toISOString(), key)\n return JSON.parse(row.value) as T\n }\n\n cacheSet<T>(key: string, value: T): T {\n this.#db\n .query(\n `INSERT INTO cache (key, value, accessed_at) VALUES (?, ?, ?)\n ON CONFLICT(key) DO UPDATE SET value = excluded.value, accessed_at = excluded.accessed_at`\n )\n .run(key, JSON.stringify(value), new Date().toISOString())\n return value\n }\n\n cachePrune(maxEntries = 10_000) {\n this.#db\n .query(\n `DELETE FROM cache WHERE key NOT IN (\n SELECT key FROM cache ORDER BY accessed_at DESC LIMIT ?\n )`\n )\n .run(maxEntries)\n }\n}\n"],"mappings":";;;AAiCA,SAAS,SAAS,GAAqB;AACrC,QAAO,IAAI,EAAE,MAAM,IAAI,GAAG,EAAE;;AAG9B,SAAS,SAAS,MAA6B;AAC7C,QAAO,KAAK,KACT,QACC,OAAO,OAAO,KAAK;EACjB,UAAU,SAAS,IAAI,SAAS;EAChC,UAAU,QAAQ,IAAI,SAAS;EAC/B,WAAW,IAAI,YAAY,IAAI,KAAK,IAAI,UAAU,GAAG,KAAA;EACrD,MAAM,SAAS,IAAI,KAAK;EACxB,YAAY,IAAI,KAAK,IAAI,WAAW;EACrC,CAAC,CACL;;AAuBH,MAAM,eAAe;AACrB,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,iBAAiB;AAEvB,SAAgB,aAAsC,GAAwC;AAC5F,QAAO,MAAM,QAAQ,EAAE,UAAU;;AAGnC,SAAgB,iBACd,QACmD;AACnD,MAAK,MAAM,KAAK,OACd,KAAI,CAAC,aAAa,EAAE,CAAE,OAAM,IAAI,MAAM,+BAA+B,KAAK,UAAU,EAAE,GAAG;;AAI7F,IAAa,KAAb,MAAa,GAAG;CACd;CACA;CAEA,YAAoB,IAAc;AAChC,QAAA,KAAW;AACX,OAAK,MAAM;;CAGb,aAAa,KAAK,QAAgB;AAChC,SAAO,IAAI,GAAG,MAAM,aAAa,OAAO,CAAC;;CAG3C,OAAe;AACb,QAAA,GAAS,IAAI,4BAA4B;AACzC,QAAA,GAAS,IAAI,2BAA2B;AACxC,QAAA,GAAS,IAAI,6BAA6B;AAE1C,QAAA,GAAS,IAAI;;;;;;;;;;;;;;;MAeX;AAEF,QAAA,GAAS,IAAI,yDAAyD;AACtE,QAAA,GAAS,IAAI,yDAAyD;AAItE,QAAA,GAAS,IAAI;;;;;;;MAOX;AAGF,QAAA,GAAS,IAAI;;;;;MAKX;AAEF,QAAA,GAAS,IAAI;;;;;MAKX;AAEF,QAAA,GAAS,IAAI;;;;;;;;;;;;;MAaX;AAGF,QAAA,GAAS,IAAI,mFAAmF;AAEhG,QAAA,GAAS,IAAI;;;;;MAKX;AAEF,QAAA,GAAS,IAAI;;;;;;MAMX;;CAGJ,QAAQ;AAEN,QAAA,GAAS,IAAI,yCAAyC;AACtD,QAAA,GAAS,IAAI,yCAAyC;AACtD,QAAA,GAAS,IAAI,yCAAyC;AACtD,QAAA,GAAS,IAAI,gCAAgC;AAC7C,QAAA,GAAS,IAAI,2BAA2B;AACxC,QAAA,GAAS,IAAI,6BAA6B;AAC1C,QAAA,GAAS,IAAI,4BAA4B;AACzC,QAAA,GAAS,IAAI,4BAA4B;AACzC,QAAA,GAAS,IAAI,SAAS;AACtB,QAAA,MAAY,EAAE,QAAQ,OAAO;AAC7B,OAAK,MAAM;;CAGb,QAAgB,MAAc;AAC5B,MAAI,KAAK,IAAI,KAAM;EACnB,MAAM,eAAe,KAAK,IAAI;AAC9B,MAAI,gBAAgB,iBAAiB,KACnC,OAAM,IAAI,MACR,qDAAqD,aAAa,qBAAqB,KAAK,sFAE7F;AACH,QAAA,GAAS,IACP;;;;0BAIoB,KAAK;SAE1B;AACD,QAAA,MAAY;GAAE;GAAM,QAAQ;GAAM,MAAM;GAAM;;CAKhD,OAAO,MAAuB;EAC5B,MAAM,QAAQ,OAAO,SAAS,WAAW,OAAO;EAChD,MAAM,MAAM,MAAA,GAAS,MAAM,4BAA4B,MAAM,MAAM,CAAC,IAAI,KAAK;AAG7E,SAAO,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,KAAA;;CAGpC,QAAQ,MAA4B;EAClC,IAAI;AAEJ,MAAI,CAAC,KAAM,OAAM,SAAS,MAAA,GAAS,MAAM,qBAAqB,CAAC,KAAK,CAAgB;OAC/E;GACH,MAAM,QAAQ,OAAO,KAAK,OAAO,WAAW,OAAO;GACnD,MAAM,eAAe,KAAK,UAAU,IAAI,CAAC,KAAK,IAAI;AAClD,SAAM,SACJ,MAAA,GACG,MAAM,4BAA4B,MAAM,OAAO,aAAa,GAAG,CAC/D,IAAI,GAAG,KAAK,CAChB;;AAEH,SAAO,IAAI,IAAI,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;;CAGjD,OAAO,KAAsC;EAC3C,MAAM,MAAM;GACV,GAAG;GACH,UAAU,IAAI;GACd,UAAU,IAAI,SAAS,KAAK,IAAI;GAEhC,WAAW,IAAI,WAAW,aAAa,IAAI;GAC3C,MAAM,IAAI,KAAK,KAAK,IAAI;GACxB,YAAY,IAAI,WAAW,aAAa;GACzC;AAkBD,SAjBe,MAAA,GACZ,MACC;;;;;;;;;;;;sBAaD,CACA,IAAI,IAAI,CACG;;CAGhB,UAAU,IAAY,SAA4C,EAAE,EAAE;AAEpE,MAAI,OAAO,IAAK,MAAK,iBAAiB,GAAG;AACzC,MAAI,OAAO,KAAM,OAAA,GAAS,MAAM,gCAAgC,CAAC,IAAI,GAAG;;CAG1E,IAAI,MAAM;AACR,MAAI,MAAA,IAAW,QAAO,MAAA;EACtB,MAAM,MAAM,MAAA,GACT,MAAM,sEAAsE,CAC5E,KAAK;EACR,MAAM,QAAQ,KAAK,IAAI,MAAM,2BAA2B;AACxD,QAAA,MAAY;GAAE,MAAM,QAAQ,SAAS,MAAM,GAAG,GAAG,KAAA;GAAW,QAAQ,CAAC,CAAC,KAAK;GAAK;AAChF,SAAO,MAAA;;CAGT,YAAY;EACV,MAAM,SAAS,QAAiB,MAAA,GAAS,MAAM,IAAI,CAAC,KAAK,CAAmB;AAC5E,SAAO;GACL,OAAO,MAAM,kCAAkC;GAC/C,QACE,MAAA,GACG,MAAM,8EAA8E,CACpF,KAAK,CACR;GACF,MAAM,MAAM,iCAAiC;GAC7C,qBAAqB,MAAM,yDAAyD;GACpF,UACE,MAAA,GAAS,MAAM,uCAAuC,CAAC,KAAK,CAC5D;GACF,YAAY,MACV,4EACD;GACD,SAAS,KAAK,IAAI;GAClB,MAAM,KAAK,IAAI,SAAS,MAAM,gCAAgC,GAAG;GACjE,YAAY,MAAM,mDAAmD;GACtE;;CAGH,YAAgC,IAAuB;AACrD,SAAO,MAAA,GAAS,YAAY,GAAG;;CAGjC,oBAA8B;AAC5B,SAAO,SACL,MAAA,GACG,MAAM;;uBAEQ,CACd,KAAK,CACT;;CAGH,SAAS,IAAY,2BAAW,IAAI,MAAM,EAAE;AAC1C,QAAA,GAAS,MAAM,6CAA6C,CAAC,IAAI,SAAS,aAAa,EAAE,GAAG;;CAG9F,aAAa,IAAY,SAAiB;AACxC,QAAA,GAAS,MAAM,4CAA4C,CAAC,IAAI,SAAS,GAAG;;;CAI9E,gBAAgB,cAAoB,QAAyB;EAC3D,IAAI,QAAQ;EACZ,MAAM,SAAS,CAAC,aAAa,aAAa,CAAC;AAC3C,MAAI,QAAQ;AACV,YAAS;AACT,UAAO,KAAK,OAAO;;EAErB,MAAM,QAAQ,MAAA,GAAS,MAAM,MAAM,CAAC,IAAI,GAAG,OAAO;AAClD,OAAK,MAAM,EAAE,QAAQ,MACnB,MAAK,UAAU,IAAI;GAAE,MAAM;GAAM,KAAK;GAAM,CAAC;AAE/C,SAAO,MAAM;;;CAOf,UAAU,OAAe,MAAqC;AAC5D,MAAI,MAAM,OAAO,WAAW,EAAG,QAAO,EAAE;EACxC,MAAM,QAAQ,MAAM,SAAS,EAAE;EAC/B,MAAM,aACJ,MAAM,WAAW,IAAI,KAAK,QAAQ,MAAM,UAAU,uBAAuB,CAAC,KAAK,OAAO,CAAC;AACzF,SAAO,MAAA,GACJ,MACC;;UAEE,MAAM,SAAS,IAAI,kCAAkC,GAAG;;YAEtD,WAAW;;iBAGhB,CACA,IAAI,OAAO,GAAG,OAAO,MAAM,SAAS,aAAa;;;;;CAMtD,eAAoC;EAElC,MAAM,YACH,MAAA,GAAS,MAAM,iCAAiC,CAAC,KAAK,EAAgC,KAAK;AAE9F,MAAI,cAAc,EAAG,wBAAO,IAAI,KAAK;EAGrC,MAAM,OAAO,MAAA,GACV,MACC;;;;gBAKD,CACA,IAAI,YAAY,oBAAoB,mBAAmB,eAAe;AAKzE,SAAO,IAAI,IACT,KAAK,KAAK,MAAM;GAEd,MAAM,MAAM,KAAK,KAAK,YAAY,EAAE,MAAM,OAAQ,EAAE,MAAM,IAAK;AAI/D,UAAO,CAAC,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,CAAC;IACjC,CACH;;CAGH,WAAW,OAA2B;AACpC,MAAI,MAAM,WAAW,EAAG,QAAO,EAAE;EACjC,MAAM,QAAS,MAAA,GAAS,MAAM,iCAAiC,CAAC,KAAK,CAAmB;EACxF,MAAM,eAAe,MAAM,UAAU,IAAI,CAAC,KAAK,IAAI;EACnD,MAAM,OAAO,MAAA,GACV,MAAM,mDAAmD,aAAa,GAAG,CACzE,IAAI,GAAG,MAAM;EAChB,MAAM,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AACpD,SAAO,MAAM,KAAK,MAAM,KAAK,KAAK,SAAS,GAAG,IAAI,EAAE,IAAI,KAAK,QAAS,GAAG,IAAI,EAAE,IAAI,KAAK,IAAK,CAAC;;;CAMhG,iBAAiB,QAAsB;AACrC,mBAAiB,OAAO;AACxB,MAAI,OAAO,WAAW,EAAG;AACzB,OAAK,QAAQ,OAAO,GAAG,UAAU,OAAO;EACxC,MAAM,OAAO,MAAA,GAAS,MAAM,oEAAoE;AAChG,OAAK,MAAM,SAAS,OAClB,MAAK,IAAI,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,UAAU,MAAM,UAAU,CAAC;;;CAKtF,iBAAiB,OAAe;AAC9B,MAAI,KAAK,IAAI,OAAQ,OAAA,GAAS,MAAM,mCAAmC,CAAC,IAAI,MAAM;;;CAIpF,UAAU,WAAqB,MAAqC;AAClE,MAAI,CAAC,KAAK,IAAI,OAAQ,QAAO,EAAE;EAC/B,MAAM,QAAQ,MAAM,SAAS;AAC7B,SAAO,MAAA,GACJ,MACC;;;;2BAKD,CACA,IAAI,KAAK,UAAU,UAAU,EAAE,MAAM;;CAK1C,MAAM,KAAsB,OAAgC;AAC1D,MAAI,OAAO,QAAQ,UAAU;GAC3B,MAAM,MAAM,KAAK,OAAO,IAAI;AAC5B,OAAI,CAAC,IAAK;AACV,SAAM;;EAER,MAAM,WAAW,SAAS,IAAI,UAAU,MAAM;AAC9C,QAAA,GAAS,MAAM,4CAA4C,CAAC,IAAI,WAAW,SAAS,EAAE,IAAI,GAAG;;CAK/F,QAAQ,KAAa;AACnB,SACE,MAAA,GAAS,MAAM,uCAAuC,CAAC,IAAI,IAAI,EAG9D;;CAGL,QAAQ,KAAa,OAAe;AAClC,QAAA,GACG,MAAM,uFAAuF,CAC7F,IAAI,KAAK,OAAO,MAAM;;CAK3B,SAAY,KAA4B;EACtC,MAAM,MAAM,MAAA,GAAS,MAAM,wCAAwC,CAAC,IAAI,IAAI;AAG5E,MAAI,CAAC,IAAK;AACV,QAAA,GACG,MAAM,iDAAiD,CACvD,qBAAI,IAAI,MAAM,EAAC,aAAa,EAAE,IAAI;AACrC,SAAO,KAAK,MAAM,IAAI,MAAM;;CAG9B,SAAY,KAAa,OAAa;AACpC,QAAA,GACG,MACC;mGAED,CACA,IAAI,KAAK,KAAK,UAAU,MAAM,mBAAE,IAAI,MAAM,EAAC,aAAa,CAAC;AAC5D,SAAO;;CAGT,WAAW,aAAa,KAAQ;AAC9B,QAAA,GACG,MACC;;WAGD,CACA,IAAI,WAAW"}
|
package/dist/doc-DnYN4jAU.mjs
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doc-DnYN4jAU.mjs","names":["#name","#isDir"],"sources":["../src/doc.ts"],"sourcesContent":["import type { Frontmatter, MarkdownDoc } from \"./md.ts\"\nimport type { VfsEntry } from \"./vfs.ts\"\n\nimport { readFile } from \"node:fs/promises\"\nimport { basename, join, resolve } from \"pathe\"\nimport { astat } from \"./fs.ts\"\nimport { parseMarkdown } from \"./md.ts\"\nimport { normUri } from \"./uri.ts\"\nimport { hash } from \"./util.ts\"\n\nconst INDEX = \"index.md\"\nconst MAX_DESC_LENGTH = 30 * 4 // roughly 30 tokens\n\nexport type DocFrontmatter = {\n description?: string\n tags?: string[]\n entities?: string[]\n} & Frontmatter\n\ntype DocHeading = {\n level: number\n text: string\n}\n\nexport class Doc {\n #isDir = false\n #name = \"\"\n hash = \"\"\n updated = new Date(0)\n headings: DocHeading[] = []\n parsed: MarkdownDoc = { body: \"\", bodyOffset: 0, frontmatter: {}, sections: [], text: \"\" }\n\n protected constructor(\n public uri: string,\n public path: string\n ) {\n this.path = resolve(path)\n this.#name = basename(this.path, \".md\")\n }\n\n /** Full original markdown text, including frontmatter. */\n get text() {\n return this.parsed.text\n }\n\n /** Markdown body without frontmatter. */\n get body() {\n return this.parsed.body\n }\n\n /** Parsed frontmatter as an object. */\n get fm() {\n return this.parsed.frontmatter as DocFrontmatter\n }\n\n /** Name of the doc, derived from the file or folder name, without extension */\n get name(): string {\n return this.#name\n }\n\n // Actual (non-empty) description from frontmatter\n get $description() {\n const ret = this.fm.description?.trim()\n return ret?.length ? ret : undefined\n }\n\n /** Desc from frontmatter or packed from headings. */\n get description() {\n const desc = this.$description\n if (desc || this.headings.length === 0) return desc\n\n const headings: (DocHeading & { used?: boolean })[] = this.headings.map((h) => ({ ...h }))\n const minLevel = Math.min(...headings.map((h) => h.level))\n const maxLevel = Math.max(...headings.map((h) => h.level))\n\n // Pack by level until we reach maximum description length\n let chars = 0\n for (let level = minLevel; level <= maxLevel; level++) {\n for (const h of headings) {\n if (h.level !== level) continue\n if (chars !== 0 && chars + h.text.length > MAX_DESC_LENGTH) continue\n h.used = true\n chars += h.text.length\n }\n }\n\n return headings\n .filter((h) => h.used)\n .map((h) => h.text)\n .join(\", \")\n .trim()\n }\n\n /** Title from fontmatter or first heading */\n get $title(): string | undefined {\n const title = this.fm.title\n if (typeof title === \"string\" && title.trim().length > 0) return title.trim()\n return this.headings[0]?.text\n }\n\n /** `$title` if it doesn't contain the name, otherwise `name - $title` */\n get title() {\n const title = this.$title\n if (!(title ?? \"\").length) return this.name\n return title?.toLowerCase().includes(this.name.toLowerCase())\n ? title\n : `${this.name} - ${title}`\n }\n\n get tags(): string[] {\n return this.fm.tags ?? []\n }\n\n get entities(): string[] {\n return this.fm.entities ?? []\n }\n\n get isDir(): boolean {\n return this.#isDir\n }\n\n protected async load(): Promise<Doc | undefined> {\n const name = basename(this.path)\n\n // Quick validation. It's up to the caller to ensure the path\n // is either a markdown file or a directory.\n if (name === INDEX) throw new Error(`Doc path cannot end with \\`${INDEX}\\`:\\n\\`${this.path}\\``)\n\n let s = await astat(this.path)\n let mdPath = this.path\n\n if (s?.isDirectory()) {\n this.#isDir = true\n mdPath = join(this.path, INDEX)\n s = await astat(mdPath)\n }\n\n if (!s && !this.#isDir) return\n\n this.uri = normUri(this.uri, this.isDir)\n\n // read file and normalize line endings to LF\n const text = (s ? await readFile(mdPath, \"utf8\") : \"\").replace(/\\r\\n/g, \"\\n\")\n\n this.updated = s?.mtime ?? new Date(0)\n this.hash = hash(text)\n this.parsed = parseMarkdown(text)\n this.headings = this.parsed.sections\n .filter((section) => section.level > 0 && section.heading.trim().length > 0)\n .map((section) => ({\n level: section.level,\n text: section.heading,\n }))\n return this\n }\n\n static async load(entry: string | VfsEntry): Promise<Doc | undefined>\n static async load(uri: string, path?: string): Promise<Doc | undefined>\n static async load(uri: string | VfsEntry, path?: string): Promise<Doc | undefined> {\n const e = typeof uri === \"string\" ? { path: path, uri } : uri\n return e.path ? await new Doc(e.uri, e.path).load() : undefined\n }\n}\n"],"mappings":";;;;;;;AAUA,MAAM,QAAQ;AACd,MAAM,kBAAkB;AAaxB,IAAa,MAAb,MAAa,IAAI;CACf,SAAS;CACT,QAAQ;CACR,OAAO;CACP,0BAAU,IAAI,KAAK,EAAE;CACrB,WAAyB,EAAE;CAC3B,SAAsB;EAAE,MAAM;EAAI,YAAY;EAAG,aAAa,EAAE;EAAE,UAAU,EAAE;EAAE,MAAM;EAAI;CAE1F,YACE,KACA,MACA;AAFO,OAAA,MAAA;AACA,OAAA,OAAA;AAEP,OAAK,OAAO,QAAQ,KAAK;AACzB,QAAA,OAAa,SAAS,KAAK,MAAM,MAAM;;;CAIzC,IAAI,OAAO;AACT,SAAO,KAAK,OAAO;;;CAIrB,IAAI,OAAO;AACT,SAAO,KAAK,OAAO;;;CAIrB,IAAI,KAAK;AACP,SAAO,KAAK,OAAO;;;CAIrB,IAAI,OAAe;AACjB,SAAO,MAAA;;CAIT,IAAI,eAAe;EACjB,MAAM,MAAM,KAAK,GAAG,aAAa,MAAM;AACvC,SAAO,KAAK,SAAS,MAAM,KAAA;;;CAI7B,IAAI,cAAc;EAChB,MAAM,OAAO,KAAK;AAClB,MAAI,QAAQ,KAAK,SAAS,WAAW,EAAG,QAAO;EAE/C,MAAM,WAAgD,KAAK,SAAS,KAAK,OAAO,EAAE,GAAG,GAAG,EAAE;EAC1F,MAAM,WAAW,KAAK,IAAI,GAAG,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC;EAC1D,MAAM,WAAW,KAAK,IAAI,GAAG,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC;EAG1D,IAAI,QAAQ;AACZ,OAAK,IAAI,QAAQ,UAAU,SAAS,UAAU,QAC5C,MAAK,MAAM,KAAK,UAAU;AACxB,OAAI,EAAE,UAAU,MAAO;AACvB,OAAI,UAAU,KAAK,QAAQ,EAAE,KAAK,SAAS,gBAAiB;AAC5D,KAAE,OAAO;AACT,YAAS,EAAE,KAAK;;AAIpB,SAAO,SACJ,QAAQ,MAAM,EAAE,KAAK,CACrB,KAAK,MAAM,EAAE,KAAK,CAClB,KAAK,KAAK,CACV,MAAM;;;CAIX,IAAI,SAA6B;EAC/B,MAAM,QAAQ,KAAK,GAAG;AACtB,MAAI,OAAO,UAAU,YAAY,MAAM,MAAM,CAAC,SAAS,EAAG,QAAO,MAAM,MAAM;AAC7E,SAAO,KAAK,SAAS,IAAI;;;CAI3B,IAAI,QAAQ;EACV,MAAM,QAAQ,KAAK;AACnB,MAAI,EAAE,SAAS,IAAI,OAAQ,QAAO,KAAK;AACvC,SAAO,OAAO,aAAa,CAAC,SAAS,KAAK,KAAK,aAAa,CAAC,GACzD,QACA,GAAG,KAAK,KAAK,KAAK;;CAGxB,IAAI,OAAiB;AACnB,SAAO,KAAK,GAAG,QAAQ,EAAE;;CAG3B,IAAI,WAAqB;AACvB,SAAO,KAAK,GAAG,YAAY,EAAE;;CAG/B,IAAI,QAAiB;AACnB,SAAO,MAAA;;CAGT,MAAgB,OAAiC;AAK/C,MAJa,SAAS,KAAK,KAAK,KAInB,MAAO,OAAM,IAAI,MAAM,8BAA8B,MAAM,SAAS,KAAK,KAAK,IAAI;EAE/F,IAAI,IAAI,MAAM,MAAM,KAAK,KAAK;EAC9B,IAAI,SAAS,KAAK;AAElB,MAAI,GAAG,aAAa,EAAE;AACpB,SAAA,QAAc;AACd,YAAS,KAAK,KAAK,MAAM,MAAM;AAC/B,OAAI,MAAM,MAAM,OAAO;;AAGzB,MAAI,CAAC,KAAK,CAAC,MAAA,MAAa;AAExB,OAAK,MAAM,QAAQ,KAAK,KAAK,KAAK,MAAM;EAGxC,MAAM,QAAQ,IAAI,MAAM,SAAS,QAAQ,OAAO,GAAG,IAAI,QAAQ,SAAS,KAAK;AAE7E,OAAK,UAAU,GAAG,yBAAS,IAAI,KAAK,EAAE;AACtC,OAAK,OAAO,KAAK,KAAK;AACtB,OAAK,SAAS,cAAc,KAAK;AACjC,OAAK,WAAW,KAAK,OAAO,SACzB,QAAQ,YAAY,QAAQ,QAAQ,KAAK,QAAQ,QAAQ,MAAM,CAAC,SAAS,EAAE,CAC3E,KAAK,aAAa;GACjB,OAAO,QAAQ;GACf,MAAM,QAAQ;GACf,EAAE;AACL,SAAO;;CAKT,aAAa,KAAK,KAAwB,MAAyC;EACjF,MAAM,IAAI,OAAO,QAAQ,WAAW;GAAQ;GAAM;GAAK,GAAG;AAC1D,SAAO,EAAE,OAAO,MAAM,IAAI,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,KAAA"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as chunkMarkdown, t as Progress } from "./progress-B1JdNapX.mjs";
|
|
2
|
-
import { n as parseModelUri, r as resolveModel, t as loadModel } from "./models-
|
|
2
|
+
import { n as parseModelUri, r as resolveModel, t as loadModel } from "./models-Bo6czhQe.mjs";
|
|
3
3
|
import { availableParallelism } from "node:os";
|
|
4
4
|
//#region src/embed/base.ts
|
|
5
5
|
const defaults = {
|
|
@@ -98,3 +98,5 @@ var Embedder = class {
|
|
|
98
98
|
};
|
|
99
99
|
//#endregion
|
|
100
100
|
export { Embedder };
|
|
101
|
+
|
|
102
|
+
//# sourceMappingURL=embed-CZI5Dz1q.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed-CZI5Dz1q.mjs","names":["#loading","#backend"],"sources":["../src/embed/base.ts"],"sourcesContent":["import type { Context } from \"../context.ts\"\nimport type {\n EmbedderBackend,\n EmbedderChunk,\n EmbedderDoc,\n EmbedderOptions,\n ModelBackend,\n ResolvedEmbedderModel,\n ResolvedEmbedderOptions,\n} from \"./index.ts\"\n\nimport { availableParallelism } from \"node:os\"\nimport { chunkMarkdown } from \"../md.ts\"\nimport { Progress } from \"../progress.ts\"\nimport { loadModel, parseModelUri, resolveModel } from \"./models.ts\"\n\nconst defaults = {\n batchSize: 0, // 0 = auto\n maxDims: 512,\n maxTokens: 512,\n useGpu: true,\n} satisfies EmbedderOptions\n\nconst backend_defaults: Record<ModelBackend, EmbedderOptions> = {\n llama: {},\n openai: { batchSize: 50 },\n transformers: {},\n}\n\nfunction isChunk(input: any): input is EmbedderChunk {\n return typeof (input as EmbedderChunk | undefined)?.prompt === \"string\"\n}\n\nexport class Embedder {\n #loading?: Promise<EmbedderBackend>\n opts: ResolvedEmbedderOptions\n model: ResolvedEmbedderModel\n status = new Progress(\"embedder\")\n #backend?: EmbedderBackend\n\n constructor(public ctx: Context) {\n const opts = ctx.opts.embedder ?? {}\n this.model = resolveModel(opts.model)\n const { backend } = parseModelUri(this.model.uri)\n const base = { ...defaults, ...backend_defaults[backend] }\n const threads = Math.max(1, opts.threads ?? Math.min(8, availableParallelism() - 2))\n this.opts = {\n threads,\n ...base,\n ...opts,\n model: this.model,\n }\n }\n\n info() {\n return parseModelUri(this.model.uri)\n }\n\n async backend() {\n this.#loading ??= (async () => {\n this.status.name = `Loading model \\`${this.model.uri}\\``\n\n const t = setTimeout(() => {\n this.opts.onProgress?.(this.status)\n }, 500)\n\n this.#backend = await loadModel({\n logger: this.ctx,\n opts: this.opts,\n root: this.ctx.root,\n status: this.status,\n })\n\n clearTimeout(t)\n\n this.opts.maxTokens = Math.min(this.opts.maxTokens, this.#backend.maxTokens)\n this.opts.maxDims = Math.min(this.opts.maxDims, this.#backend.dims)\n if (this.opts.batchSize === 0) this.opts.batchSize = this.#backend.device === \"gpu\" ? 50 : 1\n this.ctx.debug({\n batchSize: this.opts.batchSize,\n device: this.#backend.device,\n threads: this.opts.threads,\n useGpu: this.opts.useGpu,\n })\n this.status.stop()\n return this.#backend\n })()\n return (this.#backend ??= await this.#loading)\n }\n\n transform(input: string | EmbedderDoc | EmbedderChunk): string {\n if (isChunk(input)) return input.prompt\n const { prompt } = this.model\n return typeof input === \"string\" ? prompt.query(input) : prompt.document(input)\n }\n\n async embed(input: string | EmbedderDoc | EmbedderChunk): Promise<number[]>\n async embed(input: (string | EmbedderDoc | EmbedderChunk)[]): Promise<number[][]>\n async embed(\n input: string | EmbedderDoc | EmbedderChunk | (string | EmbedderDoc | EmbedderChunk)[]\n ): Promise<number[][] | number[]> {\n const single = !Array.isArray(input)\n const todo = single ? [input] : input\n const backend = await this.backend()\n const ret = await backend.embed(todo.map((item) => this.transform(item)))\n return single ? ret[0] : ret\n }\n\n async chunk(input: string | EmbedderDoc): Promise<EmbedderChunk[]>\n async chunk(input: string | EmbedderDoc): Promise<EmbedderChunk[]> {\n const backend = await this.backend()\n const isQuery = typeof input === \"string\"\n const fixed = this.transform(isQuery ? \"\" : { text: \"\", title: input.title })\n const chunkText = isQuery ? input : input.text\n const tokens = this.opts.maxTokens - backend.toks(fixed)\n return chunkMarkdown(chunkText, backend, tokens).map((text, seq) => ({\n prompt: isQuery ? this.transform(text) : this.transform({ text, title: input.title }),\n seq,\n text,\n }))\n }\n}\n"],"mappings":";;;;AAgBA,MAAM,WAAW;CACf,WAAW;CACX,SAAS;CACT,WAAW;CACX,QAAQ;CACT;AAED,MAAM,mBAA0D;CAC9D,OAAO,EAAE;CACT,QAAQ,EAAE,WAAW,IAAI;CACzB,cAAc,EAAE;CACjB;AAED,SAAS,QAAQ,OAAoC;AACnD,QAAO,OAAQ,OAAqC,WAAW;;AAGjE,IAAa,WAAb,MAAsB;CACpB;CACA;CACA;CACA,SAAS,IAAI,SAAS,WAAW;CACjC;CAEA,YAAY,KAAqB;AAAd,OAAA,MAAA;EACjB,MAAM,OAAO,IAAI,KAAK,YAAY,EAAE;AACpC,OAAK,QAAQ,aAAa,KAAK,MAAM;EACrC,MAAM,EAAE,YAAY,cAAc,KAAK,MAAM,IAAI;EACjD,MAAM,OAAO;GAAE,GAAG;GAAU,GAAG,iBAAiB;GAAU;AAE1D,OAAK,OAAO;GACV,SAFc,KAAK,IAAI,GAAG,KAAK,WAAW,KAAK,IAAI,GAAG,sBAAsB,GAAG,EAAE,CAAC;GAGlF,GAAG;GACH,GAAG;GACH,OAAO,KAAK;GACb;;CAGH,OAAO;AACL,SAAO,cAAc,KAAK,MAAM,IAAI;;CAGtC,MAAM,UAAU;AACd,QAAA,aAAmB,YAAY;AAC7B,QAAK,OAAO,OAAO,mBAAmB,KAAK,MAAM,IAAI;GAErD,MAAM,IAAI,iBAAiB;AACzB,SAAK,KAAK,aAAa,KAAK,OAAO;MAClC,IAAI;AAEP,SAAA,UAAgB,MAAM,UAAU;IAC9B,QAAQ,KAAK;IACb,MAAM,KAAK;IACX,MAAM,KAAK,IAAI;IACf,QAAQ,KAAK;IACd,CAAC;AAEF,gBAAa,EAAE;AAEf,QAAK,KAAK,YAAY,KAAK,IAAI,KAAK,KAAK,WAAW,MAAA,QAAc,UAAU;AAC5E,QAAK,KAAK,UAAU,KAAK,IAAI,KAAK,KAAK,SAAS,MAAA,QAAc,KAAK;AACnE,OAAI,KAAK,KAAK,cAAc,EAAG,MAAK,KAAK,YAAY,MAAA,QAAc,WAAW,QAAQ,KAAK;AAC3F,QAAK,IAAI,MAAM;IACb,WAAW,KAAK,KAAK;IACrB,QAAQ,MAAA,QAAc;IACtB,SAAS,KAAK,KAAK;IACnB,QAAQ,KAAK,KAAK;IACnB,CAAC;AACF,QAAK,OAAO,MAAM;AAClB,UAAO,MAAA;MACL;AACJ,SAAQ,MAAA,YAAkB,MAAM,MAAA;;CAGlC,UAAU,OAAqD;AAC7D,MAAI,QAAQ,MAAM,CAAE,QAAO,MAAM;EACjC,MAAM,EAAE,WAAW,KAAK;AACxB,SAAO,OAAO,UAAU,WAAW,OAAO,MAAM,MAAM,GAAG,OAAO,SAAS,MAAM;;CAKjF,MAAM,MACJ,OACgC;EAChC,MAAM,SAAS,CAAC,MAAM,QAAQ,MAAM;EACpC,MAAM,OAAO,SAAS,CAAC,MAAM,GAAG;EAEhC,MAAM,MAAM,OADI,MAAM,KAAK,SAAS,EACV,MAAM,KAAK,KAAK,SAAS,KAAK,UAAU,KAAK,CAAC,CAAC;AACzE,SAAO,SAAS,IAAI,KAAK;;CAI3B,MAAM,MAAM,OAAuD;EACjE,MAAM,UAAU,MAAM,KAAK,SAAS;EACpC,MAAM,UAAU,OAAO,UAAU;EACjC,MAAM,QAAQ,KAAK,UAAU,UAAU,KAAK;GAAE,MAAM;GAAI,OAAO,MAAM;GAAO,CAAC;AAG7E,SAAO,cAFW,UAAU,QAAQ,MAAM,MAEV,SADjB,KAAK,KAAK,YAAY,QAAQ,KAAK,MAAM,CACR,CAAC,KAAK,MAAM,SAAS;GACnE,QAAQ,UAAU,KAAK,UAAU,KAAK,GAAG,KAAK,UAAU;IAAE;IAAM,OAAO,MAAM;IAAO,CAAC;GACrF;GACA;GACD,EAAE"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
//#region src/frecency.ts
|
|
2
|
+
const LAMBDA = Math.LN2 / (720 * 3600);
|
|
3
|
+
const VALUES = {
|
|
4
|
+
new: 1,
|
|
5
|
+
updated: .8,
|
|
6
|
+
visit: 1
|
|
7
|
+
};
|
|
8
|
+
const now = () => Date.now() / 1e3;
|
|
9
|
+
/** Convert a frecency score to a deadline timestamp (for DB storage). */
|
|
10
|
+
function toDeadline(frecency) {
|
|
11
|
+
return now() + Math.log(frecency) / LAMBDA;
|
|
12
|
+
}
|
|
13
|
+
/** Convert a deadline timestamp back to a frecency score. */
|
|
14
|
+
function toScore(deadline) {
|
|
15
|
+
return Math.exp(LAMBDA * (deadline - now()));
|
|
16
|
+
}
|
|
17
|
+
/** Add a weighted visit to a frecency score.
|
|
18
|
+
* @param frecency - current score
|
|
19
|
+
* @param value - points to add (default: 1). Use higher values for stronger signals.
|
|
20
|
+
* @param at - timestamp of the visit in seconds (default: now).
|
|
21
|
+
* Use file mtime for seeding from disk changes. */
|
|
22
|
+
function addVisit(frecency, value = "visit", at) {
|
|
23
|
+
const n = now();
|
|
24
|
+
value = typeof value === "number" ? value : VALUES[value];
|
|
25
|
+
return frecency + value * Math.exp(LAMBDA * ((at ?? n) - n));
|
|
26
|
+
}
|
|
27
|
+
//#endregion
|
|
28
|
+
export { toDeadline as n, toScore as r, addVisit as t };
|
|
29
|
+
|
|
30
|
+
//# sourceMappingURL=frecency-CiaqPIOy.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frecency-CiaqPIOy.mjs","names":[],"sources":["../src/frecency.ts"],"sourcesContent":["// Exponential decay frecency, based on:\n// https://wiki.mozilla.org/User:Jesse/NewFrecency\n// Ported from snacks.nvim picker frecency\n\nconst HALF_LIFE = 30 * 24 * 3600 // 30 days in seconds\nconst LAMBDA = Math.LN2 / HALF_LIFE // λ = ln(2) / half_life\nconst VALUES = {\n new: 1,\n updated: 0.8,\n visit: 1,\n} as const\n\nexport type FrecencyScore = keyof typeof VALUES\n\nconst now = () => Date.now() / 1000\n\n/** Convert a frecency score to a deadline timestamp (for DB storage). */\nexport function toDeadline(frecency: number): number {\n return now() + Math.log(frecency) / LAMBDA\n}\n\n/** Convert a deadline timestamp back to a frecency score. */\nexport function toScore(deadline: number): number {\n return Math.exp(LAMBDA * (deadline - now()))\n}\n\n/** Add a weighted visit to a frecency score.\n * @param frecency - current score\n * @param value - points to add (default: 1). Use higher values for stronger signals.\n * @param at - timestamp of the visit in seconds (default: now).\n * Use file mtime for seeding from disk changes. */\nexport function addVisit(\n frecency: number,\n value: number | FrecencyScore = \"visit\",\n at?: number\n): number {\n const n = now()\n value = typeof value === \"number\" ? value : VALUES[value]\n const decayed = value * Math.exp(LAMBDA * ((at ?? n) - n))\n return frecency + decayed\n}\n"],"mappings":";AAKA,MAAM,SAAS,KAAK,OADF,MAAU;AAE5B,MAAM,SAAS;CACb,KAAK;CACL,SAAS;CACT,OAAO;CACR;AAID,MAAM,YAAY,KAAK,KAAK,GAAG;;AAG/B,SAAgB,WAAW,UAA0B;AACnD,QAAO,KAAK,GAAG,KAAK,IAAI,SAAS,GAAG;;;AAItC,SAAgB,QAAQ,UAA0B;AAChD,QAAO,KAAK,IAAI,UAAU,WAAW,KAAK,EAAE;;;;;;;AAQ9C,SAAgB,SACd,UACA,QAAgC,SAChC,IACQ;CACR,MAAM,IAAI,KAAK;AACf,SAAQ,OAAO,UAAU,WAAW,QAAQ,OAAO;AAEnD,QAAO,WADS,QAAQ,KAAK,IAAI,WAAW,MAAM,KAAK,GAAG"}
|
package/dist/fs-DMp26Byo.mjs
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs-DMp26Byo.mjs","names":[],"sources":["../src/fs.ts"],"sourcesContent":["import { statSync, existsSync } from \"node:fs\"\nimport { stat } from \"node:fs/promises\"\nimport { homedir } from \"node:os\"\nimport { dirname, join, resolve } from \"pathe\"\n\nexport function sstat(path: string) {\n try {\n return statSync(path)\n } catch {}\n}\n\nexport async function astat(path: string) {\n return await stat(path).catch(() => undefined)\n}\n\nexport function findUp(root: string, name: string, stop?: string) {\n let current = resolve(root)\n // oxlint-disable-next-line typescript/no-unnecessary-condition\n while (true) {\n const check = join(current, name)\n if (sstat(check)?.isFile()) return check\n if (stop && existsSync(join(current, stop))) return // reached stop directory without finding the file\n const next = dirname(current)\n if (next === current) break // reached filesystem root\n current = next\n }\n}\n\n// Similar to path.resolve but also expands ~ to the user home directory\nexport function normPath(...paths: string[]) {\n return resolve(...paths.map((p) => p.replace(/^~(?=\\/|\\\\|$)/, homedir())))\n}\n\nexport function gitRoot(path: string) {\n return findUp(path, \".git\")\n}\n"],"mappings":";;;;;AAKA,SAAgB,MAAM,MAAc;AAClC,KAAI;AACF,SAAO,SAAS,KAAK;SACf;;AAGV,eAAsB,MAAM,MAAc;AACxC,QAAO,MAAM,KAAK,KAAK,CAAC,YAAY,KAAA,EAAU;;AAGhD,SAAgB,OAAO,MAAc,MAAc,MAAe;CAChE,IAAI,UAAU,QAAQ,KAAK;AAE3B,QAAO,MAAM;EACX,MAAM,QAAQ,KAAK,SAAS,KAAK;AACjC,MAAI,MAAM,MAAM,EAAE,QAAQ,CAAE,QAAO;AACnC,MAAI,QAAQ,WAAW,KAAK,SAAS,KAAK,CAAC,CAAE;EAC7C,MAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,SAAS,QAAS;AACtB,YAAU;;;AAKd,SAAgB,SAAS,GAAG,OAAiB;AAC3C,QAAO,QAAQ,GAAG,MAAM,KAAK,MAAM,EAAE,QAAQ,iBAAiB,SAAS,CAAC,CAAC,CAAC;;AAG5E,SAAgB,QAAQ,MAAc;AACpC,QAAO,OAAO,MAAM,OAAO"}
|
package/dist/glob.d.mts
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"glob.d.mts","names":[],"sources":["../src/glob.ts"],"mappings":";;;KASY,QAAA,IAAY,CAAA,EAAG,MAAA,EAAQ,CAAA,EAAG,MAAA;AAAA,cAEhC,OAAA;;;;;KAUM,WAAA;EACV,GAAA;EACA,IAAA;EACA,MAAA;EACA,MAAA;EACA,MAAA;EACA,IAAA;EACA,KAAA;EACA,KAAA;EACA,WAAA;EACA,OAAA;EACA,OAAA,IAAW,GAAA;EACX,OAAA,IAAW,IAAA,UAAc,KAAA,EAAO,KAAA;EAChC,IAAA,GAAO,QAAA,gBAAwB,OAAA;AAAA;AAAA,iBA6CV,IAAA,CAAK,IAAA,GAAM,OAAA,CAAQ,WAAA,IAAoB,cAAA"}
|
package/dist/glob.mjs
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"glob.mjs","names":[],"sources":["../src/glob.ts"],"sourcesContent":["import type { Ignore } from \"ignore\"\nimport type { Dirent } from \"node:fs\"\n\nimport { readFileSync } from \"node:fs\"\nimport { readdir } from \"node:fs/promises\"\nimport { join } from \"pathe\"\nimport { findUp, normPath, sstat } from \"./fs.ts\"\nimport { toError } from \"./util.ts\"\n\nexport type GlobSort = (a: Dirent, b: Dirent) => number\n\nconst sorters = {\n name: (a, b) => a.name.localeCompare(b.name),\n none: () => 0,\n type: (a, b) => {\n if (a.isDirectory() && !b.isDirectory()) return -1\n if (!a.isDirectory() && b.isDirectory()) return 1\n return a.name.localeCompare(b.name)\n },\n} satisfies Record<string, GlobSort>\n\nexport type GlobOptions = {\n cwd: string | string[]\n glob?: string | string[] // optional glob patterns to filter files (e.g. \"*.js\")\n follow: boolean // follow symlinks\n hidden: boolean // include hidden files (those starting with a dot)\n ignore: boolean // respect ignore files\n type?: \"file\" | \"directory\" // filter by type\n empty: boolean // include empty directories\n depth: number // maximum depth to traverse\n ignoreFiles: string[] // names of ignore files to look for in each directory\n exclude: string[] // additional ignore rules to apply globally\n onVisit?: (rel: string) => void\n onError?: (path: string, error: Error) => void\n sort?: GlobSort | keyof typeof sorters\n}\n\nconst defaults: GlobOptions = {\n cwd: \".\",\n depth: Infinity,\n empty: false,\n exclude: [\".git\", \"node_modules/\"],\n follow: false,\n hidden: false,\n ignore: true,\n ignoreFiles: [\".gitignore\", \".ignore\"],\n sort: \"name\",\n}\n\ntype GlobEntry = {\n path: string\n rel: string\n ignore?: IgnoreTree\n depth: number\n dir: boolean\n}\n\nclass IgnoreTree {\n parent?: IgnoreTree\n\n constructor(\n public ig: Ignore,\n public rel = \"\"\n ) {}\n\n extend(ig: Ignore, rel: string) {\n const child = new IgnoreTree(ig, rel)\n child.parent = this\n return child\n }\n\n ignores(rel: string): boolean {\n const test = this.ig.test(rel.slice(this.rel.length))\n if (test.ignored) return true\n if (test.unignored) return false\n return this.parent?.ignores(rel) ?? false\n }\n}\n\nexport async function* glob(opts: Partial<GlobOptions> = {}): AsyncGenerator<string> {\n if (opts.depth && opts.depth < 1) return // fast path for zero results\n\n const { default: ignore } = await import(\"ignore\")\n const o: GlobOptions = { ...defaults, ...opts }\n if (Array.isArray(o.cwd)) {\n for (const cwd of o.cwd) yield* glob({ ...o, cwd })\n return\n }\n const root = normPath(o.cwd)\n const ignoreFiles = new Set(o.ignoreFiles)\n const rootIgnore = ignore().add([...o.exclude, ...ignoreFiles])\n const globIgnore = ignore().add(o.glob ?? [])\n const sorter = (typeof o.sort === \"string\" ? sorters[o.sort] : o.sort) ?? sorters.name\n const visited = new Set<string>()\n\n if (o.ignore)\n for (const igf of ignoreFiles) {\n const igPath = findUp(root, igf, \".git\")\n if (igPath) rootIgnore.add(readFileSync(igPath, \"utf8\"))\n }\n\n async function ls(dir: GlobEntry) {\n if (visited.has(dir.path)) return\n visited.add(dir.path)\n let entries\n try {\n const dirents = await readdir(dir.path, { withFileTypes: true })\n entries = dirents.toSorted(sorter).toReversed()\n } catch (error) {\n return o.onError?.(dir.path, toError(error))\n }\n\n let ig = dir.ignore\n const children: GlobEntry[] = []\n\n for (const entry of entries) {\n const path = join(entry.parentPath, entry.name)\n if (o.ignore && entry.isFile() && ignoreFiles.has(entry.name)) {\n const fig = ignore().add(readFileSync(path, \"utf8\"))\n ig = ig ? ig.extend(fig, dir.rel) : new IgnoreTree(fig, dir.rel)\n } else if (!o.hidden && entry.name.startsWith(\".\")) {\n continue\n } else {\n let isDirectory = entry.isDirectory()\n isDirectory ||= o.follow && entry.isSymbolicLink() && (sstat(path)?.isDirectory() ?? false)\n const rel = dir.rel + entry.name + (isDirectory ? \"/\" : \"\")\n const depth = dir.depth + 1\n children.push({ depth, dir: isDirectory, path, rel })\n }\n }\n\n for (const child of children) {\n o.onVisit?.(child.rel)\n if (o.ignore && ig?.ignores(child.rel)) continue\n if (o.glob && !child.dir && !globIgnore.ignores(child.rel)) continue\n stack.push({ ...child, ignore: ig })\n }\n }\n\n const stack: GlobEntry[] = [\n { depth: 0, dir: true, ignore: new IgnoreTree(rootIgnore), path: root, rel: \"\" },\n ]\n const parents: GlobEntry[] = []\n\n while (stack.length > 0) {\n const entry = stack.pop()!\n\n if (o.type !== \"file\" && entry.depth !== 0) {\n while (!o.empty && parents.length > 0 && parents[parents.length - 1].depth >= entry.depth)\n parents.pop()\n if (entry.dir && entry.depth < o.depth) {\n parents.push(entry)\n } else {\n for (const p of parents) yield p.rel\n parents.length = 0\n if (o.type !== \"directory\") yield entry.rel\n }\n } else if (!entry.dir) yield entry.rel\n\n // oxlint-disable-next-line no-await-in-loop\n if (entry.dir && entry.depth < o.depth) await ls(entry)\n }\n}\n"],"mappings":";;;;;;AAWA,MAAM,UAAU;CACd,OAAO,GAAG,MAAM,EAAE,KAAK,cAAc,EAAE,KAAK;CAC5C,YAAY;CACZ,OAAO,GAAG,MAAM;AACd,MAAI,EAAE,aAAa,IAAI,CAAC,EAAE,aAAa,CAAE,QAAO;AAChD,MAAI,CAAC,EAAE,aAAa,IAAI,EAAE,aAAa,CAAE,QAAO;AAChD,SAAO,EAAE,KAAK,cAAc,EAAE,KAAK;;CAEtC;AAkBD,MAAM,WAAwB;CAC5B,KAAK;CACL,OAAO;CACP,OAAO;CACP,SAAS,CAAC,QAAQ,gBAAgB;CAClC,QAAQ;CACR,QAAQ;CACR,QAAQ;CACR,aAAa,CAAC,cAAc,UAAU;CACtC,MAAM;CACP;AAUD,IAAM,aAAN,MAAM,WAAW;CACf;CAEA,YACE,IACA,MAAa,IACb;AAFO,OAAA,KAAA;AACA,OAAA,MAAA;;CAGT,OAAO,IAAY,KAAa;EAC9B,MAAM,QAAQ,IAAI,WAAW,IAAI,IAAI;AACrC,QAAM,SAAS;AACf,SAAO;;CAGT,QAAQ,KAAsB;EAC5B,MAAM,OAAO,KAAK,GAAG,KAAK,IAAI,MAAM,KAAK,IAAI,OAAO,CAAC;AACrD,MAAI,KAAK,QAAS,QAAO;AACzB,MAAI,KAAK,UAAW,QAAO;AAC3B,SAAO,KAAK,QAAQ,QAAQ,IAAI,IAAI;;;AAIxC,gBAAuB,KAAK,OAA6B,EAAE,EAA0B;AACnF,KAAI,KAAK,SAAS,KAAK,QAAQ,EAAG;CAElC,MAAM,EAAE,SAAS,WAAW,MAAM,OAAO;CACzC,MAAM,IAAiB;EAAE,GAAG;EAAU,GAAG;EAAM;AAC/C,KAAI,MAAM,QAAQ,EAAE,IAAI,EAAE;AACxB,OAAK,MAAM,OAAO,EAAE,IAAK,QAAO,KAAK;GAAE,GAAG;GAAG;GAAK,CAAC;AACnD;;CAEF,MAAM,OAAO,SAAS,EAAE,IAAI;CAC5B,MAAM,cAAc,IAAI,IAAI,EAAE,YAAY;CAC1C,MAAM,aAAa,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,YAAY,CAAC;CAC/D,MAAM,aAAa,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;CAC7C,MAAM,UAAU,OAAO,EAAE,SAAS,WAAW,QAAQ,EAAE,QAAQ,EAAE,SAAS,QAAQ;CAClF,MAAM,0BAAU,IAAI,KAAa;AAEjC,KAAI,EAAE,OACJ,MAAK,MAAM,OAAO,aAAa;EAC7B,MAAM,SAAS,OAAO,MAAM,KAAK,OAAO;AACxC,MAAI,OAAQ,YAAW,IAAI,aAAa,QAAQ,OAAO,CAAC;;CAG5D,eAAe,GAAG,KAAgB;AAChC,MAAI,QAAQ,IAAI,IAAI,KAAK,CAAE;AAC3B,UAAQ,IAAI,IAAI,KAAK;EACrB,IAAI;AACJ,MAAI;AAEF,cADgB,MAAM,QAAQ,IAAI,MAAM,EAAE,eAAe,MAAM,CAAC,EAC9C,SAAS,OAAO,CAAC,YAAY;WACxC,OAAO;AACd,UAAO,EAAE,UAAU,IAAI,MAAM,QAAQ,MAAM,CAAC;;EAG9C,IAAI,KAAK,IAAI;EACb,MAAM,WAAwB,EAAE;AAEhC,OAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,OAAO,KAAK,MAAM,YAAY,MAAM,KAAK;AAC/C,OAAI,EAAE,UAAU,MAAM,QAAQ,IAAI,YAAY,IAAI,MAAM,KAAK,EAAE;IAC7D,MAAM,MAAM,QAAQ,CAAC,IAAI,aAAa,MAAM,OAAO,CAAC;AACpD,SAAK,KAAK,GAAG,OAAO,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK,IAAI,IAAI;cACvD,CAAC,EAAE,UAAU,MAAM,KAAK,WAAW,IAAI,CAChD;QACK;IACL,IAAI,cAAc,MAAM,aAAa;AACrC,oBAAgB,EAAE,UAAU,MAAM,gBAAgB,KAAK,MAAM,KAAK,EAAE,aAAa,IAAI;IACrF,MAAM,MAAM,IAAI,MAAM,MAAM,QAAQ,cAAc,MAAM;IACxD,MAAM,QAAQ,IAAI,QAAQ;AAC1B,aAAS,KAAK;KAAE;KAAO,KAAK;KAAa;KAAM;KAAK,CAAC;;;AAIzD,OAAK,MAAM,SAAS,UAAU;AAC5B,KAAE,UAAU,MAAM,IAAI;AACtB,OAAI,EAAE,UAAU,IAAI,QAAQ,MAAM,IAAI,CAAE;AACxC,OAAI,EAAE,QAAQ,CAAC,MAAM,OAAO,CAAC,WAAW,QAAQ,MAAM,IAAI,CAAE;AAC5D,SAAM,KAAK;IAAE,GAAG;IAAO,QAAQ;IAAI,CAAC;;;CAIxC,MAAM,QAAqB,CACzB;EAAE,OAAO;EAAG,KAAK;EAAM,QAAQ,IAAI,WAAW,WAAW;EAAE,MAAM;EAAM,KAAK;EAAI,CACjF;CACD,MAAM,UAAuB,EAAE;AAE/B,QAAO,MAAM,SAAS,GAAG;EACvB,MAAM,QAAQ,MAAM,KAAK;AAEzB,MAAI,EAAE,SAAS,UAAU,MAAM,UAAU,GAAG;AAC1C,UAAO,CAAC,EAAE,SAAS,QAAQ,SAAS,KAAK,QAAQ,QAAQ,SAAS,GAAG,SAAS,MAAM,MAClF,SAAQ,KAAK;AACf,OAAI,MAAM,OAAO,MAAM,QAAQ,EAAE,MAC/B,SAAQ,KAAK,MAAM;QACd;AACL,SAAK,MAAM,KAAK,QAAS,OAAM,EAAE;AACjC,YAAQ,SAAS;AACjB,QAAI,EAAE,SAAS,YAAa,OAAM,MAAM;;aAEjC,CAAC,MAAM,IAAK,OAAM,MAAM;AAGnC,MAAI,MAAM,OAAO,MAAM,QAAQ,EAAE,MAAO,OAAM,GAAG,MAAM"}
|
package/dist/index.d.mts
CHANGED
|
@@ -1177,7 +1177,7 @@ declare class Store {
|
|
|
1177
1177
|
sync(opts?: {
|
|
1178
1178
|
embed?: boolean;
|
|
1179
1179
|
}): Promise<void>;
|
|
1180
|
-
prune(syncStart:
|
|
1180
|
+
prune(syncStart: Date): Promise<void>;
|
|
1181
1181
|
}
|
|
1182
1182
|
//#endregion
|
|
1183
1183
|
//#region src/context.d.ts
|
|
@@ -1279,6 +1279,14 @@ type ResolvedEmbedderOptions = Merge<SetOptional<Required<EmbedderOptions>, "onP
|
|
|
1279
1279
|
model: ResolvedEmbedderModel;
|
|
1280
1280
|
}>;
|
|
1281
1281
|
//#endregion
|
|
1282
|
+
//#region src/frecency.d.ts
|
|
1283
|
+
declare const VALUES: {
|
|
1284
|
+
readonly new: 1;
|
|
1285
|
+
readonly updated: 0.8;
|
|
1286
|
+
readonly visit: 1;
|
|
1287
|
+
};
|
|
1288
|
+
type FrecencyScore = keyof typeof VALUES;
|
|
1289
|
+
//#endregion
|
|
1282
1290
|
//#region src/db.d.ts
|
|
1283
1291
|
type DocRow = {
|
|
1284
1292
|
id: number;
|
|
@@ -1288,11 +1296,12 @@ type DocRow = {
|
|
|
1288
1296
|
vec_hash?: string;
|
|
1289
1297
|
description: string;
|
|
1290
1298
|
title: string;
|
|
1291
|
-
tags: string;
|
|
1292
|
-
entities: string;
|
|
1293
|
-
updated_at:
|
|
1294
|
-
synced_at?:
|
|
1295
|
-
deadline
|
|
1299
|
+
tags: string[];
|
|
1300
|
+
entities: string[];
|
|
1301
|
+
updated_at: Date;
|
|
1302
|
+
synced_at?: Date;
|
|
1303
|
+
deadline: number;
|
|
1304
|
+
frecency: number;
|
|
1296
1305
|
};
|
|
1297
1306
|
type VecResult = {
|
|
1298
1307
|
doc_id: number;
|
|
@@ -1326,7 +1335,7 @@ declare class Db {
|
|
|
1326
1335
|
private initVec;
|
|
1327
1336
|
getDoc(from: string | number): DocRow | undefined;
|
|
1328
1337
|
getDocs(from?: (string | number)[]): Map<number, DocRow>;
|
|
1329
|
-
addDoc(row: Omit<DocRow, "id">): number;
|
|
1338
|
+
addDoc(row: Omit<DocRow, "id" | "frecency">): number;
|
|
1330
1339
|
deleteDoc(id: number, tables?: {
|
|
1331
1340
|
docs?: boolean;
|
|
1332
1341
|
vec?: boolean;
|
|
@@ -1354,10 +1363,10 @@ declare class Db {
|
|
|
1354
1363
|
exclusive: (...args: A) => T;
|
|
1355
1364
|
};
|
|
1356
1365
|
getUnembeddedDocs(): DocRow[];
|
|
1357
|
-
touchDoc(id: number): void;
|
|
1366
|
+
touchDoc(id: number, syncedAt?: Date): void;
|
|
1358
1367
|
markEmbedded(id: number, docHash: string): void;
|
|
1359
1368
|
/** Delete docs not seen since the given sync timestamp, optionally scoped to a path prefix. */
|
|
1360
|
-
deleteStaleDocs(syncedBefore:
|
|
1369
|
+
deleteStaleDocs(syncedBefore: Date, prefix?: string): number;
|
|
1361
1370
|
/** Scoped FTS search: only match docs whose path starts with one of the given prefixes */
|
|
1362
1371
|
searchFts(query: string, opts?: DbSearchOptions): FTSResult[];
|
|
1363
1372
|
/** * Gets weights for high-frequency terms.
|
|
@@ -1371,7 +1380,7 @@ declare class Db {
|
|
|
1371
1380
|
deleteEmbeddings(docId: number): void;
|
|
1372
1381
|
/** Global KNN search, returns top results across all docs */
|
|
1373
1382
|
searchVec(embedding: number[], opts?: DbSearchOptions): VecResult[];
|
|
1374
|
-
|
|
1383
|
+
visit(doc: DocRow | number, value?: FrecencyScore | number): void;
|
|
1375
1384
|
getMeta(key: string): string | undefined;
|
|
1376
1385
|
setMeta(key: string, value: string): void;
|
|
1377
1386
|
cacheGet<T>(key: string): T | undefined;
|
|
@@ -1462,4 +1471,5 @@ declare class Snippet {
|
|
|
1462
1471
|
highlight(text: string, hl: (word: string, offset: number) => string): string;
|
|
1463
1472
|
}
|
|
1464
1473
|
//#endregion
|
|
1465
|
-
export { Context, ContextOptions, type Database, Db, DbSearchOptions, Doc, DocFrontmatter, DocRow, Embedder, EmbedderBackend, EmbedderChunk, EmbedderContext, EmbedderDevice, EmbedderDoc, EmbedderModel, EmbedderOptions, EmbedderPrompt, Events, FTSResult, Frontmatter, FtsSR, FtsSearchOptions, HybridSR, LOG_LEVELS, LogFn, LogLevel, Logger, LoggerBase, MarkdownDoc, MarkdownSection, ModelBackend, Node, Progress, ProgressOpts, ResolvedEmbedderModel, ResolvedEmbedderOptions, Search, SearchMode, SearchOptions, SearchResult, SearchScore, Snippet, SnippetOptions, SnippetResult, SnippetWindow, Store, StoreChunk, Token, TokenCounter, TokenWithScore, TypedEmitter, URI_PREFIX, VecResult, VecSR, Vfs, VfsEntry, VfsFindOptions, VfsFolder, VfsNode, VfsPath, VfsScope, VfsView, WORD_REGEX, assertEmbeddings, assertUri, astat, chunkMarkdown, chunkText, findUp, gitRoot, hasEmbedding, hash, isLogLevel, isStopWord, normPath, normUri, parentUri, parseFrontmatter, parseMarkdown, parseSections, parseYaml, shouldLog, sstat, toError, toFts, tokenize };
|
|
1474
|
+
export { Context, ContextOptions, type Database, Db, DbSearchOptions, Doc, DocFrontmatter, DocRow, Embedder, EmbedderBackend, EmbedderChunk, EmbedderContext, EmbedderDevice, EmbedderDoc, EmbedderModel, EmbedderOptions, EmbedderPrompt, Events, FTSResult, Frontmatter, FtsSR, FtsSearchOptions, HybridSR, LOG_LEVELS, LogFn, LogLevel, Logger, LoggerBase, MarkdownDoc, MarkdownSection, ModelBackend, Node, Progress, ProgressOpts, ResolvedEmbedderModel, ResolvedEmbedderOptions, Search, SearchMode, SearchOptions, SearchResult, SearchScore, Snippet, SnippetOptions, SnippetResult, SnippetWindow, Store, StoreChunk, Token, TokenCounter, TokenWithScore, TypedEmitter, URI_PREFIX, VecResult, VecSR, Vfs, VfsEntry, VfsFindOptions, VfsFolder, VfsNode, VfsPath, VfsScope, VfsView, WORD_REGEX, assertEmbeddings, assertUri, astat, chunkMarkdown, chunkText, findUp, gitRoot, hasEmbedding, hash, isLogLevel, isStopWord, normPath, normUri, parentUri, parseFrontmatter, parseMarkdown, parseSections, parseYaml, shouldLog, sstat, toError, toFts, tokenize };
|
|
1475
|
+
//# sourceMappingURL=index.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":["UnionToIntersection","Union","Intersection","distributedUnion","mergedIntersection","UnionToIntersection","KeysOfUnion","ObjectType","Record","IsAny","T","NoInfer","IsAny","IsOptionalKeyOf","Type","Key","Record","IsOptionalKeyOf","OptionalKeysOf","Type","Key","OptionalKeysOf","RequiredKeysOf","Type","Exclude","IsNever","T","IsNever","If","Type","IfBranch","ElseBranch","Simplify","T","KeyType","IsEqual","A","B","_IsEqual","G","OmitIndexSignature","ObjectType","KeyType","Record","PickIndexSignature","ObjectType","KeyType","Record","OmitIndexSignature","PickIndexSignature","Simplify","If","IsEqual","SimpleMerge","Destination","Source","Key","Merge","_Merge","Simplify","IsEqual","KeysOfUnion","RequiredKeysOf","Merge","OptionalKeysOf","IsAny","If","IsNever","FilterDefinedKeys","FilterOptionalKeys","MapsSetsOrArrays","NonRecursiveType","StringToNumber","ToString","BuildObject","Key","Value","CopiedFrom","PropertyKey","Pick","NumberKey","_","IsPlainObject","T","ObjectValue","K","NumberK","UndefinedToOptional","Exclude","HomomorphicPick","Keys","P","Extract","ValueOfUnion","Union","ReadonlyKeysOfUnion","ApplyDefaultOptions","Options","Defaults","SpecifiedOptions","Required","Omit","Record","Partial","CollapseLiterals","U","NormalizedKeys","ApplyDefaultOptions","IsEqual","Filter","KeyType","ExcludeType","ExceptOptions","requireExactProps","DefaultExceptOptions","Except","ObjectType","KeysType","Options","_Except","Required","Record","Partial","Except","HomomorphicPick","Simplify","SetOptional","BaseType","Keys","Parameters","ReturnType","_SetOptional","arguments_","Partial"],"sources":["../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/union-to-intersection.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/keys-of-union.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/is-any.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/is-optional-key-of.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/optional-keys-of.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/required-keys-of.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/is-never.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/if.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/simplify.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/is-equal.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/omit-index-signature.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/pick-index-signature.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/merge.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/internal/object.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/except.d.ts","../../../node_modules/.bun/type-fest@5.5.0/node_modules/type-fest/source/set-optional.d.ts","../src/log.ts","../src/runtime.node.ts","../src/util.ts","../src/progress.ts","../src/md.ts","../src/doc.ts","../src/vfs.ts","../src/search.ts","../src/store.ts","../src/context.ts","../src/embed/base.ts","../src/embed/index.ts","../src/frecency.ts","../src/db.ts","../src/fs.ts","../src/query.ts","../src/uri.ts","../src/snippet.ts"],"x_google_ignoreList":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],"mappings":";;;;;;;;;;;;AAiBA;;;;;;;;;;;KAAYA,mBAAAA;AAAAA;AAAAA;AAIXC,KAAAA;AAAAA;AAAAA,GAGIE,gBAAAA,EAAkBF,KAAAA;AAAAA;AAAAA;AAAAA,YAKXG,kBAAAA;AAAAA,EAETF,YAAAA,GAAeD,KAAAA;;;;;;;AAdlB;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;KAAYK,WAAAA;AAAAA,MAEJD,mBAAAA,CAAoBE,UAAAA,mBAA6BC,MAAAA,OAAaD,UAAAA;;;;;;;;;ADxBtE;;;;;;;;;;;;;;;;;;;;;;KEWYE,KAAAA,oBAAyBE,OAAAA,CAAQD,CAAAA;;;;;;;AFX7C;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;;;;KEGYG,eAAAA,wCAAuDC,IAAAA,IAClEF,KAAAA,CAAME,IAAAA,GAAOC,GAAAA,yBACVA,GAAAA,eAAkBD,IAAAA,GACjBA,IAAAA,SAAaE,MAAAA,CAAOD,GAAAA,EAAKD,IAAAA,CAAKC,GAAAA;;;;;;;AH5BnC;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;KGJYG,cAAAA,wBACXC,IAAAA;AAAAA,yBACyBA,IAAAA,IACvBF,eAAAA,CAAgBE,IAAAA,EAAMC,GAAAA,0BAEnBA,GAAAA,oBAEOD,IAAAA;AAAAA;;;;;;;AJzBb;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;KIPYG,cAAAA,wBACXC,IAAAA;AAAAA,EACGC,OAAAA,OAAcD,IAAAA,EAAMF,cAAAA,CAAeE,IAAAA;;;;;;;;;ALjBvC;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;;;;;;;;;ACXA;;;;;;KIyBYE,OAAAA,OAAcC,CAAAA;;;;;;;ANpC1B;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;;;;;;;;;ACXA;;;;;;;;;;;;ACcA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACPA;KG2DYE,EAAAA,+CACXD,OAAAA,CAAQE,IAAAA,iBACLE,UAAAA,GACAF,IAAAA,gBACCC,QAAAA,GACAC,UAAAA;;;;;;;;;APlFL;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;;;;;;;;;ACXA;;;;;;;;;;;KM8BYC,QAAAA,0BAAiCC,CAAAA,GAAIA,CAAAA,CAAEC,OAAAA;;;;;;;;;ARzCnD;;;;;;;;;;;;;;;;;;;;KSSYC,OAAAA,UACVC,CAAAA,WAAYC,CAAAA,KACTA,CAAAA,WAAYD,CAAAA,IACZE,QAAAA,CAASF,CAAAA,EAAGC,CAAAA;AAAAA;AAAAA,KAKZC,QAAAA,mBACMC,CAAAA,SAAUH,CAAAA,GAAIG,CAAAA,GAAIA,CAAAA,4BAClBA,CAAAA,SAAUF,CAAAA,GAAIE,CAAAA,GAAIA,CAAAA;;;;;;;;;ATnB7B;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;;;;;;;;;ACXA;;;;;;;;;;;;ACcA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KOiDYC,kBAAAA,mCACOC,UAAAA,eAAyBE,MAAAA,CAAOD,OAAAA,qBAE/CA,OAAAA,GAAUD,UAAAA,CAAWC,OAAAA;;;;;;;;;AV7EzB;;;;;;;;;;;;;;;;;;;;;;ACsBA;;;;;;;;;;;;;;;;;KUMYE,kBAAAA,mCACOC,UAAAA,eAAyBE,MAAAA,CAAOD,OAAAA,aAC/CA,OAAAA,WACQD,UAAAA,CAAWC,OAAAA;;;;KCzClBO,WAAAA,wBAAmCH,QAAAA,iBACzBI,WAAAA,IAAeE,GAAAA,eAAkBD,MAAAA,WAAiBC,GAAAA,GAAMF,WAAAA,CAAYE,GAAAA,MAC/ED,MAAAA;;;;;;;;;;;;;;;;;;AX8BJ;;;;;;;;;;;;;;;;;;;ACXA;;;;;;;;;;;;ACcA;;;;;;;;;;;;;;KS+BYE,KAAAA,wBACXH,WAAAA;AAAAA,EACGC,MAAAA;AAAAA,EACCJ,EAAAA,CAAGC,OAAAA,CAAQE,WAAAA,EAAaC,MAAAA,GAASD,WAAAA,EAAaI,MAAAA,CAAOJ,WAAAA,EAAaC,MAAAA;AAAAA;AAAAA;AAAAA,KAI3DG,MAAAA,wBACXR,QAAAA,CACCG,WAAAA,CAAYJ,kBAAAA,CAAmBK,WAAAA,GAAcL,kBAAAA,CAAmBM,MAAAA,KAC9DF,WAAAA,CAAYL,kBAAAA,CAAmBM,WAAAA,GAAcN,kBAAAA,CAAmBO,MAAAA;;;;ARhDpE;;;;;;;;;;;;;;;;;;;;;;;;;;ACHA;;;;;;;;;;KQiGYkC,eAAAA,iBAAgC5B,WAAAA,CAAYsB,CAAAA,mBAC3CA,CAAAA,IAAKS,OAAAA,CAAQD,CAAAA,EAAGD,IAAAA,IAAQP,CAAAA,CAAEQ,CAAAA;;;;;ALxEvC;;;;;;;;;;;;;;;;;;AChCA;;;;;;;;;;;;;;;;;;;;;;;AAKU;;;;;;;KI8LEK,mBAAAA,0CAEMrC,QAAAA,CAAS0C,IAAAA,CAAKD,QAAAA,CAASH,OAAAA,GAAUnC,cAAAA,CAAemC,OAAAA,KAAYM,OAAAA,CAAQD,MAAAA,CAAOxC,cAAAA,CAAemC,OAAAA,sCAClFA,OAAAA,IAEzB/B,EAAAA,CAAGD,KAAAA,CAAMkC,gBAAAA,GAAmBD,QAAAA,EAC3BhC,EAAAA,CAAGC,OAAAA,CAAQgC,gBAAAA,GAAmBD,QAAAA,EAC7BvC,QAAAA,CAASI,KAAAA,CAAMmC,QAAAA,kBACAC,gBAAAA,IACXxB,GAAAA,SAAYX,cAAAA,CAAeiC,OAAAA,sBAA6BE,gBAAAA,CAAiBxB,GAAAA,YAAeA,GAAAA,GAAMA,GAAAA,GAC9FwB,gBAAAA,CAAiBxB,GAAAA,OAChByB,QAAAA,CAASH,OAAAA;;;;;;AbvNjB;;;;;;;;;;;;;;;;;;;;;;ACsBA;;KaTKY,MAAAA,yBAA+BD,OAAAA,CAAQE,OAAAA,EAASC,WAAAA,0BAAqCD,OAAAA,SAAgBC,WAAAA,WAAsBD,OAAAA;AAAAA,KAEpHE,aAAAA;;;;;;EAQXC,iBAAAA;AAAAA;AAAAA,KAGIC,oBAAAA;EACJD,iBAAAA;AAAAA;AZhBD;;;;;;;;;;;;ACcA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACPA;;;;;;;;;AFPA,KY0EYE,MAAAA,oCAA0CC,UAAAA,kBAA4BJ,aAAAA,SACjFO,OAAAA,CAAQH,UAAAA,EAAYC,QAAAA,EAAUV,mBAAAA,CAAoBK,aAAAA,EAAeE,oBAAAA,EAAsBI,OAAAA;AAAAA,KAEnFC,OAAAA,oCAA2CH,UAAAA,kBAA4BI,QAAAA,CAASR,aAAAA,yBAClEI,UAAAA,IAAcP,MAAAA,CAAOC,OAAAA,EAASO,QAAAA,IAAYD,UAAAA,CAAWN,OAAAA,OACnEQ,OAAAA,qCACFI,OAAAA,CAAQD,MAAAA,CAAOJ,QAAAA;;;;;Ad3FlB;;;;;;;;;;;;;;;;;;;KeQYS,WAAAA,8BAAyCC,QAAAA,KACnDA,QAAAA,cAAqBK,UAAAA,uBACfA,UAAAA,EAAYH,UAAAA,CAAWF,QAAAA,MAAcG,UAAAA,CAAWH,QAAAA,eAErDI,YAAAA,CAAaJ,QAAAA,EAAUC,IAAAA;AAAAA,KAErBG,YAAAA,8BAA0CJ,QAAAA,IAC9CA,QAAAA;AAAAA,EACGF,QAAAA;AAEDF,MAAAA,CAAOI,QAAAA,EAAUC,IAAAA;AAEjBK,OAAAA,CAAQT,eAAAA,CAAgBG,QAAAA,EAAUC,IAAAA;;;KCrCzB,QAAA,WAAmB,UAAA;AAAA,KACnB,KAAA,iBAAsB,GAAA,gBAAmB,CAAA;AAAA,KACzC,MAAA,aAAmB,MAAA,CAAO,QAAA,EAAU,KAAA,CAAM,CAAA;AAAA,cAEzC,UAAA;AAAA,iBA2BG,UAAA,CAAW,KAAA,WAAgB,KAAA,IAAS,QAAA;AAAA,iBAIpC,SAAA,CAAU,KAAA,UAAe,QAAA,GAAW,QAAA;AAAA,uBAK9B,UAAA,sBAAgC,MAAA,CAAO,CAAA;EAC3D,MAAA,EAAS,KAAA,CAAM,CAAA;EACf,IAAA,EAAO,KAAA,CAAM,CAAA;EACb,OAAA,EAAU,KAAA,CAAM,CAAA;EAChB,IAAA,EAAO,KAAA,CAAM,CAAA;EACb,KAAA,EAAQ,KAAA,CAAM,CAAA;EACd,KAAA,EAAQ,KAAA,CAAM,CAAA;EACd,KAAA,EAAQ,KAAA,CAAM,CAAA;EACd,MAAA,EAAS,KAAA,CAAM,CAAA;EACf,GAAA,EAAM,KAAA,CAAM,CAAA;EACZ,KAAA,EAAQ,KAAA,CAAM,CAAA;;qBAQK,IAAA,CAAK,KAAA,EAAO,QAAA,KAAa,GAAA,cAAiB,CAAA;AAAA;;;iBCd/C,SAAA,CAAU,OAAA;;;iBCvCV,IAAA,CAAK,OAAA;AAAA,iBAIL,OAAA,CAAQ,GAAA,YAAe,KAAA;AAAA,KAI3B,MAAA,GAAS,MAAA;AAAA,KAET,YAAA,WAAuB,MAAA;EACjC,EAAA,iBAAmB,CAAA,EAAG,KAAA,EAAO,CAAA,EAAG,EAAA,MAAQ,IAAA,EAAM,CAAA,CAAE,CAAA,aAAc,YAAA,CAAa,CAAA;EAC3E,GAAA,iBAAoB,CAAA,EAAG,KAAA,EAAO,CAAA,EAAG,EAAA,MAAQ,IAAA,EAAM,CAAA,CAAE,CAAA,aAAc,YAAA,CAAa,CAAA;EAC5E,IAAA,iBAAqB,CAAA,EAAG,KAAA,EAAO,CAAA,EAAG,EAAA,MAAQ,IAAA,EAAM,CAAA,CAAE,CAAA,aAAc,YAAA,CAAa,CAAA;EAC7E,IAAA,iBAAqB,CAAA,EAAG,KAAA,EAAO,CAAA,KAAM,IAAA,EAAM,CAAA,CAAE,CAAA;AAAA;;;KCdnC,YAAA;EAAiB,GAAA;EAAc,MAAA;EAAiB,KAAA;AAAA;AAAA,KAEvD,cAAA;EACH,MAAA,GAAS,QAAA,EAAU,QAAA;EACnB,IAAA,GAAO,QAAA,EAAU,QAAA;AAAA;AAAA,cAAQ,aAAA,YAG8B,YAAA,CAAa,cAAA;AAAA,cAAzD,QAAA,SAAiB,aAAA;EAAA;EAQnB,IAAA;cAAA,IAAA,UACP,IAAA,GAAM,YAAA;EAAA,IAMJ,KAAA,CAAA;EAIJ,GAAA,CAAI,IAAA,EAAM,YAAA;EAAA,IAaN,MAAA,CAAA;EAAA,IAIA,MAAA,CAAO,MAAA;EAAA,IAIP,KAAA,CAAM,KAAA;EAAA,IAIN,KAAA,CAAA;EAAA,IAIA,GAAA,CAAI,GAAA;EAAA,IAIJ,GAAA,CAAA;EAAA,IAIA,IAAA,CAAA;EAAA,IAIA,KAAA,CAAA;EAAA,IAIA,GAAA,CAAA;EAIJ,IAAA,CAAA;EAQA,QAAA,CAAA,GAAQ,QAAA;EAIR,KAAA,CAAM,IAAA,UAAc,IAAA,GAAM,YAAA,GAAoB,QAAA;EAAA,CAe7C,OAAA,CAAQ,MAAA,EAAQ,MAAA,UAAgB,QAAA;EAIxB,QAAA,CAAS,MAAA;AAAA;;;KCrGR,eAAA;EACV,OAAA;EACA,OAAA;EAEA,WAAA,UpBGUhI;EoBDV,OAAA;EACA,KAAA;EACA,MAAA;AAAA;AAAA,KAGU,WAAA;EACV,IAAA;EACA,UAAA;EACA,WAAA,EAAa,WAAA;EACb,eAAA;EACA,QAAA,EAAU,eAAA;EACV,IAAA;AAAA;AAAA,KAGU,WAAA,GAAc,MAAA;AAAA,iBAEV,gBAAA,CAAiB,IAAA,WAAe,IAAA,CAAK,WAAA;AAAA,iBAYrC,aAAA,CAAc,IAAA,WAAe,WAAA;AAAA,iBAK7B,aAAA,CAAc,EAAA,WAAa,eAAA;AAAA,iBAyF3B,SAAA,CAAU,IAAA,UAAc,GAAA,EAAK,YAAA,EAAc,IAAA;AAAA,iBA4B3C,aAAA,CAAc,EAAA,UAAY,GAAA,EAAK,YAAA,EAAc,IAAA;;;KCzJjD,cAAA;EACV,WAAA;EACA,IAAA;EACA,QAAA;AAAA,IACE,WAAA;AAAA,KAEC,UAAA;EACH,KAAA;EACA,IAAA;AAAA;AAAA,cAGW,GAAA;EAAA;EASF,GAAA;EACA,IAAA;EAPT,IAAA;EACA,OAAA,EAAO,IAAA;EACP,QAAA,EAAU,UAAA;EACV,MAAA,EAAQ,WAAA;EAAA,UAEC,WAAA,CACA,GAAA,UACA,IAAA;ErBLCI;EAAAA,IqBYN,IAAA,CAAA;ErBVYH;EAAAA,IqBeZ,IAAA,CAAA;ErBfiB;EAAA,IqBoBjB,EAAA,CAAA,GACgC,cAAA;;MAIhC,IAAA,CAAA;EAAA,IAKA,YAAA,CAAA;EpBtBiB;EAAA,IoB4BjB,WAAA,CAAA;EpB1BsBM;EAAAA,IoBqDtB,MAAA,CAAA;EpBrDmDC;EAAAA,IoB4DnD,KAAA,CAAA;EAAA,IAQA,IAAA,CAAA;EAAA,IAIA,QAAA,CAAA;EAAA,IAIA,KAAA,CAAA;EAAA,UAIY,IAAA,CAAA,GAAQ,OAAA,CAAQ,GAAA;EAAA,OAmCnB,IAAA,CAAK,KAAA,WAAgB,QAAA,GAAW,OAAA,CAAQ,GAAA;EAAA,OACxC,IAAA,CAAK,GAAA,UAAa,IAAA,YAAgB,OAAA,CAAQ,GAAA;AAAA;;;cCtJ5C,IAAA;EAEF,GAAA;EACA,GAAA,EAAK,GAAA;cADL,GAAA,UACA,GAAA,EAAK,GAAA;AAAA;AAAA,KAMJ,SAAA;EACV,GAAA;EACA,IAAA;EACA,KAAA;AAAA;AAAA,KAGU,QAAA;EACV,GAAA;EACA,IAAA;AAAA;AAAA,KAGU,OAAA;EACV,IAAA,EAAM,OAAA;EACN,IAAA;AAAA;AAAA,KAGU,OAAA;EACV,IAAA;EACA,MAAA,GAAS,OAAA;EACT,GAAA;EACA,KAAA;EACA,KAAA;EACA,QAAA,EAAU,GAAA,SAAY,OAAA;AAAA;AAAA,KAGZ,cAAA;ErBFW,0CqBIrB,GAAA;EACA,KAAA;EACA,OAAA;EACA,UAAA;EACA,KAAA;EACA,IAAA;AAAA;AAAA,KAGU,OAAA;EACV,GAAA;EACA,IAAA,EAAM,OAAA;EACN,KAAA,EAAO,OAAA;AAAA;AAAA,KAGG,QAAA,GAAW,OAAA;EAErB,GAAA,GAAM,IAAA;AAAA;AAAA,cAGK,GAAA;EAAA;EAIe,GAAA,EAAK,OAAA;cAAL,GAAA,EAAK,OAAA;EAAA,IAI3B,OAAA,CAAA,GAAW,SAAA;EAIf,QAAA,CAAS,IAAA;EAKT,QAAA,CAAS,GAAA,WAAc,IAAA;IAAS,QAAA;EAAA,IAAuB,QAAA;EAmBvD,OAAA,CAAQ,GAAA,UAAa,MAAA,aAAc,OAAA;EAsBnC,OAAA,CAAQ,IAAA,EAAM,OAAA,WAAkB,IAAA;IAAS,QAAA;EAAA,IAAuB,OAAA;EAoChE,SAAA,CAAU,MAAA,EAAQ,SAAA;EASlB,OAAA,CAAQ,IAAA,GAAO,cAAA,IAAkB,GAAA;EAQ1B,IAAA,CAAK,IAAA,GAAM,cAAA,GAAsB,cAAA,CAAe,QAAA;EA4DhD,EAAA,CAAG,IAAA,GAAO,IAAA,CAAK,cAAA,aAAwB,cAAA,CAAA,QAAA;EnB/LmBM;EmBoMjE,QAAA,CAAS,CAAA,EAAG,OAAA,GAAU,QAAA;AAAA;;;KCtOZ,UAAA;AAAA,KAEA,WAAA;EACV,KAAA;EACA,aAAA;EACA,IAAA;AAAA;AAAA,KAGU,YAAA;EACV,GAAA;EACA,IAAA;EACA,GAAA,EAAK,MAAA;EACL,MAAA,EAAQ,OAAA,CAAQ,MAAA,CAAO,UAAA,EAAY,WAAA;EACnC,KAAA;IAAS,GAAA,GAAM,SAAA;IAAW,GAAA,GAAM,SAAA;EAAA;AAAA,IAC9B,QAAA;AAAA,KAEC,eAAA;EACH,MAAA,EAAQ,QAAA;EACR,GAAA,EAAK,KAAA;EACL,GAAA,EAAK,KAAA;AAAA;AAAA,KAGK,QAAA,GAAW,YAAA;EAAiB,MAAA;IAAU,MAAA,EAAQ,WAAA;EAAA;AAAA;AAAA,KAC9C,KAAA,GAAQ,YAAA;EAAiB,MAAA;IAAU,GAAA,EAAK,WAAA;EAAA;EAAe,KAAA;IAAS,GAAA,EAAK,SAAA;EAAA;AAAA;AAAA,KACrE,KAAA,GAAQ,YAAA;EAAiB,MAAA;IAAU,GAAA,EAAK,WAAA;EAAA;EAAe,KAAA;IAAS,GAAA,EAAK,SAAA;EAAA;AAAA;AAAA,KAErE,aAAA;EACV,KAAA;EACA,GAAA;EACA,IAAA,GAAO,UAAA;AAAA;AAAA,KAGG,gBAAA,GAAmB,IAAA,CAAK,aAAA;EAClC,EAAA;AAAA;AAAA,cAUW,MAAA;EAEF,EAAA,EAAI,EAAA;EACJ,GAAA,EAAK,OAAA;EAAA,QAFP,WAAA,CAAA;EAAA,OAKM,IAAA,CAAK,GAAA,EAAK,OAAA,GAAO,OAAA,CAAA,MAAA;EAIxB,MAAA,CAAO,KAAA,UAAe,IAAA,GAAM,aAAA,GAAqB,OAAA,CAAQ,YAAA;EAiBzD,SAAA,CACJ,KAAA,UACA,IAAA,GAAM,IAAA,CAAK,aAAA;IAA2B,KAAA;EAAA,IACrC,OAAA,CAAQ,KAAA;EA+DL,SAAA,CAAU,KAAA,UAAe,IAAA,GAAM,gBAAA,GAAwB,OAAA,CAAQ,KAAA;EAyBrE,IAAA,WAAe,UAAA,CAAA,CAAY,IAAA,EAAM,CAAA,EAAG,OAAA,EAAS,eAAA,CAAgB,CAAA,MAAO,eAAA,CAAgB,CAAA;EpB9HxEC;EAAAA,QoB6IJ,IAAA;AAAA;;;KChLE,UAAA,GAAa,aAAA;EACvB,MAAA;EACA,GAAA,EAAK,GAAA;AAAA;AAAA,cAGM,KAAA;EAEF,EAAA,EAAI,EAAA;EACJ,GAAA,EAAK,OAAA;EAAA,QAFP,WAAA,CAAA;EAAA,OAKM,IAAA,CAAK,GAAA,EAAK,OAAA,GAAO,OAAA,CAAA,KAAA;EAK9B,GAAA,CAAI,GAAA,EAAK,GAAA;EAiCH,KAAA,CAAM,EAAA,UAAY,GAAA,EAAK,GAAA,GAAM,OAAA,CAAQ,UAAA;EAyBrC,KAAA,CAAA,GAAK,OAAA,CAAA,GAAA,SAAA,GAAA;EAoBL,KAAA,CAAM,IAAA,EAAM,GAAA,SAAY,GAAA,IAAI,OAAA;EAmE5B,IAAA,CAAK,IAAA;IAAS,KAAA;EAAA,IAAiB,OAAA;EAQ/B,KAAA,CAAM,SAAA,EAAW,IAAA,GAAI,OAAA;AAAA;;;KClKxB,aAAA;EACH,GAAA,GAAM,KAAA,EAAO,QAAA,KAAa,GAAA;EAC1B,QAAA,GAAW,QAAA,EAAU,QAAA;AAAA;AAAA,KAGX,cAAA;EACV,QAAA,GAAW,eAAA;EACX,OAAA,GAAU,SAAA;EACV,IAAA;AAAA;AAAA,cAGW,OAAA,SAAgB,UAAA;EAAA;EASR,IAAA,EAAM,cAAA;EAFzB,MAAA,EAA+B,YAAA,CAAa,aAAA,IAAiB,YAAA;cAE1C,IAAA,GAAM,cAAA;EAAA,UASf,IAAA,CAAK,KAAA,EAAO,QAAA,KAAa,GAAA;EAAA,IAI/B,IAAA,CAAA;EAIE,EAAA,CAAA,GAAE,OAAA,CAAA,EAAA;EAKF,MAAA,CAAA,GAAM,OAAA,CAAA,MAAA;EAKN,KAAA,CAAA,GAAK,OAAA,CAAA,KAAA;EAKL,QAAA,CAAA,GAAQ,OAAA,CAAA,QAAA;EAKR,GAAA,CAAA,GAAG,OAAA,CAAA,GAAA;AAAA;;;cCvCE,QAAA;EAAA;EAOQ,GAAA,EAAK,OAAA;EALxB,IAAA,EAAM,uBAAA;EACN,KAAA,EAAO,qBAAA;EACP,MAAA,EAAM,QAAA;cAGa,GAAA,EAAK,OAAA;EAcxB,IAAA,CAAA;;;;;EAIM,OAAA,CAAA,GAAO,OAAA,CAAA,eAAA;EAgCb,SAAA,CAAU,KAAA,WAAgB,WAAA,GAAc,aAAA;EAMlC,KAAA,CAAM,KAAA,WAAgB,WAAA,GAAc,aAAA,GAAgB,OAAA;EACpD,KAAA,CAAM,KAAA,YAAiB,WAAA,GAAc,aAAA,MAAmB,OAAA;EAWxD,KAAA,CAAM,KAAA,WAAgB,WAAA,GAAc,OAAA,CAAQ,aAAA;AAAA;;;KCtGxC,YAAA;EACV,IAAA,CAAK,KAAA;AAAA;AAAA,KAGK,cAAA;AAAA,UACK,eAAA,SAAwB,YAAA;EACvC,MAAA,EAAQ,cAAA;EACR,SAAA;EACA,IAAA;EACA,KAAA,CAAM,KAAA,aAAkB,OAAA;AAAA;AAAA,KAGd,WAAA;EACV,IAAA;EACA,KAAA;AAAA;AAAA,KAGU,cAAA;EACV,QAAA,IAAY,GAAA,EAAK,WAAA;EACjB,KAAA,IAAS,KAAA;AAAA;AAAA,KAGC,aAAA;EACV,GAAA;EACA,IAAA;EACA,MAAA;EACA,MAAA,GAAS,cAAA;EACT,OAAA;AAAA;AAAA,KAGU,YAAA;AAAA,KAEA,eAAA;EACV,IAAA,EAAM,uBAAA;EACN,IAAA;EACA,MAAA,EAAQ,QAAA;EACR,MAAA,EAAQ,MAAA;AAAA;AAAA,KAGE,aAAA;EACV,GAAA;EACA,IAAA;EACA,MAAA;EACA,SAAA;AAAA;AAAA,KAGU,eAAA;EACV,SAAA;EACA,KAAA,GAAQ,aAAA;EACR,SAAA;EACA,OAAA;EACA,OAAA;EACA,MAAA;EACA,UAAA,IAAc,MAAA,EAAQ,QAAA;AAAA;AAAA,KAGZ,qBAAA,GAAwB,KAAA,CAAM,aAAA;EAAiB,MAAA,EAAQ,QAAA,CAAS,cAAA;AAAA;AAAA,KAChE,uBAAA,GAA0B,KAAA,CACpC,WAAA,CAAY,QAAA,CAAS,eAAA;EACnB,KAAA,EAAO,qBAAA;AAAA;;;cC3DL,MAAA;EAAA;;;;KAMM,aAAA,gBAA6B,MAAA;;;KCF7B,MAAA;EACV,EAAA;EACA,IAAA;EACA,IAAA;EACA,IAAA;EACA,QAAA;EACA,WAAA;EACA,KAAA;EACA,IAAA;EACA,QAAA;EACA,UAAA,EAAY,IAAA;EACZ,SAAA,GAAY,IAAA;EACZ,QAAA;EACA,QAAA;AAAA;AAAA,KA2BU,SAAA;EACV,MAAA;EACA,IAAA;EACA,GAAA;EACA,QAAA;EACA,KAAA;EACA,IAAA;AAAA;AAAA,KAGU,SAAA;EACV,KAAA;EACA,KAAA;EACA,IAAA;AAAA;AAAA,KAGU,eAAA;EACV,KAAA;EACA,KAAA;AAAA;AAAA,iBAQc,YAAA,WAAuB,aAAA,CAAA,CAAe,CAAA,EAAG,CAAA,GAAI,CAAA,IAAK,CAAA;EAAM,SAAA;AAAA;AAAA,iBAIxD,gBAAA,WAA2B,aAAA,CAAA,CACzC,MAAA,EAAQ,CAAA,aACC,MAAA,KAAW,CAAA;EAAM,SAAA;AAAA;AAAA,cAMf,EAAA;EAAA;UAIJ,WAAA,CAAA;EAAA,OAKM,IAAA,CAAK,MAAA,WAAc,OAAA,CAAA,EAAA;EAAA,QAIxB,IAAA;EAqFR,KAAA,CAAA;EAAA,QAeQ,OAAA;EAqBR,MAAA,CAAO,IAAA,oBAAqB,MAAA;EAQ5B,OAAA,CAAQ,IAAA,yBAA0B,GAAA,SAAA,MAAA;EAgBlC,MAAA,CAAO,GAAA,EAAK,IAAA,CAAK,MAAA;EA8BjB,SAAA,CAAU,EAAA,UAAY,MAAA;IAAU,IAAA;IAAgB,GAAA;EAAA;EAAA,IAM5C,GAAA,CAAA;;;;;EAUJ,SAAA,CAAA;;;;;;;;;;;EAuBA,WAAA,oBAAA,CAAgC,EAAA,MAAQ,IAAA,EAAM,CAAA,KAAM,CAAA;IAAA;;;;;EAIpD,iBAAA,CAAA,GAAqB,MAAA;EAUrB,QAAA,CAAS,EAAA,UAAY,QAAA,GAAQ,IAAA;EAI7B,YAAA,CAAa,EAAA,UAAY,OAAA;E1BhSLD;E0BqSpB,eAAA,CAAgB,YAAA,EAAc,IAAA,EAAM,MAAA;E1BpSpBE;E0BsThB,SAAA,CAAU,KAAA,UAAe,IAAA,GAAO,eAAA,GAAkB,SAAA;E1BtTtBF;;;E0B2U5B,YAAA,CAAA,GAAgB,GAAA;EAiChB,UAAA,CAAW,KAAA;;EAcX,gBAAA,CAAiB,MAAA,EAAQ,UAAA;EzBpYfI;EyB+YV,gBAAA,CAAiB,KAAA;EzB/YOC;EyBoZxB,SAAA,CAAU,SAAA,YAAqB,IAAA,GAAO,eAAA,GAAkB,SAAA;EAgBxD,KAAA,CAAM,GAAA,EAAK,MAAA,WAAiB,KAAA,GAAQ,aAAA;EAYpC,OAAA,CAAQ,GAAA;EAQR,OAAA,CAAQ,GAAA,UAAa,KAAA;EAQrB,QAAA,GAAA,CAAY,GAAA,WAAc,CAAA;EAW1B,QAAA,GAAA,CAAY,GAAA,UAAa,KAAA,EAAO,CAAA,GAAI,CAAA;EAUpC,UAAA,CAAW,UAAA;AAAA;;;iBCnfG,KAAA,CAAM,IAAA,WAAD,UAAA,CAAa,KAAA;AAAA,iBAMZ,KAAA,CAAM,IAAA,WAAY,OAAA,CAAb,UAAA,CAAa,KAAA;AAAA,iBAIxB,MAAA,CAAO,IAAA,UAAc,IAAA,UAAc,IAAA;AAAA,iBAcnC,QAAA,CAAA,GAAY,KAAA;AAAA,iBAIZ,OAAA,CAAQ,IAAA;;;KCjCnB,OAAA;EACC,IAAA;EAAc,KAAA;EAAe,GAAA;EAAe,GAAA;EAAe,KAAA;AAAA;EAC3D,IAAA;EAAY,KAAA;AAAA;EACZ,IAAA;EAAe,KAAA;AAAA;AAAA,iBAIL,QAAA,CAAS,KAAA,WAAgB,OAAA;;iBAoEzB,KAAA,CAAM,KAAA,UAAe,SAAA;;;cC3ExB,UAAA;AAAA,iBAEG,SAAA,CAAU,GAAA;AAAA,iBAIV,OAAA,CAAQ,GAAA,WAAc,GAAA;AAAA,iBAetB,SAAA,CAAU,GAAA;;;KCpBd,KAAA;EACV,IAAA;EACA,KAAA;AAAA;AAAA,KAGU,cAAA,GAAiB,KAAA;EAAU,KAAA;AAAA;AAAA,KAE3B,cAAA;EACV,KAAA;EACA,KAAA;EACA,SAAA,GAAY,GAAA;AAAA;AAAA,KAGF,aAAA;EACV,KAAA;EACA,IAAA;EACA,QAAA;EACA,KAAA;AAAA;AAAA,KAGU,aAAA;EACV,KAAA;EACA,MAAA,EAAQ,KAAA;EACR,MAAA;EACA,OAAA,EAAS,aAAA;EACT,IAAA,EAAM,aAAA;EACN,IAAA;EACA,OAAA;AAAA;AAAA,cAGW,UAAA,EAAU,MAAA;AAAA,iBA8BP,UAAA,CAAW,IAAA;AAAA,cAId,OAAA;EACX,KAAA,GAAQ,KAAA;IAAU,KAAA;EAAA;EAClB,QAAA,EAAQ,GAAA;EACR,WAAA,EAAa,MAAA;EACb,IAAA,EAAM,QAAA,CAAS,cAAA;cAEH,IAAA,EAAM,cAAA;EAkBlB,SAAA,CAAU,IAAA;EAUV,QAAA,CAAS,IAAA,UAAc,SAAA,aAAmB,KAAA;EAY1C,KAAA,CAAM,KAAA,EAAO,KAAA,EAAO,UAAA,EAAY,KAAA;EAgBhC,KAAA,CAAM,KAAA,EAAO,KAAA,YAAiB,cAAA;EAoB9B,MAAA,CAAO,MAAA,EAAQ,KAAA;;;;EAkBf,IAAA,CAAK,KAAA,YAAiB,MAAA;EAsBtB,OAAA,CAAQ,IAAA,WAAe,aAAA;EAiCvB,KAAA,CAAM,MAAA,EAAQ,aAAA;EAwBd,SAAA,CAAU,IAAA,UAAc,EAAA,GAAK,IAAA,UAAc,MAAA;AAAA"}
|
package/dist/index.mjs
CHANGED
|
@@ -80,23 +80,23 @@ var Context = class extends LoggerBase {
|
|
|
80
80
|
return this.#root;
|
|
81
81
|
}
|
|
82
82
|
async db() {
|
|
83
|
-
const { Db } = await import("./db-
|
|
83
|
+
const { Db } = await import("./db-CHpq7OOi.mjs");
|
|
84
84
|
return this.#db ??= await Db.load(join(this.root, "index.sqlite3"));
|
|
85
85
|
}
|
|
86
86
|
async search() {
|
|
87
|
-
const { Search } = await import("./search-
|
|
87
|
+
const { Search } = await import("./search-DsVjB-9f.mjs");
|
|
88
88
|
return this.#search ??= await Search.load(this);
|
|
89
89
|
}
|
|
90
90
|
async store() {
|
|
91
|
-
const { Store } = await import("./store-
|
|
91
|
+
const { Store } = await import("./store-I5nVEYxK.mjs");
|
|
92
92
|
return this.#store ??= await Store.load(this);
|
|
93
93
|
}
|
|
94
94
|
async embedder() {
|
|
95
|
-
const { Embedder } = await import("./embed-
|
|
95
|
+
const { Embedder } = await import("./embed-CZI5Dz1q.mjs");
|
|
96
96
|
return this.#embedder ??= new Embedder(this);
|
|
97
97
|
}
|
|
98
98
|
async vfs() {
|
|
99
|
-
const { Vfs } = await import("./vfs-
|
|
99
|
+
const { Vfs } = await import("./vfs-QUP1rnSI.mjs");
|
|
100
100
|
return this.#vfs ??= new Vfs(this);
|
|
101
101
|
}
|
|
102
102
|
};
|
|
@@ -349,3 +349,5 @@ var Snippet = class {
|
|
|
349
349
|
};
|
|
350
350
|
//#endregion
|
|
351
351
|
export { Context, Doc, LOG_LEVELS, LoggerBase, Progress, Snippet, URI_PREFIX, WORD_REGEX, assertUri, astat, chunkMarkdown, chunkText, findUp, gitRoot, hash, isLogLevel, isStopWord, normPath, normUri, parentUri, parseFrontmatter, parseMarkdown, parseSections, parseYaml, shouldLog, sstat, toError, toFts, tokenize };
|
|
352
|
+
|
|
353
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.mjs","names":["#root","#db","#search","#store","#embedder","#vfs"],"sources":["../src/log.ts","../src/context.ts","../src/snippet.ts"],"sourcesContent":["export type LogLevel = (typeof LOG_LEVELS)[number]\nexport type LogFn<T = void> = (...msg: unknown[]) => T\nexport type Logger<T = void> = Record<LogLevel, LogFn<T>>\n\nexport const LOG_LEVELS = [\n \"cancel\",\n \"info\",\n \"success\",\n \"warn\",\n \"error\",\n \"debug\",\n \"fatal\",\n \"prompt\",\n \"log\",\n \"trace\",\n] as const\n\n// oxlint-disable-next-line sort-keys\nconst LOG_PRIORITY: Record<LogLevel, number> = {\n trace: 0,\n debug: 1,\n log: 2,\n info: 2,\n prompt: 2,\n success: 2,\n cancel: 2,\n warn: 3,\n error: 4,\n fatal: 5,\n}\n\nexport function isLogLevel(level: string): level is LogLevel {\n return LOG_LEVELS.includes(level as LogLevel)\n}\n\nexport function shouldLog(level: string, minLevel?: LogLevel): boolean {\n if (!isLogLevel(level)) return true\n return LOG_PRIORITY[level] >= LOG_PRIORITY[minLevel ?? \"log\"]\n}\n\nexport abstract class LoggerBase<T = void> implements Logger<T> {\n cancel!: LogFn<T>\n info!: LogFn<T>\n success!: LogFn<T>\n warn!: LogFn<T>\n error!: LogFn<T>\n debug!: LogFn<T>\n fatal!: LogFn<T>\n prompt!: LogFn<T>\n log!: LogFn<T>\n trace!: LogFn<T>\n\n constructor() {\n for (const level of LOG_LEVELS) {\n this[level] = (...msg: unknown[]) => this._log(level, ...msg)\n }\n }\n\n protected abstract _log(level: LogLevel, ...msg: unknown[]): T\n}\n","import type { Db } from \"./db.ts\"\nimport type { Embedder, EmbedderOptions } from \"./embed/index.ts\"\nimport type { LogLevel } from \"./log.ts\"\nimport type { Progress } from \"./progress.ts\"\nimport type { Search } from \"./search.ts\"\nimport type { Store } from \"./store.ts\"\nimport type { TypedEmitter } from \"./util.ts\"\nimport type { Vfs, VfsFolder } from \"./vfs.ts\"\n\nimport { EventEmitter } from \"node:events\"\nimport { mkdirSync } from \"node:fs\"\nimport { join } from \"pathe\"\nimport { normPath } from \"./fs.ts\"\nimport { LoggerBase } from \"./log.ts\"\n\ntype ContextEvents = {\n log: [level: LogLevel, ...msg: unknown[]]\n progress: [progress: Progress]\n}\n\nexport type ContextOptions = {\n embedder?: EmbedderOptions\n folders?: VfsFolder[]\n root?: string // path to data folder, defaults to `~/.rekal-data/`\n}\n\nexport class Context extends LoggerBase {\n #embedder?: Embedder\n #root: string\n #db?: Db\n #search?: Search\n #store?: Store\n #vfs?: Vfs\n events = new EventEmitter() as TypedEmitter<ContextEvents> & EventEmitter\n\n constructor(public opts: ContextOptions = {}) {\n super()\n this.opts.embedder ??= {}\n this.opts.embedder.onProgress ??= (progress) => this.events.emit(\"progress\", progress)\n this.#root = normPath(this.opts.root ?? \"~/.rekal-data\")\n mkdirSync(this.#root, { recursive: true })\n }\n\n // Emit log events instead of logging directly\n protected _log(level: LogLevel, ...msg: unknown[]) {\n this.events.emit(\"log\", level, ...msg)\n }\n\n get root() {\n return this.#root\n }\n\n async db() {\n const { Db } = await import(\"./db.ts\")\n return (this.#db ??= await Db.load(join(this.root, \"index.sqlite3\")))\n }\n\n async search() {\n const { Search } = await import(\"./search.ts\")\n return (this.#search ??= await Search.load(this))\n }\n\n async store() {\n const { Store } = await import(\"./store.ts\")\n return (this.#store ??= await Store.load(this))\n }\n\n async embedder() {\n const { Embedder } = await import(\"./embed/index.ts\")\n return (this.#embedder ??= new Embedder(this))\n }\n\n async vfs() {\n const { Vfs } = await import(\"./vfs.ts\")\n return (this.#vfs ??= new Vfs(this))\n }\n}\n","// oxfmt-ignore\nexport type Token = {\n text: string\n lower: string\n}\n\nexport type TokenWithScore = Token & { score: number }\n\nexport type SnippetOptions = {\n query: string\n lines?: number\n stopWords?: Map<string, number>\n}\n\nexport type SnippetWindow = {\n start: number\n heat: number\n coverage: number\n score: number\n}\n\nexport type SnippetResult = {\n lines: string[]\n tokens: Token[][]\n scores: number[]\n windows: SnippetWindow[]\n best: SnippetWindow\n heat: number[]\n snippet: string[]\n}\n\nexport const WORD_REGEX = /[\\p{L}\\p{N}]+/gu // regex to match words (unicode letters and numbers)\n\nconst SCORE_EXACT = 3\nconst SCORE_LOWER = 2\nconst SCORE_QUERY_PREFIX = 1.5 // query \"child\" matches doc \"children\" — query is prefix of doc\nconst SCORE_DOC_PREFIX = 1 // query \"children\" matches doc \"child\" — doc is prefix of query\nconst SCORE_OVERLAP = 1\nconst MIN_PREFIX_LENGTH = 3 // minimum prefix length to consider for scoring\nconst STOPWORD_MIN = 0 // min stopword score for a term\nconst STOPWORD_MAX = 1 // max stopword score for a term\nconst HEAT_SPREAD = 5 // how many lines to spread the heat on each side of a match (should be < lines/2)\nconst WEIGHT_EMPTY = 0.1 // score multiplier for empty lines\nconst WEIGHT_NONWORD = 0.3 // score multiplier for lines without any word characters\nconst WEIGHT_REPETITION = 0.7 // score multiplier for repeated terms in the same line (to de-emphasize boilerplate)\nconst decayLinear = (d: number, r: number) => Math.max(0, 1 - d / r) // linear decay function\n// const decayHyperbolic = (d: number, _r: number) => 1 / (1 + d) // hyperbolic decay function\n// const decayExponential = (d: number, r: number) => Math.exp(-d / r) // exponential decay function\n\n// Common English stop words — used for post-processing snippets.\n// Kept minimal: only the highest-frequency words that add no search value.\n// oxfmt-ignore\nconst STOP_WORDS = new Set([\n \"a\", \"an\", \"and\", \"are\", \"as\", \"at\", \"be\", \"but\", \"by\", \"do\", \"for\", \"from\",\n \"had\", \"has\", \"have\", \"he\", \"her\", \"his\", \"how\", \"i\", \"if\", \"in\", \"is\", \"it\",\n \"its\", \"my\", \"no\", \"not\", \"of\", \"on\", \"or\", \"our\", \"she\", \"so\", \"than\",\n \"that\", \"the\", \"their\", \"them\", \"then\", \"there\", \"these\", \"they\", \"this\",\n \"to\", \"up\", \"us\", \"was\", \"we\", \"what\", \"when\", \"which\", \"who\", \"will\",\n \"with\", \"you\", \"your\",\n])\n\nexport function isStopWord(word: string): boolean {\n return STOP_WORDS.has(word.toLowerCase())\n}\n\nexport class Snippet {\n query: (Token & { score: number })[] = []\n prefixes = new Set<string>()\n prefixRegex: RegExp\n opts: Required<SnippetOptions>\n\n constructor(opts: SnippetOptions) {\n this.opts = { ...opts, lines: 5, stopWords: new Map() }\n const stopwords = new Map<string, number>([...STOP_WORDS].map((w) => [w, 0]))\n this.opts.stopWords.forEach((s, w) =>\n stopwords.set(w, Math.max(STOPWORD_MIN, Math.min(STOPWORD_MAX, s)))\n )\n\n const tokens = this.tokenize(opts.query, false)\n for (const tok of tokens) {\n const score = stopwords.get(tok.lower) ?? 2\n if (score === 0) continue\n this.prefixes.add(tok.lower.slice(0, 2))\n this.query.push({ ...tok, score })\n }\n this.prefixRegex =\n this.prefixes.size > 0 ? new RegExp(`(${[...this.prefixes].join(\"|\")})`, \"i\") : /(?!)/ // never matches\n }\n\n normalize(text: string): string {\n // Handle diacritics\n text = text.normalize(\"NFD\").replace(/\\p{M}/gu, \"\")\n // Drop possessive 's (and smart quote ’s) completely to avoid orphaned \"s\" tokens\n text = text.replace(/['’]s\\b/gi, \"\")\n // Globally replace any remaining single quotes/apostrophes with a space\n text = text.replace(/['’]/g, \" \")\n return text\n }\n\n tokenize(text: string, queryOnly = true): Token[] {\n if (queryOnly && !this.prefixRegex.test(text)) return []\n // only keep unicode letters and numbers\n const tokens = this.normalize(text).match(WORD_REGEX) ?? []\n const ret: Token[] = []\n for (const token of tokens) {\n const lower = token.toLowerCase()\n ret.push({ lower, text: token })\n }\n return ret\n }\n\n score(token: Token, queryToken: Token): number {\n const tl = token.lower.length\n if (token.text === queryToken.text) return SCORE_EXACT\n if (token.lower === queryToken.lower) return SCORE_LOWER\n if (token.lower.startsWith(queryToken.lower)) return SCORE_QUERY_PREFIX\n if (queryToken.lower.startsWith(token.lower) && tl >= MIN_PREFIX_LENGTH) return SCORE_DOC_PREFIX\n let prefix = 0\n for (let i = 0; i < token.lower.length; i++) {\n if (token.lower[i] !== queryToken.lower[i]) break\n prefix++\n }\n return prefix >= MIN_PREFIX_LENGTH\n ? SCORE_OVERLAP * (prefix / Math.max(token.lower.length, queryToken.lower.length))\n : 0\n }\n\n match(input: Token | string): TokenWithScore | undefined {\n let token: Token\n if (typeof input === \"string\") {\n const tok = this.normalize(input)\n token = { lower: tok.toLowerCase(), text: input }\n } else token = input\n\n let [bestScore, bestTok] = [0, this.query[0]]\n // oxlint-disable-next-line typescript/prefer-for-of\n for (let t = 0; t < this.query.length; t++) {\n const queryTok = this.query[t]\n const s = this.score(token, queryTok)\n if (s > bestScore) {\n ;[bestScore, bestTok] = [s, queryTok]\n }\n }\n if (bestScore > 0) return { ...bestTok, score: bestTok.score * bestScore }\n }\n\n // get initial scores/coverage for each line\n scores(tokens: Token[][]) {\n const coverage: Set<string>[] = tokens.map(() => new Set())\n const scores = tokens.map((line, l) => {\n let lineScore = 0\n for (const token of line) {\n const queryTok = this.match(token)\n if (!queryTok) continue\n let score = queryTok.score\n if (coverage[l].has(queryTok.lower)) score *= WEIGHT_REPETITION\n coverage[l].add(queryTok.lower)\n lineScore += score\n }\n return lineScore\n })\n return { coverage, scores }\n }\n\n // Build heatmap using a bounded spread (O(N * radius))\n heat(lines: string[], scores: number[]): number[] {\n const spread = Math.max(HEAT_SPREAD, Math.ceil(this.opts.lines / 2))\n const heat = new Float64Array(scores.length)\n for (let i = 0; i < scores.length; i++) {\n if (scores[i] === 0) continue\n const spreadStart = Math.max(0, i - spread)\n const spreadEnd = Math.min(scores.length - 1, i + spread)\n for (let j = spreadStart; j <= spreadEnd; j++) {\n let weight = 1\n\n // NOTE: de-emphasize lines without any word characters (e.g. code blocks, separators)\n if (!lines[j].trim()) weight *= WEIGHT_EMPTY\n else if (!lines[j].match(/\\p{L}/u)) weight *= WEIGHT_NONWORD\n\n weight *= decayLinear(Math.abs(i - j), spread)\n\n heat[j] += scores[i] * weight\n }\n }\n return [...heat]\n }\n\n extract(text: string): SnippetResult {\n const lines = text.split(\"\\n\")\n const radius = Math.min(this.opts.lines, lines.length)\n const tokens = lines.map((line) => this.tokenize(line))\n\n const { scores, coverage } = this.scores(tokens)\n const heat = this.heat(lines, scores)\n\n // Find the window with highest heat × term coverage\n const windows: SnippetWindow[] = []\n\n for (let i = 0; i <= scores.length - radius; i++) {\n if (heat[i] === 0 && windows.length > 0) continue // skip windows that don't start with any heat to save computation\n let heatSum = 0\n // Count how many distinct query terms appear in this window\n const covered = new Set<string>()\n for (let j = i; j < i + radius; j++) {\n heatSum += heat[j]\n coverage[j].forEach((t) => covered.add(t))\n }\n const cov = this.query.length === 0 ? 1 : covered.size / this.query.length\n windows.push({ coverage: cov, heat: heatSum, score: heatSum * cov, start: i })\n }\n\n let best = windows[0] ?? { coverage: 0, heat: 0, score: 0, start: 0 }\n for (let i = 1; i < windows.length; i++) {\n if (windows[i].score > best.score) best = windows[i]\n }\n\n const snippet = lines.slice(best.start, best.start + radius)\n return { best, heat: [...heat], lines, scores, snippet, tokens, windows }\n }\n\n debug(result: SnippetResult) {\n // oxlint-disable-next-line no-console\n console.info(\"Options:\", this.opts)\n // oxlint-disable-next-line no-console\n console.info(\"Query :\", this.query)\n // oxlint-disable-next-line unicorn/consistent-function-scoping\n const score = (n?: number, f = 1) => (n !== undefined ? n.toFixed(f).padEnd(4) : \" \".repeat(4))\n result.lines.forEach((line, i) => {\n const isBest = (l: number) =>\n l >= result.best.start && l < result.best.start + this.opts.lines\n\n const lineScore = score(result.scores[i])\n const lineHeat = score(result.heat[i])\n const windowHeat = score(result.windows[i]?.heat ?? 0)\n const windowScore = score(result.windows[i]?.score ?? 0)\n const coverage = ((result.windows[i]?.coverage ?? 0) * 100).toFixed(0).padStart(3)\n\n // oxlint-disable-next-line no-console\n console.log(\n `s:${lineScore} h:${lineHeat} wh:${windowHeat} ws:${windowScore} ${coverage}% ${isBest(i) ? \">\" : \" \"} ${line}`\n )\n })\n }\n\n highlight(text: string, hl: (word: string, offset: number) => string): string {\n return text.replace(WORD_REGEX, (word, offset) => (this.match(word) ? hl(word, offset) : word))\n }\n}\n"],"mappings":";;;;;;;;;;;AAIA,MAAa,aAAa;CACxB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAGD,MAAM,eAAyC;CAC7C,OAAO;CACP,OAAO;CACP,KAAK;CACL,MAAM;CACN,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,MAAM;CACN,OAAO;CACP,OAAO;CACR;AAED,SAAgB,WAAW,OAAkC;AAC3D,QAAO,WAAW,SAAS,MAAkB;;AAG/C,SAAgB,UAAU,OAAe,UAA8B;AACrE,KAAI,CAAC,WAAW,MAAM,CAAE,QAAO;AAC/B,QAAO,aAAa,UAAU,aAAa,YAAY;;AAGzD,IAAsB,aAAtB,MAAgE;CAC9D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,cAAc;AACZ,OAAK,MAAM,SAAS,WAClB,MAAK,UAAU,GAAG,QAAmB,KAAK,KAAK,OAAO,GAAG,IAAI;;;;;AC5BnE,IAAa,UAAb,cAA6B,WAAW;CACtC;CACA;CACA;CACA;CACA;CACA;CACA,SAAS,IAAI,cAAc;CAE3B,YAAY,OAA8B,EAAE,EAAE;AAC5C,SAAO;AADU,OAAA,OAAA;AAEjB,OAAK,KAAK,aAAa,EAAE;AACzB,OAAK,KAAK,SAAS,gBAAgB,aAAa,KAAK,OAAO,KAAK,YAAY,SAAS;AACtF,QAAA,OAAa,SAAS,KAAK,KAAK,QAAQ,gBAAgB;AACxD,YAAU,MAAA,MAAY,EAAE,WAAW,MAAM,CAAC;;CAI5C,KAAe,OAAiB,GAAG,KAAgB;AACjD,OAAK,OAAO,KAAK,OAAO,OAAO,GAAG,IAAI;;CAGxC,IAAI,OAAO;AACT,SAAO,MAAA;;CAGT,MAAM,KAAK;EACT,MAAM,EAAE,OAAO,MAAM,OAAO;AAC5B,SAAQ,MAAA,OAAa,MAAM,GAAG,KAAK,KAAK,KAAK,MAAM,gBAAgB,CAAC;;CAGtE,MAAM,SAAS;EACb,MAAM,EAAE,WAAW,MAAM,OAAO;AAChC,SAAQ,MAAA,WAAiB,MAAM,OAAO,KAAK,KAAK;;CAGlD,MAAM,QAAQ;EACZ,MAAM,EAAE,UAAU,MAAM,OAAO;AAC/B,SAAQ,MAAA,UAAgB,MAAM,MAAM,KAAK,KAAK;;CAGhD,MAAM,WAAW;EACf,MAAM,EAAE,aAAa,MAAM,OAAO;AAClC,SAAQ,MAAA,aAAmB,IAAI,SAAS,KAAK;;CAG/C,MAAM,MAAM;EACV,MAAM,EAAE,QAAQ,MAAM,OAAO;AAC7B,SAAQ,MAAA,QAAc,IAAI,IAAI,KAAK;;;;;AC3CvC,MAAa,aAAa;AAE1B,MAAM,cAAc;AACpB,MAAM,cAAc;AACpB,MAAM,qBAAqB;AAC3B,MAAM,mBAAmB;AACzB,MAAM,gBAAgB;AACtB,MAAM,oBAAoB;AAC1B,MAAM,eAAe;AACrB,MAAM,eAAe;AACrB,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,iBAAiB;AACvB,MAAM,oBAAoB;AAC1B,MAAM,eAAe,GAAW,MAAc,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE;AAOpE,MAAM,aAAa,IAAI,IAAI;CACzB;CAAK;CAAM;CAAO;CAAO;CAAM;CAAM;CAAM;CAAO;CAAM;CAAM;CAAO;CACrE;CAAO;CAAO;CAAQ;CAAM;CAAO;CAAO;CAAO;CAAK;CAAM;CAAM;CAAM;CACxE;CAAO;CAAM;CAAM;CAAO;CAAM;CAAM;CAAM;CAAO;CAAO;CAAM;CAChE;CAAQ;CAAO;CAAS;CAAQ;CAAQ;CAAS;CAAS;CAAQ;CAClE;CAAM;CAAM;CAAM;CAAO;CAAM;CAAQ;CAAQ;CAAS;CAAO;CAC/D;CAAQ;CAAO;CAChB,CAAC;AAEF,SAAgB,WAAW,MAAuB;AAChD,QAAO,WAAW,IAAI,KAAK,aAAa,CAAC;;AAG3C,IAAa,UAAb,MAAqB;CACnB,QAAuC,EAAE;CACzC,2BAAW,IAAI,KAAa;CAC5B;CACA;CAEA,YAAY,MAAsB;AAChC,OAAK,OAAO;GAAE,GAAG;GAAM,OAAO;GAAG,2BAAW,IAAI,KAAK;GAAE;EACvD,MAAM,YAAY,IAAI,IAAoB,CAAC,GAAG,WAAW,CAAC,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;AAC7E,OAAK,KAAK,UAAU,SAAS,GAAG,MAC9B,UAAU,IAAI,GAAG,KAAK,IAAI,cAAc,KAAK,IAAI,cAAc,EAAE,CAAC,CAAC,CACpE;EAED,MAAM,SAAS,KAAK,SAAS,KAAK,OAAO,MAAM;AAC/C,OAAK,MAAM,OAAO,QAAQ;GACxB,MAAM,QAAQ,UAAU,IAAI,IAAI,MAAM,IAAI;AAC1C,OAAI,UAAU,EAAG;AACjB,QAAK,SAAS,IAAI,IAAI,MAAM,MAAM,GAAG,EAAE,CAAC;AACxC,QAAK,MAAM,KAAK;IAAE,GAAG;IAAK;IAAO,CAAC;;AAEpC,OAAK,cACH,KAAK,SAAS,OAAO,IAAI,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,GAAG;;CAGpF,UAAU,MAAsB;AAE9B,SAAO,KAAK,UAAU,MAAM,CAAC,QAAQ,WAAW,GAAG;AAEnD,SAAO,KAAK,QAAQ,aAAa,GAAG;AAEpC,SAAO,KAAK,QAAQ,SAAS,IAAI;AACjC,SAAO;;CAGT,SAAS,MAAc,YAAY,MAAe;AAChD,MAAI,aAAa,CAAC,KAAK,YAAY,KAAK,KAAK,CAAE,QAAO,EAAE;EAExD,MAAM,SAAS,KAAK,UAAU,KAAK,CAAC,MAAM,WAAW,IAAI,EAAE;EAC3D,MAAM,MAAe,EAAE;AACvB,OAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,QAAQ,MAAM,aAAa;AACjC,OAAI,KAAK;IAAE;IAAO,MAAM;IAAO,CAAC;;AAElC,SAAO;;CAGT,MAAM,OAAc,YAA2B;EAC7C,MAAM,KAAK,MAAM,MAAM;AACvB,MAAI,MAAM,SAAS,WAAW,KAAM,QAAO;AAC3C,MAAI,MAAM,UAAU,WAAW,MAAO,QAAO;AAC7C,MAAI,MAAM,MAAM,WAAW,WAAW,MAAM,CAAE,QAAO;AACrD,MAAI,WAAW,MAAM,WAAW,MAAM,MAAM,IAAI,MAAM,kBAAmB,QAAO;EAChF,IAAI,SAAS;AACb,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,MAAM,QAAQ,KAAK;AAC3C,OAAI,MAAM,MAAM,OAAO,WAAW,MAAM,GAAI;AAC5C;;AAEF,SAAO,UAAU,oBACb,iBAAiB,SAAS,KAAK,IAAI,MAAM,MAAM,QAAQ,WAAW,MAAM,OAAO,IAC/E;;CAGN,MAAM,OAAmD;EACvD,IAAI;AACJ,MAAI,OAAO,UAAU,SAEnB,SAAQ;GAAE,OADE,KAAK,UAAU,MAAM,CACZ,aAAa;GAAE,MAAM;GAAO;MAC5C,SAAQ;EAEf,IAAI,CAAC,WAAW,WAAW,CAAC,GAAG,KAAK,MAAM,GAAG;AAE7C,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;GAC1C,MAAM,WAAW,KAAK,MAAM;GAC5B,MAAM,IAAI,KAAK,MAAM,OAAO,SAAS;AACrC,OAAI,IAAI,UACL,EAAC,WAAW,WAAW,CAAC,GAAG,SAAS;;AAGzC,MAAI,YAAY,EAAG,QAAO;GAAE,GAAG;GAAS,OAAO,QAAQ,QAAQ;GAAW;;CAI5E,OAAO,QAAmB;EACxB,MAAM,WAA0B,OAAO,0BAAU,IAAI,KAAK,CAAC;AAa3D,SAAO;GAAE;GAAU,QAZJ,OAAO,KAAK,MAAM,MAAM;IACrC,IAAI,YAAY;AAChB,SAAK,MAAM,SAAS,MAAM;KACxB,MAAM,WAAW,KAAK,MAAM,MAAM;AAClC,SAAI,CAAC,SAAU;KACf,IAAI,QAAQ,SAAS;AACrB,SAAI,SAAS,GAAG,IAAI,SAAS,MAAM,CAAE,UAAS;AAC9C,cAAS,GAAG,IAAI,SAAS,MAAM;AAC/B,kBAAa;;AAEf,WAAO;KACP;GACyB;;CAI7B,KAAK,OAAiB,QAA4B;EAChD,MAAM,SAAS,KAAK,IAAI,aAAa,KAAK,KAAK,KAAK,KAAK,QAAQ,EAAE,CAAC;EACpE,MAAM,OAAO,IAAI,aAAa,OAAO,OAAO;AAC5C,OAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,OAAI,OAAO,OAAO,EAAG;GACrB,MAAM,cAAc,KAAK,IAAI,GAAG,IAAI,OAAO;GAC3C,MAAM,YAAY,KAAK,IAAI,OAAO,SAAS,GAAG,IAAI,OAAO;AACzD,QAAK,IAAI,IAAI,aAAa,KAAK,WAAW,KAAK;IAC7C,IAAI,SAAS;AAGb,QAAI,CAAC,MAAM,GAAG,MAAM,CAAE,WAAU;aACvB,CAAC,MAAM,GAAG,MAAM,SAAS,CAAE,WAAU;AAE9C,cAAU,YAAY,KAAK,IAAI,IAAI,EAAE,EAAE,OAAO;AAE9C,SAAK,MAAM,OAAO,KAAK;;;AAG3B,SAAO,CAAC,GAAG,KAAK;;CAGlB,QAAQ,MAA6B;EACnC,MAAM,QAAQ,KAAK,MAAM,KAAK;EAC9B,MAAM,SAAS,KAAK,IAAI,KAAK,KAAK,OAAO,MAAM,OAAO;EACtD,MAAM,SAAS,MAAM,KAAK,SAAS,KAAK,SAAS,KAAK,CAAC;EAEvD,MAAM,EAAE,QAAQ,aAAa,KAAK,OAAO,OAAO;EAChD,MAAM,OAAO,KAAK,KAAK,OAAO,OAAO;EAGrC,MAAM,UAA2B,EAAE;AAEnC,OAAK,IAAI,IAAI,GAAG,KAAK,OAAO,SAAS,QAAQ,KAAK;AAChD,OAAI,KAAK,OAAO,KAAK,QAAQ,SAAS,EAAG;GACzC,IAAI,UAAU;GAEd,MAAM,0BAAU,IAAI,KAAa;AACjC,QAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,eAAW,KAAK;AAChB,aAAS,GAAG,SAAS,MAAM,QAAQ,IAAI,EAAE,CAAC;;GAE5C,MAAM,MAAM,KAAK,MAAM,WAAW,IAAI,IAAI,QAAQ,OAAO,KAAK,MAAM;AACpE,WAAQ,KAAK;IAAE,UAAU;IAAK,MAAM;IAAS,OAAO,UAAU;IAAK,OAAO;IAAG,CAAC;;EAGhF,IAAI,OAAO,QAAQ,MAAM;GAAE,UAAU;GAAG,MAAM;GAAG,OAAO;GAAG,OAAO;GAAG;AACrE,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAClC,KAAI,QAAQ,GAAG,QAAQ,KAAK,MAAO,QAAO,QAAQ;EAGpD,MAAM,UAAU,MAAM,MAAM,KAAK,OAAO,KAAK,QAAQ,OAAO;AAC5D,SAAO;GAAE;GAAM,MAAM,CAAC,GAAG,KAAK;GAAE;GAAO;GAAQ;GAAS;GAAQ;GAAS;;CAG3E,MAAM,QAAuB;AAE3B,UAAQ,KAAK,YAAY,KAAK,KAAK;AAEnC,UAAQ,KAAK,WAAW,KAAK,MAAM;EAEnC,MAAM,SAAS,GAAY,IAAI,MAAO,MAAM,KAAA,IAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,OAAO,EAAE;AAC9F,SAAO,MAAM,SAAS,MAAM,MAAM;GAChC,MAAM,UAAU,MACd,KAAK,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,QAAQ,KAAK,KAAK;GAE9D,MAAM,YAAY,MAAM,OAAO,OAAO,GAAG;GACzC,MAAM,WAAW,MAAM,OAAO,KAAK,GAAG;GACtC,MAAM,aAAa,MAAM,OAAO,QAAQ,IAAI,QAAQ,EAAE;GACtD,MAAM,cAAc,MAAM,OAAO,QAAQ,IAAI,SAAS,EAAE;GACxD,MAAM,aAAa,OAAO,QAAQ,IAAI,YAAY,KAAK,KAAK,QAAQ,EAAE,CAAC,SAAS,EAAE;AAGlF,WAAQ,IACN,KAAK,UAAU,KAAK,SAAS,MAAM,WAAW,MAAM,YAAY,GAAG,SAAS,IAAI,OAAO,EAAE,GAAG,MAAM,IAAI,GAAG,OAC1G;IACD;;CAGJ,UAAU,MAAc,IAAsD;AAC5E,SAAO,KAAK,QAAQ,aAAa,MAAM,WAAY,KAAK,MAAM,KAAK,GAAG,GAAG,MAAM,OAAO,GAAG,KAAM"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as parseModelUri } from "./models-
|
|
1
|
+
import { n as parseModelUri } from "./models-Bo6czhQe.mjs";
|
|
2
2
|
import { join } from "pathe";
|
|
3
3
|
import { availableParallelism } from "node:os";
|
|
4
4
|
import { LlamaLogLevel } from "node-llama-cpp";
|
|
@@ -73,3 +73,5 @@ var LlamaBackend = class LlamaBackend {
|
|
|
73
73
|
};
|
|
74
74
|
//#endregion
|
|
75
75
|
export { LlamaBackend };
|
|
76
|
+
|
|
77
|
+
//# sourceMappingURL=llama-CpNV7Lh9.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llama-CpNV7Lh9.mjs","names":["#model","#ctx","#threadsPerCtx","#poolSize","#contexts"],"sources":["../src/embed/llama.ts"],"sourcesContent":["import type { Llama, LlamaEmbeddingContext, LlamaModel } from \"node-llama-cpp\"\nimport type { LogLevel } from \"../log.ts\"\nimport type { EmbedderBackend, EmbedderContext, EmbedderDevice } from \"./index.ts\"\n\nimport { LlamaLogLevel } from \"node-llama-cpp\"\nimport { availableParallelism } from \"node:os\"\nimport { join } from \"pathe\"\nimport { parseModelUri } from \"./models.ts\"\n\nexport class LlamaBackend implements EmbedderBackend {\n static llama?: Llama\n device: EmbedderDevice\n maxTokens: number\n dims: number\n #contexts: LlamaEmbeddingContext[] = []\n #model: LlamaModel\n #poolSize: number\n #threadsPerCtx: number\n #ctx: EmbedderContext\n\n private constructor(llama: Llama, model: LlamaModel, ctx: EmbedderContext) {\n this.#model = model\n this.#ctx = ctx\n this.maxTokens = model.trainContextSize\n this.dims = model.embeddingVectorSize\n this.device = llama.gpu ? \"gpu\" : \"cpu\"\n\n const useGpu = llama.gpu && ctx.opts.useGpu\n const cores = availableParallelism()\n const poolSize = useGpu\n ? Math.min(8, Math.max(1, Math.floor(cores / 4)))\n : Math.min(8, Math.max(1, Math.floor(cores / 4)))\n\n this.#threadsPerCtx = useGpu ? 0 : Math.max(1, Math.floor(ctx.opts.threads / poolSize))\n this.#poolSize = poolSize\n ctx.logger.debug({\n cores,\n gpu: llama.gpu,\n poolSize,\n threadsPerCtx: this.#threadsPerCtx,\n })\n }\n\n static async load(this: void, ctx: EmbedderContext) {\n const { model, variant } = parseModelUri(ctx.opts.model.uri)\n const { getLlama, resolveModelFile } = await import(\"node-llama-cpp\")\n const modelsDir = join(ctx.root, \"models\")\n const uri = `hf:${model}${variant ? `:${variant}` : \"\"}`\n const modelPath = await resolveModelFile(uri, modelsDir)\n\n LlamaBackend.llama ??= await getLlama({\n gpu: ctx.opts.useGpu ? \"auto\" : false,\n logLevel: LlamaLogLevel.error,\n logger: (level, message) => {\n const l = level as LogLevel\n // oxlint-disable-next-line typescript/no-unnecessary-condition\n ;(ctx.logger[l] ?? ctx.logger.log)(level.toString(), message)\n },\n })\n\n const lm = await LlamaBackend.llama.loadModel({\n // the below makes GPU super slow\n // defaultContextFlashAttention: true,\n modelPath,\n })\n return new LlamaBackend(LlamaBackend.llama, lm, ctx)\n }\n\n /** Get or create up to `count` embedding contexts */\n private async acquire(count: number): Promise<LlamaEmbeddingContext[]> {\n const needed = Math.min(count, this.#poolSize) - this.#contexts.length\n for (let i = 0; i < needed; i++) {\n try {\n this.#ctx.logger.debug(\n `Creating embedding context ${this.#contexts.length + 1}/${this.#poolSize}...`\n )\n this.#contexts.push(\n // oxlint-disable-next-line no-await-in-loop\n await this.#model.createEmbeddingContext({\n contextSize: this.#ctx.opts.maxTokens,\n threads: this.#threadsPerCtx,\n })\n )\n } catch {\n this.#ctx.logger.warn(\n `Failed to create embedding context ${this.#contexts.length + 1}. Adjusting pool size down to ${this.#contexts.length}.`\n )\n // adjust pool size down if we fail to create contexts,\n // which can happen if we run out of VRAM or hit some other resource limit\n this.#poolSize = this.#contexts.length\n break\n }\n }\n return this.#contexts\n }\n\n async embed(texts: string[]): Promise<number[][]> {\n const contexts = await this.acquire(texts.length)\n return Promise.all(\n texts.map((text, idx) =>\n contexts[idx % contexts.length]\n .getEmbeddingFor(text)\n .then((embedding) => [...embedding.vector])\n )\n )\n }\n\n toks(input: string) {\n return this.#model.tokenize(input).length\n }\n}\n"],"mappings":";;;;;AASA,IAAa,eAAb,MAAa,aAAwC;CACnD,OAAO;CACP;CACA;CACA;CACA,YAAqC,EAAE;CACvC;CACA;CACA;CACA;CAEA,YAAoB,OAAc,OAAmB,KAAsB;AACzE,QAAA,QAAc;AACd,QAAA,MAAY;AACZ,OAAK,YAAY,MAAM;AACvB,OAAK,OAAO,MAAM;AAClB,OAAK,SAAS,MAAM,MAAM,QAAQ;EAElC,MAAM,SAAS,MAAM,OAAO,IAAI,KAAK;EACrC,MAAM,QAAQ,sBAAsB;EACpC,MAAM,WAAW,SACb,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,EAAE,CAAC,CAAC,GAC/C,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,EAAE,CAAC,CAAC;AAEnD,QAAA,gBAAsB,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,KAAK,UAAU,SAAS,CAAC;AACvF,QAAA,WAAiB;AACjB,MAAI,OAAO,MAAM;GACf;GACA,KAAK,MAAM;GACX;GACA,eAAe,MAAA;GAChB,CAAC;;CAGJ,aAAa,KAAiB,KAAsB;EAClD,MAAM,EAAE,OAAO,YAAY,cAAc,IAAI,KAAK,MAAM,IAAI;EAC5D,MAAM,EAAE,UAAU,qBAAqB,MAAM,OAAO;EACpD,MAAM,YAAY,KAAK,IAAI,MAAM,SAAS;EAE1C,MAAM,YAAY,MAAM,iBADZ,MAAM,QAAQ,UAAU,IAAI,YAAY,MACN,UAAU;AAExD,eAAa,UAAU,MAAM,SAAS;GACpC,KAAK,IAAI,KAAK,SAAS,SAAS;GAChC,UAAU,cAAc;GACxB,SAAS,OAAO,YAAY;IAC1B,MAAM,IAAI;AAET,KAAC,IAAI,OAAO,MAAM,IAAI,OAAO,KAAK,MAAM,UAAU,EAAE,QAAQ;;GAEhE,CAAC;EAEF,MAAM,KAAK,MAAM,aAAa,MAAM,UAAU,EAG5C,WACD,CAAC;AACF,SAAO,IAAI,aAAa,aAAa,OAAO,IAAI,IAAI;;;CAItD,MAAc,QAAQ,OAAiD;EACrE,MAAM,SAAS,KAAK,IAAI,OAAO,MAAA,SAAe,GAAG,MAAA,SAAe;AAChE,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAC1B,KAAI;AACF,SAAA,IAAU,OAAO,MACf,8BAA8B,MAAA,SAAe,SAAS,EAAE,GAAG,MAAA,SAAe,KAC3E;AACD,SAAA,SAAe,KAEb,MAAM,MAAA,MAAY,uBAAuB;IACvC,aAAa,MAAA,IAAU,KAAK;IAC5B,SAAS,MAAA;IACV,CAAC,CACH;UACK;AACN,SAAA,IAAU,OAAO,KACf,sCAAsC,MAAA,SAAe,SAAS,EAAE,gCAAgC,MAAA,SAAe,OAAO,GACvH;AAGD,SAAA,WAAiB,MAAA,SAAe;AAChC;;AAGJ,SAAO,MAAA;;CAGT,MAAM,MAAM,OAAsC;EAChD,MAAM,WAAW,MAAM,KAAK,QAAQ,MAAM,OAAO;AACjD,SAAO,QAAQ,IACb,MAAM,KAAK,MAAM,QACf,SAAS,MAAM,SAAS,QACrB,gBAAgB,KAAK,CACrB,MAAM,cAAc,CAAC,GAAG,UAAU,OAAO,CAAC,CAC9C,CACF;;CAGH,KAAK,OAAe;AAClB,SAAO,MAAA,MAAY,SAAS,MAAM,CAAC"}
|
|
@@ -17,9 +17,9 @@ function resolveModel(opts) {
|
|
|
17
17
|
}
|
|
18
18
|
function loadModel(ctx) {
|
|
19
19
|
const { backend } = parseModelUri(ctx.opts.model.uri);
|
|
20
|
-
if (backend === "transformers") return import("./transformers-
|
|
21
|
-
else if (backend === "llama") return import("./llama-
|
|
22
|
-
else if (backend === "openai") return import("./openai-
|
|
20
|
+
if (backend === "transformers") return import("./transformers-Df56Nq9G.mjs").then(({ TransformersBackend }) => TransformersBackend.load(ctx));
|
|
21
|
+
else if (backend === "llama") return import("./llama-CpNV7Lh9.mjs").then(({ LlamaBackend }) => LlamaBackend.load(ctx));
|
|
22
|
+
else if (backend === "openai") return import("./openai-ALl6_YhI.mjs").then(({ OpenAIBackend }) => OpenAIBackend.load(ctx));
|
|
23
23
|
else throw new Error(`Unsupported model backend: ${String(backend)}`);
|
|
24
24
|
}
|
|
25
25
|
const DEFAULTS = {
|
|
@@ -75,3 +75,5 @@ for (const [base, options] of Object.entries(MODELS)) for (const model of option
|
|
|
75
75
|
};
|
|
76
76
|
//#endregion
|
|
77
77
|
export { parseModelUri as n, resolveModel as r, loadModel as t };
|
|
78
|
+
|
|
79
|
+
//# sourceMappingURL=models-Bo6czhQe.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models-Bo6czhQe.mjs","names":[],"sources":["../src/embed/models.ts"],"sourcesContent":["import type { SetOptional } from \"type-fest\"\nimport type {\n EmbedderBackend,\n EmbedderContext,\n EmbedderModel,\n ModelBackend,\n ResolvedEmbedderModel,\n} from \"./index.ts\"\n\nimport { defu } from \"defu\"\n\nexport function parseModelUri(uri: string) {\n const [backend, model, variant] = uri.split(\":\")\n if (!backend || !model)\n throw new Error(`Invalid model URI: ${uri}. Expected format \"provider:model[:variant]\"`)\n return { backend, model, variant } as { model: string; backend: ModelBackend; variant?: string }\n}\n\nexport function resolveModel(opts?: string | EmbedderModel): ResolvedEmbedderModel {\n const uri = typeof opts === \"string\" ? opts : (opts?.uri ?? DEFAULTS.uri)\n const options = typeof opts === \"string\" ? { uri } : (opts ?? { uri })\n const base = MODELS[options.base ?? MODELS[uri]?.base ?? \"\"] ?? {}\n const model = MODELS[uri] ?? {}\n return defu(options, model, base, DEFAULTS) as ResolvedEmbedderModel\n}\n\nexport function loadModel(ctx: EmbedderContext): Promise<EmbedderBackend> {\n const { backend } = parseModelUri(ctx.opts.model.uri) as { backend: string }\n if (backend === \"transformers\") {\n return import(\"./transformers.ts\").then(({ TransformersBackend }) =>\n TransformersBackend.load(ctx)\n )\n } else if (backend === \"llama\") {\n return import(\"./llama.ts\").then(({ LlamaBackend }) => LlamaBackend.load(ctx))\n } else if (backend === \"openai\") {\n return import(\"./openai.ts\").then(({ OpenAIBackend }) => OpenAIBackend.load(ctx))\n } else {\n throw new Error(`Unsupported model backend: ${String(backend)}`)\n }\n}\n\nconst DEFAULTS: ResolvedEmbedderModel = {\n prompt: {\n document: (doc) => {\n // If the title isn't found near the start of the text,\n // prepend it to ensure it's included in the embedding.\n const title = doc.title?.trim()\n if (title?.length) {\n const idx = doc.text.indexOf(title)\n if (idx === -1 || idx > 10) return `${title}\\n\\n${doc.text}`\n }\n return doc.text\n },\n query: (query) => query,\n },\n uri: \"transformers:Snowflake/snowflake-arctic-embed-s:q8\",\n}\n\nconst MODELS: Record<string, SetOptional<EmbedderModel, \"uri\">> = {\n BAAI: {\n models: [\n \"transformers:BAAI/bge-large-en-v1.5\", // 55.44%, 335M params, 304M active, 1024 dims, 512 context\n \"transformers:BAAI/bge-base-en-v1.5\", // 54.75%, 109M params, 86M active, 768 dims, 512 context\n \"transformers:BAAI/bge-small-en-v1.5\", // 53.86%, 33M params, 22M active, 512 dims, 512 context\n ],\n pooling: \"cls\",\n prompt: {\n query: (query) => `Represent this sentence for searching relevant passages: ${query}`,\n },\n },\n openai: {\n models: [\n \"openai:text-embedding-3-small\",\n \"openai:text-embedding-3-large\",\n \"openai:text-embedding-ada-002\",\n ],\n },\n snowflake: {\n models: [\n \"transformers:Snowflake/snowflake-arctic-embed-m-v2.0:q8\", // 58.41%, 305M params, 113M active, 768 dims, 8192 context\n \"transformers:Snowflake/snowflake-arctic-embed-s:q8\", // 54.85%, 33M params, 22M active, 384 dims, 512 context\n \"transformers:Snowflake/snowflake-arctic-embed-xs:q8\", // 52.65%, 23M params, 11M active, 384 dims, 512 context\n \"llama:mradermacher/snowflake-arctic-embed-s-GGUF:Q4_K_M\",\n ],\n pooling: \"cls\",\n prompt: {\n query: (query) => `Represent this sentence for searching relevant passages: ${query}`,\n },\n },\n \"transformers:onnx-community/embeddinggemma-300m-ONNX:q8\": {\n pooling: \"mean\",\n // 55.69%, 308M params, 106M active, 768 dims, 2048 context\n prompt: {\n document: (doc) => `title: ${doc.title ?? \"none\"} | ${doc.text}`,\n query: (query) => `task: search result | query: ${query}`,\n },\n },\n}\n\nfor (const [base, options] of Object.entries(MODELS)) {\n for (const model of options.models ?? []) {\n MODELS[model] = { base, uri: model }\n }\n}\n"],"mappings":";;AAWA,SAAgB,cAAc,KAAa;CACzC,MAAM,CAAC,SAAS,OAAO,WAAW,IAAI,MAAM,IAAI;AAChD,KAAI,CAAC,WAAW,CAAC,MACf,OAAM,IAAI,MAAM,sBAAsB,IAAI,8CAA8C;AAC1F,QAAO;EAAE;EAAS;EAAO;EAAS;;AAGpC,SAAgB,aAAa,MAAsD;CACjF,MAAM,MAAM,OAAO,SAAS,WAAW,OAAQ,MAAM,OAAO,SAAS;CACrE,MAAM,UAAU,OAAO,SAAS,WAAW,EAAE,KAAK,GAAI,QAAQ,EAAE,KAAK;CACrE,MAAM,OAAO,OAAO,QAAQ,QAAQ,OAAO,MAAM,QAAQ,OAAO,EAAE;AAElE,QAAO,KAAK,SADE,OAAO,QAAQ,EAAE,EACH,MAAM,SAAS;;AAG7C,SAAgB,UAAU,KAAgD;CACxE,MAAM,EAAE,YAAY,cAAc,IAAI,KAAK,MAAM,IAAI;AACrD,KAAI,YAAY,eACd,QAAO,OAAO,+BAAqB,MAAM,EAAE,0BACzC,oBAAoB,KAAK,IAAI,CAC9B;UACQ,YAAY,QACrB,QAAO,OAAO,wBAAc,MAAM,EAAE,mBAAmB,aAAa,KAAK,IAAI,CAAC;UACrE,YAAY,SACrB,QAAO,OAAO,yBAAe,MAAM,EAAE,oBAAoB,cAAc,KAAK,IAAI,CAAC;KAEjF,OAAM,IAAI,MAAM,8BAA8B,OAAO,QAAQ,GAAG;;AAIpE,MAAM,WAAkC;CACtC,QAAQ;EACN,WAAW,QAAQ;GAGjB,MAAM,QAAQ,IAAI,OAAO,MAAM;AAC/B,OAAI,OAAO,QAAQ;IACjB,MAAM,MAAM,IAAI,KAAK,QAAQ,MAAM;AACnC,QAAI,QAAQ,MAAM,MAAM,GAAI,QAAO,GAAG,MAAM,MAAM,IAAI;;AAExD,UAAO,IAAI;;EAEb,QAAQ,UAAU;EACnB;CACD,KAAK;CACN;AAED,MAAM,SAA4D;CAChE,MAAM;EACJ,QAAQ;GACN;GACA;GACA;GACD;EACD,SAAS;EACT,QAAQ,EACN,QAAQ,UAAU,4DAA4D,SAC/E;EACF;CACD,QAAQ,EACN,QAAQ;EACN;EACA;EACA;EACD,EACF;CACD,WAAW;EACT,QAAQ;GACN;GACA;GACA;GACA;GACD;EACD,SAAS;EACT,QAAQ,EACN,QAAQ,UAAU,4DAA4D,SAC/E;EACF;CACD,2DAA2D;EACzD,SAAS;EAET,QAAQ;GACN,WAAW,QAAQ,UAAU,IAAI,SAAS,OAAO,KAAK,IAAI;GAC1D,QAAQ,UAAU,gCAAgC;GACnD;EACF;CACF;AAED,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,OAAO,CAClD,MAAK,MAAM,SAAS,QAAQ,UAAU,EAAE,CACtC,QAAO,SAAS;CAAE;CAAM,KAAK;CAAO"}
|