@claritylabs/cl-sdk 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -218
- package/dist/index.d.mts +8376 -1209
- package/dist/index.d.ts +8376 -1209
- package/dist/index.js +3308 -2136
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +3459 -2404
- package/dist/index.mjs.map +1 -1
- package/dist/storage-sqlite.d.mts +2804 -0
- package/dist/storage-sqlite.d.ts +2804 -0
- package/dist/storage-sqlite.js +238 -0
- package/dist/storage-sqlite.js.map +1 -0
- package/dist/storage-sqlite.mjs +218 -0
- package/dist/storage-sqlite.mjs.map +1 -0
- package/package.json +16 -7
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/storage/sqlite/index.ts
|
|
21
|
+
var sqlite_exports = {};
|
|
22
|
+
__export(sqlite_exports, {
|
|
23
|
+
createSqliteStore: () => createSqliteStore
|
|
24
|
+
});
|
|
25
|
+
module.exports = __toCommonJS(sqlite_exports);
|
|
26
|
+
|
|
27
|
+
// src/storage/sqlite/migrations.ts
|
|
28
|
+
var CREATE_TABLES = `
|
|
29
|
+
CREATE TABLE IF NOT EXISTS documents (
|
|
30
|
+
id TEXT PRIMARY KEY,
|
|
31
|
+
type TEXT NOT NULL,
|
|
32
|
+
data TEXT NOT NULL,
|
|
33
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
37
|
+
id TEXT PRIMARY KEY,
|
|
38
|
+
document_id TEXT NOT NULL,
|
|
39
|
+
type TEXT NOT NULL,
|
|
40
|
+
text TEXT NOT NULL,
|
|
41
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
42
|
+
embedding BLOB,
|
|
43
|
+
FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
CREATE TABLE IF NOT EXISTS conversation_turns (
|
|
47
|
+
id TEXT PRIMARY KEY,
|
|
48
|
+
conversation_id TEXT NOT NULL,
|
|
49
|
+
role TEXT NOT NULL,
|
|
50
|
+
content TEXT NOT NULL,
|
|
51
|
+
tool_name TEXT,
|
|
52
|
+
tool_result TEXT,
|
|
53
|
+
timestamp INTEGER NOT NULL,
|
|
54
|
+
embedding BLOB
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id);
|
|
58
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_turns_conversation_id ON conversation_turns(conversation_id);
|
|
60
|
+
CREATE INDEX IF NOT EXISTS idx_turns_timestamp ON conversation_turns(timestamp);
|
|
61
|
+
`;
|
|
62
|
+
|
|
63
|
+
// src/storage/sqlite/document-store.ts
|
|
64
|
+
function createSqliteDocumentStore(db) {
|
|
65
|
+
return {
|
|
66
|
+
async save(doc) {
|
|
67
|
+
db.prepare("INSERT OR REPLACE INTO documents (id, type, data) VALUES (?, ?, ?)").run(
|
|
68
|
+
doc.id,
|
|
69
|
+
doc.type,
|
|
70
|
+
JSON.stringify(doc)
|
|
71
|
+
);
|
|
72
|
+
},
|
|
73
|
+
async get(id) {
|
|
74
|
+
const row = db.prepare("SELECT data FROM documents WHERE id = ?").get(id);
|
|
75
|
+
return row ? JSON.parse(row.data) : null;
|
|
76
|
+
},
|
|
77
|
+
async query(filters) {
|
|
78
|
+
let sql = "SELECT data FROM documents WHERE 1=1";
|
|
79
|
+
const params = [];
|
|
80
|
+
if (filters.type) {
|
|
81
|
+
sql += " AND type = ?";
|
|
82
|
+
params.push(filters.type);
|
|
83
|
+
}
|
|
84
|
+
if (filters.carrier) {
|
|
85
|
+
sql += " AND json_extract(data, '$.carrier') LIKE ?";
|
|
86
|
+
params.push(`%${filters.carrier}%`);
|
|
87
|
+
}
|
|
88
|
+
if (filters.insuredName) {
|
|
89
|
+
sql += " AND json_extract(data, '$.insuredName') LIKE ?";
|
|
90
|
+
params.push(`%${filters.insuredName}%`);
|
|
91
|
+
}
|
|
92
|
+
if (filters.policyNumber) {
|
|
93
|
+
sql += " AND json_extract(data, '$.policyNumber') = ?";
|
|
94
|
+
params.push(filters.policyNumber);
|
|
95
|
+
}
|
|
96
|
+
if (filters.quoteNumber) {
|
|
97
|
+
sql += " AND json_extract(data, '$.quoteNumber') = ?";
|
|
98
|
+
params.push(filters.quoteNumber);
|
|
99
|
+
}
|
|
100
|
+
const rows = db.prepare(sql).all(...params);
|
|
101
|
+
return rows.map((r) => JSON.parse(r.data));
|
|
102
|
+
},
|
|
103
|
+
async delete(id) {
|
|
104
|
+
db.prepare("DELETE FROM documents WHERE id = ?").run(id);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// src/storage/sqlite/memory-store.ts
|
|
110
|
+
function cosineSimilarity(a, b) {
|
|
111
|
+
let dot = 0, magA = 0, magB = 0;
|
|
112
|
+
for (let i = 0; i < a.length; i++) {
|
|
113
|
+
dot += a[i] * b[i];
|
|
114
|
+
magA += a[i] * a[i];
|
|
115
|
+
magB += b[i] * b[i];
|
|
116
|
+
}
|
|
117
|
+
return dot / (Math.sqrt(magA) * Math.sqrt(magB));
|
|
118
|
+
}
|
|
119
|
+
function createSqliteMemoryStore(db, embed) {
|
|
120
|
+
return {
|
|
121
|
+
async addChunks(chunks) {
|
|
122
|
+
const stmt = db.prepare("INSERT OR REPLACE INTO chunks (id, document_id, type, text, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)");
|
|
123
|
+
const insertMany = db.transaction((items) => {
|
|
124
|
+
for (const chunk of items) {
|
|
125
|
+
stmt.run(chunk.id, chunk.documentId, chunk.type, chunk.text, JSON.stringify(chunk.metadata), null);
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
insertMany(chunks);
|
|
129
|
+
for (const chunk of chunks) {
|
|
130
|
+
try {
|
|
131
|
+
const embedding = await embed(chunk.text);
|
|
132
|
+
const buf = Buffer.from(new Float64Array(embedding).buffer);
|
|
133
|
+
db.prepare("UPDATE chunks SET embedding = ? WHERE id = ?").run(buf, chunk.id);
|
|
134
|
+
} catch {
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
async search(query, options) {
|
|
139
|
+
const queryEmbedding = await embed(query);
|
|
140
|
+
const resultLimit = options?.limit ?? 10;
|
|
141
|
+
let sql = "SELECT id, document_id, type, text, metadata, embedding FROM chunks WHERE embedding IS NOT NULL";
|
|
142
|
+
const params = [];
|
|
143
|
+
if (options?.filter?.documentId) {
|
|
144
|
+
sql += " AND document_id = ?";
|
|
145
|
+
params.push(options.filter.documentId);
|
|
146
|
+
}
|
|
147
|
+
if (options?.filter?.type) {
|
|
148
|
+
sql += " AND type = ?";
|
|
149
|
+
params.push(options.filter.type);
|
|
150
|
+
}
|
|
151
|
+
const rows = db.prepare(sql).all(...params);
|
|
152
|
+
const scored = rows.map((row) => {
|
|
153
|
+
const stored = Array.from(new Float64Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 8));
|
|
154
|
+
return {
|
|
155
|
+
chunk: {
|
|
156
|
+
id: row.id,
|
|
157
|
+
documentId: row.document_id,
|
|
158
|
+
type: row.type,
|
|
159
|
+
text: row.text,
|
|
160
|
+
metadata: JSON.parse(row.metadata)
|
|
161
|
+
},
|
|
162
|
+
score: cosineSimilarity(queryEmbedding, stored)
|
|
163
|
+
};
|
|
164
|
+
});
|
|
165
|
+
scored.sort((a, b) => b.score - a.score);
|
|
166
|
+
return scored.slice(0, resultLimit).map((s) => s.chunk);
|
|
167
|
+
},
|
|
168
|
+
async addTurn(turn) {
|
|
169
|
+
const embedding = await embed(turn.content).catch(() => null);
|
|
170
|
+
const buf = embedding ? Buffer.from(new Float64Array(embedding).buffer) : null;
|
|
171
|
+
db.prepare(
|
|
172
|
+
"INSERT INTO conversation_turns (id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
173
|
+
).run(turn.id, turn.conversationId, turn.role, turn.content, turn.toolName ?? null, turn.toolResult ?? null, turn.timestamp, buf);
|
|
174
|
+
},
|
|
175
|
+
async getHistory(conversationId, options) {
|
|
176
|
+
const resultLimit = options?.limit ?? 50;
|
|
177
|
+
const rows = db.prepare(
|
|
178
|
+
"SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp FROM conversation_turns WHERE conversation_id = ? ORDER BY timestamp DESC LIMIT ?"
|
|
179
|
+
).all(conversationId, resultLimit);
|
|
180
|
+
return rows.reverse().map((r) => ({
|
|
181
|
+
id: r.id,
|
|
182
|
+
conversationId: r.conversation_id,
|
|
183
|
+
role: r.role,
|
|
184
|
+
content: r.content,
|
|
185
|
+
toolName: r.tool_name,
|
|
186
|
+
toolResult: r.tool_result,
|
|
187
|
+
timestamp: r.timestamp
|
|
188
|
+
}));
|
|
189
|
+
},
|
|
190
|
+
async searchHistory(query, conversationId) {
|
|
191
|
+
const queryEmbedding = await embed(query);
|
|
192
|
+
let sql = "SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding FROM conversation_turns WHERE embedding IS NOT NULL";
|
|
193
|
+
const params = [];
|
|
194
|
+
if (conversationId) {
|
|
195
|
+
sql += " AND conversation_id = ?";
|
|
196
|
+
params.push(conversationId);
|
|
197
|
+
}
|
|
198
|
+
const rows = db.prepare(sql).all(...params);
|
|
199
|
+
const scored = rows.map((row) => {
|
|
200
|
+
const buf = row.embedding;
|
|
201
|
+
const stored = Array.from(new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8));
|
|
202
|
+
return {
|
|
203
|
+
turn: {
|
|
204
|
+
id: row.id,
|
|
205
|
+
conversationId: row.conversation_id,
|
|
206
|
+
role: row.role,
|
|
207
|
+
content: row.content,
|
|
208
|
+
toolName: row.tool_name,
|
|
209
|
+
toolResult: row.tool_result,
|
|
210
|
+
timestamp: row.timestamp
|
|
211
|
+
},
|
|
212
|
+
score: cosineSimilarity(queryEmbedding, stored)
|
|
213
|
+
};
|
|
214
|
+
});
|
|
215
|
+
scored.sort((a, b) => b.score - a.score);
|
|
216
|
+
return scored.slice(0, 10).map((s) => s.turn);
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// src/storage/sqlite/index.ts
|
|
222
|
+
function createSqliteStore(options) {
|
|
223
|
+
const BetterSqlite3 = require("better-sqlite3");
|
|
224
|
+
const db = new BetterSqlite3(options.path);
|
|
225
|
+
db.pragma("journal_mode = WAL");
|
|
226
|
+
db.pragma("foreign_keys = ON");
|
|
227
|
+
db.exec(CREATE_TABLES);
|
|
228
|
+
return {
|
|
229
|
+
documents: createSqliteDocumentStore(db),
|
|
230
|
+
memory: createSqliteMemoryStore(db, options.embed),
|
|
231
|
+
close: () => db.close()
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
235
|
+
0 && (module.exports = {
|
|
236
|
+
createSqliteStore
|
|
237
|
+
});
|
|
238
|
+
//# sourceMappingURL=storage-sqlite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/storage/sqlite/index.ts","../src/storage/sqlite/migrations.ts","../src/storage/sqlite/document-store.ts","../src/storage/sqlite/memory-store.ts"],"sourcesContent":["import type { EmbedText } from \"../../core/types\";\nimport type { DocumentStore } from \"../interfaces\";\nimport type { MemoryStore } from \"../interfaces\";\nimport { CREATE_TABLES } from \"./migrations\";\nimport { createSqliteDocumentStore } from \"./document-store\";\nimport { createSqliteMemoryStore } from \"./memory-store\";\n\nexport interface SqliteStoreOptions {\n path: string;\n embed: EmbedText;\n}\n\nexport function createSqliteStore(options: SqliteStoreOptions): {\n documents: DocumentStore;\n memory: MemoryStore;\n close: () => void;\n} {\n // Dynamic import to keep better-sqlite3 optional\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const BetterSqlite3 = require(\"better-sqlite3\");\n const db = new BetterSqlite3(options.path);\n\n db.pragma(\"journal_mode = WAL\");\n db.pragma(\"foreign_keys = ON\");\n db.exec(CREATE_TABLES);\n\n return {\n documents: createSqliteDocumentStore(db),\n memory: createSqliteMemoryStore(db, options.embed),\n close: () => db.close(),\n };\n}\n\nexport type { DocumentStore, MemoryStore } from \"../interfaces\";\n","export const CREATE_TABLES = `\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n type TEXT NOT NULL,\n data TEXT NOT NULL,\n created_at INTEGER NOT NULL DEFAULT (unixepoch())\n);\n\nCREATE TABLE IF NOT EXISTS chunks (\n id TEXT PRIMARY KEY,\n document_id TEXT NOT NULL,\n type TEXT NOT NULL,\n text TEXT NOT NULL,\n metadata TEXT NOT NULL DEFAULT '{}',\n embedding BLOB,\n FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS conversation_turns (\n id TEXT PRIMARY KEY,\n conversation_id TEXT NOT NULL,\n role TEXT NOT NULL,\n content TEXT NOT NULL,\n tool_name TEXT,\n tool_result TEXT,\n timestamp INTEGER NOT NULL,\n embedding BLOB\n);\n\nCREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id);\nCREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);\nCREATE INDEX IF NOT EXISTS idx_turns_conversation_id ON conversation_turns(conversation_id);\nCREATE INDEX IF NOT EXISTS idx_turns_timestamp ON conversation_turns(timestamp);\n`;\n","import type Database from \"better-sqlite3\";\nimport type { DocumentStore } from \"../interfaces\";\nimport type { InsuranceDocument } from \"../../schemas/document\";\nimport type { DocumentFilters } from \"../chunk-types\";\n\nexport function createSqliteDocumentStore(db: Database.Database): DocumentStore {\n return {\n async save(doc: InsuranceDocument): Promise<void> {\n db.prepare(\"INSERT OR REPLACE INTO documents (id, type, data) VALUES (?, ?, ?)\").run(\n doc.id, doc.type, JSON.stringify(doc),\n );\n },\n\n async get(id: string): Promise<InsuranceDocument | null> {\n const row = db.prepare(\"SELECT data FROM documents WHERE id = ?\").get(id) as { data: string } | undefined;\n return row ? JSON.parse(row.data) : null;\n },\n\n async query(filters: DocumentFilters): Promise<InsuranceDocument[]> {\n let sql = \"SELECT data FROM documents WHERE 1=1\";\n const params: unknown[] = [];\n\n if (filters.type) {\n sql += \" AND type = ?\";\n params.push(filters.type);\n }\n if (filters.carrier) {\n sql += \" AND json_extract(data, '$.carrier') LIKE ?\";\n params.push(`%${filters.carrier}%`);\n }\n if (filters.insuredName) {\n sql += \" AND json_extract(data, '$.insuredName') LIKE ?\";\n params.push(`%${filters.insuredName}%`);\n }\n if (filters.policyNumber) {\n sql += \" AND json_extract(data, '$.policyNumber') = ?\";\n params.push(filters.policyNumber);\n }\n if (filters.quoteNumber) {\n sql += \" AND json_extract(data, '$.quoteNumber') = ?\";\n params.push(filters.quoteNumber);\n }\n\n const rows = db.prepare(sql).all(...params) as { data: string }[];\n return rows.map((r) => JSON.parse(r.data));\n },\n\n async delete(id: string): Promise<void> {\n db.prepare(\"DELETE FROM documents WHERE id = ?\").run(id);\n },\n };\n}\n","import type Database from \"better-sqlite3\";\nimport type { MemoryStore } from \"../interfaces\";\nimport type { DocumentChunk, ConversationTurn, ChunkFilter } from \"../chunk-types\";\nimport type { EmbedText } from \"../../core/types\";\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dot = 0, magA = 0, magB = 0;\n for (let i = 0; i < a.length; i++) {\n dot += a[i] * b[i];\n magA += a[i] * a[i];\n magB += b[i] * b[i];\n }\n return dot / (Math.sqrt(magA) * Math.sqrt(magB));\n}\n\nexport function createSqliteMemoryStore(db: Database.Database, embed: EmbedText): MemoryStore {\n return {\n async addChunks(chunks: DocumentChunk[]): Promise<void> {\n const stmt = db.prepare(\"INSERT OR REPLACE INTO chunks (id, document_id, type, text, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)\");\n const insertMany = db.transaction((items: DocumentChunk[]) => {\n for (const chunk of items) {\n stmt.run(chunk.id, chunk.documentId, chunk.type, chunk.text, JSON.stringify(chunk.metadata), null);\n }\n });\n insertMany(chunks);\n\n for (const chunk of chunks) {\n try {\n const embedding = await embed(chunk.text);\n const buf = Buffer.from(new Float64Array(embedding).buffer);\n db.prepare(\"UPDATE chunks SET embedding = ? WHERE id = ?\").run(buf, chunk.id);\n } catch {\n // Embedding failure is non-fatal\n }\n }\n },\n\n async search(query: string, options?: { limit?: number; filter?: ChunkFilter }): Promise<DocumentChunk[]> {\n const queryEmbedding = await embed(query);\n const resultLimit = options?.limit ?? 10;\n\n let sql = \"SELECT id, document_id, type, text, metadata, embedding FROM chunks WHERE embedding IS NOT NULL\";\n const params: unknown[] = [];\n\n if (options?.filter?.documentId) {\n sql += \" AND document_id = ?\";\n params.push(options.filter.documentId);\n }\n if (options?.filter?.type) {\n sql += \" AND type = ?\";\n params.push(options.filter.type);\n }\n\n const rows = db.prepare(sql).all(...params) as Array<{\n id: string; document_id: string; type: string; text: string; metadata: string; embedding: Buffer;\n }>;\n\n const scored = rows.map((row) => {\n const stored = Array.from(new Float64Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 8));\n return {\n chunk: {\n id: row.id,\n documentId: row.document_id,\n type: row.type as DocumentChunk[\"type\"],\n text: row.text,\n metadata: JSON.parse(row.metadata),\n },\n score: cosineSimilarity(queryEmbedding, stored),\n };\n });\n\n scored.sort((a, b) => b.score - a.score);\n return scored.slice(0, resultLimit).map((s) => s.chunk);\n },\n\n async addTurn(turn: ConversationTurn): Promise<void> {\n const embedding = await embed(turn.content).catch(() => null);\n const buf = embedding ? Buffer.from(new Float64Array(embedding).buffer) : null;\n db.prepare(\n \"INSERT INTO conversation_turns (id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?)\"\n ).run(turn.id, turn.conversationId, turn.role, turn.content, turn.toolName ?? null, turn.toolResult ?? null, turn.timestamp, buf);\n },\n\n async getHistory(conversationId: string, options?: { limit?: number }): Promise<ConversationTurn[]> {\n const resultLimit = options?.limit ?? 50;\n const rows = db.prepare(\n \"SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp FROM conversation_turns WHERE conversation_id = ? ORDER BY timestamp DESC LIMIT ?\"\n ).all(conversationId, resultLimit) as Array<Record<string, unknown>>;\n\n return rows.reverse().map((r) => ({\n id: r.id as string,\n conversationId: r.conversation_id as string,\n role: r.role as ConversationTurn[\"role\"],\n content: r.content as string,\n toolName: r.tool_name as string | undefined,\n toolResult: r.tool_result as string | undefined,\n timestamp: r.timestamp as number,\n }));\n },\n\n async searchHistory(query: string, conversationId?: string): Promise<ConversationTurn[]> {\n const queryEmbedding = await embed(query);\n\n let sql = \"SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding FROM conversation_turns WHERE embedding IS NOT NULL\";\n const params: unknown[] = [];\n if (conversationId) {\n sql += \" AND conversation_id = ?\";\n params.push(conversationId);\n }\n\n const rows = db.prepare(sql).all(...params) as Array<Record<string, unknown>>;\n\n const scored = rows.map((row) => {\n const buf = row.embedding as Buffer;\n const stored = Array.from(new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8));\n return {\n turn: {\n id: row.id as string,\n conversationId: row.conversation_id as string,\n role: row.role as ConversationTurn[\"role\"],\n content: row.content as string,\n toolName: row.tool_name as string | undefined,\n toolResult: row.tool_result as string | undefined,\n timestamp: row.timestamp as number,\n },\n score: cosineSimilarity(queryEmbedding, stored),\n };\n });\n\n scored.sort((a, b) => b.score - a.score);\n return scored.slice(0, 10).map((s) => s.turn);\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAO,IAAM,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACKtB,SAAS,0BAA0B,IAAsC;AAC9E,SAAO;AAAA,IACL,MAAM,KAAK,KAAuC;AAChD,SAAG,QAAQ,oEAAoE,EAAE;AAAA,QAC/E,IAAI;AAAA,QAAI,IAAI;AAAA,QAAM,KAAK,UAAU,GAAG;AAAA,MACtC;AAAA,IACF;AAAA,IAEA,MAAM,IAAI,IAA+C;AACvD,YAAM,MAAM,GAAG,QAAQ,yCAAyC,EAAE,IAAI,EAAE;AACxE,aAAO,MAAM,KAAK,MAAM,IAAI,IAAI,IAAI;AAAA,IACtC;AAAA,IAEA,MAAM,MAAM,SAAwD;AAClE,UAAI,MAAM;AACV,YAAM,SAAoB,CAAC;AAE3B,UAAI,QAAQ,MAAM;AAChB,eAAO;AACP,eAAO,KAAK,QAAQ,IAAI;AAAA,MAC1B;AACA,UAAI,QAAQ,SAAS;AACnB,eAAO;AACP,eAAO,KAAK,IAAI,QAAQ,OAAO,GAAG;AAAA,MACpC;AACA,UAAI,QAAQ,aAAa;AACvB,eAAO;AACP,eAAO,KAAK,IAAI,QAAQ,WAAW,GAAG;AAAA,MACxC;AACA,UAAI,QAAQ,cAAc;AACxB,eAAO;AACP,eAAO,KAAK,QAAQ,YAAY;AAAA,MAClC;AACA,UAAI,QAAQ,aAAa;AACvB,eAAO;AACP,eAAO,KAAK,QAAQ,WAAW;AAAA,MACjC;AAEA,YAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAC1C,aAAO,KAAK,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,IAAI,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,OAAO,IAA2B;AACtC,SAAG,QAAQ,oCAAoC,EAAE,IAAI,EAAE;AAAA,IACzD;AAAA,EACF;AACF;;;AC9CA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,MAAM,GAAG,OAAO,GAAG,OAAO;AAC9B,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,WAAO,EAAE,CAAC,IAAI,EAAE,CAAC;AACjB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAClB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpB;AACA,SAAO,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;AAChD;AAEO,SAAS,wBAAwB,IAAuB,OAA+B;AAC5F,SAAO;AAAA,IACL,MAAM,UAAU,QAAwC;AACtD,YAAM,OAAO,GAAG,QAAQ,4GAA4G;AACpI,YAAM,aAAa,GAAG,YAAY,CAAC,UAA2B;AAC5D,mBAAW,SAAS,OAAO;AACzB,eAAK,IAAI,MAAM,IAAI,MAAM,YAAY,MAAM,MAAM,MAAM,MAAM,KAAK,UAAU,MAAM,QAAQ,GAAG,IAAI;AAAA,QACnG;AAAA,MACF,CAAC;AACD,iBAAW,MAAM;AAEjB,iBAAW,SAAS,QAAQ;AAC1B,YAAI;AACF,gBAAM,YAAY,MAAM,MAAM,MAAM,IAAI;AACxC,gBAAM,MAAM,OAAO,KAAK,IAAI,aAAa,SAAS,EAAE,MAAM;AAC1D,aAAG,QAAQ,8CAA8C,EAAE,IAAI,KAAK,MAAM,EAAE;AAAA,QAC9E,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,OAAO,OAAe,SAA8E;AACxG,YAAM,iBAAiB,MAAM,MAAM,KAAK;AACxC,YAAM,cAAc,SAAS,SAAS;AAEtC,UAAI,MAAM;AACV,YAAM,SAAoB,CAAC;AAE3B,UAAI,SAAS,QAAQ,YAAY;AAC/B,eAAO;AACP,eAAO,KAAK,QAAQ,OAAO,UAAU;AAAA,MACvC;AACA,UAAI,SAAS,QAAQ,MAAM;AACzB,eAAO;AACP,eAAO,KAAK,QAAQ,OAAO,IAAI;AAAA,MACjC;AAEA,YAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAI1C,YAAM,SAAS,KAAK,IAAI,CAAC,QAAQ;AAC/B,cAAM,SAAS,MAAM,KAAK,IAAI,aAAa,IAAI,UAAU,QAAQ,IAAI,UAAU,YAAY,IAAI,UAAU,aAAa,CAAC,CAAC;AACxH,eAAO;AAAA,UACL,OAAO;AAAA,YACL,IAAI,IAAI;AAAA,YACR,YAAY,IAAI;AAAA,YAChB,MAAM,IAAI;AAAA,YACV,MAAM,IAAI;AAAA,YACV,UAAU,KAAK,MAAM,IAAI,QAAQ;AAAA,UACnC;AAAA,UACA,OAAO,iBAAiB,gBAAgB,MAAM;AAAA,QAChD;AAAA,MACF,CAAC;AAED,aAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,aAAO,OAAO,MAAM,GAAG,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK;AAAA,IACxD;AAAA,IAEA,MAAM,QAAQ,MAAuC;AACnD,YAAM,YAAY,MAAM,MAAM,KAAK,OAAO,EAAE,MAAM,MAAM,IAAI;AAC5D,YAAM,MAAM,YAAY,OAAO,KAAK,IAAI,aAAa,SAAS,EAAE,MAAM,IAAI;AAC1E,SAAG;AAAA,QACD;AAAA,MACF,EAAE,IAAI,KAAK,IAAI,KAAK,gBAAgB,KAAK,MAAM,KAAK,SAAS,KAAK,YAAY,MAAM,KAAK,cAAc,MAAM,KAAK,WAAW,GAAG;AAAA,IAClI;AAAA,IAEA,MAAM,WAAW,gBAAwB,SAA2D;AAClG,YAAM,cAAc,SAAS,SAAS;AACtC,YAAM,OAAO,GAAG;AAAA,QACd;AAAA,MACF,EAAE,IAAI,gBAAgB,WAAW;AAEjC,aAAO,KAAK,QAAQ,EAAE,IAAI,CAAC,OAAO;AAAA,QAChC,IAAI,EAAE;AAAA,QACN,gBAAgB,EAAE;AAAA,QAClB,MAAM,EAAE;AAAA,QACR,SAAS,EAAE;AAAA,QACX,UAAU,EAAE;AAAA,QACZ,YAAY,EAAE;AAAA,QACd,WAAW,EAAE;AAAA,MACf,EAAE;AAAA,IACJ;AAAA,IAEA,MAAM,cAAc,OAAe,gBAAsD;AACvF,YAAM,iBAAiB,MAAM,MAAM,KAAK;AAExC,UAAI,MAAM;AACV,YAAM,SAAoB,CAAC;AAC3B,UAAI,gBAAgB;AAClB,eAAO;AACP,eAAO,KAAK,cAAc;AAAA,MAC5B;AAEA,YAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAE1C,YAAM,SAAS,KAAK,IAAI,CAAC,QAAQ;AAC/B,cAAM,MAAM,IAAI;AAChB,cAAM,SAAS,MAAM,KAAK,IAAI,aAAa,IAAI,QAAQ,IAAI,YAAY,IAAI,aAAa,CAAC,CAAC;AAC1F,eAAO;AAAA,UACL,MAAM;AAAA,YACJ,IAAI,IAAI;AAAA,YACR,gBAAgB,IAAI;AAAA,YACpB,MAAM,IAAI;AAAA,YACV,SAAS,IAAI;AAAA,YACb,UAAU,IAAI;AAAA,YACd,YAAY,IAAI;AAAA,YAChB,WAAW,IAAI;AAAA,UACjB;AAAA,UACA,OAAO,iBAAiB,gBAAgB,MAAM;AAAA,QAChD;AAAA,MACF,CAAC;AAED,aAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,aAAO,OAAO,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,IAC9C;AAAA,EACF;AACF;;;AHzHO,SAAS,kBAAkB,SAIhC;AAGA,QAAM,gBAAgB,QAAQ,gBAAgB;AAC9C,QAAM,KAAK,IAAI,cAAc,QAAQ,IAAI;AAEzC,KAAG,OAAO,oBAAoB;AAC9B,KAAG,OAAO,mBAAmB;AAC7B,KAAG,KAAK,aAAa;AAErB,SAAO;AAAA,IACL,WAAW,0BAA0B,EAAE;AAAA,IACvC,QAAQ,wBAAwB,IAAI,QAAQ,KAAK;AAAA,IACjD,OAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AACF;","names":[]}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
2
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
3
|
+
}) : x)(function(x) {
|
|
4
|
+
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
5
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
// src/storage/sqlite/migrations.ts
|
|
9
|
+
var CREATE_TABLES = `
|
|
10
|
+
CREATE TABLE IF NOT EXISTS documents (
|
|
11
|
+
id TEXT PRIMARY KEY,
|
|
12
|
+
type TEXT NOT NULL,
|
|
13
|
+
data TEXT NOT NULL,
|
|
14
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
18
|
+
id TEXT PRIMARY KEY,
|
|
19
|
+
document_id TEXT NOT NULL,
|
|
20
|
+
type TEXT NOT NULL,
|
|
21
|
+
text TEXT NOT NULL,
|
|
22
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
23
|
+
embedding BLOB,
|
|
24
|
+
FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE TABLE IF NOT EXISTS conversation_turns (
|
|
28
|
+
id TEXT PRIMARY KEY,
|
|
29
|
+
conversation_id TEXT NOT NULL,
|
|
30
|
+
role TEXT NOT NULL,
|
|
31
|
+
content TEXT NOT NULL,
|
|
32
|
+
tool_name TEXT,
|
|
33
|
+
tool_result TEXT,
|
|
34
|
+
timestamp INTEGER NOT NULL,
|
|
35
|
+
embedding BLOB
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id);
|
|
39
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);
|
|
40
|
+
CREATE INDEX IF NOT EXISTS idx_turns_conversation_id ON conversation_turns(conversation_id);
|
|
41
|
+
CREATE INDEX IF NOT EXISTS idx_turns_timestamp ON conversation_turns(timestamp);
|
|
42
|
+
`;
|
|
43
|
+
|
|
44
|
+
// src/storage/sqlite/document-store.ts
|
|
45
|
+
function createSqliteDocumentStore(db) {
|
|
46
|
+
return {
|
|
47
|
+
async save(doc) {
|
|
48
|
+
db.prepare("INSERT OR REPLACE INTO documents (id, type, data) VALUES (?, ?, ?)").run(
|
|
49
|
+
doc.id,
|
|
50
|
+
doc.type,
|
|
51
|
+
JSON.stringify(doc)
|
|
52
|
+
);
|
|
53
|
+
},
|
|
54
|
+
async get(id) {
|
|
55
|
+
const row = db.prepare("SELECT data FROM documents WHERE id = ?").get(id);
|
|
56
|
+
return row ? JSON.parse(row.data) : null;
|
|
57
|
+
},
|
|
58
|
+
async query(filters) {
|
|
59
|
+
let sql = "SELECT data FROM documents WHERE 1=1";
|
|
60
|
+
const params = [];
|
|
61
|
+
if (filters.type) {
|
|
62
|
+
sql += " AND type = ?";
|
|
63
|
+
params.push(filters.type);
|
|
64
|
+
}
|
|
65
|
+
if (filters.carrier) {
|
|
66
|
+
sql += " AND json_extract(data, '$.carrier') LIKE ?";
|
|
67
|
+
params.push(`%${filters.carrier}%`);
|
|
68
|
+
}
|
|
69
|
+
if (filters.insuredName) {
|
|
70
|
+
sql += " AND json_extract(data, '$.insuredName') LIKE ?";
|
|
71
|
+
params.push(`%${filters.insuredName}%`);
|
|
72
|
+
}
|
|
73
|
+
if (filters.policyNumber) {
|
|
74
|
+
sql += " AND json_extract(data, '$.policyNumber') = ?";
|
|
75
|
+
params.push(filters.policyNumber);
|
|
76
|
+
}
|
|
77
|
+
if (filters.quoteNumber) {
|
|
78
|
+
sql += " AND json_extract(data, '$.quoteNumber') = ?";
|
|
79
|
+
params.push(filters.quoteNumber);
|
|
80
|
+
}
|
|
81
|
+
const rows = db.prepare(sql).all(...params);
|
|
82
|
+
return rows.map((r) => JSON.parse(r.data));
|
|
83
|
+
},
|
|
84
|
+
async delete(id) {
|
|
85
|
+
db.prepare("DELETE FROM documents WHERE id = ?").run(id);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// src/storage/sqlite/memory-store.ts
|
|
91
|
+
function cosineSimilarity(a, b) {
|
|
92
|
+
let dot = 0, magA = 0, magB = 0;
|
|
93
|
+
for (let i = 0; i < a.length; i++) {
|
|
94
|
+
dot += a[i] * b[i];
|
|
95
|
+
magA += a[i] * a[i];
|
|
96
|
+
magB += b[i] * b[i];
|
|
97
|
+
}
|
|
98
|
+
return dot / (Math.sqrt(magA) * Math.sqrt(magB));
|
|
99
|
+
}
|
|
100
|
+
function createSqliteMemoryStore(db, embed) {
|
|
101
|
+
return {
|
|
102
|
+
async addChunks(chunks) {
|
|
103
|
+
const stmt = db.prepare("INSERT OR REPLACE INTO chunks (id, document_id, type, text, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)");
|
|
104
|
+
const insertMany = db.transaction((items) => {
|
|
105
|
+
for (const chunk of items) {
|
|
106
|
+
stmt.run(chunk.id, chunk.documentId, chunk.type, chunk.text, JSON.stringify(chunk.metadata), null);
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
insertMany(chunks);
|
|
110
|
+
for (const chunk of chunks) {
|
|
111
|
+
try {
|
|
112
|
+
const embedding = await embed(chunk.text);
|
|
113
|
+
const buf = Buffer.from(new Float64Array(embedding).buffer);
|
|
114
|
+
db.prepare("UPDATE chunks SET embedding = ? WHERE id = ?").run(buf, chunk.id);
|
|
115
|
+
} catch {
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
async search(query, options) {
|
|
120
|
+
const queryEmbedding = await embed(query);
|
|
121
|
+
const resultLimit = options?.limit ?? 10;
|
|
122
|
+
let sql = "SELECT id, document_id, type, text, metadata, embedding FROM chunks WHERE embedding IS NOT NULL";
|
|
123
|
+
const params = [];
|
|
124
|
+
if (options?.filter?.documentId) {
|
|
125
|
+
sql += " AND document_id = ?";
|
|
126
|
+
params.push(options.filter.documentId);
|
|
127
|
+
}
|
|
128
|
+
if (options?.filter?.type) {
|
|
129
|
+
sql += " AND type = ?";
|
|
130
|
+
params.push(options.filter.type);
|
|
131
|
+
}
|
|
132
|
+
const rows = db.prepare(sql).all(...params);
|
|
133
|
+
const scored = rows.map((row) => {
|
|
134
|
+
const stored = Array.from(new Float64Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 8));
|
|
135
|
+
return {
|
|
136
|
+
chunk: {
|
|
137
|
+
id: row.id,
|
|
138
|
+
documentId: row.document_id,
|
|
139
|
+
type: row.type,
|
|
140
|
+
text: row.text,
|
|
141
|
+
metadata: JSON.parse(row.metadata)
|
|
142
|
+
},
|
|
143
|
+
score: cosineSimilarity(queryEmbedding, stored)
|
|
144
|
+
};
|
|
145
|
+
});
|
|
146
|
+
scored.sort((a, b) => b.score - a.score);
|
|
147
|
+
return scored.slice(0, resultLimit).map((s) => s.chunk);
|
|
148
|
+
},
|
|
149
|
+
async addTurn(turn) {
|
|
150
|
+
const embedding = await embed(turn.content).catch(() => null);
|
|
151
|
+
const buf = embedding ? Buffer.from(new Float64Array(embedding).buffer) : null;
|
|
152
|
+
db.prepare(
|
|
153
|
+
"INSERT INTO conversation_turns (id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
154
|
+
).run(turn.id, turn.conversationId, turn.role, turn.content, turn.toolName ?? null, turn.toolResult ?? null, turn.timestamp, buf);
|
|
155
|
+
},
|
|
156
|
+
async getHistory(conversationId, options) {
|
|
157
|
+
const resultLimit = options?.limit ?? 50;
|
|
158
|
+
const rows = db.prepare(
|
|
159
|
+
"SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp FROM conversation_turns WHERE conversation_id = ? ORDER BY timestamp DESC LIMIT ?"
|
|
160
|
+
).all(conversationId, resultLimit);
|
|
161
|
+
return rows.reverse().map((r) => ({
|
|
162
|
+
id: r.id,
|
|
163
|
+
conversationId: r.conversation_id,
|
|
164
|
+
role: r.role,
|
|
165
|
+
content: r.content,
|
|
166
|
+
toolName: r.tool_name,
|
|
167
|
+
toolResult: r.tool_result,
|
|
168
|
+
timestamp: r.timestamp
|
|
169
|
+
}));
|
|
170
|
+
},
|
|
171
|
+
async searchHistory(query, conversationId) {
|
|
172
|
+
const queryEmbedding = await embed(query);
|
|
173
|
+
let sql = "SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding FROM conversation_turns WHERE embedding IS NOT NULL";
|
|
174
|
+
const params = [];
|
|
175
|
+
if (conversationId) {
|
|
176
|
+
sql += " AND conversation_id = ?";
|
|
177
|
+
params.push(conversationId);
|
|
178
|
+
}
|
|
179
|
+
const rows = db.prepare(sql).all(...params);
|
|
180
|
+
const scored = rows.map((row) => {
|
|
181
|
+
const buf = row.embedding;
|
|
182
|
+
const stored = Array.from(new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8));
|
|
183
|
+
return {
|
|
184
|
+
turn: {
|
|
185
|
+
id: row.id,
|
|
186
|
+
conversationId: row.conversation_id,
|
|
187
|
+
role: row.role,
|
|
188
|
+
content: row.content,
|
|
189
|
+
toolName: row.tool_name,
|
|
190
|
+
toolResult: row.tool_result,
|
|
191
|
+
timestamp: row.timestamp
|
|
192
|
+
},
|
|
193
|
+
score: cosineSimilarity(queryEmbedding, stored)
|
|
194
|
+
};
|
|
195
|
+
});
|
|
196
|
+
scored.sort((a, b) => b.score - a.score);
|
|
197
|
+
return scored.slice(0, 10).map((s) => s.turn);
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// src/storage/sqlite/index.ts
|
|
203
|
+
function createSqliteStore(options) {
|
|
204
|
+
const BetterSqlite3 = __require("better-sqlite3");
|
|
205
|
+
const db = new BetterSqlite3(options.path);
|
|
206
|
+
db.pragma("journal_mode = WAL");
|
|
207
|
+
db.pragma("foreign_keys = ON");
|
|
208
|
+
db.exec(CREATE_TABLES);
|
|
209
|
+
return {
|
|
210
|
+
documents: createSqliteDocumentStore(db),
|
|
211
|
+
memory: createSqliteMemoryStore(db, options.embed),
|
|
212
|
+
close: () => db.close()
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
export {
|
|
216
|
+
createSqliteStore
|
|
217
|
+
};
|
|
218
|
+
//# sourceMappingURL=storage-sqlite.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/storage/sqlite/migrations.ts","../src/storage/sqlite/document-store.ts","../src/storage/sqlite/memory-store.ts","../src/storage/sqlite/index.ts"],"sourcesContent":["export const CREATE_TABLES = `\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n type TEXT NOT NULL,\n data TEXT NOT NULL,\n created_at INTEGER NOT NULL DEFAULT (unixepoch())\n);\n\nCREATE TABLE IF NOT EXISTS chunks (\n id TEXT PRIMARY KEY,\n document_id TEXT NOT NULL,\n type TEXT NOT NULL,\n text TEXT NOT NULL,\n metadata TEXT NOT NULL DEFAULT '{}',\n embedding BLOB,\n FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE\n);\n\nCREATE TABLE IF NOT EXISTS conversation_turns (\n id TEXT PRIMARY KEY,\n conversation_id TEXT NOT NULL,\n role TEXT NOT NULL,\n content TEXT NOT NULL,\n tool_name TEXT,\n tool_result TEXT,\n timestamp INTEGER NOT NULL,\n embedding BLOB\n);\n\nCREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id);\nCREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);\nCREATE INDEX IF NOT EXISTS idx_turns_conversation_id ON conversation_turns(conversation_id);\nCREATE INDEX IF NOT EXISTS idx_turns_timestamp ON conversation_turns(timestamp);\n`;\n","import type Database from \"better-sqlite3\";\nimport type { DocumentStore } from \"../interfaces\";\nimport type { InsuranceDocument } from \"../../schemas/document\";\nimport type { DocumentFilters } from \"../chunk-types\";\n\nexport function createSqliteDocumentStore(db: Database.Database): DocumentStore {\n return {\n async save(doc: InsuranceDocument): Promise<void> {\n db.prepare(\"INSERT OR REPLACE INTO documents (id, type, data) VALUES (?, ?, ?)\").run(\n doc.id, doc.type, JSON.stringify(doc),\n );\n },\n\n async get(id: string): Promise<InsuranceDocument | null> {\n const row = db.prepare(\"SELECT data FROM documents WHERE id = ?\").get(id) as { data: string } | undefined;\n return row ? JSON.parse(row.data) : null;\n },\n\n async query(filters: DocumentFilters): Promise<InsuranceDocument[]> {\n let sql = \"SELECT data FROM documents WHERE 1=1\";\n const params: unknown[] = [];\n\n if (filters.type) {\n sql += \" AND type = ?\";\n params.push(filters.type);\n }\n if (filters.carrier) {\n sql += \" AND json_extract(data, '$.carrier') LIKE ?\";\n params.push(`%${filters.carrier}%`);\n }\n if (filters.insuredName) {\n sql += \" AND json_extract(data, '$.insuredName') LIKE ?\";\n params.push(`%${filters.insuredName}%`);\n }\n if (filters.policyNumber) {\n sql += \" AND json_extract(data, '$.policyNumber') = ?\";\n params.push(filters.policyNumber);\n }\n if (filters.quoteNumber) {\n sql += \" AND json_extract(data, '$.quoteNumber') = ?\";\n params.push(filters.quoteNumber);\n }\n\n const rows = db.prepare(sql).all(...params) as { data: string }[];\n return rows.map((r) => JSON.parse(r.data));\n },\n\n async delete(id: string): Promise<void> {\n db.prepare(\"DELETE FROM documents WHERE id = ?\").run(id);\n },\n };\n}\n","import type Database from \"better-sqlite3\";\nimport type { MemoryStore } from \"../interfaces\";\nimport type { DocumentChunk, ConversationTurn, ChunkFilter } from \"../chunk-types\";\nimport type { EmbedText } from \"../../core/types\";\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dot = 0, magA = 0, magB = 0;\n for (let i = 0; i < a.length; i++) {\n dot += a[i] * b[i];\n magA += a[i] * a[i];\n magB += b[i] * b[i];\n }\n return dot / (Math.sqrt(magA) * Math.sqrt(magB));\n}\n\nexport function createSqliteMemoryStore(db: Database.Database, embed: EmbedText): MemoryStore {\n return {\n async addChunks(chunks: DocumentChunk[]): Promise<void> {\n const stmt = db.prepare(\"INSERT OR REPLACE INTO chunks (id, document_id, type, text, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)\");\n const insertMany = db.transaction((items: DocumentChunk[]) => {\n for (const chunk of items) {\n stmt.run(chunk.id, chunk.documentId, chunk.type, chunk.text, JSON.stringify(chunk.metadata), null);\n }\n });\n insertMany(chunks);\n\n for (const chunk of chunks) {\n try {\n const embedding = await embed(chunk.text);\n const buf = Buffer.from(new Float64Array(embedding).buffer);\n db.prepare(\"UPDATE chunks SET embedding = ? WHERE id = ?\").run(buf, chunk.id);\n } catch {\n // Embedding failure is non-fatal\n }\n }\n },\n\n async search(query: string, options?: { limit?: number; filter?: ChunkFilter }): Promise<DocumentChunk[]> {\n const queryEmbedding = await embed(query);\n const resultLimit = options?.limit ?? 10;\n\n let sql = \"SELECT id, document_id, type, text, metadata, embedding FROM chunks WHERE embedding IS NOT NULL\";\n const params: unknown[] = [];\n\n if (options?.filter?.documentId) {\n sql += \" AND document_id = ?\";\n params.push(options.filter.documentId);\n }\n if (options?.filter?.type) {\n sql += \" AND type = ?\";\n params.push(options.filter.type);\n }\n\n const rows = db.prepare(sql).all(...params) as Array<{\n id: string; document_id: string; type: string; text: string; metadata: string; embedding: Buffer;\n }>;\n\n const scored = rows.map((row) => {\n const stored = Array.from(new Float64Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 8));\n return {\n chunk: {\n id: row.id,\n documentId: row.document_id,\n type: row.type as DocumentChunk[\"type\"],\n text: row.text,\n metadata: JSON.parse(row.metadata),\n },\n score: cosineSimilarity(queryEmbedding, stored),\n };\n });\n\n scored.sort((a, b) => b.score - a.score);\n return scored.slice(0, resultLimit).map((s) => s.chunk);\n },\n\n async addTurn(turn: ConversationTurn): Promise<void> {\n const embedding = await embed(turn.content).catch(() => null);\n const buf = embedding ? Buffer.from(new Float64Array(embedding).buffer) : null;\n db.prepare(\n \"INSERT INTO conversation_turns (id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding) VALUES (?, ?, ?, ?, ?, ?, ?, ?)\"\n ).run(turn.id, turn.conversationId, turn.role, turn.content, turn.toolName ?? null, turn.toolResult ?? null, turn.timestamp, buf);\n },\n\n async getHistory(conversationId: string, options?: { limit?: number }): Promise<ConversationTurn[]> {\n const resultLimit = options?.limit ?? 50;\n const rows = db.prepare(\n \"SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp FROM conversation_turns WHERE conversation_id = ? ORDER BY timestamp DESC LIMIT ?\"\n ).all(conversationId, resultLimit) as Array<Record<string, unknown>>;\n\n return rows.reverse().map((r) => ({\n id: r.id as string,\n conversationId: r.conversation_id as string,\n role: r.role as ConversationTurn[\"role\"],\n content: r.content as string,\n toolName: r.tool_name as string | undefined,\n toolResult: r.tool_result as string | undefined,\n timestamp: r.timestamp as number,\n }));\n },\n\n async searchHistory(query: string, conversationId?: string): Promise<ConversationTurn[]> {\n const queryEmbedding = await embed(query);\n\n let sql = \"SELECT id, conversation_id, role, content, tool_name, tool_result, timestamp, embedding FROM conversation_turns WHERE embedding IS NOT NULL\";\n const params: unknown[] = [];\n if (conversationId) {\n sql += \" AND conversation_id = ?\";\n params.push(conversationId);\n }\n\n const rows = db.prepare(sql).all(...params) as Array<Record<string, unknown>>;\n\n const scored = rows.map((row) => {\n const buf = row.embedding as Buffer;\n const stored = Array.from(new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8));\n return {\n turn: {\n id: row.id as string,\n conversationId: row.conversation_id as string,\n role: row.role as ConversationTurn[\"role\"],\n content: row.content as string,\n toolName: row.tool_name as string | undefined,\n toolResult: row.tool_result as string | undefined,\n timestamp: row.timestamp as number,\n },\n score: cosineSimilarity(queryEmbedding, stored),\n };\n });\n\n scored.sort((a, b) => b.score - a.score);\n return scored.slice(0, 10).map((s) => s.turn);\n },\n };\n}\n","import type { EmbedText } from \"../../core/types\";\nimport type { DocumentStore } from \"../interfaces\";\nimport type { MemoryStore } from \"../interfaces\";\nimport { CREATE_TABLES } from \"./migrations\";\nimport { createSqliteDocumentStore } from \"./document-store\";\nimport { createSqliteMemoryStore } from \"./memory-store\";\n\nexport interface SqliteStoreOptions {\n path: string;\n embed: EmbedText;\n}\n\nexport function createSqliteStore(options: SqliteStoreOptions): {\n documents: DocumentStore;\n memory: MemoryStore;\n close: () => void;\n} {\n // Dynamic import to keep better-sqlite3 optional\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const BetterSqlite3 = require(\"better-sqlite3\");\n const db = new BetterSqlite3(options.path);\n\n db.pragma(\"journal_mode = WAL\");\n db.pragma(\"foreign_keys = ON\");\n db.exec(CREATE_TABLES);\n\n return {\n documents: createSqliteDocumentStore(db),\n memory: createSqliteMemoryStore(db, options.embed),\n close: () => db.close(),\n };\n}\n\nexport type { DocumentStore, MemoryStore } from \"../interfaces\";\n"],"mappings":";;;;;;;;AAAO,IAAM,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACKtB,SAAS,0BAA0B,IAAsC;AAC9E,SAAO;AAAA,IACL,MAAM,KAAK,KAAuC;AAChD,SAAG,QAAQ,oEAAoE,EAAE;AAAA,QAC/E,IAAI;AAAA,QAAI,IAAI;AAAA,QAAM,KAAK,UAAU,GAAG;AAAA,MACtC;AAAA,IACF;AAAA,IAEA,MAAM,IAAI,IAA+C;AACvD,YAAM,MAAM,GAAG,QAAQ,yCAAyC,EAAE,IAAI,EAAE;AACxE,aAAO,MAAM,KAAK,MAAM,IAAI,IAAI,IAAI;AAAA,IACtC;AAAA,IAEA,MAAM,MAAM,SAAwD;AAClE,UAAI,MAAM;AACV,YAAM,SAAoB,CAAC;AAE3B,UAAI,QAAQ,MAAM;AAChB,eAAO;AACP,eAAO,KAAK,QAAQ,IAAI;AAAA,MAC1B;AACA,UAAI,QAAQ,SAAS;AACnB,eAAO;AACP,eAAO,KAAK,IAAI,QAAQ,OAAO,GAAG;AAAA,MACpC;AACA,UAAI,QAAQ,aAAa;AACvB,eAAO;AACP,eAAO,KAAK,IAAI,QAAQ,WAAW,GAAG;AAAA,MACxC;AACA,UAAI,QAAQ,cAAc;AACxB,eAAO;AACP,eAAO,KAAK,QAAQ,YAAY;AAAA,MAClC;AACA,UAAI,QAAQ,aAAa;AACvB,eAAO;AACP,eAAO,KAAK,QAAQ,WAAW;AAAA,MACjC;AAEA,YAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAC1C,aAAO,KAAK,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,IAAI,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,OAAO,IAA2B;AACtC,SAAG,QAAQ,oCAAoC,EAAE,IAAI,EAAE;AAAA,IACzD;AAAA,EACF;AACF;;;AC9CA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,MAAM,GAAG,OAAO,GAAG,OAAO;AAC9B,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,WAAO,EAAE,CAAC,IAAI,EAAE,CAAC;AACjB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAClB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpB;AACA,SAAO,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;AAChD;AAEO,SAAS,wBAAwB,IAAuB,OAA+B;AAC5F,SAAO;AAAA,IACL,MAAM,UAAU,QAAwC;AACtD,YAAM,OAAO,GAAG,QAAQ,4GAA4G;AACpI,YAAM,aAAa,GAAG,YAAY,CAAC,UAA2B;AAC5D,mBAAW,SAAS,OAAO;AACzB,eAAK,IAAI,MAAM,IAAI,MAAM,YAAY,MAAM,MAAM,MAAM,MAAM,KAAK,UAAU,MAAM,QAAQ,GAAG,IAAI;AAAA,QACnG;AAAA,MACF,CAAC;AACD,iBAAW,MAAM;AAEjB,iBAAW,SAAS,QAAQ;AAC1B,YAAI;AACF,gBAAM,YAAY,MAAM,MAAM,MAAM,IAAI;AACxC,gBAAM,MAAM,OAAO,KAAK,IAAI,aAAa,SAAS,EAAE,MAAM;AAC1D,aAAG,QAAQ,8CAA8C,EAAE,IAAI,KAAK,MAAM,EAAE;AAAA,QAC9E,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,OAAO,OAAe,SAA8E;AACxG,YAAM,iBAAiB,MAAM,MAAM,KAAK;AACxC,YAAM,cAAc,SAAS,SAAS;AAEtC,UAAI,MAAM;AACV,YAAM,SAAoB,CAAC;AAE3B,UAAI,SAAS,QAAQ,YAAY;AAC/B,eAAO;AACP,eAAO,KAAK,QAAQ,OAAO,UAAU;AAAA,MACvC;AACA,UAAI,SAAS,QAAQ,MAAM;AACzB,eAAO;AACP,eAAO,KAAK,QAAQ,OAAO,IAAI;AAAA,MACjC;AAEA,YAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAI1C,YAAM,SAAS,KAAK,IAAI,CAAC,QAAQ;AAC/B,cAAM,SAAS,MAAM,KAAK,IAAI,aAAa,IAAI,UAAU,QAAQ,IAAI,UAAU,YAAY,IAAI,UAAU,aAAa,CAAC,CAAC;AACxH,eAAO;AAAA,UACL,OAAO;AAAA,YACL,IAAI,IAAI;AAAA,YACR,YAAY,IAAI;AAAA,YAChB,MAAM,IAAI;AAAA,YACV,MAAM,IAAI;AAAA,YACV,UAAU,KAAK,MAAM,IAAI,QAAQ;AAAA,UACnC;AAAA,UACA,OAAO,iBAAiB,gBAAgB,MAAM;AAAA,QAChD;AAAA,MACF,CAAC;AAED,aAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,aAAO,OAAO,MAAM,GAAG,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK;AAAA,IACxD;AAAA,IAEA,MAAM,QAAQ,MAAuC;AACnD,YAAM,YAAY,MAAM,MAAM,KAAK,OAAO,EAAE,MAAM,MAAM,IAAI;AAC5D,YAAM,MAAM,YAAY,OAAO,KAAK,IAAI,aAAa,SAAS,EAAE,MAAM,IAAI;AAC1E,SAAG;AAAA,QACD;AAAA,MACF,EAAE,IAAI,KAAK,IAAI,KAAK,gBAAgB,KAAK,MAAM,KAAK,SAAS,KAAK,YAAY,MAAM,KAAK,cAAc,MAAM,KAAK,WAAW,GAAG;AAAA,IAClI;AAAA,IAEA,MAAM,WAAW,gBAAwB,SAA2D;AAClG,YAAM,cAAc,SAAS,SAAS;AACtC,YAAM,OAAO,GAAG;AAAA,QACd;AAAA,MACF,EAAE,IAAI,gBAAgB,WAAW;AAEjC,aAAO,KAAK,QAAQ,EAAE,IAAI,CAAC,OAAO;AAAA,QAChC,IAAI,EAAE;AAAA,QACN,gBAAgB,EAAE;AAAA,QAClB,MAAM,EAAE;AAAA,QACR,SAAS,EAAE;AAAA,QACX,UAAU,EAAE;AAAA,QACZ,YAAY,EAAE;AAAA,QACd,WAAW,EAAE;AAAA,MACf,EAAE;AAAA,IACJ;AAAA,IAEA,MAAM,cAAc,OAAe,gBAAsD;AACvF,YAAM,iBAAiB,MAAM,MAAM,KAAK;AAExC,UAAI,MAAM;AACV,YAAM,SAAoB,CAAC;AAC3B,UAAI,gBAAgB;AAClB,eAAO;AACP,eAAO,KAAK,cAAc;AAAA,MAC5B;AAEA,YAAM,OAAO,GAAG,QAAQ,GAAG,EAAE,IAAI,GAAG,MAAM;AAE1C,YAAM,SAAS,KAAK,IAAI,CAAC,QAAQ;AAC/B,cAAM,MAAM,IAAI;AAChB,cAAM,SAAS,MAAM,KAAK,IAAI,aAAa,IAAI,QAAQ,IAAI,YAAY,IAAI,aAAa,CAAC,CAAC;AAC1F,eAAO;AAAA,UACL,MAAM;AAAA,YACJ,IAAI,IAAI;AAAA,YACR,gBAAgB,IAAI;AAAA,YACpB,MAAM,IAAI;AAAA,YACV,SAAS,IAAI;AAAA,YACb,UAAU,IAAI;AAAA,YACd,YAAY,IAAI;AAAA,YAChB,WAAW,IAAI;AAAA,UACjB;AAAA,UACA,OAAO,iBAAiB,gBAAgB,MAAM;AAAA,QAChD;AAAA,MACF,CAAC;AAED,aAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,aAAO,OAAO,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,IAC9C;AAAA,EACF;AACF;;;ACzHO,SAAS,kBAAkB,SAIhC;AAGA,QAAM,gBAAgB,UAAQ,gBAAgB;AAC9C,QAAM,KAAK,IAAI,cAAc,QAAQ,IAAI;AAEzC,KAAG,OAAO,oBAAoB;AAC9B,KAAG,OAAO,mBAAmB;AAC7B,KAAG,KAAK,aAAa;AAErB,SAAO;AAAA,IACL,WAAW,0BAA0B,EAAE;AAAA,IACvC,QAAQ,wBAAwB,IAAI,QAAQ,KAAK;AAAA,IACjD,OAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AACF;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@claritylabs/cl-sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "CL-0 SDK — open infrastructure for building AI agents that work with insurance",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -10,6 +10,11 @@
|
|
|
10
10
|
"types": "./dist/index.d.ts",
|
|
11
11
|
"import": "./dist/index.mjs",
|
|
12
12
|
"require": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./storage/sqlite": {
|
|
15
|
+
"types": "./dist/storage-sqlite.d.ts",
|
|
16
|
+
"import": "./dist/storage-sqlite.mjs",
|
|
17
|
+
"require": "./dist/storage-sqlite.js"
|
|
13
18
|
}
|
|
14
19
|
},
|
|
15
20
|
"files": [
|
|
@@ -19,27 +24,31 @@
|
|
|
19
24
|
"build": "tsup",
|
|
20
25
|
"dev": "tsup --watch",
|
|
21
26
|
"typecheck": "tsc --noEmit",
|
|
27
|
+
"test": "vitest run",
|
|
28
|
+
"test:watch": "vitest",
|
|
22
29
|
"release": "semantic-release"
|
|
23
30
|
},
|
|
24
31
|
"peerDependencies": {
|
|
25
|
-
"
|
|
26
|
-
"
|
|
32
|
+
"pdf-lib": ">=1.17.0",
|
|
33
|
+
"zod": ">=3.22.0"
|
|
27
34
|
},
|
|
28
35
|
"peerDependenciesMeta": {
|
|
29
|
-
"
|
|
36
|
+
"better-sqlite3": {
|
|
30
37
|
"optional": true
|
|
31
38
|
}
|
|
32
39
|
},
|
|
33
40
|
"devDependencies": {
|
|
34
|
-
"@ai-sdk/anthropic": "^3.0.58",
|
|
35
41
|
"@semantic-release/changelog": "^6.0.3",
|
|
36
42
|
"@semantic-release/git": "^10.0.1",
|
|
43
|
+
"@types/better-sqlite3": "^7.6.12",
|
|
37
44
|
"@types/node": "^25.5.0",
|
|
38
|
-
"
|
|
45
|
+
"better-sqlite3": "^11.7.0",
|
|
39
46
|
"pdf-lib": "^1.17.1",
|
|
40
47
|
"semantic-release": "^25.0.3",
|
|
41
48
|
"tsup": "^8.4.0",
|
|
42
|
-
"typescript": "^5.8.0"
|
|
49
|
+
"typescript": "^5.8.0",
|
|
50
|
+
"vitest": "^3.1.0",
|
|
51
|
+
"zod": "^3.24.0"
|
|
43
52
|
},
|
|
44
53
|
"repository": {
|
|
45
54
|
"type": "git",
|